├── hack ├── cert-gen │ ├── .gitignore │ └── gen.sh ├── local.kubeconfig ├── samples │ ├── clustermanagementaddon.yaml │ ├── ocm-secret.yaml │ ├── clusterproxy-secret.yaml │ ├── cluster-gateway-secret-serviceaccount-token.yaml │ ├── cluster-gateway-secret-x509.yaml │ └── clustergatewayconfiguration.yaml ├── local-impersonated.kubeconfig ├── boilerplate.go.txt ├── patch │ └── main.go └── crd │ └── addon │ └── clustermanagementaddon.yaml ├── docs ├── images │ └── arch.png └── local-run.md ├── pkg ├── util │ ├── scheme │ │ └── scheme.go │ ├── context │ │ └── context.go │ ├── namespace.go │ ├── cluster │ │ └── cluster.go │ ├── cert │ │ ├── cert.go │ │ ├── secret_test.go │ │ └── secret.go │ ├── exec │ │ └── exec.go │ └── singleton │ │ └── loopback.go ├── config │ ├── meta.go │ ├── args_authorization.go │ ├── args_secret.go │ ├── args_log.go │ ├── args_proxyconfig.go │ ├── args_virtualcluster.go │ ├── args_useragent.go │ └── args_clusterproxy.go ├── options │ └── flags.go ├── apis │ ├── cluster │ │ ├── transport │ │ │ ├── context.go │ │ │ ├── prependroundtripper.go │ │ │ └── roundtripper.go │ │ ├── doc.go │ │ └── v1alpha1 │ │ │ ├── doc.go │ │ │ ├── printer.go │ │ │ ├── register.go │ │ │ ├── virtualcluster_errors.go │ │ │ ├── clustergateway_health.go │ │ │ ├── validation.go │ │ │ ├── transport.go │ │ │ ├── clustergateway_proxy_configuration.go │ │ │ └── virtualcluster_client.go │ ├── doc.go │ ├── proxy │ │ └── v1alpha1 │ │ │ ├── doc.go │ │ │ ├── groupversion_info.go │ │ │ └── clustergatewayconfigurations.go │ └── generate.go ├── metrics │ ├── register.go │ └── proxy.go ├── generated │ └── clientset │ │ └── versioned │ │ ├── typed │ │ └── cluster │ │ │ └── v1alpha1 │ │ │ ├── generated_expansion.go │ │ │ ├── doc.go │ │ │ ├── cluster_client.go │ │ │ └── clustergateway_expansion.go │ │ ├── doc.go │ │ ├── scheme │ │ ├── doc.go │ │ └── register.go │ │ └── clientset.go ├── common │ └── constants.go ├── event │ ├── secret_handler.go │ ├── apiservice_handler.go │ ├── clustergatewayconfiguration_handler.go │ └── resync.go ├── featuregates │ └── featue_gate.go └── addon │ ├── agent │ └── addon.go │ └── controllers │ └── health.go ├── charts ├── cluster-gateway │ ├── templates │ │ ├── secret-namespace.yaml │ │ ├── serviceaccount.yaml │ │ ├── cluster-gateway-service.yaml │ │ ├── secret-roles.yaml │ │ ├── secret-rolebindings.yaml │ │ ├── rolebindings.yaml │ │ ├── clusterrolebindings.yaml │ │ ├── apiservice.yaml │ │ ├── clusterroles.yaml │ │ └── cluster-gateway-apiserver.yaml │ ├── Chart.yaml │ └── values.yaml └── addon-manager │ ├── templates │ ├── serviceaccount.yaml │ ├── clustermanagementaddon.yaml │ ├── clusterrolebindings.yaml │ ├── rolebinder-kubesystem.yaml │ ├── rolebinder.yaml │ ├── addon-manager.yaml │ ├── clustergatewayconfiguration.yaml │ └── clusterroles.yaml │ ├── Chart.yaml │ └── values.yaml ├── codecov.yml ├── e2e ├── e2e.go ├── kubernetes │ ├── non_resource.go │ └── e2e_test.go ├── e2e_test.go ├── ocm │ ├── e2e_test.go │ └── clustergateway.go ├── roundtrip │ ├── e2e_test.go │ └── clustergateway.go ├── framework │ ├── scheme.go │ ├── context.go │ └── framework.go ├── benchmark │ ├── e2e_test.go │ └── configmap.go └── env │ └── prepare │ └── main.go ├── examples ├── client-identity-exchanger │ └── config.yaml ├── dynamic-multicluster-client │ └── main.go └── single-cluster-informer │ └── main.go ├── .gitignore ├── cmd ├── addon-manager │ ├── Dockerfile │ └── main.go └── apiserver │ ├── Dockerfile │ └── main.go ├── .github └── workflows │ ├── build-image.yml │ └── ci.yaml ├── Makefile └── go.mod /hack/cert-gen/.gitignore: -------------------------------------------------------------------------------- 1 | *.yaml 2 | cert/* -------------------------------------------------------------------------------- /docs/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oam-dev/cluster-gateway/HEAD/docs/images/arch.png -------------------------------------------------------------------------------- /pkg/util/scheme/scheme.go: -------------------------------------------------------------------------------- 1 | package scheme 2 | 3 | import "k8s.io/apimachinery/pkg/runtime" 4 | 5 | var Scheme = runtime.NewScheme() 6 | -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/secret-namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: {{ .Values.secretNamespace }} -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: cluster-gateway 5 | namespace: {{ .Release.Namespace }} 6 | -------------------------------------------------------------------------------- /pkg/config/meta.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | var MetaApiGroupName = "cluster.core.oam.dev" 4 | var MetaApiVersionName = "v1alpha1" 5 | var MetaApiResourceName = "clustergateways" 6 | -------------------------------------------------------------------------------- /charts/addon-manager/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: cluster-gateway-addon-manager 5 | namespace: {{ .Release.Namespace }} 6 | -------------------------------------------------------------------------------- /charts/cluster-gateway/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: cluster-gateway 3 | description: A Helm chart for Cluster-Gateway 4 | type: application 5 | version: 0.1.0 6 | appVersion: 1.0.0 7 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | threshold: 1% 6 | patch: 7 | default: 8 | target: 70% 9 | ignore: 10 | - "hack/**" 11 | -------------------------------------------------------------------------------- /charts/addon-manager/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: cluster-gateway-addon-manager 3 | description: A Helm chart for Cluster-Gateway Addon-Manager 4 | type: application 5 | version: 0.1.0 6 | appVersion: 1.0.0 7 | -------------------------------------------------------------------------------- /e2e/e2e.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/onsi/ginkgo/v2" 7 | ) 8 | 9 | func RunE2ETests(t *testing.T) { 10 | ginkgo.RunSpecs(t, "ClusterGateway e2e suite") 11 | } 12 | -------------------------------------------------------------------------------- /e2e/kubernetes/non_resource.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | ) 6 | 7 | const ( 8 | kubernetesTestBasename = "kubernetes" 9 | ) 10 | 11 | var _ = Describe("Basic RoundTrip Test", 12 | func() { 13 | }) 14 | -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/cluster-gateway-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: gateway-service 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | selector: 8 | app: gateway 9 | ports: 10 | - protocol: TCP 11 | port: 9443 12 | targetPort: 9443 -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/secret-roles.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: cluster-gateway-secret-reader 5 | namespace: {{ .Values.secretNamespace }} 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - "secrets" 11 | verbs: 12 | - "*" 13 | -------------------------------------------------------------------------------- /pkg/options/flags.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | var ( 4 | // OCMIntegration indicates whether to load cluster information from 5 | // the hosting cluster via OCM's cluster api. After enabling this option, 6 | // no caBundle and apiserver's URL are required in the cluster secret. 7 | // NOTE: This option only works in "non-etcd" mode. 8 | OCMIntegration = false 9 | ) 10 | -------------------------------------------------------------------------------- /charts/addon-manager/values.yaml: -------------------------------------------------------------------------------- 1 | # Image of the cluster-gateway instances 2 | image: oamdev/cluster-gateway-addon-manager 3 | 4 | tag: 5 | 6 | clusterGateway: 7 | image: oamdev/cluster-gateway 8 | installNamespace: vela-system 9 | secretNamespace: open-cluster-management-credentials 10 | # Number of replicas 11 | replicas: 1 12 | 13 | manualSecretManagement: true 14 | konnectivityEgress: false -------------------------------------------------------------------------------- /hack/local.kubeconfig: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Config 3 | preferences: {} 4 | clusters: 5 | - cluster: 6 | insecure-skip-tls-verify: true 7 | server: https://127.0.0.1:9443 8 | name: local 9 | contexts: 10 | - context: 11 | cluster: local 12 | user: local 13 | name: local 14 | current-context: local 15 | users: 16 | - name: local 17 | user: 18 | username: foo 19 | password: foo -------------------------------------------------------------------------------- /hack/samples/clustermanagementaddon.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: addon.open-cluster-management.io/v1alpha1 2 | kind: ClusterManagementAddOn 3 | metadata: 4 | name: cluster-gateway 5 | spec: 6 | addOnMeta: 7 | displayName: cluster-gateway 8 | description: cluster-gateway 9 | addOnConfiguration: 10 | crdName: clustergatewayconfigurations.proxy.open-cluster-management.io 11 | crName: cluster-gateway 12 | -------------------------------------------------------------------------------- /hack/samples/ocm-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | ca.crt: <...> 4 | namespace: a3ViZS1zeXN0ZW0= 5 | token: <...> 6 | # no endpoint are required 7 | kind: Secret 8 | metadata: 9 | labels: 10 | cluster.core.oam.dev/cluster-credential-type: ServiceAccountToken 11 | cluster.core.oam.dev/cluster-endpoint-type: Const 12 | name: foo1 13 | namespace: open-cluster-management-credentials 14 | type: Opaque -------------------------------------------------------------------------------- /charts/addon-manager/templates/clustermanagementaddon.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: addon.open-cluster-management.io/v1alpha1 2 | kind: ClusterManagementAddOn 3 | metadata: 4 | name: cluster-gateway 5 | spec: 6 | addOnMeta: 7 | displayName: cluster-gateway 8 | description: cluster-gateway 9 | addOnConfiguration: 10 | crdName: clustergatewayconfigurations.proxy.open-cluster-management.io 11 | crName: cluster-gateway 12 | -------------------------------------------------------------------------------- /pkg/util/context/context.go: -------------------------------------------------------------------------------- 1 | package context 2 | 3 | import "context" 4 | 5 | type contextKeyClusterName string 6 | 7 | var ( 8 | key contextKeyClusterName = "" 9 | ) 10 | 11 | func WithClusterName(ctx context.Context, clusterName string) context.Context { 12 | return context.WithValue(ctx, key, clusterName) 13 | } 14 | 15 | func GetClusterName(ctx context.Context) string { 16 | return ctx.Value(key).(string) 17 | } 18 | -------------------------------------------------------------------------------- /hack/local-impersonated.kubeconfig: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Config 3 | preferences: {} 4 | clusters: 5 | - cluster: 6 | insecure-skip-tls-verify: true 7 | server: https://127.0.0.1:9443 8 | name: local 9 | contexts: 10 | - context: 11 | cluster: local 12 | user: local 13 | name: local 14 | current-context: local 15 | users: 16 | - name: local 17 | user: 18 | username: foo 19 | password: foo 20 | as: foo -------------------------------------------------------------------------------- /hack/samples/clusterproxy-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | ca.crt: <...> 4 | namespace: a3ViZS1zeXN0ZW0= 5 | token: <...> 6 | # no endpoint are required 7 | kind: Secret 8 | metadata: 9 | labels: 10 | cluster.core.oam.dev/cluster-credential-type: ServiceAccountToken 11 | cluster.core.oam.dev/cluster-endpoint-type: ClusterProxy 12 | name: foo1 13 | namespace: open-cluster-management-credentials 14 | type: Opaque -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/secret-rolebindings.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: cluster-gateway-secret-reader 5 | namespace: {{ .Values.secretNamespace }} 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: Role 9 | name: cluster-gateway-secret-reader 10 | subjects: 11 | - kind: ServiceAccount 12 | name: cluster-gateway 13 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /pkg/config/args_authorization.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/spf13/pflag" 5 | ) 6 | 7 | var AuthorizateProxySubpath bool 8 | 9 | func AddProxyAuthorizationFlags(set *pflag.FlagSet) { 10 | set.BoolVarP(&AuthorizateProxySubpath, "authorize-proxy-subpath", "", false, 11 | "perform an additional delegated authorization against the hub cluster for the target proxying path when invoking clustergateway/proxy subresource") 12 | } 13 | -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/rolebindings.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: system:extension-apiserver-authentication-reader:cluster-gateway 5 | namespace: kube-system 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: Role 9 | name: extension-apiserver-authentication-reader 10 | subjects: 11 | - kind: ServiceAccount 12 | name: cluster-gateway 13 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /hack/samples/cluster-gateway-secret-serviceaccount-token.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | labels: 5 | cluster.core.oam.dev/cluster-credential-type: ServiceAccountToken 6 | cluster.core.oam.dev/cluster-endpoint-type: Const 7 | name: foo1 8 | namespace: open-cluster-management-credentials 9 | type: Opaque 10 | data: 11 | ca.crt: <...> 12 | token: <...> 13 | endpoint: "https://127.0.0.1:6443" # Optional upon ClusterProxy endpoint type 14 | -------------------------------------------------------------------------------- /hack/samples/cluster-gateway-secret-x509.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | labels: 5 | cluster.core.oam.dev/cluster-credential-type: X509Certificate 6 | cluster.core.oam.dev/cluster-endpoint-type: Const 7 | name: foo1 8 | namespace: open-cluster-management-credentials 9 | type: Opaque 10 | data: 11 | ca.crt: <...> 12 | tls.crt: <...> 13 | tls.key: <...> 14 | endpoint: "https://127.0.0.1:6443" # Optional upon ClusterProxy endpoint type 15 | -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/clusterrolebindings.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: open-cluster-management:cluster-gateway:managedcluster-reader 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: open-cluster-management:cluster-gateway:managedcluster-reader 9 | subjects: 10 | - kind: ServiceAccount 11 | name: cluster-gateway 12 | namespace: {{ .Release.Namespace }} 13 | -------------------------------------------------------------------------------- /charts/addon-manager/templates/clusterrolebindings.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: open-cluster-management:cluster-gateway:managedcluster-reader 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: open-cluster-management:cluster-gateway:managedcluster-reader 9 | subjects: 10 | - kind: ServiceAccount 11 | name: cluster-gateway-addon-manager 12 | namespace: {{ .Release.Namespace }} 13 | -------------------------------------------------------------------------------- /charts/addon-manager/templates/rolebinder-kubesystem.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: open-cluster-management:cluster-gateway:role-grantor 5 | namespace: kube-system 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: open-cluster-management:cluster-gateway:managedcluster-reader 10 | subjects: 11 | - kind: ServiceAccount 12 | name: cluster-gateway-addon-manager 13 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /charts/addon-manager/templates/rolebinder.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: open-cluster-management:cluster-gateway:role-grantor 5 | namespace: {{ .Release.Namespace }} 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: open-cluster-management:cluster-gateway:managedcluster-reader 10 | subjects: 11 | - kind: ServiceAccount 12 | name: cluster-gateway-addon-manager 13 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /e2e/e2e_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/onsi/ginkgo/v2" 8 | "github.com/onsi/gomega" 9 | 10 | "github.com/oam-dev/cluster-gateway/e2e/framework" 11 | // per-package e2e suite 12 | _ "github.com/oam-dev/cluster-gateway/e2e/roundtrip" 13 | ) 14 | 15 | func TestMain(m *testing.M) { 16 | gomega.RegisterFailHandler(ginkgo.Fail) 17 | framework.ParseFlags() 18 | os.Exit(m.Run()) 19 | } 20 | 21 | func TestE2E(t *testing.T) { 22 | RunE2ETests(t) 23 | } 24 | -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/apiservice.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiregistration.k8s.io/v1 2 | kind: APIService 3 | metadata: 4 | name: v1alpha1.cluster.core.oam.dev 5 | labels: 6 | api: cluster-extension-apiserver 7 | apiserver: "true" 8 | spec: 9 | version: v1alpha1 10 | group: cluster.core.oam.dev 11 | groupPriorityMinimum: 2000 12 | service: 13 | name: gateway-service 14 | namespace: {{ .Release.Namespace }} 15 | port: 9443 16 | versionPriority: 10 17 | insecureSkipTLSVerify: true -------------------------------------------------------------------------------- /examples/client-identity-exchanger/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cluster.core.oam.dev/v1alpha1 2 | kind: ClusterGatewayProxyConfiguration 3 | spec: 4 | clientIdentityExchanger: 5 | rules: 6 | - name: super-user 7 | source: 8 | group: sudoer 9 | type: PrivilegedIdentityExchanger 10 | - name: mapping 11 | source: 12 | user: user-12345 13 | cluster: cluster-34567 14 | target: 15 | user: user-34567 16 | type: StaticMappingIdentityExchanger -------------------------------------------------------------------------------- /e2e/ocm/e2e_test.go: -------------------------------------------------------------------------------- 1 | package roundtrip 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/onsi/ginkgo/v2" 8 | "github.com/onsi/gomega" 9 | 10 | "github.com/oam-dev/cluster-gateway/e2e/framework" 11 | ) 12 | 13 | func TestMain(m *testing.M) { 14 | gomega.RegisterFailHandler(ginkgo.Fail) 15 | framework.ParseFlags() 16 | os.Exit(m.Run()) 17 | } 18 | 19 | func RunE2ETests(t *testing.T) { 20 | ginkgo.RunSpecs(t, "ClusterGateway e2e suite -- OCM addon") 21 | } 22 | 23 | func TestE2E(t *testing.T) { 24 | RunE2ETests(t) 25 | } 26 | -------------------------------------------------------------------------------- /charts/cluster-gateway/values.yaml: -------------------------------------------------------------------------------- 1 | # Image of the cluster-gateway instances 2 | image: oamdev/cluster-gateway 3 | 4 | tag: 5 | 6 | # Number of replicas 7 | replicas: 1 8 | # A secured namespace for reading cluster secrets 9 | secretNamespace: open-cluster-management-credentials 10 | 11 | ocmIntegration: 12 | enabled: false 13 | clusterProxy: 14 | enabled: false 15 | endpoint: 16 | host: proxy-entrypoint.open-cluster-management-cluster-proxy 17 | port: 8090 18 | 19 | featureGate: 20 | healthiness: false 21 | secretCache: false -------------------------------------------------------------------------------- /e2e/roundtrip/e2e_test.go: -------------------------------------------------------------------------------- 1 | package roundtrip 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/onsi/ginkgo/v2" 8 | "github.com/onsi/gomega" 9 | 10 | "github.com/oam-dev/cluster-gateway/e2e/framework" 11 | ) 12 | 13 | func TestMain(m *testing.M) { 14 | gomega.RegisterFailHandler(ginkgo.Fail) 15 | framework.ParseFlags() 16 | os.Exit(m.Run()) 17 | } 18 | 19 | func RunE2ETests(t *testing.T) { 20 | ginkgo.RunSpecs(t, "ClusterGateway e2e suite -- basic api round-tripping") 21 | } 22 | 23 | func TestE2E(t *testing.T) { 24 | RunE2ETests(t) 25 | } 26 | -------------------------------------------------------------------------------- /e2e/kubernetes/e2e_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/onsi/ginkgo/v2" 8 | "github.com/onsi/gomega" 9 | 10 | "github.com/oam-dev/cluster-gateway/e2e/framework" 11 | ) 12 | 13 | func TestMain(m *testing.M) { 14 | gomega.RegisterFailHandler(ginkgo.Fail) 15 | framework.ParseFlags() 16 | os.Exit(m.Run()) 17 | } 18 | 19 | func RunE2ETests(t *testing.T) { 20 | ginkgo.RunSpecs(t, "ClusterGateway e2e suite -- kubernetes api manipulation") 21 | } 22 | 23 | func TestE2E(t *testing.T) { 24 | RunE2ETests(t) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/config/args_secret.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/pflag" 7 | ) 8 | 9 | // SecretNamespace the namespace to search cluster credentials 10 | var SecretNamespace = "vela-system" 11 | 12 | func ValidateSecret() error { 13 | if len(SecretNamespace) == 0 { 14 | return fmt.Errorf("must specify --secret-namespace") 15 | } 16 | return nil 17 | } 18 | 19 | func AddSecretFlags(set *pflag.FlagSet) { 20 | set.StringVarP(&SecretNamespace, "secret-namespace", "", SecretNamespace, 21 | "the namespace to reading secrets") 22 | } 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | bin 8 | 9 | # Test binary, build with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Kubernetes Generated files - skip generated files, except for vendored files 16 | 17 | !vendor/**/zz_generated.* 18 | 19 | # editor and IDE paraphernalia 20 | .idea 21 | *.swp 22 | *.swo 23 | *~ 24 | apiserver.local.config/** 25 | 26 | config/crd/** 27 | kubeconfig 28 | default.etcd/** 29 | 30 | *.tgz 31 | index.yaml 32 | *.test -------------------------------------------------------------------------------- /pkg/apis/cluster/transport/context.go: -------------------------------------------------------------------------------- 1 | package multicluster 2 | 3 | import "context" 4 | 5 | type contextKey string 6 | 7 | const ( 8 | // ClusterContextKey is the name of cluster using in client http context 9 | clusterContextKey = contextKey("ClusterName") 10 | ) 11 | 12 | func WithMultiClusterContext(ctx context.Context, clusterName string) context.Context { 13 | return context.WithValue(ctx, clusterContextKey, clusterName) 14 | } 15 | 16 | func GetMultiClusterContext(ctx context.Context) (string, bool) { 17 | clusterName, ok := ctx.Value(clusterContextKey).(string) 18 | return clusterName, ok 19 | } 20 | -------------------------------------------------------------------------------- /e2e/framework/scheme.go: -------------------------------------------------------------------------------- 1 | package framework 2 | 3 | import ( 4 | clusterv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 5 | proxyv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/proxy/v1alpha1" 6 | "k8s.io/apimachinery/pkg/runtime" 7 | addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" 8 | clusterv1 "open-cluster-management.io/api/cluster/v1" 9 | ) 10 | 11 | var scheme = runtime.NewScheme() 12 | 13 | func init() { 14 | clusterv1alpha1.AddToScheme(scheme) 15 | proxyv1alpha1.AddToScheme(scheme) 16 | clusterv1.AddToScheme(scheme) 17 | addonv1alpha1.AddToScheme(scheme) 18 | } 19 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ -------------------------------------------------------------------------------- /pkg/metrics/register.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "sync" 5 | 6 | compbasemetrics "k8s.io/component-base/metrics" 7 | 8 | "k8s.io/component-base/metrics/legacyregistry" 9 | ) 10 | 11 | var registerMetrics sync.Once 12 | 13 | var metrics = []compbasemetrics.Registerable{ 14 | ocmProxiedRequestsByResourceTotal, 15 | ocmProxiedRequestsByClusterTotal, 16 | ocmProxiedRequestsDurationHistogram, 17 | ocmProxiedClusterEscalationRequestDurationHistogram, 18 | } 19 | 20 | func Register() { 21 | registerMetrics.Do(func() { 22 | for _, metric := range metrics { 23 | legacyregistry.MustRegister(metric) 24 | } 25 | }) 26 | } 27 | -------------------------------------------------------------------------------- /e2e/benchmark/e2e_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "testing" 7 | 8 | "github.com/onsi/ginkgo/v2" 9 | "github.com/onsi/gomega" 10 | 11 | "github.com/oam-dev/cluster-gateway/e2e/framework" 12 | ) 13 | 14 | func TestMain(m *testing.M) { 15 | gomega.RegisterFailHandler(ginkgo.Fail) 16 | flag.BoolVar(&direct, "direct", false, "Indicating direct access to the spoke cluster") 17 | framework.ParseFlags() 18 | 19 | os.Exit(m.Run()) 20 | } 21 | 22 | func RunE2ETests(t *testing.T) { 23 | ginkgo.RunSpecs(t, "ClusterGateway e2e suite -- gateway benchmark") 24 | } 25 | 26 | func TestE2E(t *testing.T) { 27 | RunE2ETests(t) 28 | } 29 | -------------------------------------------------------------------------------- /hack/samples/clustergatewayconfiguration.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: proxy.open-cluster-management.io/v1alpha1 2 | kind: ClusterGatewayConfiguration 3 | metadata: 4 | name: cluster-gateway 5 | spec: 6 | image: oamdev/cluster-gateway:v1.1.6.ocm 7 | installNamespace: open-cluster-management-addon 8 | secretNamespace: open-cluster-management-credentials 9 | egress: 10 | type: ClusterProxy 11 | clusterProxy: 12 | proxyServerHost: "proxy-entrypoint.open-cluster-management-addon" 13 | proxyServerPort: 8090 14 | credentials: 15 | namespace: open-cluster-management-addon 16 | proxyClientCASecretName: proxy-server-ca 17 | proxyClientSecretName: proxy-client 18 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/cluster/v1alpha1/generated_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | // Code generated by client-gen. DO NOT EDIT. 15 | 16 | package v1alpha1 17 | -------------------------------------------------------------------------------- /pkg/apis/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | //go:generate apiregister-gen -g client-gen -h ../../boilerplate.go.txt 16 | 17 | // 18 | // +domain=core.oam.dev 19 | 20 | package apis 21 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | // Code generated by client-gen. DO NOT EDIT. 15 | 16 | // This package has the automatically generated clientset. 17 | package versioned 18 | -------------------------------------------------------------------------------- /pkg/apis/cluster/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | // +k8s:deepcopy-gen=package,register 16 | // +groupName=core.oam.dev 17 | 18 | // Package api is the internal version of the API. 19 | package cluster 20 | -------------------------------------------------------------------------------- /charts/addon-manager/templates/addon-manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: cluster-gateway-addon-manager 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | app: cluster-gateway-addon-manager 8 | spec: 9 | replicas: {{ .Values.replicas }} 10 | selector: 11 | matchLabels: 12 | app: cluster-gateway-addon-manager 13 | template: 14 | metadata: 15 | labels: 16 | app: cluster-gateway-addon-manager 17 | spec: 18 | serviceAccount: cluster-gateway-addon-manager 19 | containers: 20 | - name: cluster-gateway-addon-manager 21 | image: {{ .Values.image }}:{{ .Values.tag | default (print "v" .Chart.Version) }} 22 | imagePullPolicy: IfNotPresent 23 | args: 24 | - --leader-elect=true -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/scheme/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | // Code generated by client-gen. DO NOT EDIT. 15 | 16 | // This package contains the scheme of the automatically generated clientset. 17 | package scheme 18 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/cluster/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | // Code generated by client-gen. DO NOT EDIT. 15 | 16 | // This package has the automatically generated typed clients. 17 | package v1alpha1 18 | -------------------------------------------------------------------------------- /pkg/util/namespace.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | const inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" 10 | 11 | func GetInClusterNamespace() (string, error) { 12 | // Check whether the namespace file exists. 13 | // If not, we are not running in cluster so can't guess the namespace. 14 | if _, err := os.Stat(inClusterNamespacePath); os.IsNotExist(err) { 15 | return "", fmt.Errorf("not running in-cluster, please specify LeaderElectionNamespace") 16 | } else if err != nil { 17 | return "", fmt.Errorf("error checking namespace file: %w", err) 18 | } 19 | 20 | // Load the namespace file and return its content 21 | namespace, err := ioutil.ReadFile(inClusterNamespacePath) 22 | if err != nil { 23 | return "", fmt.Errorf("error reading namespace file: %w", err) 24 | } 25 | return string(namespace), nil 26 | } 27 | -------------------------------------------------------------------------------- /pkg/apis/proxy/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | // Api versions allow the api contract for a resource to be changed while keeping 16 | // backward compatibility by support multiple concurrent versions 17 | // of the same resource 18 | 19 | // +k8s:openapi-gen=true 20 | // +k8s:deepcopy-gen=package,register 21 | // +groupName=proxy.open-cluster-management.io 22 | package v1alpha1 23 | -------------------------------------------------------------------------------- /pkg/apis/generate.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Generate deepcopy methodsets 18 | //go:generate go run -tags generate sigs.k8s.io/controller-tools/cmd/controller-gen object:headerFile=../../hack/boilerplate.go.txt paths=./... 19 | 20 | package apis 21 | 22 | import ( 23 | _ "sigs.k8s.io/controller-tools/pkg/version" //nolint:typecheck 24 | ) 25 | -------------------------------------------------------------------------------- /pkg/config/args_log.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "flag" 21 | 22 | "github.com/spf13/pflag" 23 | "k8s.io/klog/v2" 24 | ) 25 | 26 | // AddLogFlags add log flags to command 27 | func AddLogFlags(set *pflag.FlagSet) { 28 | fs := flag.NewFlagSet("", 0) 29 | klog.InitFlags(fs) 30 | set.AddGoFlagSet(fs) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/common/constants.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/oam-dev/cluster-gateway/pkg/config" 4 | 5 | const ( 6 | AddonName = "cluster-gateway" 7 | ) 8 | 9 | const ( 10 | LabelKeyOpenClusterManagementAddon = "proxy.open-cluster-management.io/addon-name" 11 | ) 12 | 13 | const ( 14 | ClusterGatewayConfigurationCRDName = "clustergatewayconfigurations.proxy.open-cluster-management.io" 15 | ClusterGatewayConfigurationCRName = "cluster-gateway" 16 | ) 17 | 18 | const ( 19 | InstallNamespace = "open-cluster-management-cluster-gateway" 20 | ) 21 | 22 | const ( 23 | ClusterGatewayAPIServiceName = "v1alpha1.cluster.core.oam.dev" 24 | ) 25 | 26 | var ( 27 | // LabelKeyClusterCredentialType describes the credential type in object label field 28 | LabelKeyClusterCredentialType = config.MetaApiGroupName + "/cluster-credential-type" 29 | // LabelKeyClusterEndpointType describes the endpoint type. 30 | LabelKeyClusterEndpointType = config.MetaApiGroupName + "/cluster-endpoint-type" 31 | ) 32 | -------------------------------------------------------------------------------- /pkg/config/args_proxyconfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "github.com/spf13/pflag" 21 | ) 22 | 23 | var ClusterGatewayProxyConfigPath string 24 | 25 | func AddClusterGatewayProxyConfig(set *pflag.FlagSet) { 26 | set.StringVarP(&ClusterGatewayProxyConfigPath, "cluster-gateway-proxy-config", "", "", 27 | "the path for cluster-gateway proxy configuration") 28 | } 29 | -------------------------------------------------------------------------------- /cmd/addon-manager/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG OS=linux 2 | ARG ARCH=amd64 3 | # Build the manager binary 4 | FROM golang:1.23 as builder 5 | ARG OS 6 | ARG ARCH 7 | 8 | WORKDIR /workspace 9 | 10 | # Copy the Go Modules manifests 11 | COPY go.mod go.mod 12 | COPY go.sum go.sum 13 | # cache deps before building and copying source so that we don't need to re-download as much 14 | # and so that source changes don't invalidate our downloaded layer 15 | RUN go mod download 16 | 17 | # Copy the go source 18 | COPY cmd/ cmd/ 19 | COPY pkg/ pkg/ 20 | COPY hack/ hack/ 21 | 22 | # Build 23 | RUN CGO_ENABLED=0 \ 24 | GOOS=${OS} \ 25 | GOARCH=${ARCH} \ 26 | GO111MODULE=on \ 27 | go build \ 28 | -a -o addon-manager \ 29 | cmd/addon-manager/main.go 30 | 31 | # Use distroless as minimal base image to package the manager binary 32 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 33 | ARG ARCH 34 | FROM multiarch/alpine:${ARCH}-v3.13 35 | 36 | WORKDIR / 37 | COPY --from=builder /workspace/addon-manager / 38 | 39 | ENTRYPOINT ["/addon-manager"] 40 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | // Api versions allow the api contract for a resource to be changed while keeping 16 | // backward compatibility by support multiple concurrent versions 17 | // of the same resource 18 | 19 | // +k8s:openapi-gen=true 20 | // +k8s:deepcopy-gen=package,register 21 | // +k8s:conversion-gen=cluster-extension/pkg/apis/cluster 22 | // +k8s:defaulter-gen=TypeMeta 23 | // +groupName=cluster.core.oam.dev 24 | package v1alpha1 // import "cluster-extension/pkg/apis/cluster/v1alpha1" 25 | -------------------------------------------------------------------------------- /charts/addon-manager/templates/clustergatewayconfiguration.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: proxy.open-cluster-management.io/v1alpha1 2 | kind: ClusterGatewayConfiguration 3 | metadata: 4 | name: cluster-gateway 5 | spec: 6 | image: {{ .Values.clusterGateway.image }}:{{ .Values.tag | default (print "v" .Chart.Version) }} 7 | installNamespace: {{ .Values.clusterGateway.installNamespace }} 8 | secretNamespace: {{ .Values.clusterGateway.secretNamespace }} 9 | secretManagement: 10 | {{ if .Values.manualSecretManagement }} 11 | type: Manual 12 | {{ else }} 13 | type: ManagedServiceAccount 14 | managedServiceAccount: 15 | name: cluster-gateway 16 | {{ end }} 17 | egress: 18 | {{ if .Values.konnectivityEgress }} 19 | type: ClusterProxy 20 | clusterProxy: 21 | proxyServerHost: "proxy-entrypoint.open-cluster-management-addon" 22 | proxyServerPort: 8090 23 | credentials: 24 | namespace: open-cluster-management-addon 25 | proxyClientCASecretName: proxy-server-ca 26 | proxyClientSecretName: proxy-client 27 | {{ else }} 28 | type: Direct 29 | {{ end }} 30 | -------------------------------------------------------------------------------- /hack/cert-gen/gen.sh: -------------------------------------------------------------------------------- 1 | SVC_NAME="${SVC_NAME:-kubevela-cluster-gateway}" 2 | SVC_NAMESPACE="${SVC_NAMESPACE:-vela-system}" 3 | OUTPUT_DIR=${OUTPUT_DIR:-./cert} 4 | 5 | rm -r $OUTPUT_DIR; 6 | mkdir -p $OUTPUT_DIR; 7 | cd $OUTPUT_DIR; 8 | echo "authorityKeyIdentifier=keyid,issuer 9 | basicConstraints=CA:FALSE 10 | subjectAltName = @alt_names 11 | [alt_names] 12 | DNS.1 = $SVC_NAME 13 | DNS.2 = $SVC_NAME.$SVC_NAMESPACE.svc" > domain.ext 14 | openssl req -x509 -sha256 -days 3650 -newkey rsa:2048 -keyout ca.key -out ca -nodes -subj '/O=kubevela' \ 15 | && openssl ecparam -name prime256v1 -genkey -noout -out apiserver.key \ 16 | && openssl req -new -key apiserver.key -out apiserver.csr -subj '/O='$SVC_NAME \ 17 | && openssl x509 -req -in apiserver.csr -CA ca -CAkey ca.key -CAcreateserial -extfile domain.ext -out apiserver.crt -days 3650 -sha256 18 | 19 | kubectl create secret generic $SVC_NAME -n $SVC_NAMESPACE \ 20 | --from-file=ca=ca \ 21 | --from-file=apiserver.key=apiserver.key \ 22 | --from-file=apiserver.crt=apiserver.crt \ 23 | --dry-run=client -oyaml > $SVC_NAME.yaml 24 | 25 | cd .. 26 | mv ./cert/$SVC_NAME.yaml ./ 27 | -------------------------------------------------------------------------------- /pkg/config/args_virtualcluster.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import "github.com/spf13/pflag" 20 | 21 | // VirtualClusterWithControlPlane if the control plane should be treated as one 22 | // default virtual cluster 23 | var VirtualClusterWithControlPlane = true 24 | 25 | func AddVirtualClusterFlags(set *pflag.FlagSet) { 26 | set.BoolVarP(&VirtualClusterWithControlPlane, "virtual-clutser-with-control-plane", "", VirtualClusterWithControlPlane, 27 | "if the control plane should be treated as one default virtual clusters") 28 | } 29 | -------------------------------------------------------------------------------- /pkg/apis/cluster/transport/prependroundtripper.go: -------------------------------------------------------------------------------- 1 | package multicluster 2 | 3 | import "net/http" 4 | 5 | var _ ProxyPathPrependingClusterGatewayRoundTripper = &proxyPathPrependingClusterGatewayRoundTripper{} 6 | 7 | type ProxyPathPrependingClusterGatewayRoundTripper interface { 8 | http.RoundTripper 9 | 10 | NewRoundTripper(delegate http.RoundTripper) http.RoundTripper 11 | } 12 | 13 | type proxyPathPrependingClusterGatewayRoundTripper struct { 14 | clusterName string 15 | delegate http.RoundTripper 16 | } 17 | 18 | func NewProxyPathPrependingClusterGatewayRoundTripper(clusterName string) ProxyPathPrependingClusterGatewayRoundTripper { 19 | return &proxyPathPrependingClusterGatewayRoundTripper{clusterName: clusterName} 20 | } 21 | 22 | func (p *proxyPathPrependingClusterGatewayRoundTripper) NewRoundTripper(delegate http.RoundTripper) http.RoundTripper { 23 | p.delegate = delegate 24 | return p 25 | } 26 | 27 | func (p *proxyPathPrependingClusterGatewayRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { 28 | request.URL.Path = formatProxyURL(p.clusterName, request.URL.Path) 29 | return p.delegate.RoundTrip(request) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/config/args_useragent.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "github.com/spf13/pflag" 21 | "k8s.io/apiserver/pkg/server" 22 | ) 23 | 24 | var UserAgent string 25 | 26 | func AddUserAgentFlags(set *pflag.FlagSet) { 27 | set.StringVarP(&UserAgent, "user-agent", "", "", 28 | "Specifying the UserAgent for communicating with the host cluster.") 29 | } 30 | 31 | func WithUserAgent(config *server.RecommendedConfig) *server.RecommendedConfig { 32 | if UserAgent != "" { 33 | config.ClientConfig.UserAgent = UserAgent 34 | } 35 | return config 36 | } 37 | -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/clusterroles.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: open-cluster-management:cluster-gateway:managedcluster-reader 5 | rules: 6 | - apiGroups: 7 | - cluster.open-cluster-management.io 8 | resources: 9 | - managedclusters 10 | verbs: 11 | - get 12 | - list 13 | - watch 14 | - apiGroups: 15 | - "" 16 | resources: 17 | - namespaces 18 | verbs: 19 | - get 20 | - list 21 | - watch 22 | - apiGroups: 23 | - admissionregistration.k8s.io 24 | resources: 25 | - mutatingwebhookconfigurations 26 | - validatingwebhookconfigurations 27 | - validatingadmissionpolicies 28 | - validatingadmissionpolicybindings 29 | verbs: 30 | - get 31 | - list 32 | - watch 33 | - apiGroups: 34 | - flowcontrol.apiserver.k8s.io 35 | resources: 36 | - prioritylevelconfigurations 37 | - flowschemas 38 | verbs: 39 | - get 40 | - list 41 | - watch 42 | - apiGroups: 43 | - authorization.k8s.io 44 | resources: 45 | - subjectaccessreviews 46 | verbs: 47 | - "*" -------------------------------------------------------------------------------- /cmd/apiserver/Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.23-alpine as builder 3 | ARG OS 4 | ARG ARCH 5 | 6 | WORKDIR /workspace 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | # cache deps before building and copying source so that we don't need to re-download as much 11 | # and so that source changes don't invalidate our downloaded layer 12 | RUN go mod download 13 | 14 | # Copy the go source 15 | COPY cmd/ cmd/ 16 | COPY pkg/ pkg/ 17 | COPY hack/ hack/ 18 | 19 | ARG API_GROUP_NAME=cluster.core.oam.dev 20 | 21 | # Build 22 | RUN CGO_ENABLED=0 \ 23 | GOOS=${OS} \ 24 | GOARCH=${ARCH} \ 25 | go build \ 26 | -ldflags="-X 'github.com/oam-dev/cluster-gateway/pkg/config.MetaApiGroupName=${API_GROUP_NAME}'" \ 27 | -o apiserver \ 28 | cmd/apiserver/main.go 29 | 30 | RUN CGO_ENABLED=0 \ 31 | GOOS=${OS} \ 32 | GOARCH=${ARCH} \ 33 | go build \ 34 | -o patch \ 35 | hack/patch/main.go 36 | 37 | # Use distroless as minimal base image to package the manager binary 38 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 39 | FROM alpine:3 40 | 41 | WORKDIR / 42 | COPY --from=builder /workspace/apiserver / 43 | COPY --from=builder /workspace/patch / 44 | 45 | ENTRYPOINT ["/apiserver"] 46 | -------------------------------------------------------------------------------- /e2e/framework/context.go: -------------------------------------------------------------------------------- 1 | package framework 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "path/filepath" 7 | 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | var context = &E2EContext{} 12 | 13 | type E2EContext struct { 14 | HubKubeConfig string 15 | TestCluster string 16 | IsOCMInstalled bool 17 | } 18 | 19 | func ParseFlags() { 20 | registerFlags() 21 | flag.Parse() 22 | defaultFlags() 23 | validateFlags() 24 | } 25 | 26 | func registerFlags() { 27 | flag.StringVar(&context.HubKubeConfig, 28 | "hub-kubeconfig", 29 | os.Getenv("KUBECONFIG"), 30 | "Path to kubeconfig of the hub cluster.") 31 | flag.StringVar(&context.TestCluster, 32 | "test-cluster", 33 | "", 34 | "The target cluster to run the e2e suite.") 35 | flag.BoolVar(&context.IsOCMInstalled, 36 | "ocm-installed", 37 | false, 38 | "Is the test running inside OCM environment") 39 | } 40 | 41 | func defaultFlags() { 42 | if len(context.HubKubeConfig) == 0 { 43 | home := os.Getenv("HOME") 44 | if len(home) > 0 { 45 | context.HubKubeConfig = filepath.Join(home, ".kube", "config") 46 | } 47 | } 48 | } 49 | 50 | func validateFlags() { 51 | if len(context.HubKubeConfig) == 0 { 52 | klog.Fatalf("--hub-kubeconfig is required") 53 | } 54 | if len(context.TestCluster) == 0 { 55 | klog.Fatalf("--test-cluster is required") 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /pkg/config/args_clusterproxy.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/spf13/pflag" 6 | ) 7 | 8 | var ClusterProxyHost string 9 | var ClusterProxyPort int 10 | var ClusterProxyCAFile string 11 | var ClusterProxyCertFile string 12 | var ClusterProxyKeyFile string 13 | 14 | func ValidateClusterProxy() error { 15 | if len(ClusterProxyHost) == 0 { 16 | return nil 17 | } 18 | if ClusterProxyPort == 0 { 19 | return errors.New("--proxy-port must be greater than 0") 20 | } 21 | if len(ClusterProxyCAFile) == 0 { 22 | return errors.New("--proxy-ca-cert must be specified") 23 | } 24 | if len(ClusterProxyCertFile) == 0 { 25 | return errors.New("--proxy-cert must be specified") 26 | } 27 | if len(ClusterProxyKeyFile) == 0 { 28 | return errors.New("--proxy-key must be specified") 29 | } 30 | return nil 31 | } 32 | 33 | func AddClusterProxyFlags(set *pflag.FlagSet) { 34 | set.StringVarP(&ClusterProxyHost, "proxy-host", "", "", 35 | "the host of the cluster proxy endpoint") 36 | set.IntVarP(&ClusterProxyPort, "proxy-port", "", 8090, 37 | "the port of the cluster proxy endpoint") 38 | set.StringVarP(&ClusterProxyCAFile, "proxy-ca-cert", "", "", 39 | "the path to ca file for connecting cluster proxy") 40 | set.StringVarP(&ClusterProxyCertFile, "proxy-cert", "", "", 41 | "the path to tls cert for connecting cluster proxy") 42 | set.StringVarP(&ClusterProxyKeyFile, "proxy-key", "", "", 43 | "the path to tls key for connecting cluster proxy") 44 | } 45 | -------------------------------------------------------------------------------- /pkg/apis/proxy/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the proxy v1alpha1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=proxy.open-cluster-management.io 20 | package v1alpha1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "proxy.open-cluster-management.io", Version: "v1alpha1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | 37 | SchemeGroupVersion = GroupVersion 38 | ) 39 | 40 | func Resource(resource string) schema.GroupResource { 41 | return schema.GroupResource{ 42 | Group: "proxy.open-cluster-management.io", 43 | Resource: resource, 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/printer.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "strconv" 5 | 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/apimachinery/pkg/runtime" 8 | ) 9 | 10 | var ( 11 | definitions = []metav1.TableColumnDefinition{ 12 | {Name: "Name", Type: "string", Format: "name", Description: "the name of the cluster"}, 13 | {Name: "Provider", Type: "string", Description: "the cluster provider type"}, 14 | {Name: "Credential-Type", Type: "string", Description: "the credential type"}, 15 | {Name: "Endpoint-Type", Type: "string", Description: "the endpoint type"}, 16 | {Name: "Healthy", Type: "string", Description: "the healthiness of the gateway"}, 17 | } 18 | ) 19 | 20 | func printClusterGateway(in *ClusterGateway) *metav1.Table { 21 | return &metav1.Table{ 22 | ColumnDefinitions: definitions, 23 | Rows: []metav1.TableRow{printClusterGatewayRow(in)}, 24 | } 25 | } 26 | 27 | func printClusterGatewayList(in *ClusterGatewayList) *metav1.Table { 28 | t := &metav1.Table{ 29 | ColumnDefinitions: definitions, 30 | } 31 | for _, c := range in.Items { 32 | t.Rows = append(t.Rows, printClusterGatewayRow(&c)) 33 | } 34 | return t 35 | } 36 | 37 | func printClusterGatewayRow(c *ClusterGateway) metav1.TableRow { 38 | name := c.Name 39 | provideType := c.Spec.Provider 40 | credType := "" 41 | if c.Spec.Access.Credential != nil { 42 | credType = string(c.Spec.Access.Credential.Type) 43 | } 44 | epType := string(c.Spec.Access.Endpoint.Type) 45 | row := metav1.TableRow{ 46 | Object: runtime.RawExtension{Object: c}, 47 | } 48 | row.Cells = append(row.Cells, name, provideType, credType, epType, strconv.FormatBool(c.Status.Healthy)) 49 | return row 50 | } 51 | -------------------------------------------------------------------------------- /pkg/apis/cluster/transport/roundtripper.go: -------------------------------------------------------------------------------- 1 | package multicluster 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | 8 | "github.com/oam-dev/cluster-gateway/pkg/config" 9 | ) 10 | 11 | var _ http.RoundTripper = &clusterGatewayRoundTripper{} 12 | 13 | type clusterGatewayRoundTripper struct { 14 | delegate http.RoundTripper 15 | // falling back to the hosting cluster 16 | // this is required when the client does implicit api discovery 17 | // e.g. controller-runtime client 18 | fallback bool 19 | } 20 | 21 | func NewClusterGatewayRoundTripper(delegate http.RoundTripper) http.RoundTripper { 22 | return &clusterGatewayRoundTripper{ 23 | delegate: delegate, 24 | fallback: true, 25 | } 26 | } 27 | 28 | func NewStrictClusterGatewayRoundTripper(delegate http.RoundTripper, fallback bool) http.RoundTripper { 29 | return &clusterGatewayRoundTripper{ 30 | delegate: delegate, 31 | fallback: false, 32 | } 33 | } 34 | 35 | func (c *clusterGatewayRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { 36 | clusterName, exists := GetMultiClusterContext(request.Context()) 37 | if !exists { 38 | if !c.fallback { 39 | return nil, fmt.Errorf("missing cluster name in the request context") 40 | } 41 | return c.delegate.RoundTrip(request) 42 | } 43 | request.URL.Path = formatProxyURL(clusterName, request.URL.Path) 44 | return c.delegate.RoundTrip(request) 45 | } 46 | 47 | func formatProxyURL(clusterName, originalPath string) string { 48 | originalPath = strings.TrimPrefix(originalPath, "/") 49 | return strings.Join([]string{ 50 | "/apis", 51 | config.MetaApiGroupName, 52 | config.MetaApiVersionName, 53 | "clustergateways", 54 | clusterName, 55 | "proxy", 56 | originalPath}, "/") 57 | } 58 | -------------------------------------------------------------------------------- /pkg/event/secret_handler.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import ( 4 | "github.com/oam-dev/cluster-gateway/pkg/common" 5 | "sigs.k8s.io/controller-runtime/pkg/client" 6 | 7 | "context" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | "k8s.io/apimachinery/pkg/types" 11 | "k8s.io/client-go/util/workqueue" 12 | "sigs.k8s.io/controller-runtime/pkg/event" 13 | "sigs.k8s.io/controller-runtime/pkg/handler" 14 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 15 | ) 16 | 17 | var _ handler.EventHandler = &SecretHandler{} 18 | 19 | type SecretHandler struct { 20 | } 21 | 22 | func (s *SecretHandler) Create(_ context.Context, event event.TypedCreateEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 23 | s.process(event.Object.(*corev1.Secret), q) 24 | } 25 | 26 | func (s *SecretHandler) Update(_ context.Context, event event.TypedUpdateEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 27 | s.process(event.ObjectNew.(*corev1.Secret), q) 28 | } 29 | 30 | func (s *SecretHandler) Delete(_ context.Context, event event.TypedDeleteEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 31 | s.process(event.Object.(*corev1.Secret), q) 32 | } 33 | 34 | func (s *SecretHandler) Generic(_ context.Context, event event.TypedGenericEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 35 | s.process(event.Object.(*corev1.Secret), q) 36 | } 37 | 38 | func (s *SecretHandler) process(secret *corev1.Secret, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 39 | for _, ref := range secret.OwnerReferences { 40 | if ref.Kind == "ManagedServiceAccount" && ref.Name == common.AddonName { 41 | q.Add(reconcile.Request{ 42 | NamespacedName: types.NamespacedName{ 43 | Name: common.AddonName, 44 | }, 45 | }) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/build-image.yml: -------------------------------------------------------------------------------- 1 | name: BuildImage 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - release-* 8 | tags: 9 | - 'v*' 10 | workflow_dispatch: {} 11 | 12 | jobs: 13 | build-push-image: 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: read 17 | packages: write 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v3 21 | - name: Docker meta 22 | id: meta 23 | uses: docker/metadata-action@v4 24 | with: 25 | images: | 26 | oamdev/cluster-gateway 27 | ghcr.io/oam-dev/cluster-gateway 28 | tags: | 29 | type=ref,event=branch 30 | type=ref,event=tag 31 | type=raw,value=latest,enable={{is_default_branch}} 32 | - name: Login docker.io 33 | uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 34 | with: 35 | registry: docker.io 36 | username: ${{ secrets.DOCKER_USER }} 37 | password: ${{ secrets.DOCKER_PASSWORD }} 38 | - name: Login to GitHub Container Registry 39 | uses: docker/login-action@v2 40 | with: 41 | registry: ghcr.io 42 | username: ${{ github.repository_owner }} 43 | password: ${{ secrets.GITHUB_TOKEN }} 44 | 45 | - name: Set up Docker Buildx 46 | uses: docker/setup-buildx-action@v2 47 | 48 | - name: Build and push 49 | uses: docker/build-push-action@v4 50 | with: 51 | context: . 52 | platforms: linux/amd64,linux/arm64 53 | file: ./cmd/apiserver/Dockerfile 54 | push: true 55 | tags: ${{ steps.meta.outputs.tags }} 56 | labels: ${{ steps.meta.outputs.labels }} 57 | cache-from: type=gha 58 | cache-to: type=gha,mode=max 59 | -------------------------------------------------------------------------------- /pkg/event/apiservice_handler.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import ( 4 | "github.com/oam-dev/cluster-gateway/pkg/common" 5 | "sigs.k8s.io/controller-runtime/pkg/client" 6 | 7 | "context" 8 | 9 | "k8s.io/apimachinery/pkg/types" 10 | "k8s.io/client-go/util/workqueue" 11 | apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" 12 | "sigs.k8s.io/controller-runtime/pkg/event" 13 | "sigs.k8s.io/controller-runtime/pkg/handler" 14 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 15 | ) 16 | 17 | var _ handler.EventHandler = &APIServiceHandler{} 18 | 19 | type APIServiceHandler struct { 20 | WatchingName string 21 | } 22 | 23 | func (a *APIServiceHandler) Create(_ context.Context, event event.TypedCreateEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 24 | a.process(event.Object.(*apiregistrationv1.APIService), q) 25 | } 26 | 27 | func (a *APIServiceHandler) Update(_ context.Context, event event.TypedUpdateEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 28 | a.process(event.ObjectNew.(*apiregistrationv1.APIService), q) 29 | } 30 | 31 | func (a *APIServiceHandler) Delete(_ context.Context, event event.TypedDeleteEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 32 | a.process(event.Object.(*apiregistrationv1.APIService), q) 33 | } 34 | 35 | func (a *APIServiceHandler) Generic(_ context.Context, event event.TypedGenericEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 36 | a.process(event.Object.(*apiregistrationv1.APIService), q) 37 | } 38 | 39 | func (a *APIServiceHandler) process(apiService *apiregistrationv1.APIService, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 40 | if apiService.Name == a.WatchingName { 41 | q.Add(reconcile.Request{ 42 | NamespacedName: types.NamespacedName{ 43 | Name: common.AddonName, 44 | }, 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /examples/dynamic-multicluster-client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | multicluster "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/transport" 8 | "github.com/spf13/cobra" 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/types" 12 | "k8s.io/client-go/kubernetes" 13 | "k8s.io/client-go/tools/clientcmd" 14 | "sigs.k8s.io/controller-runtime/pkg/client" 15 | ) 16 | 17 | var kubeconfig string 18 | var clusterName string 19 | 20 | func main() { 21 | 22 | cmd := cobra.Command{ 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) 25 | if err != nil { 26 | return err 27 | } 28 | cfg.Wrap(multicluster.NewClusterGatewayRoundTripper) 29 | 30 | // Native kubernetes client example 31 | nativeClient := kubernetes.NewForConfigOrDie(cfg) 32 | defaultNs, err := nativeClient.CoreV1().Namespaces().Get( 33 | multicluster.WithMultiClusterContext(context.TODO(), clusterName), 34 | "default", 35 | metav1.GetOptions{}) 36 | fmt.Printf("Native client get default namespace: %v\n", defaultNs) 37 | 38 | // Controller-runtime client example 39 | controllerRuntimeClient, err := client.New(cfg, client.Options{}) 40 | if err != nil { 41 | panic(err) 42 | } 43 | ns := &corev1.Namespace{} 44 | err = controllerRuntimeClient.Get( 45 | multicluster.WithMultiClusterContext(context.TODO(), clusterName), 46 | types.NamespacedName{Name: "default"}, 47 | ns) 48 | if err != nil { 49 | panic(err) 50 | } 51 | fmt.Printf("Controller-runtime client get default namespace: %v\n", ns) 52 | return nil 53 | }, 54 | } 55 | 56 | cmd.Flags().StringVarP(&kubeconfig, "kubeconfig", "", "", "the client kubeconfig") 57 | cmd.Flags().StringVarP(&clusterName, "cluster-name", "", "", "the target cluster name") 58 | 59 | if err := cmd.Execute(); err != nil { 60 | panic(err) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /e2e/env/prepare/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 7 | "github.com/oam-dev/cluster-gateway/pkg/common" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/ghodss/yaml" 12 | corev1 "k8s.io/api/core/v1" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/client-go/tools/clientcmd" 15 | "k8s.io/klog/v2" 16 | ) 17 | 18 | func main() { 19 | 20 | var clusterName string 21 | var secretNamespace string 22 | var dryRun bool 23 | 24 | flag.StringVar(&secretNamespace, "secret-namespace", "open-cluster-management-credentials", 25 | "Namespace of the cluster secret.") 26 | flag.StringVar(&clusterName, "cluster-name", "loopback", 27 | "Target name of the secret.") 28 | flag.BoolVar(&dryRun, "dry-run", false, 29 | "Whether to dry run") 30 | flag.Parse() 31 | 32 | kubeconfigPath := os.Getenv("KUBECONFIG") 33 | if len(kubeconfigPath) == 0 { 34 | kubeconfigPath = filepath.Join(os.Getenv("HOME"), ".kube", "config") 35 | } 36 | restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) 37 | if err != nil { 38 | klog.Fatal(err) 39 | } 40 | 41 | secret := &corev1.Secret{ 42 | TypeMeta: metav1.TypeMeta{ 43 | APIVersion: "v1", 44 | Kind: "Secret", 45 | }, 46 | ObjectMeta: metav1.ObjectMeta{ 47 | Namespace: secretNamespace, 48 | Name: clusterName, 49 | Labels: map[string]string{}, 50 | }, 51 | Data: map[string][]byte{ 52 | "ca.crt": restConfig.CAData, 53 | "endpoint": []byte("https://kubernetes.default.svc.cluster.local:443"), 54 | }, 55 | } 56 | if len(restConfig.BearerToken) > 0 { 57 | // TODO 58 | } else { 59 | secret.Labels[common.LabelKeyClusterCredentialType] = string(v1alpha1.CredentialTypeX509Certificate) 60 | secret.Data["tls.crt"] = restConfig.CertData 61 | secret.Data["tls.key"] = restConfig.KeyData 62 | } 63 | 64 | secretYamlData, err := yaml.Marshal(secret) 65 | if err != nil { 66 | klog.Fatal(err) 67 | } 68 | 69 | fmt.Print(string(secretYamlData)) 70 | } 71 | -------------------------------------------------------------------------------- /pkg/featuregates/featue_gate.go: -------------------------------------------------------------------------------- 1 | package featuregates 2 | 3 | import ( 4 | utilfeature "k8s.io/apiserver/pkg/util/feature" 5 | "k8s.io/component-base/featuregate" 6 | "k8s.io/klog/v2" 7 | ) 8 | 9 | func init() { 10 | if err := utilfeature.DefaultMutableFeatureGate.Add(DefaultKubeFedFeatureGates); err != nil { 11 | klog.Fatalf("Unexpected error: %v", err) 12 | } 13 | } 14 | 15 | const ( 16 | // owner: @yue9944882 17 | // alpha: v1.1.12 18 | // 19 | // HealthinessCheck enables the "/health" subresource on the ClusterGateway 20 | // by which we can read/update the healthiness related status under the 21 | // ".status". 22 | // 23 | // Additionally, OCM cluster-gateway addon will enable a health-check controller 24 | // in the background which periodically checks the healthiness for each managed 25 | // cluster by dialing "/healthz" api path. 26 | HealthinessCheck featuregate.Feature = "HealthinessCheck" 27 | 28 | // owner: @yue9944882 29 | // alpha: v1.1.15 30 | // 31 | // SecretCache runs a namespaced secret informer inside the apiserver which 32 | // provides a cache for reading secret data. 33 | SecretCache featuregate.Feature = "SecretCache" 34 | 35 | // owner: @somefive 36 | // alpha: v1.4.0 37 | // 38 | // ClientIdentityPenetration enforce impersonate as the original request user 39 | // when accessing apiserver in ManagedCluster 40 | ClientIdentityPenetration featuregate.Feature = "ClientIdentityPenetration" 41 | 42 | // owner: @ivan-cai 43 | // beta: v1.6.0 44 | // 45 | // SecretCache runs a OCM ManagedCluster informer inside the apiserver which 46 | // provides a cache for reading ManagedCluster. 47 | OCMClusterCache featuregate.Feature = "OCMClusterCache" 48 | ) 49 | 50 | var DefaultKubeFedFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ 51 | HealthinessCheck: {Default: false, PreRelease: featuregate.Alpha}, 52 | SecretCache: {Default: true, PreRelease: featuregate.Beta}, 53 | ClientIdentityPenetration: {Default: false, PreRelease: featuregate.Alpha}, 54 | OCMClusterCache: {Default: true, PreRelease: featuregate.Beta}, 55 | } 56 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/scheme/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | // Code generated by client-gen. DO NOT EDIT. 15 | 16 | package scheme 17 | 18 | import ( 19 | clusterv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 20 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | runtime "k8s.io/apimachinery/pkg/runtime" 22 | schema "k8s.io/apimachinery/pkg/runtime/schema" 23 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 24 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 25 | ) 26 | 27 | var Scheme = runtime.NewScheme() 28 | var Codecs = serializer.NewCodecFactory(Scheme) 29 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 30 | var localSchemeBuilder = runtime.SchemeBuilder{ 31 | clusterv1alpha1.AddToScheme, 32 | } 33 | 34 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 35 | // of clientsets, like in: 36 | // 37 | // import ( 38 | // "k8s.io/client-go/kubernetes" 39 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 40 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 41 | // ) 42 | // 43 | // kclientset, _ := kubernetes.NewForConfig(c) 44 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 45 | // 46 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 47 | // correctly. 48 | var AddToScheme = localSchemeBuilder.AddToScheme 49 | 50 | func init() { 51 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 52 | utilruntime.Must(AddToScheme(Scheme)) 53 | } 54 | -------------------------------------------------------------------------------- /charts/cluster-gateway/templates/cluster-gateway-apiserver.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: gateway-deployment 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | app: gateway 8 | spec: 9 | replicas: {{ .Values.replicas }} 10 | selector: 11 | matchLabels: 12 | app: gateway 13 | template: 14 | metadata: 15 | labels: 16 | app: gateway 17 | spec: 18 | serviceAccount: cluster-gateway 19 | volumes: 20 | - name: proxy-client 21 | secret: 22 | secretName: proxy-client 23 | - name: proxy-server-ca 24 | secret: 25 | secretName: proxy-server-ca 26 | containers: 27 | - name: gateway 28 | image: {{ .Values.image }}:{{ .Values.tag | default (print "v" .Chart.Version) }} 29 | imagePullPolicy: IfNotPresent 30 | args: 31 | - --secure-port=9443 32 | - --secret-namespace={{ .Values.secretNamespace }} 33 | {{ if .Values.ocmIntegration.enabled }} 34 | - --ocm-integration=true 35 | {{ if .Values.ocmIntegration.clusterProxy.enabled }} 36 | - --proxy-host={{ .Values.ocmIntegration.clusterProxy.endpoint.host }} 37 | - --proxy-port={{ .Values.ocmIntegration.clusterProxy.endpoint.port }} 38 | - --proxy-ca-cert=/etc/ca/ca.crt 39 | - --proxy-cert=/etc/tls/tls.crt 40 | - --proxy-key=/etc/tls/tls.key 41 | {{ end }} 42 | {{ end }} 43 | - --feature-gates={{ if .Values.featureGate.healthiness }}HealthinessCheck=true,{{ end }}{{ if .Values.featureGate.secretCache }}SecretCache=true,{{ end }} 44 | # TODO: certificate rotation, otherwise the self-signed will expire in 1 year 45 | {{ if .Values.ocmIntegration.clusterProxy.enabled }} 46 | volumeMounts: 47 | - name: proxy-client 48 | mountPath: "/etc/tls/" 49 | readOnly: true 50 | - name: proxy-server-ca 51 | mountPath: "/etc/ca/" 52 | readOnly: true 53 | {{ end }} 54 | ports: 55 | - containerPort: 9443 56 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "k8s.io/klog" 25 | ocmclusterv1 "open-cluster-management.io/api/cluster/v1" 26 | 27 | "github.com/oam-dev/cluster-gateway/pkg/config" 28 | 29 | "github.com/oam-dev/cluster-gateway/pkg/util/scheme" 30 | ) 31 | 32 | func init() { 33 | for _, fn := range []func(*runtime.Scheme) error{ 34 | AddToScheme, 35 | corev1.AddToScheme, 36 | ocmclusterv1.Install, 37 | } { 38 | if err := fn(scheme.Scheme); err != nil { 39 | klog.Fatalf("failed registering core api types") 40 | } 41 | } 42 | } 43 | 44 | var AddToScheme = func(scheme *runtime.Scheme) error { 45 | metav1.AddToGroupVersion(scheme, schema.GroupVersion{ 46 | Group: config.MetaApiGroupName, 47 | Version: config.MetaApiVersionName, 48 | }) 49 | // +kubebuilder:scaffold:install 50 | 51 | scheme.AddKnownTypes(schema.GroupVersion{ 52 | Group: config.MetaApiGroupName, 53 | Version: config.MetaApiVersionName, 54 | }, &ClusterGateway{}, &ClusterGatewayList{}) 55 | scheme.AddKnownTypes(schema.GroupVersion{ 56 | Group: config.MetaApiGroupName, 57 | Version: config.MetaApiVersionName, 58 | }, &ClusterGatewayProxyOptions{}) 59 | 60 | scheme.AddKnownTypes(schema.GroupVersion{ 61 | Group: config.MetaApiGroupName, 62 | Version: config.MetaApiVersionName, 63 | }, &VirtualCluster{}, &VirtualClusterList{}) 64 | 65 | return nil 66 | } 67 | 68 | var SchemeGroupVersion = schema.GroupVersion{Group: config.MetaApiGroupName, Version: config.MetaApiVersionName} 69 | -------------------------------------------------------------------------------- /e2e/framework/framework.go: -------------------------------------------------------------------------------- 1 | package framework 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | "k8s.io/apimachinery/pkg/util/rand" 7 | "k8s.io/client-go/kubernetes" 8 | "k8s.io/client-go/rest" 9 | "k8s.io/client-go/tools/clientcmd" 10 | "sigs.k8s.io/controller-runtime/pkg/client" 11 | 12 | "github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned" 13 | ) 14 | 15 | // unique identifier of the e2e run 16 | var RunID = rand.String(6) 17 | 18 | type Framework interface { 19 | HubRESTConfig() *rest.Config 20 | TestClusterName() string 21 | IsOCMInstalled() bool 22 | 23 | HubNativeClient() kubernetes.Interface 24 | HubRuntimeClient() client.Client 25 | HubGatewayClient() versioned.Interface 26 | } 27 | 28 | var _ Framework = &framework{} 29 | 30 | type framework struct { 31 | basename string 32 | ctx *E2EContext 33 | } 34 | 35 | func NewE2EFramework(basename string) Framework { 36 | f := &framework{ 37 | basename: basename, 38 | ctx: context, 39 | } 40 | AfterEach(f.AfterEach) 41 | BeforeEach(f.BeforeEach) 42 | return f 43 | } 44 | 45 | func (f *framework) HubRESTConfig() *rest.Config { 46 | restConfig, err := clientcmd.BuildConfigFromFlags("", f.ctx.HubKubeConfig) 47 | Expect(err).NotTo(HaveOccurred()) 48 | return restConfig 49 | } 50 | 51 | func (f *framework) HubNativeClient() kubernetes.Interface { 52 | cfg := f.HubRESTConfig() 53 | nativeClient, err := kubernetes.NewForConfig(cfg) 54 | Expect(err).NotTo(HaveOccurred()) 55 | return nativeClient 56 | } 57 | 58 | func (f *framework) HubRuntimeClient() client.Client { 59 | cfg := f.HubRESTConfig() 60 | runtimeClient, err := client.New(cfg, client.Options{ 61 | Scheme: scheme, 62 | }) 63 | Expect(err).NotTo(HaveOccurred()) 64 | return runtimeClient 65 | } 66 | 67 | func (f *framework) HubGatewayClient() versioned.Interface { 68 | cfg := f.HubRESTConfig() 69 | gatewayClient, err := versioned.NewForConfig(cfg) 70 | Expect(err).NotTo(HaveOccurred()) 71 | return gatewayClient 72 | } 73 | 74 | func (f *framework) IsOCMInstalled() bool { 75 | return f.ctx.IsOCMInstalled 76 | } 77 | 78 | func (f *framework) TestClusterName() string { 79 | return f.ctx.TestCluster 80 | } 81 | 82 | func (f *framework) BeforeEach() { 83 | 84 | } 85 | 86 | func (f *framework) AfterEach() { 87 | 88 | } 89 | -------------------------------------------------------------------------------- /e2e/ocm/clustergateway.go: -------------------------------------------------------------------------------- 1 | package roundtrip 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | apierrors "k8s.io/apimachinery/pkg/api/errors" 10 | "k8s.io/apimachinery/pkg/api/meta" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/types" 13 | addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" 14 | 15 | "github.com/oam-dev/cluster-gateway/e2e/framework" 16 | "github.com/oam-dev/cluster-gateway/pkg/common" 17 | ) 18 | 19 | const ( 20 | ocmTestBasename = "ocm-addon" 21 | ) 22 | 23 | var _ = Describe("Addon Manager Test", func() { 24 | f := framework.NewE2EFramework(ocmTestBasename) 25 | It("ClusterGateway addon installation should work", 26 | func() { 27 | c := f.HubRuntimeClient() 28 | By("Polling addon and gateway healthiness") 29 | Eventually( 30 | func() (bool, error) { 31 | addon := &addonapiv1alpha1.ManagedClusterAddOn{} 32 | if err := c.Get(context.TODO(), types.NamespacedName{ 33 | Namespace: f.TestClusterName(), 34 | Name: common.AddonName, 35 | }, addon); err != nil { 36 | if apierrors.IsNotFound(err) { 37 | return false, nil 38 | } 39 | return false, err 40 | } 41 | if addon.Status.HealthCheck.Mode != addonapiv1alpha1.HealthCheckModeCustomized { 42 | return false, nil 43 | } 44 | addonHealthy := meta.IsStatusConditionTrue( 45 | addon.Status.Conditions, 46 | addonapiv1alpha1.ManagedClusterAddOnConditionAvailable) 47 | gw, err := f.HubGatewayClient(). 48 | ClusterV1alpha1(). 49 | ClusterGateways(). 50 | GetHealthiness(context.TODO(), f.TestClusterName(), metav1.GetOptions{}) 51 | if err != nil { 52 | return false, err 53 | } 54 | gwHealthy := gw.Status.Healthy 55 | return addonHealthy && gwHealthy, nil 56 | }). 57 | WithTimeout(time.Minute). 58 | Should(BeTrue()) 59 | }) 60 | It("Manual probe healthiness should work", 61 | func() { 62 | resp, err := f.HubNativeClient().Discovery(). 63 | RESTClient(). 64 | Get(). 65 | AbsPath( 66 | "apis/cluster.core.oam.dev/v1alpha1/clustergateways", 67 | f.TestClusterName(), 68 | "proxy", 69 | "healthz", 70 | ).DoRaw(context.TODO()) 71 | Expect(err).NotTo(HaveOccurred()) 72 | Expect(string(resp)).To(Equal("ok")) 73 | }) 74 | }) 75 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/virtualcluster_errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import "errors" 20 | 21 | type emptyCredentialTypeClusterSecretError struct{} 22 | 23 | func (e emptyCredentialTypeClusterSecretError) Error() string { 24 | return "secret is not a valid cluster secret, no credential type found" 25 | } 26 | 27 | // NewEmptyCredentialTypeClusterSecretError create an invalid cluster secret error due to empty credential type 28 | func NewEmptyCredentialTypeClusterSecretError() error { 29 | return emptyCredentialTypeClusterSecretError{} 30 | } 31 | 32 | type emptyEndpointClusterSecretError struct{} 33 | 34 | func (e emptyEndpointClusterSecretError) Error() string { 35 | return "secret is not a valid cluster secret, no credential type found" 36 | } 37 | 38 | // NewEmptyEndpointClusterSecretError create an invalid cluster secret error due to empty endpoint 39 | func NewEmptyEndpointClusterSecretError() error { 40 | return emptyEndpointClusterSecretError{} 41 | } 42 | 43 | // IsInvalidClusterSecretError check if an error is an invalid cluster secret error 44 | func IsInvalidClusterSecretError(err error) bool { 45 | return errors.As(err, &emptyCredentialTypeClusterSecretError{}) || errors.As(err, &emptyEndpointClusterSecretError{}) 46 | } 47 | 48 | type invalidManagedClusterError struct{} 49 | 50 | func (e invalidManagedClusterError) Error() string { 51 | return "managed cluster has no client config" 52 | } 53 | 54 | // NewInvalidManagedClusterError create an invalid managed cluster error 55 | func NewInvalidManagedClusterError() error { 56 | return invalidManagedClusterError{} 57 | } 58 | 59 | // IsInvalidManagedClusterError check if an error is an invalid managed cluster error 60 | func IsInvalidManagedClusterError(err error) bool { 61 | return errors.As(err, &invalidManagedClusterError{}) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/event/clustergatewayconfiguration_handler.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import ( 4 | "context" 5 | 6 | "k8s.io/apimachinery/pkg/types" 7 | "k8s.io/client-go/util/workqueue" 8 | addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" 9 | ctrl "sigs.k8s.io/controller-runtime" 10 | "sigs.k8s.io/controller-runtime/pkg/client" 11 | "sigs.k8s.io/controller-runtime/pkg/event" 12 | "sigs.k8s.io/controller-runtime/pkg/handler" 13 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 14 | 15 | proxyv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/proxy/v1alpha1" 16 | "github.com/oam-dev/cluster-gateway/pkg/common" 17 | ) 18 | 19 | var _ handler.EventHandler = &ClusterGatewayConfigurationHandler{} 20 | 21 | type ClusterGatewayConfigurationHandler struct { 22 | client.Client 23 | } 24 | 25 | func (c *ClusterGatewayConfigurationHandler) Create(ctx context.Context, event event.TypedCreateEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 26 | cfg := event.Object.(*proxyv1alpha1.ClusterGatewayConfiguration) 27 | c.process(ctx, cfg, q) 28 | } 29 | 30 | func (c *ClusterGatewayConfigurationHandler) Update(ctx context.Context, event event.TypedUpdateEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 31 | cfg := event.ObjectNew.(*proxyv1alpha1.ClusterGatewayConfiguration) 32 | c.process(ctx, cfg, q) 33 | } 34 | 35 | func (c *ClusterGatewayConfigurationHandler) Delete(ctx context.Context, event event.TypedDeleteEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 36 | cfg := event.Object.(*proxyv1alpha1.ClusterGatewayConfiguration) 37 | c.process(ctx, cfg, q) 38 | } 39 | 40 | func (c *ClusterGatewayConfigurationHandler) Generic(ctx context.Context, event event.TypedGenericEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 41 | cfg := event.Object.(*proxyv1alpha1.ClusterGatewayConfiguration) 42 | c.process(ctx, cfg, q) 43 | } 44 | 45 | func (c *ClusterGatewayConfigurationHandler) process(ctx context.Context, config *proxyv1alpha1.ClusterGatewayConfiguration, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { 46 | list := addonv1alpha1.ClusterManagementAddOnList{} 47 | 48 | if err := c.Client.List(ctx, &list); err != nil { 49 | ctrl.Log.WithName("ClusterGatewayConfiguration").Error(err, "failed list addons") 50 | return 51 | } 52 | 53 | for _, addon := range list.Items { 54 | if addon.Spec.AddOnConfiguration.CRDName != common.ClusterGatewayConfigurationCRDName { 55 | continue 56 | } 57 | if addon.Spec.AddOnConfiguration.CRName == config.Name { 58 | q.Add(reconcile.Request{ 59 | NamespacedName: types.NamespacedName{ 60 | Name: addon.Name, 61 | }, 62 | }) 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /examples/single-cluster-informer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | multicluster "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/transport" 9 | "github.com/spf13/cobra" 10 | corev1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | "k8s.io/client-go/informers" 13 | "k8s.io/client-go/kubernetes" 14 | "k8s.io/client-go/kubernetes/scheme" 15 | "k8s.io/client-go/tools/cache" 16 | "k8s.io/client-go/tools/clientcmd" 17 | controllers "sigs.k8s.io/controller-runtime" 18 | "sigs.k8s.io/controller-runtime/pkg/manager" 19 | ) 20 | 21 | var kubeconfig string 22 | var clusterName string 23 | 24 | func main() { 25 | 26 | cmd := cobra.Command{ 27 | RunE: func(cmd *cobra.Command, args []string) error { 28 | cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) 29 | if err != nil { 30 | return err 31 | } 32 | cfg.Wrap(multicluster.NewProxyPathPrependingClusterGatewayRoundTripper(clusterName).NewRoundTripper) 33 | 34 | // Native kubernetes client informer 35 | nativeClient := kubernetes.NewForConfigOrDie(cfg) 36 | 37 | sharedInformer := informers.NewSharedInformerFactory(nativeClient, 0) 38 | podInformer := sharedInformer.Core().V1().Pods().Informer() 39 | 40 | fmt.Printf("Native client cache pod info:\n") 41 | podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: addFunc}) 42 | 43 | ctx, cancel := context.WithCancel(context.TODO()) 44 | go sharedInformer.Start(ctx.Done()) 45 | for !podInformer.HasSynced() { 46 | time.Sleep(time.Millisecond * 100) 47 | } 48 | cancel() 49 | 50 | // Controller-runtime client informer 51 | s := runtime.NewScheme() 52 | scheme.AddToScheme(s) 53 | 54 | mgr, err := controllers.NewManager(cfg, manager.Options{Scheme: s}) 55 | 56 | runtimePodInformer, err := mgr.GetCache().GetInformer(context.TODO(), &corev1.Pod{}) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | fmt.Printf("Controller-runtime cache pod info:\n") 62 | runtimePodInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: addFunc}) 63 | 64 | ctx, cancel = context.WithCancel(context.TODO()) 65 | go mgr.Start(ctx) 66 | for !runtimePodInformer.HasSynced() { 67 | time.Sleep(time.Millisecond * 100) 68 | } 69 | cancel() 70 | 71 | return nil 72 | }, 73 | } 74 | 75 | cmd.Flags().StringVarP(&kubeconfig, "kubeconfig", "", "", "the client kubeconfig") 76 | cmd.Flags().StringVarP(&clusterName, "cluster-name", "", "", "the target cluster name") 77 | 78 | if err := cmd.Execute(); err != nil { 79 | panic(err) 80 | } 81 | } 82 | 83 | func addFunc(obj interface{}) { 84 | pod, ok := obj.(*corev1.Pod) 85 | if !ok { 86 | return 87 | } 88 | if pod.Namespace == "kube-system" { 89 | fmt.Printf("%s\t%s\t%s\t%s\n", pod.Namespace, pod.Name, pod.Status.PodIP, pod.Status.HostIP) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/clustergateway_health.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | 8 | "github.com/oam-dev/cluster-gateway/pkg/config" 9 | "github.com/oam-dev/cluster-gateway/pkg/util/singleton" 10 | 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/runtime" 13 | "k8s.io/apiserver/pkg/registry/rest" 14 | "sigs.k8s.io/apiserver-runtime/pkg/builder/resource" 15 | contextutil "sigs.k8s.io/apiserver-runtime/pkg/util/context" 16 | ) 17 | 18 | var _ resource.ArbitrarySubResource = &ClusterGatewayHealth{} 19 | var _ rest.Getter = &ClusterGatewayHealth{} 20 | var _ rest.Updater = &ClusterGatewayHealth{} 21 | 22 | type ClusterGatewayHealth ClusterGateway 23 | 24 | func (in *ClusterGatewayHealth) New() runtime.Object { 25 | return &ClusterGateway{} 26 | } 27 | 28 | func (in *ClusterGatewayHealth) SubResourceName() string { 29 | return "health" 30 | } 31 | 32 | func (in *ClusterGatewayHealth) Destroy() {} 33 | 34 | func (in *ClusterGatewayHealth) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { 35 | parentStorage, ok := contextutil.GetParentStorageGetter(ctx) 36 | if !ok { 37 | return nil, fmt.Errorf("no parent storage found") 38 | } 39 | parentObj, err := parentStorage.Get(ctx, name, options) 40 | if err != nil { 41 | return nil, fmt.Errorf("no such cluster %v", name) 42 | } 43 | clusterGateway := parentObj.(*ClusterGateway) 44 | return clusterGateway, nil 45 | } 46 | 47 | func (in *ClusterGatewayHealth) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) { 48 | if singleton.GetSecretControl() == nil { 49 | return nil, false, fmt.Errorf("loopback clients are not inited") 50 | } 51 | 52 | latestSecret, err := singleton.GetSecretControl().Get(ctx, name) 53 | if err != nil { 54 | return nil, false, err 55 | } 56 | updating, err := objInfo.UpdatedObject(ctx, nil) 57 | if err != nil { 58 | return nil, false, err 59 | } 60 | updatingClusterGateway := updating.(*ClusterGateway) 61 | if latestSecret.Annotations == nil { 62 | latestSecret.Annotations = make(map[string]string) 63 | } 64 | latestSecret.Annotations[AnnotationKeyClusterGatewayStatusHealthy] = strconv.FormatBool(updatingClusterGateway.Status.Healthy) 65 | latestSecret.Annotations[AnnotationKeyClusterGatewayStatusHealthyReason] = string(updatingClusterGateway.Status.HealthyReason) 66 | updated, err := singleton.GetKubeClient(). 67 | CoreV1(). 68 | Secrets(config.SecretNamespace). 69 | Update(ctx, latestSecret, metav1.UpdateOptions{}) 70 | if err != nil { 71 | return nil, false, err 72 | } 73 | clusterGateway, err := convertFromSecret(updated) 74 | if err != nil { 75 | return nil, false, err 76 | } 77 | return clusterGateway, false, nil 78 | } 79 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/cluster/v1alpha1/cluster_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | // Code generated by client-gen. DO NOT EDIT. 15 | 16 | package v1alpha1 17 | 18 | import ( 19 | v1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 20 | "github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned/scheme" 21 | rest "k8s.io/client-go/rest" 22 | ) 23 | 24 | type ClusterV1alpha1Interface interface { 25 | RESTClient() rest.Interface 26 | ClusterGatewaysGetter 27 | } 28 | 29 | // ClusterV1alpha1Client is used to interact with features provided by the cluster.core.oam.dev group. 30 | type ClusterV1alpha1Client struct { 31 | restClient rest.Interface 32 | } 33 | 34 | func (c *ClusterV1alpha1Client) ClusterGateways() ClusterGatewayInterface { 35 | return newClusterGateways(c) 36 | } 37 | 38 | // NewForConfig creates a new ClusterV1alpha1Client for the given config. 39 | func NewForConfig(c *rest.Config) (*ClusterV1alpha1Client, error) { 40 | config := *c 41 | if err := setConfigDefaults(&config); err != nil { 42 | return nil, err 43 | } 44 | client, err := rest.RESTClientFor(&config) 45 | if err != nil { 46 | return nil, err 47 | } 48 | return &ClusterV1alpha1Client{client}, nil 49 | } 50 | 51 | // NewForConfigOrDie creates a new ClusterV1alpha1Client for the given config and 52 | // panics if there is an error in the config. 53 | func NewForConfigOrDie(c *rest.Config) *ClusterV1alpha1Client { 54 | client, err := NewForConfig(c) 55 | if err != nil { 56 | panic(err) 57 | } 58 | return client 59 | } 60 | 61 | // New creates a new ClusterV1alpha1Client for the given RESTClient. 62 | func New(c rest.Interface) *ClusterV1alpha1Client { 63 | return &ClusterV1alpha1Client{c} 64 | } 65 | 66 | func setConfigDefaults(config *rest.Config) error { 67 | gv := v1alpha1.SchemeGroupVersion 68 | config.GroupVersion = &gv 69 | config.APIPath = "/apis" 70 | config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() 71 | 72 | if config.UserAgent == "" { 73 | config.UserAgent = rest.DefaultKubernetesUserAgent() 74 | } 75 | 76 | return nil 77 | } 78 | 79 | // RESTClient returns a RESTClient that is used to communicate 80 | // with API server by this client implementation. 81 | func (c *ClusterV1alpha1Client) RESTClient() rest.Interface { 82 | if c == nil { 83 | return nil 84 | } 85 | return c.restClient 86 | } 87 | -------------------------------------------------------------------------------- /pkg/metrics/proxy.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | 7 | compbasemetrics "k8s.io/component-base/metrics" 8 | ) 9 | 10 | const ( 11 | namespace = "ocm" 12 | subsystem = "proxy" 13 | ) 14 | 15 | // labels 16 | const ( 17 | proxiedResource = "resource" 18 | proxiedVerb = "verb" 19 | proxiedCluster = "cluster" 20 | success = "success" 21 | code = "code" 22 | ) 23 | 24 | var ( 25 | requestDurationSecondsBuckets = []float64{0, 0.005, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30} 26 | ) 27 | 28 | var ( 29 | ocmProxiedRequestsByResourceTotal = compbasemetrics.NewCounterVec( 30 | &compbasemetrics.CounterOpts{ 31 | Namespace: namespace, 32 | Subsystem: subsystem, 33 | Name: "proxied_resource_requests_by_resource_total", 34 | Help: "Number of requests proxied requests", 35 | StabilityLevel: compbasemetrics.ALPHA, 36 | }, 37 | []string{proxiedResource, proxiedVerb, code}, 38 | ) 39 | ocmProxiedRequestsByClusterTotal = compbasemetrics.NewCounterVec( 40 | &compbasemetrics.CounterOpts{ 41 | Namespace: namespace, 42 | Subsystem: subsystem, 43 | Name: "proxied_requests_by_cluster_total", 44 | Help: "Number of requests proxied requests", 45 | StabilityLevel: compbasemetrics.ALPHA, 46 | }, 47 | []string{proxiedCluster, code}, 48 | ) 49 | ocmProxiedRequestsDurationHistogram = compbasemetrics.NewHistogramVec( 50 | &compbasemetrics.HistogramOpts{ 51 | Namespace: namespace, 52 | Subsystem: subsystem, 53 | Name: "proxied_request_duration_seconds", 54 | Help: "Cluster proxy request time cost", 55 | Buckets: requestDurationSecondsBuckets, 56 | StabilityLevel: compbasemetrics.ALPHA, 57 | }, 58 | []string{proxiedResource, proxiedVerb, proxiedCluster, code}, 59 | ) 60 | ocmProxiedClusterEscalationRequestDurationHistogram = compbasemetrics.NewHistogramVec( 61 | &compbasemetrics.HistogramOpts{ 62 | Namespace: namespace, 63 | Subsystem: subsystem, 64 | Name: "cluster_escalation_access_review_duration_seconds", 65 | Help: "Cluster escalation access review time cost", 66 | Buckets: requestDurationSecondsBuckets, 67 | StabilityLevel: compbasemetrics.ALPHA, 68 | }, 69 | []string{success}, 70 | ) 71 | ) 72 | 73 | func RecordProxiedRequestsByResource(resource string, verb string, code int) { 74 | ocmProxiedRequestsByResourceTotal. 75 | WithLabelValues(resource, verb, strconv.Itoa(code)). 76 | Inc() 77 | } 78 | 79 | func RecordProxiedRequestsByCluster(cluster string, code int) { 80 | ocmProxiedRequestsByClusterTotal. 81 | WithLabelValues(cluster, strconv.Itoa(code)). 82 | Inc() 83 | } 84 | 85 | func RecordProxiedRequestsDuration(resource string, verb string, cluster string, code int, ts time.Duration) { 86 | ocmProxiedRequestsDurationHistogram. 87 | WithLabelValues(resource, verb, cluster, strconv.Itoa(code)). 88 | Observe(ts.Seconds()) 89 | } 90 | -------------------------------------------------------------------------------- /pkg/util/cluster/cluster.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/apimachinery/pkg/labels" 9 | "k8s.io/apimachinery/pkg/runtime/schema" 10 | ocmclient "open-cluster-management.io/api/client/cluster/clientset/versioned" 11 | clusterv1Lister "open-cluster-management.io/api/client/cluster/listers/cluster/v1" 12 | clusterv1 "open-cluster-management.io/api/cluster/v1" 13 | ) 14 | 15 | type OCMClusterControl interface { 16 | Get(ctx context.Context, name string) (*clusterv1.ManagedCluster, error) 17 | List(ctx context.Context) ([]*clusterv1.ManagedCluster, error) 18 | } 19 | 20 | var _ OCMClusterControl = &directOCMClusterControl{} 21 | 22 | type directOCMClusterControl struct { 23 | ocmClient ocmclient.Interface 24 | } 25 | 26 | func NewDirectOCMClusterControl(ocmClient ocmclient.Interface) OCMClusterControl { 27 | return &directOCMClusterControl{ 28 | ocmClient: ocmClient, 29 | } 30 | } 31 | 32 | func (c *directOCMClusterControl) Get(ctx context.Context, name string) (*clusterv1.ManagedCluster, error) { 33 | return c.ocmClient.ClusterV1().ManagedClusters().Get(ctx, name, metav1.GetOptions{}) 34 | } 35 | 36 | func (c *directOCMClusterControl) List(ctx context.Context) ([]*clusterv1.ManagedCluster, error) { 37 | clusterList, err := c.ocmClient.ClusterV1().ManagedClusters().List(ctx, metav1.ListOptions{}) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | clusters := make([]*clusterv1.ManagedCluster, len(clusterList.Items)) 43 | for _, item := range clusters { 44 | clusters = append(clusters, item) 45 | } 46 | 47 | return clusters, nil 48 | } 49 | 50 | var _ OCMClusterControl = &cacheOCMClusterControl{} 51 | 52 | type cacheOCMClusterControl struct { 53 | clusterLister clusterv1Lister.ManagedClusterLister 54 | } 55 | 56 | func NewCacheOCMClusterControl(clusterLister clusterv1Lister.ManagedClusterLister) OCMClusterControl { 57 | return &cacheOCMClusterControl{ 58 | clusterLister: clusterLister, 59 | } 60 | } 61 | 62 | func (c *cacheOCMClusterControl) Get(ctx context.Context, name string) (*clusterv1.ManagedCluster, error) { 63 | return c.clusterLister.Get(name) 64 | } 65 | 66 | func (c *cacheOCMClusterControl) List(ctx context.Context) ([]*clusterv1.ManagedCluster, error) { 67 | return c.clusterLister.List(labels.Everything()) 68 | } 69 | 70 | // IsOCMManagedClusterInstalled check if managed cluster is installed in the cluster 71 | func IsOCMManagedClusterInstalled(ocmClient ocmclient.Interface) (bool, error) { 72 | _, resources, err := ocmClient.Discovery().ServerGroupsAndResources() 73 | if err != nil { 74 | return false, fmt.Errorf("unable to get api-resources: %w", err) 75 | } 76 | for _, resource := range resources { 77 | if gv, _ := schema.ParseGroupVersion(resource.GroupVersion); gv.Group == clusterv1.GroupName { 78 | for _, rsc := range resource.APIResources { 79 | if rsc.Kind == "ManagedCluster" { 80 | return true, nil 81 | } 82 | } 83 | } 84 | } 85 | return false, nil 86 | } 87 | -------------------------------------------------------------------------------- /pkg/util/cert/cert.go: -------------------------------------------------------------------------------- 1 | package cert 2 | 3 | import ( 4 | "context" 5 | "crypto/rand" 6 | "crypto/rsa" 7 | "crypto/x509" 8 | "encoding/pem" 9 | 10 | "github.com/oam-dev/cluster-gateway/pkg/common" 11 | "github.com/openshift/library-go/pkg/crypto" 12 | "github.com/pkg/errors" 13 | corev1 "k8s.io/api/core/v1" 14 | apierrors "k8s.io/apimachinery/pkg/api/errors" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "k8s.io/client-go/kubernetes" 17 | "k8s.io/client-go/rest" 18 | "k8s.io/client-go/util/cert" 19 | ) 20 | 21 | var ( 22 | rsaKeySize = 2048 // a decent number, as of 2019 23 | ) 24 | 25 | func EnsureCAPair(cfg *rest.Config, namespace, name string) (*crypto.CA, error) { 26 | c, err := kubernetes.NewForConfig(cfg) 27 | if err != nil { 28 | return nil, err 29 | } 30 | generate := false 31 | current, err := c.CoreV1(). 32 | Secrets(namespace). 33 | Get(context.TODO(), name, metav1.GetOptions{}) 34 | if err != nil { 35 | if !apierrors.IsNotFound(err) { 36 | return nil, err 37 | } 38 | generate = true 39 | } 40 | 41 | if !generate { 42 | caCertData := current.Data["ca.crt"] 43 | caKeyData := current.Data["ca.key"] 44 | certBlock, _ := pem.Decode(caCertData) 45 | caCert, err := x509.ParseCertificate(certBlock.Bytes) 46 | if err != nil { 47 | return nil, errors.Wrapf(err, "failed to parse ca certificate") 48 | } 49 | keyBlock, _ := pem.Decode(caKeyData) 50 | caKey, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes) 51 | if err != nil { 52 | return nil, errors.Wrapf(err, "failed to parse ca key") 53 | } 54 | return &crypto.CA{ 55 | Config: &crypto.TLSCertificateConfig{ 56 | Certs: []*x509.Certificate{caCert}, 57 | Key: caKey, 58 | }, 59 | SerialGenerator: &crypto.RandomSerialGenerator{}, 60 | }, nil 61 | } 62 | 63 | privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeySize) 64 | if err != nil { 65 | return nil, err 66 | } 67 | caCert, err := cert.NewSelfSignedCACert(cert.Config{ 68 | CommonName: common.AddonName, 69 | }, privateKey) 70 | if err != nil { 71 | return nil, err 72 | } 73 | rawKeyData, err := x509.MarshalPKCS8PrivateKey(privateKey) 74 | if err != nil { 75 | return nil, err 76 | } 77 | if _, err := c.CoreV1(). 78 | Secrets(namespace). 79 | Create(context.TODO(), &corev1.Secret{ 80 | ObjectMeta: metav1.ObjectMeta{ 81 | Namespace: namespace, 82 | Name: name, 83 | }, 84 | Data: map[string][]byte{ 85 | "ca.crt": pem.EncodeToMemory(&pem.Block{ 86 | Type: "CERTIFICATE", 87 | Bytes: caCert.Raw, 88 | }), 89 | "ca.key": pem.EncodeToMemory(&pem.Block{ 90 | Type: "PRIVATE KEY", 91 | Bytes: rawKeyData, 92 | }), 93 | }, 94 | }, metav1.CreateOptions{}); err != nil { 95 | return nil, err 96 | } 97 | return &crypto.CA{ 98 | Config: &crypto.TLSCertificateConfig{ 99 | Certs: []*x509.Certificate{caCert}, 100 | Key: privateKey, 101 | }, 102 | SerialGenerator: &crypto.RandomSerialGenerator{}, 103 | }, nil 104 | } 105 | -------------------------------------------------------------------------------- /charts/addon-manager/templates/clusterroles.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: open-cluster-management:cluster-gateway:managedcluster-reader 5 | rules: 6 | - apiGroups: 7 | - cluster.open-cluster-management.io 8 | resources: 9 | - managedclusters 10 | verbs: 11 | - get 12 | - list 13 | - watch 14 | - apiGroups: 15 | - authentication.open-cluster-management.io 16 | resources: 17 | - managedserviceaccounts 18 | verbs: 19 | - "*" 20 | - apiGroups: 21 | - proxy.open-cluster-management.io 22 | resources: 23 | - clustergatewayconfigurations 24 | verbs: 25 | - "*" 26 | - apiGroups: 27 | - cluster.core.oam.dev 28 | resources: 29 | - clustergateways/health 30 | - clustergateways/proxy 31 | verbs: 32 | - "*" 33 | - apiGroups: 34 | - "" 35 | resources: 36 | - namespaces 37 | - secrets 38 | - configmaps 39 | - events 40 | - serviceaccounts 41 | - services 42 | verbs: 43 | - "*" 44 | - apiGroups: 45 | - apps 46 | resources: 47 | - deployments 48 | verbs: 49 | - "*" 50 | - apiGroups: 51 | - work.open-cluster-management.io 52 | resources: 53 | - manifestworks 54 | verbs: 55 | - "*" 56 | - apiGroups: 57 | - addon.open-cluster-management.io 58 | resources: 59 | - clustermanagementaddons 60 | - managedclusteraddons 61 | - clustermanagementaddons/status 62 | - managedclusteraddons/status 63 | verbs: 64 | - get 65 | - list 66 | - watch 67 | - create 68 | - update 69 | - patch 70 | - apiGroups: 71 | - certificates.k8s.io 72 | resources: 73 | - certificatesigningrequests 74 | verbs: 75 | - get 76 | - list 77 | - watch 78 | - apiGroups: 79 | - admissionregistration.k8s.io 80 | resources: 81 | - mutatingwebhookconfigurations 82 | - validatingwebhookconfigurations 83 | - validatingadmissionpolicies 84 | - validatingadmissionpolicybindings 85 | verbs: 86 | - get 87 | - list 88 | - watch 89 | - apiGroups: 90 | - flowcontrol.apiserver.k8s.io 91 | resources: 92 | - prioritylevelconfigurations 93 | - flowschemas 94 | verbs: 95 | - get 96 | - list 97 | - watch 98 | - apiGroups: 99 | - rbac.authorization.k8s.io 100 | resources: 101 | - clusterroles 102 | - clusterrolebindings 103 | verbs: 104 | - create 105 | - bind 106 | - apiGroups: 107 | - rbac.authorization.k8s.io 108 | resources: 109 | - roles 110 | - rolebindings 111 | verbs: 112 | - create 113 | - apiGroups: 114 | - coordination.k8s.io 115 | resources: 116 | - leases 117 | verbs: 118 | - "*" 119 | - apiGroups: 120 | - apiregistration.k8s.io 121 | resources: 122 | - apiservices 123 | verbs: 124 | - "*" 125 | - apiGroups: 126 | - authorization.k8s.io 127 | resources: 128 | - subjectaccessreviews 129 | verbs: 130 | - "*" -------------------------------------------------------------------------------- /pkg/event/resync.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "k8s.io/apimachinery/pkg/types" 8 | "k8s.io/client-go/util/workqueue" 9 | "k8s.io/klog/v2" 10 | addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | "sigs.k8s.io/controller-runtime/pkg/event" 13 | "sigs.k8s.io/controller-runtime/pkg/handler" 14 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 15 | "sigs.k8s.io/controller-runtime/pkg/source" 16 | 17 | "github.com/oam-dev/cluster-gateway/pkg/common" 18 | ) 19 | 20 | func AddOnHealthResyncHandler(c client.Client, interval time.Duration) source.TypedSource[reconcile.Request] { 21 | src := StartBackgroundExternalTimerResync(func() ([]event.TypedGenericEvent[client.Object], error) { 22 | addonList := &addonv1alpha1.ManagedClusterAddOnList{} 23 | if err := c.List(context.TODO(), addonList); err != nil { 24 | return nil, err 25 | } 26 | evs := make([]event.TypedGenericEvent[client.Object], 0) 27 | for _, addon := range addonList.Items { 28 | if addon.Name != common.AddonName { 29 | continue 30 | } 31 | addon := addon 32 | evs = append(evs, event.TypedGenericEvent[client.Object]{ 33 | Object: &addon, 34 | }) 35 | } 36 | return evs, nil 37 | }, interval) 38 | return src 39 | } 40 | 41 | type GeneratorFunc func() ([]event.TypedGenericEvent[client.Object], error) 42 | 43 | func StartBackgroundExternalTimerResync(g GeneratorFunc, interval time.Duration) source.TypedSource[reconcile.Request] { 44 | events := make(chan event.TypedGenericEvent[client.Object]) 45 | ch := source.Channel[client.Object](events, AddonHealthHandler{}) 46 | ticker := time.NewTicker(interval) 47 | go func() { 48 | for { 49 | _, ok := <-ticker.C 50 | if !ok { 51 | return 52 | } 53 | evs, err := g() 54 | if err != nil { 55 | klog.Errorf("Encountered an error when getting periodic events: %v", err) 56 | continue 57 | } 58 | for _, ev := range evs { 59 | events <- ev 60 | } 61 | } 62 | }() 63 | return ch 64 | } 65 | 66 | var _ handler.EventHandler = AddonHealthHandler{} 67 | 68 | type AddonHealthHandler struct { 69 | } 70 | 71 | func (a AddonHealthHandler) Generic(_ context.Context, genericEvent event.TypedGenericEvent[client.Object], limitingInterface workqueue.TypedRateLimitingInterface[reconcile.Request]) { 72 | limitingInterface.Add(reconcile.Request{ 73 | NamespacedName: types.NamespacedName{ 74 | Namespace: genericEvent.Object.GetNamespace(), 75 | Name: genericEvent.Object.GetName(), 76 | }, 77 | }) 78 | } 79 | 80 | func (a AddonHealthHandler) Create(_ context.Context, createEvent event.TypedCreateEvent[client.Object], limitingInterface workqueue.TypedRateLimitingInterface[reconcile.Request]) { 81 | panic("implement me") // unreachable 82 | } 83 | 84 | func (a AddonHealthHandler) Update(_ context.Context, updateEvent event.TypedUpdateEvent[client.Object], limitingInterface workqueue.TypedRateLimitingInterface[reconcile.Request]) { 85 | panic("implement me") // unreachable 86 | } 87 | 88 | func (a AddonHealthHandler) Delete(_ context.Context, deleteEvent event.TypedDeleteEvent[client.Object], limitingInterface workqueue.TypedRateLimitingInterface[reconcile.Request]) { 89 | panic("implement me") // unreachable 90 | } 91 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/clientset.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | // Code generated by client-gen. DO NOT EDIT. 15 | 16 | package versioned 17 | 18 | import ( 19 | "fmt" 20 | 21 | clusterv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned/typed/cluster/v1alpha1" 22 | discovery "k8s.io/client-go/discovery" 23 | rest "k8s.io/client-go/rest" 24 | flowcontrol "k8s.io/client-go/util/flowcontrol" 25 | ) 26 | 27 | type Interface interface { 28 | Discovery() discovery.DiscoveryInterface 29 | ClusterV1alpha1() clusterv1alpha1.ClusterV1alpha1Interface 30 | } 31 | 32 | // Clientset contains the clients for groups. Each group has exactly one 33 | // version included in a Clientset. 34 | type Clientset struct { 35 | *discovery.DiscoveryClient 36 | clusterV1alpha1 *clusterv1alpha1.ClusterV1alpha1Client 37 | } 38 | 39 | // ClusterV1alpha1 retrieves the ClusterV1alpha1Client 40 | func (c *Clientset) ClusterV1alpha1() clusterv1alpha1.ClusterV1alpha1Interface { 41 | return c.clusterV1alpha1 42 | } 43 | 44 | // Discovery retrieves the DiscoveryClient 45 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 46 | if c == nil { 47 | return nil 48 | } 49 | return c.DiscoveryClient 50 | } 51 | 52 | // NewForConfig creates a new Clientset for the given config. 53 | // If config's RateLimiter is not set and QPS and Burst are acceptable, 54 | // NewForConfig will generate a rate-limiter in configShallowCopy. 55 | func NewForConfig(c *rest.Config) (*Clientset, error) { 56 | configShallowCopy := *c 57 | if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { 58 | if configShallowCopy.Burst <= 0 { 59 | return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") 60 | } 61 | configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) 62 | } 63 | var cs Clientset 64 | var err error 65 | cs.clusterV1alpha1, err = clusterv1alpha1.NewForConfig(&configShallowCopy) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) 71 | if err != nil { 72 | return nil, err 73 | } 74 | return &cs, nil 75 | } 76 | 77 | // NewForConfigOrDie creates a new Clientset for the given config and 78 | // panics if there is an error in the config. 79 | func NewForConfigOrDie(c *rest.Config) *Clientset { 80 | var cs Clientset 81 | cs.clusterV1alpha1 = clusterv1alpha1.NewForConfigOrDie(c) 82 | 83 | cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) 84 | return &cs 85 | } 86 | 87 | // New creates a new Clientset for the given RESTClient. 88 | func New(c rest.Interface) *Clientset { 89 | var cs Clientset 90 | cs.clusterV1alpha1 = clusterv1alpha1.New(c) 91 | 92 | cs.DiscoveryClient = discovery.NewDiscoveryClient(c) 93 | return &cs 94 | } 95 | -------------------------------------------------------------------------------- /pkg/util/cert/secret_test.go: -------------------------------------------------------------------------------- 1 | package cert 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | "k8s.io/client-go/kubernetes/fake" 13 | ) 14 | 15 | func TestCopySecret(t *testing.T) { 16 | cases := []struct { 17 | name string 18 | sourceNamespace string 19 | sourceName string 20 | targetNamespace string 21 | targetName string 22 | source *corev1.Secret 23 | existing *corev1.Secret 24 | expected *corev1.Secret 25 | errAssert func(err error) bool 26 | }{ 27 | { 28 | name: "target should be created", 29 | sourceNamespace: "ns1", 30 | sourceName: "s1", 31 | targetNamespace: "ns2", 32 | targetName: "s2", 33 | source: newSecret("ns1", "s1", map[string][]byte{ 34 | "k1": []byte("v1"), 35 | }), 36 | existing: nil, 37 | expected: newSecret("ns2", "s2", map[string][]byte{ 38 | "k1": []byte("v1"), 39 | }), 40 | }, 41 | { 42 | name: "diff should be reconciled", 43 | sourceNamespace: "ns1", 44 | sourceName: "s1", 45 | targetNamespace: "ns2", 46 | targetName: "s2", 47 | source: newSecret("ns1", "s1", map[string][]byte{ 48 | "k1": []byte("v1"), 49 | }), 50 | existing: newSecret("ns2", "s2", map[string][]byte{ 51 | "k1": []byte("v2"), 52 | }), 53 | expected: newSecret("ns2", "s2", map[string][]byte{ 54 | "k1": []byte("v1"), 55 | }), 56 | }, 57 | { 58 | name: "extra content should be kept", 59 | sourceNamespace: "ns1", 60 | sourceName: "s1", 61 | targetNamespace: "ns2", 62 | targetName: "s2", 63 | source: newSecret("ns1", "s1", map[string][]byte{ 64 | "k1": []byte("v1"), 65 | }), 66 | existing: newSecret("ns2", "s2", map[string][]byte{ 67 | "k1": []byte("v1"), 68 | "k2": []byte("v2"), 69 | }), 70 | expected: newSecret("ns2", "s2", map[string][]byte{ 71 | "k1": []byte("v1"), 72 | "k2": []byte("v2"), 73 | }), 74 | }, 75 | { 76 | name: "no source should error", 77 | sourceNamespace: "ns1", 78 | sourceName: "s1", 79 | targetNamespace: "ns2", 80 | targetName: "s2", 81 | errAssert: func(err error) bool { 82 | return strings.HasPrefix(err.Error(), "failed getting source secret") 83 | }, 84 | }, 85 | } 86 | for _, c := range cases { 87 | t.Run(c.name, func(t *testing.T) { 88 | objs := []runtime.Object{} 89 | if c.source != nil { 90 | objs = append(objs, c.source) 91 | } 92 | if c.existing != nil { 93 | objs = append(objs, c.existing) 94 | } 95 | client := fake.NewSimpleClientset(objs...) 96 | err := CopySecret(client, c.sourceNamespace, c.sourceName, c.targetNamespace, c.targetName) 97 | if c.errAssert != nil { 98 | assert.True(t, c.errAssert(err)) 99 | return 100 | } 101 | assert.NoError(t, err) 102 | actual, err := client.CoreV1().Secrets(c.targetNamespace). 103 | Get(context.TODO(), c.targetName, metav1.GetOptions{}) 104 | assert.NoError(t, err) 105 | assert.Equal(t, c.expected, actual) 106 | }) 107 | } 108 | } 109 | 110 | func newSecret(namespace, name string, data map[string][]byte) *corev1.Secret { 111 | return &corev1.Secret{ 112 | ObjectMeta: metav1.ObjectMeta{ 113 | Namespace: namespace, 114 | Name: name, 115 | }, 116 | Data: data, 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /e2e/benchmark/configmap.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | corev1 "k8s.io/api/core/v1" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/client-go/kubernetes" 11 | 12 | "github.com/oam-dev/cluster-gateway/e2e/framework" 13 | multicluster "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/transport" 14 | ) 15 | 16 | const ( 17 | configmapTestBasename = "configmap-benchmark" 18 | ) 19 | 20 | var direct bool 21 | 22 | var _ = Describe("Basic RoundTrip Test", 23 | func() { 24 | f := framework.NewE2EFramework(configmapTestBasename) 25 | 26 | var multiClusterClient kubernetes.Interface 27 | var err error 28 | 29 | cfg := f.HubRESTConfig() 30 | cfg.RateLimiter = nil 31 | if !direct { 32 | cfg.WrapTransport = multicluster.NewClusterGatewayRoundTripper 33 | } 34 | multiClusterClient, err = kubernetes.NewForConfig(cfg) 35 | Expect(err).NotTo(HaveOccurred()) 36 | 37 | var targetConfigMapName = "cluster-gateway-e2e-" + framework.RunID 38 | var targetConfigMapNamespace = "default" 39 | 40 | Measure("it should do something hard efficiently", func(b Benchmarker) { 41 | runtime := b.Time("create-update-delete", func() { 42 | 43 | creatingConfigMap := &corev1.ConfigMap{ 44 | ObjectMeta: metav1.ObjectMeta{ 45 | Namespace: targetConfigMapNamespace, 46 | Name: targetConfigMapName, 47 | }, 48 | Data: map[string]string{ 49 | "version": "1", 50 | }, 51 | } 52 | createdConfigMap, err := multiClusterClient.CoreV1(). 53 | ConfigMaps(targetConfigMapNamespace). 54 | Create( 55 | multicluster.WithMultiClusterContext(context.TODO(), f.TestClusterName()), 56 | creatingConfigMap, 57 | metav1.CreateOptions{}) 58 | Expect(err).NotTo(HaveOccurred()) 59 | 60 | createdConfigMap.Data["version"] = "2" 61 | _, err = multiClusterClient.CoreV1(). 62 | ConfigMaps(targetConfigMapNamespace). 63 | Update( 64 | multicluster.WithMultiClusterContext(context.TODO(), f.TestClusterName()), 65 | createdConfigMap, 66 | metav1.UpdateOptions{}) 67 | Expect(err).NotTo(HaveOccurred()) 68 | 69 | err = multiClusterClient.CoreV1(). 70 | ConfigMaps(targetConfigMapNamespace). 71 | Delete( 72 | multicluster.WithMultiClusterContext(context.TODO(), f.TestClusterName()), 73 | targetConfigMapName, 74 | metav1.DeleteOptions{}) 75 | Expect(err).NotTo(HaveOccurred()) 76 | }) 77 | 78 | Ω(runtime.Seconds()). 79 | Should( 80 | BeNumerically("<", 15), 81 | "shouldn't take too long.") 82 | }, 100) 83 | 84 | Measure("get namespace kube-system from managed cluster", func(b Benchmarker) { 85 | runtime := b.Time("runtime", func() { 86 | _, err = multiClusterClient.CoreV1().Namespaces().Get( 87 | multicluster.WithMultiClusterContext(context.TODO(), f.TestClusterName()), "kube-system", metav1.GetOptions{}) 88 | Expect(err).NotTo(HaveOccurred()) 89 | }) 90 | 91 | Ω(runtime.Seconds()).Should(BeNumerically("<", 15)) 92 | 93 | }, 1000) 94 | 95 | Measure("list namespace from managed cluster", func(b Benchmarker) { 96 | runtime := b.Time("runtime", func() { 97 | _, err = multiClusterClient.CoreV1().Namespaces().List( 98 | multicluster.WithMultiClusterContext(context.TODO(), f.TestClusterName()), metav1.ListOptions{Limit: 100}) 99 | Expect(err).NotTo(HaveOccurred()) 100 | }) 101 | 102 | Ω(runtime.Seconds()).Should(BeNumerically("<", 15)) 103 | 104 | }, 1000) 105 | }) 106 | -------------------------------------------------------------------------------- /pkg/apis/proxy/v1alpha1/clustergatewayconfigurations.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 4 | 5 | func init() { 6 | SchemeBuilder.Register(&ClusterGatewayConfiguration{}, &ClusterGatewayConfigurationList{}) 7 | } 8 | 9 | //+kubebuilder:object:root=true 10 | //+kubebuilder:subresource:status 11 | //+kubebuilder:resource:scope=Cluster 12 | 13 | // +genclient 14 | // +genclient:nonNamespaced 15 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 16 | type ClusterGatewayConfiguration struct { 17 | metav1.TypeMeta `json:",inline"` 18 | metav1.ObjectMeta `json:"metadata,omitempty"` 19 | 20 | Spec ClusterGatewayConfigurationSpec `json:"spec,omitempty"` 21 | Status ClusterGatewayConfigurationStatus `json:"status,omitempty"` 22 | } 23 | 24 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 25 | type ClusterGatewayConfigurationList struct { 26 | metav1.TypeMeta `json:",inline"` 27 | metav1.ListMeta `json:"metadata,omitempty"` 28 | Items []ClusterGatewayConfiguration `json:"items"` 29 | } 30 | 31 | type ClusterGatewayConfigurationSpec struct { 32 | // +required 33 | Image string `json:"image"` 34 | // +required 35 | SecretNamespace string `json:"secretNamespace"` 36 | // +required 37 | InstallNamespace string `json:"installNamespace"` 38 | // +required 39 | SecretManagement ClusterGatewaySecretManagement `json:"secretManagement"` 40 | // +required 41 | Egress ClusterGatewayTrafficEgress `json:"egress"` 42 | } 43 | 44 | type ClusterGatewayConfigurationStatus struct { 45 | // +optional 46 | LastObservedGeneration int64 `json:"lastObservedGeneration,omitempty"` 47 | // +optional 48 | Conditions []metav1.Condition `json:"conditions,omitempty"` 49 | } 50 | 51 | type ClusterGatewayTrafficEgress struct { 52 | Type EgressType `json:"type"` 53 | ClusterProxy *ClusterGatewayTrafficEgressClusterProxy `json:"clusterProxy,omitempty"` 54 | } 55 | 56 | type EgressType string 57 | 58 | const ( 59 | EgressTypeDirect = "Direct" 60 | EgressTypeClusterProxy = "ClusterProxy" 61 | ) 62 | 63 | type ClusterGatewayTrafficEgressClusterProxy struct { 64 | ProxyServerHost string `json:"proxyServerHost"` 65 | ProxyServerPort int32 `json:"proxyServerPort"` 66 | Credentials ClusterGatewayTrafficEgressClusterProxyCredential `json:"credentials"` 67 | } 68 | 69 | type ClusterGatewayTrafficEgressClusterProxyCredential struct { 70 | Namespace string `json:"namespace"` 71 | ProxyClientSecretName string `json:"proxyClientSecretName"` 72 | ProxyClientCASecretName string `json:"proxyClientCASecretName"` 73 | } 74 | 75 | type ClusterGatewaySecretManagement struct { 76 | // +optional 77 | // +kubebuilder:default=ManagedServiceAccount 78 | Type SecretManagementType `json:"type"` 79 | // +optional 80 | ManagedServiceAccount *SecretManagementManagedServiceAccount `json:"managedServiceAccount,omitempty"` 81 | } 82 | 83 | // +kubebuilder:validation:Enum=Manual;ManagedServiceAccount 84 | type SecretManagementType string 85 | 86 | const ( 87 | SecretManagementTypeManual = "Manual" 88 | SecretManagementTypeManagedServiceAccount = "ManagedServiceAccount" 89 | ) 90 | 91 | type SecretManagementManagedServiceAccount struct { 92 | // +optional 93 | // +kubebuilder:default=cluster-gateway 94 | Name string `json:"name"` 95 | } 96 | 97 | const ( 98 | ConditionTypeClusterGatewayDeployed = "ClusterGatewayDeployed" 99 | ) 100 | -------------------------------------------------------------------------------- /cmd/apiserver/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "net/http" 19 | 20 | "k8s.io/apimachinery/pkg/util/sets" 21 | "k8s.io/apiserver/pkg/endpoints/request" 22 | "k8s.io/apiserver/pkg/server" 23 | genericfilters "k8s.io/apiserver/pkg/server/filters" 24 | "k8s.io/klog/v2" 25 | "sigs.k8s.io/apiserver-runtime/pkg/builder" 26 | 27 | "github.com/oam-dev/cluster-gateway/pkg/config" 28 | "github.com/oam-dev/cluster-gateway/pkg/metrics" 29 | "github.com/oam-dev/cluster-gateway/pkg/options" 30 | "github.com/oam-dev/cluster-gateway/pkg/util/singleton" 31 | 32 | // +kubebuilder:scaffold:resource-imports 33 | clusterv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 34 | "github.com/oam-dev/cluster-gateway/pkg/apis/generated" 35 | 36 | _ "github.com/oam-dev/cluster-gateway/pkg/featuregates" 37 | ) 38 | 39 | func main() { 40 | 41 | // registering metrics 42 | metrics.Register() 43 | 44 | cmd, err := builder.APIServer. 45 | // +kubebuilder:scaffold:resource-register 46 | WithResource(&clusterv1alpha1.ClusterGateway{}). 47 | WithResource(&clusterv1alpha1.VirtualCluster{}). 48 | WithLocalDebugExtension(). 49 | ExposeLoopbackMasterClientConfig(). 50 | ExposeLoopbackAuthorizer(). 51 | WithoutEtcd(). 52 | WithConfigFns(func(config *server.RecommendedConfig) *server.RecommendedConfig { 53 | config.LongRunningFunc = func(r *http.Request, requestInfo *request.RequestInfo) bool { 54 | if requestInfo.Resource == "clustergateways" && requestInfo.Subresource == "proxy" { 55 | return true 56 | } 57 | return genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString())(r, requestInfo) 58 | } 59 | return config 60 | }, config.WithUserAgent). 61 | WithOptionsFns(func(options *builder.ServerOptions) *builder.ServerOptions { 62 | if err := config.ValidateSecret(); err != nil { 63 | klog.Fatal(err) 64 | } 65 | if err := config.ValidateClusterProxy(); err != nil { 66 | klog.Fatal(err) 67 | } 68 | if err := clusterv1alpha1.LoadGlobalClusterGatewayProxyConfig(); err != nil { 69 | klog.Fatal(err) 70 | } 71 | return options 72 | }). 73 | WithServerFns(func(server *builder.GenericAPIServer) *builder.GenericAPIServer { 74 | server.Handler.FullHandlerChain = clusterv1alpha1.NewClusterGatewayProxyRequestEscaper(server.Handler.FullHandlerChain) 75 | return server 76 | }). 77 | WithPostStartHook("init-master-loopback-client", singleton.InitLoopbackClient). 78 | WithOpenAPIDefinitions("Cluster Gateway", "1.0.0", generated.GetOpenAPIDefinitions). 79 | Build() 80 | if err != nil { 81 | klog.Fatal(err) 82 | } 83 | config.AddLogFlags(cmd.Flags()) 84 | config.AddSecretFlags(cmd.Flags()) 85 | config.AddVirtualClusterFlags(cmd.Flags()) 86 | config.AddClusterProxyFlags(cmd.Flags()) 87 | config.AddProxyAuthorizationFlags(cmd.Flags()) 88 | config.AddUserAgentFlags(cmd.Flags()) 89 | config.AddClusterGatewayProxyConfig(cmd.Flags()) 90 | cmd.Flags().BoolVarP(&options.OCMIntegration, "ocm-integration", "", false, 91 | "Enabling OCM integration, reading cluster CA and api endpoint from managed "+ 92 | "cluster.") 93 | if err := cmd.Execute(); err != nil { 94 | klog.Fatal(err) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/validation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | package v1alpha1 16 | 17 | import ( 18 | "encoding/base64" 19 | "fmt" 20 | "net/url" 21 | 22 | "k8s.io/apimachinery/pkg/util/sets" 23 | "k8s.io/apimachinery/pkg/util/validation/field" 24 | ) 25 | 26 | func ValidateClusterGateway(c *ClusterGateway) field.ErrorList { 27 | var errs field.ErrorList 28 | errs = append(errs, ValidateClusterGatewaySpec(&c.Spec, field.NewPath("spec"))...) 29 | return errs 30 | } 31 | 32 | func ValidateClusterGatewaySpec(c *ClusterGatewaySpec, path *field.Path) field.ErrorList { 33 | var errs field.ErrorList 34 | if len(c.Provider) == 0 { 35 | errs = append(errs, field.Required(path.Child("provider"), "should set provider")) 36 | } 37 | errs = append(errs, ValidateClusterGatewaySpecAccess(&c.Access, path.Child("access"))...) 38 | return errs 39 | } 40 | 41 | func ValidateClusterGatewaySpecAccess(c *ClusterAccess, path *field.Path) field.ErrorList { 42 | var errs field.ErrorList 43 | switch c.Endpoint.Type { 44 | case ClusterEndpointTypeConst: 45 | if len(c.Endpoint.Const.Address) == 0 { 46 | errs = append(errs, field.Required(path.Child("endpoint"), "should provide cluster endpoint")) 47 | } 48 | u, err := url.Parse(c.Endpoint.Const.Address) 49 | if err != nil { 50 | errs = append(errs, field.Invalid(path.Child("endpoint"), c.Endpoint, fmt.Sprintf("failed parsing as URL: %v", err))) 51 | return errs 52 | } 53 | if u.Scheme != "https" { 54 | errs = append(errs, field.Invalid(path.Child("endpoint"), c.Endpoint, "scheme must be https")) 55 | } 56 | if len(c.Endpoint.Const.CABundle) == 0 && 57 | (c.Endpoint.Const.Insecure == nil || *c.Endpoint.Const.Insecure == false) { 58 | errs = append(errs, field.Required(path.Child("caBundle"), "required for non-insecure endpoint")) 59 | } 60 | } 61 | if c.Credential != nil { 62 | errs = append(errs, ValidateClusterGatewaySpecAccessCredential(c.Credential, path.Child("credential"))...) 63 | } 64 | return errs 65 | } 66 | 67 | func ValidateClusterGatewaySpecAccessCredential(c *ClusterAccessCredential, path *field.Path) field.ErrorList { 68 | var errs field.ErrorList 69 | supportedCredTypes := sets.NewString(string(CredentialTypeServiceAccountToken), string(CredentialTypeX509Certificate)) 70 | if !supportedCredTypes.Has(string(c.Type)) { 71 | errs = append(errs, field.NotSupported(path.Child("type"), c.Type, supportedCredTypes.List())) 72 | } 73 | switch c.Type { 74 | case CredentialTypeServiceAccountToken: 75 | if _, err := base64.StdEncoding.DecodeString(c.ServiceAccountToken); err == nil { 76 | errs = append(errs, field.Invalid(path.Child("serviceAccountToken"), c.ServiceAccountToken, "should not be base64 encoded")) 77 | } 78 | if len(c.ServiceAccountToken) == 0 { 79 | errs = append(errs, field.Required(path.Child("serviceAccountToken"), "should provide service-account token")) 80 | } 81 | case CredentialTypeX509Certificate: 82 | if c.X509 == nil { 83 | errs = append(errs, field.Required(path.Child("x509"), "should provide x509 certificate and private-key")) 84 | } else { 85 | if len(c.X509.Certificate) == 0 { 86 | errs = append(errs, field.Required(path.Child("x509").Child("certificate"), "should provide x509 certificate")) 87 | } 88 | if len(c.X509.PrivateKey) == 0 { 89 | errs = append(errs, field.Required(path.Child("x509").Child("privateKey"), "should provide x509 private key")) 90 | } 91 | // TODO: test if certificate and private-key matches modulus 92 | } 93 | } 94 | return errs 95 | } 96 | -------------------------------------------------------------------------------- /hack/patch/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | /* 4 | Copyright 2021 The KubeVela Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | 22 | "github.com/pkg/errors" 23 | "github.com/spf13/cobra" 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/types" 27 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 28 | apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" 29 | "sigs.k8s.io/controller-runtime/pkg/client" 30 | "sigs.k8s.io/controller-runtime/pkg/client/config" 31 | 32 | "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 33 | ) 34 | 35 | const ( 36 | FlagAPIServiceName = "target-APIService" 37 | FlagSecretName = "secret-name" 38 | FlagSecretNamespace = "secret-namespace" 39 | FlagSecretCABundleKey = "secret-ca-bundle-key" 40 | ) 41 | 42 | func buildSchemeOrDie() *runtime.Scheme { 43 | scheme := runtime.NewScheme() 44 | if err := clientgoscheme.AddToScheme(scheme); err != nil { 45 | fmt.Printf("build client-go scheme error: %v\n", err) 46 | os.Exit(1) 47 | } 48 | if err := apiregistrationv1.AddToScheme(scheme); err != nil { 49 | fmt.Printf("build api-registration scheme error: %v\n", err) 50 | os.Exit(1) 51 | } 52 | return scheme 53 | } 54 | 55 | func main() { 56 | var APIServiceName string 57 | var secretName string 58 | var secretNamespace string 59 | var secretCABundleKey string 60 | cmd := &cobra.Command{ 61 | Use: "patch", 62 | Short: "patch APIService CABundle from given secret", 63 | RunE: func(cmd *cobra.Command, args []string) error { 64 | c, err := client.New(config.GetConfigOrDie(), client.Options{Scheme: buildSchemeOrDie()}) 65 | if err != nil { 66 | return errors.Wrapf(err, "get k8s client error") 67 | } 68 | ctx := context.Background() 69 | secret := &v1.Secret{} 70 | if err = c.Get(ctx, types.NamespacedName{Namespace: secretNamespace, Name: secretName}, secret); err != nil { 71 | return errors.Wrapf(err, "failed to get source secret") 72 | } 73 | apiService := &apiregistrationv1.APIService{} 74 | if err = c.Get(ctx, types.NamespacedName{Name: APIServiceName}, apiService); err != nil { 75 | return errors.Wrapf(err, "failed to get APIService") 76 | } 77 | caBundle, ok := secret.Data[secretCABundleKey] 78 | if !ok { 79 | return fmt.Errorf("failed to find caBundle in secret(%s/%s), key: %s", secretNamespace, secretName, secretCABundleKey) 80 | } 81 | apiService.Spec.InsecureSkipTLSVerify = false 82 | apiService.Spec.CABundle = caBundle 83 | if err = c.Update(ctx, apiService); err != nil { 84 | return errors.Wrapf(err, "failed to update APIService") 85 | } 86 | fmt.Printf("successfully update APIService %s caBundle: \n%s\n", APIServiceName, caBundle) 87 | return nil 88 | }, 89 | } 90 | gv := v1alpha1.SchemeGroupVersion 91 | apiServiceName := gv.Version + "." + gv.Group 92 | cmd.Flags().StringVar(&APIServiceName, FlagAPIServiceName, apiServiceName, "specify the target APIService to patch caBundle") 93 | cmd.Flags().StringVar(&secretName, FlagSecretName, "", "specify the source secret name") 94 | cmd.Flags().StringVar(&secretNamespace, FlagSecretNamespace, "", "specify the source secret namespace") 95 | cmd.Flags().StringVar(&secretCABundleKey, FlagSecretCABundleKey, "ca", "specify the CABundle key in source secret") 96 | if err := cmd.Execute(); err != nil { 97 | fmt.Printf("%v\n", err) 98 | os.Exit(1) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /pkg/addon/agent/addon.go: -------------------------------------------------------------------------------- 1 | package agent 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/pkg/errors" 7 | rbacv1 "k8s.io/api/rbac/v1" 8 | apierrors "k8s.io/apimachinery/pkg/api/errors" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/apimachinery/pkg/runtime" 11 | "k8s.io/apimachinery/pkg/types" 12 | "k8s.io/client-go/rest" 13 | "open-cluster-management.io/addon-framework/pkg/agent" 14 | addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" 15 | clusterv1 "open-cluster-management.io/api/cluster/v1" 16 | "sigs.k8s.io/controller-runtime/pkg/client" 17 | 18 | proxyv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/proxy/v1alpha1" 19 | "github.com/oam-dev/cluster-gateway/pkg/common" 20 | ) 21 | 22 | var _ agent.AgentAddon = &clusterGatewayAddonManager{} 23 | 24 | func NewClusterGatewayAddonManager(cfg *rest.Config, c client.Client) agent.AgentAddon { 25 | return &clusterGatewayAddonManager{ 26 | clientConfig: cfg, 27 | client: c, 28 | } 29 | } 30 | 31 | type clusterGatewayAddonManager struct { 32 | clientConfig *rest.Config 33 | client client.Client 34 | } 35 | 36 | func (c *clusterGatewayAddonManager) Manifests(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) { 37 | if len(addon.Status.AddOnConfiguration.CRName) == 0 { 38 | return nil, nil 39 | } 40 | cfg := &proxyv1alpha1.ClusterGatewayConfiguration{} 41 | if err := c.client.Get( 42 | context.TODO(), types.NamespacedName{ 43 | Name: addon.Status.AddOnConfiguration.CRName, 44 | }, 45 | cfg); err != nil { 46 | if apierrors.IsNotFound(err) { 47 | return nil, nil 48 | } 49 | return nil, errors.Wrapf(err, "failed getting gateway configuration") 50 | } 51 | 52 | if cfg.Spec.SecretManagement.Type == proxyv1alpha1.SecretManagementTypeManual { 53 | return nil, nil 54 | } 55 | switch cfg.Spec.SecretManagement.Type { 56 | case proxyv1alpha1.SecretManagementTypeManagedServiceAccount: 57 | managedServiceAccountAddon := &addonv1alpha1.ManagedClusterAddOn{} 58 | if err := c.client.Get( 59 | context.TODO(), 60 | types.NamespacedName{ 61 | Namespace: cluster.Name, 62 | Name: "managed-serviceaccount", 63 | }, 64 | managedServiceAccountAddon); err != nil { 65 | if apierrors.IsNotFound(err) { 66 | return nil, nil 67 | } 68 | return nil, err 69 | } 70 | return buildClusterGatewayOutboundPermission( 71 | managedServiceAccountAddon.Spec.InstallNamespace, 72 | cfg.Spec.SecretManagement.ManagedServiceAccount.Name), nil 73 | case proxyv1alpha1.SecretManagementTypeManual: 74 | fallthrough 75 | default: 76 | return nil, nil 77 | } 78 | } 79 | 80 | func (c *clusterGatewayAddonManager) GetAgentAddonOptions() agent.AgentAddonOptions { 81 | return agent.AgentAddonOptions{ 82 | AddonName: common.AddonName, 83 | InstallStrategy: agent.InstallAllStrategy(common.InstallNamespace), 84 | HealthProber: &agent.HealthProber{ 85 | Type: agent.HealthProberTypeNone, // TODO: switch to ManifestWork-based prober 86 | }, 87 | } 88 | } 89 | 90 | func buildClusterGatewayOutboundPermission(serviceAccountNamespace, serviceAccountName string) []runtime.Object { 91 | const clusterRoleName = "open-cluster-management:cluster-gateway:default" 92 | clusterGatewayClusterRole := &rbacv1.ClusterRole{ 93 | TypeMeta: metav1.TypeMeta{ 94 | APIVersion: "rbac.authorization.k8s.io/v1", 95 | Kind: "ClusterRole", 96 | }, 97 | ObjectMeta: metav1.ObjectMeta{ 98 | Name: clusterRoleName, 99 | }, 100 | Rules: []rbacv1.PolicyRule{ 101 | { 102 | APIGroups: []string{"*"}, 103 | Verbs: []string{"*"}, 104 | Resources: []string{"*"}, 105 | }, 106 | }, 107 | } 108 | clusterGatewayClusterRoleBinding := &rbacv1.ClusterRoleBinding{ 109 | TypeMeta: metav1.TypeMeta{ 110 | APIVersion: "rbac.authorization.k8s.io/v1", 111 | Kind: "ClusterRoleBinding", 112 | }, 113 | ObjectMeta: metav1.ObjectMeta{ 114 | Name: clusterRoleName, 115 | }, 116 | RoleRef: rbacv1.RoleRef{ 117 | Kind: "ClusterRole", 118 | Name: clusterRoleName, 119 | }, 120 | Subjects: []rbacv1.Subject{ 121 | { 122 | Kind: rbacv1.ServiceAccountKind, 123 | Namespace: serviceAccountNamespace, 124 | Name: serviceAccountName, 125 | }, 126 | }, 127 | } 128 | return []runtime.Object{ 129 | clusterGatewayClusterRole, 130 | clusterGatewayClusterRoleBinding, 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/transport.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "net/http" 7 | "net/url" 8 | "strconv" 9 | "time" 10 | 11 | "github.com/pkg/errors" 12 | "google.golang.org/grpc" 13 | grpccredentials "google.golang.org/grpc/credentials" 14 | "google.golang.org/grpc/keepalive" 15 | k8snet "k8s.io/apimachinery/pkg/util/net" 16 | restclient "k8s.io/client-go/rest" 17 | konnectivity "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" 18 | "sigs.k8s.io/apiserver-network-proxy/pkg/util" 19 | 20 | "github.com/oam-dev/cluster-gateway/pkg/config" 21 | ) 22 | 23 | var DialerGetter = func(ctx context.Context) (k8snet.DialFunc, error) { 24 | tlsCfg, err := util.GetClientTLSConfig( 25 | config.ClusterProxyCAFile, 26 | config.ClusterProxyCertFile, 27 | config.ClusterProxyKeyFile, 28 | config.ClusterProxyHost, 29 | nil) 30 | if err != nil { 31 | return nil, err 32 | } 33 | dialerTunnel, err := konnectivity.CreateSingleUseGrpcTunnelWithContext( 34 | ctx, 35 | context.Background(), 36 | net.JoinHostPort(config.ClusterProxyHost, strconv.Itoa(config.ClusterProxyPort)), 37 | grpc.WithTransportCredentials(grpccredentials.NewTLS(tlsCfg)), 38 | grpc.WithKeepaliveParams(keepalive.ClientParameters{ 39 | Time: 20 * time.Second, 40 | }), 41 | ) 42 | if err != nil { 43 | return nil, err 44 | } 45 | return dialerTunnel.DialContext, nil 46 | } 47 | 48 | func NewConfigFromCluster(ctx context.Context, c *ClusterGateway) (*restclient.Config, error) { 49 | cfg := &restclient.Config{ 50 | Timeout: time.Second * 40, 51 | } 52 | // setting up endpoint 53 | switch c.Spec.Access.Endpoint.Type { 54 | case ClusterEndpointTypeConst: 55 | cfg.Host = c.Spec.Access.Endpoint.Const.Address 56 | cfg.CAData = c.Spec.Access.Endpoint.Const.CABundle 57 | if c.Spec.Access.Endpoint.Const.Insecure != nil && *c.Spec.Access.Endpoint.Const.Insecure { 58 | cfg.TLSClientConfig = restclient.TLSClientConfig{Insecure: true} 59 | } 60 | u, err := url.Parse(c.Spec.Access.Endpoint.Const.Address) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | const missingPort = "missing port in address" 66 | host, _, err := net.SplitHostPort(u.Host) 67 | if err != nil { 68 | addrErr, ok := err.(*net.AddrError) 69 | if !ok { 70 | return nil, err 71 | } 72 | if addrErr.Err != missingPort { 73 | return nil, err 74 | } 75 | host = u.Host 76 | } 77 | cfg.ServerName = host // apiserver may listen on SNI cert 78 | 79 | if c.Spec.Access.Endpoint.Const.ProxyURL != nil { 80 | _url, _err := url.Parse(*c.Spec.Access.Endpoint.Const.ProxyURL) 81 | if _err != nil { 82 | return nil, _err 83 | } 84 | cfg.Proxy = http.ProxyURL(_url) 85 | } 86 | case ClusterEndpointTypeClusterProxy: 87 | cfg.Host = c.Name // the same as the cluster name 88 | cfg.Insecure = true 89 | cfg.CAData = nil 90 | dail, err := DialerGetter(ctx) 91 | if err != nil { 92 | return nil, err 93 | } 94 | cfg.Dial = dail 95 | } 96 | // setting up credentials 97 | switch c.Spec.Access.Credential.Type { 98 | case CredentialTypeDynamic: 99 | if token := c.Spec.Access.Credential.ServiceAccountToken; token != "" { 100 | cfg.BearerToken = token 101 | } 102 | 103 | if c.Spec.Access.Credential.X509 != nil && len(c.Spec.Access.Credential.X509.Certificate) > 0 && len(c.Spec.Access.Credential.X509.PrivateKey) > 0 { 104 | cfg.CertData = c.Spec.Access.Credential.X509.Certificate 105 | cfg.KeyData = c.Spec.Access.Credential.X509.PrivateKey 106 | } 107 | 108 | case CredentialTypeServiceAccountToken: 109 | cfg.BearerToken = c.Spec.Access.Credential.ServiceAccountToken 110 | 111 | case CredentialTypeX509Certificate: 112 | cfg.CertData = c.Spec.Access.Credential.X509.Certificate 113 | cfg.KeyData = c.Spec.Access.Credential.X509.PrivateKey 114 | } 115 | return cfg, nil 116 | } 117 | 118 | func GetEndpointURL(c *ClusterGateway) (*url.URL, error) { 119 | switch c.Spec.Access.Endpoint.Type { 120 | case ClusterEndpointTypeConst: 121 | urlAddr, err := url.Parse(c.Spec.Access.Endpoint.Const.Address) 122 | if err != nil { 123 | return nil, errors.Wrapf(err, "failed parsing url from cluster %s invalid value %s", 124 | c.Name, c.Spec.Access.Endpoint.Const.Address) 125 | } 126 | return urlAddr, nil 127 | case ClusterEndpointTypeClusterProxy: 128 | return &url.URL{ 129 | Scheme: "https", 130 | Host: c.Name, 131 | }, nil 132 | default: 133 | return nil, errors.New("unsupported cluster gateway endpoint type") 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /hack/crd/addon/clustermanagementaddon.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | name: clustermanagementaddons.addon.open-cluster-management.io 5 | spec: 6 | group: addon.open-cluster-management.io 7 | names: 8 | kind: ClusterManagementAddOn 9 | listKind: ClusterManagementAddOnList 10 | plural: clustermanagementaddons 11 | singular: clustermanagementaddon 12 | scope: Cluster 13 | preserveUnknownFields: false 14 | versions: 15 | - additionalPrinterColumns: 16 | - jsonPath: .spec.addOnMeta.displayName 17 | name: DISPLAY NAME 18 | type: string 19 | - jsonPath: .spec.addOnConfiguration.crdName 20 | name: CRD NAME 21 | type: string 22 | name: v1alpha1 23 | schema: 24 | openAPIV3Schema: 25 | description: ClusterManagementAddOn represents the registration of an add-on 26 | to the cluster manager. This resource allows the user to discover which 27 | add-on is available for the cluster manager and also provides metadata information 28 | about the add-on. This resource also provides a linkage to ManagedClusterAddOn, 29 | the name of the ClusterManagementAddOn resource will be used for the namespace-scoped 30 | ManagedClusterAddOn resource. ClusterManagementAddOn is a cluster-scoped 31 | resource. 32 | type: object 33 | properties: 34 | apiVersion: 35 | description: 'APIVersion defines the versioned schema of this representation 36 | of an object. Servers should convert recognized schemas to the latest 37 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 38 | type: string 39 | kind: 40 | description: 'Kind is a string value representing the REST resource this 41 | object represents. Servers may infer this from the endpoint the client 42 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 43 | type: string 44 | metadata: 45 | type: object 46 | spec: 47 | description: spec represents a desired configuration for the agent on 48 | the cluster management add-on. 49 | type: object 50 | properties: 51 | addOnConfiguration: 52 | description: addOnConfiguration is a reference to configuration information 53 | for the add-on. In scenario where a multiple add-ons share the same 54 | add-on CRD, multiple ClusterManagementAddOn resources need to be 55 | created and reference the same AddOnConfiguration. 56 | type: object 57 | properties: 58 | crName: 59 | description: crName is the name of the CR used to configure instances 60 | of the managed add-on. This field should be configured if add-on 61 | CR have a consistent name across the all of the ManagedCluster 62 | instaces. 63 | type: string 64 | crdName: 65 | description: crdName is the name of the CRD used to configure 66 | instances of the managed add-on. This field should be configured 67 | if the add-on have a CRD that controls the configuration of 68 | the add-on. 69 | type: string 70 | addOnMeta: 71 | description: addOnMeta is a reference to the metadata information 72 | for the add-on. 73 | type: object 74 | properties: 75 | description: 76 | description: description represents the detailed description of 77 | the add-on. 78 | type: string 79 | displayName: 80 | description: displayName represents the name of add-on that will 81 | be displayed. 82 | type: string 83 | status: 84 | description: status represents the current status of cluster management 85 | add-on. 86 | type: object 87 | served: true 88 | storage: true 89 | subresources: 90 | status: {} 91 | status: 92 | acceptedNames: 93 | kind: "" 94 | plural: "" 95 | conditions: [] 96 | storedVersions: [] -------------------------------------------------------------------------------- /e2e/roundtrip/clustergateway.go: -------------------------------------------------------------------------------- 1 | package roundtrip 2 | 3 | import ( 4 | "context" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/apimachinery/pkg/types" 10 | "k8s.io/client-go/kubernetes" 11 | clusterv1 "open-cluster-management.io/api/cluster/v1" 12 | 13 | "github.com/oam-dev/cluster-gateway/e2e/framework" 14 | multicluster "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/transport" 15 | clusterv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 16 | ) 17 | 18 | const ( 19 | roundtripTestBasename = "roundtrip" 20 | ) 21 | 22 | var _ = Describe("Basic RoundTrip Test", func() { 23 | f := framework.NewE2EFramework(roundtripTestBasename) 24 | 25 | It("ClusterGateway in the API discovery", 26 | func() { 27 | By("Discovering ClusterGateway") 28 | nativeClient := f.HubNativeClient() 29 | resources, err := nativeClient.Discovery(). 30 | ServerResourcesForGroupVersion("cluster.core.oam.dev/v1alpha1") 31 | Expect(err).NotTo(HaveOccurred()) 32 | apiFound := false 33 | for _, resource := range resources.APIResources { 34 | if resource.Kind == "ClusterGateway" { 35 | apiFound = true 36 | } 37 | } 38 | if !apiFound { 39 | Fail(`Api ClusterGateway not found`) 40 | } 41 | }) 42 | 43 | It("ManagedCluster present", 44 | func() { 45 | By("Getting ManagedCluster") 46 | if f.IsOCMInstalled() { 47 | runtimeClient := f.HubRuntimeClient() 48 | cluster := &clusterv1.ManagedCluster{} 49 | err := runtimeClient.Get(context.TODO(), types.NamespacedName{ 50 | Name: f.TestClusterName(), 51 | }, cluster) 52 | Expect(err).NotTo(HaveOccurred()) 53 | } 54 | }) 55 | 56 | It("ClusterGateway can be read via GET", 57 | func() { 58 | By("Getting ClusterGateway") 59 | runtimeClient := f.HubRuntimeClient() 60 | clusterGateway := &clusterv1alpha1.ClusterGateway{} 61 | err := runtimeClient.Get(context.TODO(), types.NamespacedName{ 62 | Name: f.TestClusterName(), 63 | }, clusterGateway) 64 | Expect(err).NotTo(HaveOccurred()) 65 | }) 66 | 67 | It("ClusterGateway can be read via LIST", 68 | func() { 69 | By("Getting ClusterGateway") 70 | runtimeClient := f.HubRuntimeClient() 71 | clusterGatewayList := &clusterv1alpha1.ClusterGatewayList{} 72 | err := runtimeClient.List(context.TODO(), clusterGatewayList) 73 | Expect(err).NotTo(HaveOccurred()) 74 | clusterFound := false 75 | for _, clusterGateway := range clusterGatewayList.Items { 76 | if clusterGateway.Name == f.TestClusterName() { 77 | clusterFound = true 78 | } 79 | } 80 | if !clusterFound { 81 | Fail(`ClusterGateway not found`) 82 | } 83 | }) 84 | 85 | It("ClusterGateway healthiness can be manipulated", 86 | func() { 87 | By("get healthiness") 88 | gw, err := f.HubGatewayClient(). 89 | ClusterV1alpha1(). 90 | ClusterGateways(). 91 | GetHealthiness(context.TODO(), f.TestClusterName(), metav1.GetOptions{}) 92 | Expect(err).NotTo(HaveOccurred()) 93 | Expect(gw).ShouldNot(BeNil()) 94 | Expect(gw.Status.Healthy).To(BeFalse()) 95 | By("update healthiness") 96 | gw.Status.Healthy = true 97 | gw.Status.HealthyReason = clusterv1alpha1.HealthyReasonTypeConnectionTimeout 98 | updated, err := f.HubGatewayClient(). 99 | ClusterV1alpha1(). 100 | ClusterGateways(). 101 | UpdateHealthiness(context.TODO(), gw, metav1.UpdateOptions{}) 102 | Expect(err).NotTo(HaveOccurred()) 103 | Expect(updated).NotTo(BeNil()) 104 | Expect(updated.Status.Healthy).To(BeTrue()) 105 | Expect(updated.Status.HealthyReason).To(Equal(clusterv1alpha1.HealthyReasonTypeConnectionTimeout)) 106 | }) 107 | 108 | It("Probing cluster health (raw)", 109 | func() { 110 | resp, err := f.HubNativeClient().Discovery(). 111 | RESTClient(). 112 | Get(). 113 | AbsPath( 114 | "apis/cluster.core.oam.dev/v1alpha1/clustergateways", 115 | f.TestClusterName(), 116 | "proxy", 117 | "healthz", 118 | ).DoRaw(context.TODO()) 119 | Expect(err).NotTo(HaveOccurred()) 120 | Expect(string(resp)).To(Equal("ok")) 121 | }) 122 | 123 | It("Probing cluster health (context)", 124 | func() { 125 | cfg := f.HubRESTConfig() 126 | cfg.WrapTransport = multicluster.NewClusterGatewayRoundTripper 127 | multiClusterClient, err := kubernetes.NewForConfig(cfg) 128 | Expect(err).NotTo(HaveOccurred()) 129 | resp, err := multiClusterClient.RESTClient(). 130 | Get(). 131 | AbsPath("healthz"). 132 | DoRaw(multicluster.WithMultiClusterContext(context.TODO(), f.TestClusterName())) 133 | Expect(err).NotTo(HaveOccurred()) 134 | Expect(string(resp)).To(Equal("ok")) 135 | }) 136 | 137 | }) 138 | -------------------------------------------------------------------------------- /cmd/addon-manager/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "os" 7 | 8 | "github.com/oam-dev/cluster-gateway/pkg/addon/agent" 9 | "github.com/oam-dev/cluster-gateway/pkg/addon/controllers" 10 | proxyv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/proxy/v1alpha1" 11 | "github.com/oam-dev/cluster-gateway/pkg/util" 12 | "github.com/oam-dev/cluster-gateway/pkg/util/cert" 13 | "k8s.io/apimachinery/pkg/runtime" 14 | "k8s.io/client-go/informers" 15 | "k8s.io/client-go/kubernetes" 16 | nativescheme "k8s.io/client-go/kubernetes/scheme" 17 | "k8s.io/klog/v2" 18 | "k8s.io/klog/v2/klogr" 19 | apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" 20 | "open-cluster-management.io/addon-framework/pkg/addonmanager" 21 | addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" 22 | ocmauthv1alpha1 "open-cluster-management.io/managed-serviceaccount/api/v1alpha1" 23 | ctrl "sigs.k8s.io/controller-runtime" 24 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 25 | "sigs.k8s.io/controller-runtime/pkg/webhook" 26 | ) 27 | 28 | var ( 29 | scheme = runtime.NewScheme() 30 | setupLog = ctrl.Log.WithName("setup") 31 | ) 32 | 33 | func init() { 34 | addonv1alpha1.AddToScheme(scheme) 35 | proxyv1alpha1.AddToScheme(scheme) 36 | nativescheme.AddToScheme(scheme) 37 | apiregistrationv1.AddToScheme(scheme) 38 | ocmauthv1alpha1.AddToScheme(scheme) 39 | } 40 | 41 | func main() { 42 | var metricsAddr string 43 | var enableLeaderElection bool 44 | var probeAddr string 45 | var signerSecretName string 46 | 47 | logger := klogr.New() 48 | klog.SetOutput(os.Stdout) 49 | klog.InitFlags(flag.CommandLine) 50 | flag.StringVar(&metricsAddr, "metrics-bind-address", ":48080", "The address the metric endpoint binds to.") 51 | flag.StringVar(&probeAddr, "health-probe-bind-address", ":48081", "The address the probe endpoint binds to.") 52 | flag.BoolVar(&enableLeaderElection, "leader-elect", false, 53 | "Enable leader election for controller manager. "+ 54 | "Enabling this will ensure there is only one active controller manager.") 55 | flag.StringVar(&signerSecretName, "signer-secret-name", "cluster-gateway-signer", 56 | "The name of the secret to store the signer CA") 57 | 58 | flag.Parse() 59 | ctrl.SetLogger(logger) 60 | 61 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 62 | Scheme: scheme, 63 | Metrics: metricsserver.Options{ 64 | BindAddress: metricsAddr, 65 | }, 66 | WebhookServer: webhook.NewServer(webhook.Options{ 67 | Port: 9443, 68 | }), 69 | HealthProbeBindAddress: probeAddr, 70 | LeaderElection: enableLeaderElection, 71 | LeaderElectionID: "cluster-gateway-addon-manager", 72 | }) 73 | if err != nil { 74 | setupLog.Error(err, "unable to start manager") 75 | os.Exit(1) 76 | } 77 | 78 | currentNamespace := os.Getenv("NAMESPACE") 79 | if len(currentNamespace) == 0 { 80 | inClusterNamespace, err := util.GetInClusterNamespace() 81 | if err != nil { 82 | klog.Fatal("the manager should be either running in a container or specify NAMESPACE environment") 83 | } 84 | currentNamespace = inClusterNamespace 85 | } 86 | 87 | caPair, err := cert.EnsureCAPair(mgr.GetConfig(), currentNamespace, signerSecretName) 88 | if err != nil { 89 | setupLog.Error(err, "unable to ensure ca signer") 90 | } 91 | nativeClient, err := kubernetes.NewForConfig(mgr.GetConfig()) 92 | if err != nil { 93 | setupLog.Error(err, "unable to instantiate legacy client") 94 | os.Exit(1) 95 | } 96 | informerFactory := informers.NewSharedInformerFactory(nativeClient, 0) 97 | if err := controllers.SetupClusterGatewayInstallerWithManager( 98 | mgr, 99 | caPair, 100 | nativeClient, 101 | informerFactory.Core().V1().Secrets().Lister()); err != nil { 102 | setupLog.Error(err, "unable to setup installer") 103 | os.Exit(1) 104 | } 105 | if err := controllers.SetupClusterGatewayHealthProberWithManager(mgr); err != nil { 106 | setupLog.Error(err, "unable to setup health prober") 107 | os.Exit(1) 108 | } 109 | 110 | ctx := context.Background() 111 | go informerFactory.Start(ctx.Done()) 112 | 113 | addonManager, err := addonmanager.New(mgr.GetConfig()) 114 | if err != nil { 115 | setupLog.Error(err, "unable to set up ready check") 116 | os.Exit(1) 117 | } 118 | if err := addonManager.AddAgent(agent.NewClusterGatewayAddonManager( 119 | mgr.GetConfig(), 120 | mgr.GetClient(), 121 | )); err != nil { 122 | setupLog.Error(err, "unable to register addon manager") 123 | os.Exit(1) 124 | } 125 | 126 | ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) 127 | defer cancel() 128 | go addonManager.Start(ctx) 129 | 130 | if err := mgr.Start(ctx); err != nil { 131 | panic(err) 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /pkg/util/cert/secret.go: -------------------------------------------------------------------------------- 1 | package cert 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | 7 | "github.com/oam-dev/cluster-gateway/pkg/common" 8 | 9 | "github.com/pkg/errors" 10 | corev1 "k8s.io/api/core/v1" 11 | apierrors "k8s.io/apimachinery/pkg/api/errors" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "k8s.io/apimachinery/pkg/labels" 14 | "k8s.io/apimachinery/pkg/selection" 15 | "k8s.io/client-go/kubernetes" 16 | corev1lister "k8s.io/client-go/listers/core/v1" 17 | ) 18 | 19 | func CopySecret(kubeClient kubernetes.Interface, sourceNamespace string, sourceName string, targetNamespace, targetName string) error { 20 | sourceSecret, err := kubeClient.CoreV1(). 21 | Secrets(sourceNamespace). 22 | Get(context.TODO(), sourceName, metav1.GetOptions{}) 23 | if err != nil { 24 | return errors.Wrapf(err, "failed getting source secret %v/%v: %v", sourceNamespace, sourceName, err) 25 | } 26 | shouldCreate := false 27 | existingTargetSecret, err := kubeClient.CoreV1(). 28 | Secrets(targetNamespace). 29 | Get(context.TODO(), targetName, metav1.GetOptions{}) 30 | if err != nil { 31 | if !apierrors.IsNotFound(err) { 32 | return errors.Wrapf(err, "failed getting target secret %v/%v: %v", targetNamespace, targetName, err) 33 | } 34 | shouldCreate = true 35 | } 36 | if shouldCreate { 37 | existingTargetSecret = sourceSecret.DeepCopy() 38 | existingTargetSecret.Namespace = targetNamespace 39 | existingTargetSecret.Name = targetName 40 | existingTargetSecret.UID = "" 41 | existingTargetSecret.ResourceVersion = "" 42 | if _, err := kubeClient.CoreV1().Secrets(targetNamespace). 43 | Create(context.TODO(), existingTargetSecret, metav1.CreateOptions{}); err != nil { 44 | if !apierrors.IsAlreadyExists(err) { 45 | return errors.Wrapf(err, "failed creating CA secret") 46 | } 47 | } 48 | return nil 49 | } 50 | 51 | if IsSubset(sourceSecret.Data, existingTargetSecret.Data) { 52 | return nil 53 | } 54 | Merge(sourceSecret.Data, existingTargetSecret.Data) 55 | _, err = kubeClient.CoreV1(). 56 | Secrets(targetNamespace). 57 | Update(context.TODO(), existingTargetSecret, metav1.UpdateOptions{}) 58 | return err 59 | } 60 | 61 | func IsSubset(subset, superset map[string][]byte) bool { 62 | for k, v := range subset { 63 | if !bytes.Equal(v, superset[k]) { 64 | return false 65 | } 66 | } 67 | return true 68 | } 69 | 70 | func Merge(l, r map[string][]byte) { 71 | for k, v := range l { 72 | r[k] = v 73 | } 74 | } 75 | 76 | type SecretControl interface { 77 | Get(ctx context.Context, name string) (*corev1.Secret, error) 78 | List(ctx context.Context) ([]*corev1.Secret, error) 79 | } 80 | 81 | var _ SecretControl = &directApiSecretControl{} 82 | 83 | type directApiSecretControl struct { 84 | secretNamespace string 85 | kubeClient kubernetes.Interface 86 | } 87 | 88 | func (d *directApiSecretControl) Get(ctx context.Context, name string) (*corev1.Secret, error) { 89 | return d.kubeClient.CoreV1().Secrets(d.secretNamespace).Get(ctx, name, metav1.GetOptions{}) 90 | } 91 | 92 | func (d *directApiSecretControl) List(ctx context.Context) ([]*corev1.Secret, error) { 93 | requirement, err := labels.NewRequirement( 94 | common.LabelKeyClusterCredentialType, 95 | selection.Exists, 96 | nil) 97 | if err != nil { 98 | return nil, err 99 | } 100 | secretList, err := d.kubeClient.CoreV1().Secrets(d.secretNamespace).List(ctx, metav1.ListOptions{ 101 | LabelSelector: labels.NewSelector().Add(*requirement).String(), 102 | }) 103 | if err != nil { 104 | return nil, err 105 | } 106 | secrets := make([]*corev1.Secret, len(secretList.Items)) 107 | for i := range secretList.Items { 108 | secrets[i] = &secretList.Items[i] 109 | } 110 | return secrets, nil 111 | } 112 | 113 | var _ SecretControl = &cachedSecretControl{} 114 | 115 | type cachedSecretControl struct { 116 | secretNamespace string 117 | secretLister corev1lister.SecretLister 118 | } 119 | 120 | func (c *cachedSecretControl) Get(ctx context.Context, name string) (*corev1.Secret, error) { 121 | return c.secretLister.Secrets(c.secretNamespace).Get(name) 122 | } 123 | 124 | func (c *cachedSecretControl) List(ctx context.Context) ([]*corev1.Secret, error) { 125 | requirement, err := labels.NewRequirement( 126 | common.LabelKeyClusterCredentialType, 127 | selection.Exists, 128 | nil) 129 | if err != nil { 130 | return nil, err 131 | } 132 | selector := labels.NewSelector().Add(*requirement) 133 | return c.secretLister.Secrets(c.secretNamespace).List(selector) 134 | } 135 | 136 | func NewDirectApiSecretControl(secretNamespace string, kubeClient kubernetes.Interface) SecretControl { 137 | return &directApiSecretControl{ 138 | secretNamespace: secretNamespace, 139 | kubeClient: kubeClient, 140 | } 141 | } 142 | 143 | func NewCachedSecretControl(secretNamespace string, secretLister corev1lister.SecretLister) SecretControl { 144 | return &cachedSecretControl{ 145 | secretNamespace: secretNamespace, 146 | secretLister: secretLister, 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /pkg/addon/controllers/health.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "k8s.io/apimachinery/pkg/api/meta" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/client-go/kubernetes" 10 | "k8s.io/client-go/rest" 11 | addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" 12 | ctrl "sigs.k8s.io/controller-runtime" 13 | "sigs.k8s.io/controller-runtime/pkg/client" 14 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 15 | 16 | multicluster "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/transport" 17 | "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 18 | "github.com/oam-dev/cluster-gateway/pkg/common" 19 | "github.com/oam-dev/cluster-gateway/pkg/event" 20 | "github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned" 21 | ) 22 | 23 | var ( 24 | healthLog = ctrl.Log.WithName("ClusterGatewayHealthProber") 25 | ) 26 | var _ reconcile.Reconciler = &ClusterGatewayHealthProber{} 27 | 28 | type ClusterGatewayHealthProber struct { 29 | multiClusterRestClient rest.Interface 30 | gatewayClient versioned.Interface 31 | runtimeClient client.Client 32 | } 33 | 34 | func SetupClusterGatewayHealthProberWithManager(mgr ctrl.Manager) error { 35 | gatewayClient, err := versioned.NewForConfig(mgr.GetConfig()) 36 | if err != nil { 37 | return err 38 | } 39 | copied := rest.CopyConfig(mgr.GetConfig()) 40 | copied.WrapTransport = multicluster.NewClusterGatewayRoundTripper 41 | multiClusterClient, err := kubernetes.NewForConfig(copied) 42 | if err != nil { 43 | return err 44 | } 45 | prober := &ClusterGatewayHealthProber{ 46 | multiClusterRestClient: multiClusterClient.Discovery().RESTClient(), 47 | gatewayClient: gatewayClient, 48 | runtimeClient: mgr.GetClient(), 49 | } 50 | src := event.AddOnHealthResyncHandler(mgr.GetClient(), time.Second) 51 | return ctrl.NewControllerManagedBy(mgr). 52 | For(&addonv1alpha1.ManagedClusterAddOn{}). 53 | WatchesRawSource(src). 54 | Complete(prober) 55 | } 56 | 57 | func (c *ClusterGatewayHealthProber) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { 58 | if request.Name != common.AddonName { 59 | return reconcile.Result{}, nil 60 | } 61 | clusterName := request.Namespace 62 | gw, err := c.gatewayClient.ClusterV1alpha1(). 63 | ClusterGateways(). 64 | GetHealthiness(ctx, clusterName, metav1.GetOptions{}) 65 | if err != nil { 66 | return reconcile.Result{}, err 67 | } 68 | resp, healthErr := c.multiClusterRestClient. 69 | Get(). 70 | AbsPath("healthz"). 71 | DoRaw(multicluster.WithMultiClusterContext(context.TODO(), clusterName)) 72 | healthy := string(resp) == "ok" && healthErr == nil 73 | if !healthy { 74 | healthErrMsg := "" 75 | if healthErr != nil { 76 | healthErrMsg = healthErr.Error() 77 | } 78 | healthLog.Info("Cluster unhealthy", "cluster", clusterName, 79 | "body", string(resp), 80 | "error", healthErrMsg) 81 | } 82 | if healthy != gw.Status.Healthy { 83 | gw.Status.Healthy = healthy 84 | if !healthy { 85 | if healthErr != nil { 86 | gw.Status.HealthyReason = v1alpha1.HealthyReasonType(healthErr.Error()) 87 | } 88 | } else { 89 | gw.Status.HealthyReason = "" 90 | } 91 | healthLog.Info("Updating cluster healthiness", 92 | "cluster", clusterName, 93 | "healthy", healthy) 94 | _, err = c.gatewayClient.ClusterV1alpha1(). 95 | ClusterGateways(). 96 | UpdateHealthiness(ctx, gw, metav1.UpdateOptions{}) 97 | if err != nil { 98 | return reconcile.Result{}, err 99 | } 100 | } 101 | 102 | addon := &addonv1alpha1.ManagedClusterAddOn{} 103 | if err := c.runtimeClient.Get(ctx, request.NamespacedName, addon); err != nil { 104 | return reconcile.Result{}, err 105 | } 106 | if healthy != meta.IsStatusConditionTrue(addon.Status.Conditions, addonv1alpha1.ManagedClusterAddOnConditionAvailable) { 107 | healthLog.Info("Updating addon healthiness", 108 | "cluster", clusterName, 109 | "healthy", healthy) 110 | healthyStatus := metav1.ConditionTrue 111 | if !healthy { 112 | healthyStatus = metav1.ConditionFalse 113 | } 114 | if healthy { 115 | meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ 116 | Type: addonv1alpha1.ManagedClusterAddOnConditionAvailable, 117 | Status: healthyStatus, 118 | Reason: "SuccessfullyProbedHealthz", 119 | Message: "Returned OK", 120 | }) 121 | } else { 122 | errMsg := "Unknown" 123 | if healthErr != nil { 124 | errMsg = healthErr.Error() 125 | } else if len(string(resp)) > 0 { 126 | errMsg = string(resp) 127 | } 128 | meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ 129 | Type: addonv1alpha1.ManagedClusterAddOnConditionAvailable, 130 | Status: healthyStatus, 131 | Reason: "FailedProbingHealthz", 132 | Message: errMsg, 133 | }) 134 | } 135 | if err := c.runtimeClient.Status().Update(ctx, addon); err != nil { 136 | return reconcile.Result{}, err 137 | } 138 | } 139 | 140 | if !healthy { 141 | return reconcile.Result{ 142 | Requeue: true, 143 | RequeueAfter: 5 * time.Second, 144 | }, nil 145 | } 146 | return reconcile.Result{}, nil 147 | } 148 | -------------------------------------------------------------------------------- /pkg/util/exec/exec.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "sync" 10 | "time" 11 | 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "k8s.io/apimachinery/pkg/runtime" 14 | "k8s.io/apimachinery/pkg/runtime/schema" 15 | "k8s.io/apimachinery/pkg/runtime/serializer" 16 | 17 | "k8s.io/client-go/pkg/apis/clientauthentication" 18 | "k8s.io/client-go/pkg/apis/clientauthentication/install" 19 | clientauthenticationv1 "k8s.io/client-go/pkg/apis/clientauthentication/v1" 20 | clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" 21 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 22 | ) 23 | 24 | var ( 25 | scheme = runtime.NewScheme() 26 | 27 | codecs = serializer.NewCodecFactory(scheme) 28 | 29 | apiVersions = map[string]schema.GroupVersion{ 30 | clientauthenticationv1beta1.SchemeGroupVersion.String(): clientauthenticationv1beta1.SchemeGroupVersion, 31 | clientauthenticationv1.SchemeGroupVersion.String(): clientauthenticationv1.SchemeGroupVersion, 32 | } 33 | 34 | credentials sync.Map 35 | ) 36 | 37 | func init() { 38 | install.Install(scheme) 39 | } 40 | 41 | func IssueClusterCredential(name string, ec *clientcmdapi.ExecConfig) (*clientauthentication.ExecCredential, error) { 42 | if name == "" { 43 | return nil, errors.New("cluster name not provided") 44 | } 45 | 46 | value, found := credentials.Load(name) 47 | if found { 48 | cred, ok := value.(*clientauthentication.ExecCredential) 49 | if !ok { 50 | return nil, errors.New("failed to convert item in cache to ExecCredential") 51 | } 52 | 53 | now := &metav1.Time{Time: time.Now().Add(time.Minute)} // expires a minute early 54 | 55 | if cred.Status != nil && cred.Status.ExpirationTimestamp.Before(now) { 56 | credentials.Delete(name) 57 | return IssueClusterCredential(name, ec) // credential expired, calling function again 58 | } 59 | 60 | return cred, nil // credential on cache still valid 61 | } 62 | 63 | cred, err := issueClusterCredential(ec) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | if cred.Status != nil && !cred.Status.ExpirationTimestamp.IsZero() { 69 | credentials.Store(name, cred) // storing credential in cache 70 | } 71 | 72 | return cred, nil 73 | } 74 | 75 | func issueClusterCredential(ec *clientcmdapi.ExecConfig) (*clientauthentication.ExecCredential, error) { 76 | if ec == nil { 77 | return nil, errors.New("exec config not provided") 78 | } 79 | 80 | if ec.Command == "" { 81 | return nil, errors.New("missing \"command\" property on exec config object") 82 | } 83 | 84 | command, err := exec.LookPath(ec.Command) 85 | if err != nil { 86 | return nil, unwrapExecCommandError(ec.Command, err) 87 | } 88 | 89 | cmd := exec.Command(command, ec.Args...) 90 | cmd.Env = os.Environ() 91 | 92 | for _, env := range ec.Env { 93 | cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", env.Name, env.Value)) 94 | } 95 | 96 | var stderr, stdout bytes.Buffer 97 | cmd.Stderr = &stderr 98 | cmd.Stdout = &stdout 99 | 100 | if err := cmd.Run(); err != nil { 101 | return nil, unwrapExecCommandError(command, err) 102 | } 103 | 104 | ecgv, err := schema.ParseGroupVersion(ec.APIVersion) 105 | if err != nil { 106 | return nil, fmt.Errorf("failed to parse exec config API version: %v", err) 107 | } 108 | 109 | cred := &clientauthentication.ExecCredential{ 110 | TypeMeta: metav1.TypeMeta{ 111 | APIVersion: ec.APIVersion, 112 | Kind: "ExecCredential", 113 | }, 114 | Spec: clientauthentication.ExecCredentialSpec{}, 115 | } 116 | 117 | gv, ok := apiVersions[ec.APIVersion] 118 | if !ok { 119 | return nil, fmt.Errorf("exec plugin: invalid apiVersion %q", ec.APIVersion) 120 | } 121 | 122 | _, gvk, err := codecs.UniversalDecoder(gv).Decode(stdout.Bytes(), nil, cred) 123 | if err != nil { 124 | return nil, fmt.Errorf("decoding stdout: %v", err) 125 | } 126 | 127 | if gvk.Group != ecgv.Group || gvk.Version != ecgv.Version { 128 | return nil, fmt.Errorf("exec plugin is configured to use API version %s, plugin returned version %s", ecgv, schema.GroupVersion{Group: gvk.Group, Version: gvk.Version}) 129 | } 130 | 131 | if cred.Status == nil { 132 | return nil, fmt.Errorf("exec plugin didn't return a status field") 133 | } 134 | 135 | if cred.Status.Token == "" && cred.Status.ClientCertificateData == "" && cred.Status.ClientKeyData == "" { 136 | return nil, fmt.Errorf("exec plugin didn't return a token or cert/key pair") 137 | } 138 | 139 | if (cred.Status.ClientCertificateData == "") != (cred.Status.ClientKeyData == "") { 140 | return nil, fmt.Errorf("exec plugin returned only certificate or key, not both") 141 | } 142 | 143 | return cred, nil 144 | } 145 | 146 | func unwrapExecCommandError(path string, err error) error { 147 | switch err.(type) { 148 | case *exec.Error: // Binary does not exist (see exec.Error). 149 | return fmt.Errorf("exec: executable %s not found", path) 150 | 151 | case *exec.ExitError: // Binary execution failed (see exec.Cmd.Run()). 152 | e := err.(*exec.ExitError) 153 | return fmt.Errorf("exec: executable %s failed with exit code %d", path, e.ProcessState.ExitCode()) 154 | 155 | default: 156 | return fmt.Errorf("exec: %v", err) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /pkg/util/singleton/loopback.go: -------------------------------------------------------------------------------- 1 | package singleton 2 | 3 | import ( 4 | "time" 5 | 6 | "k8s.io/apimachinery/pkg/util/wait" 7 | "k8s.io/apiserver/pkg/server" 8 | utilfeature "k8s.io/apiserver/pkg/util/feature" 9 | corev1informer "k8s.io/client-go/informers/core/v1" 10 | "k8s.io/client-go/kubernetes" 11 | corev1lister "k8s.io/client-go/listers/core/v1" 12 | clientgorest "k8s.io/client-go/rest" 13 | "k8s.io/client-go/tools/cache" 14 | "k8s.io/klog/v2" 15 | ocmclient "open-cluster-management.io/api/client/cluster/clientset/versioned" 16 | clusterinformers "open-cluster-management.io/api/client/cluster/informers/externalversions" 17 | clusterv1Lister "open-cluster-management.io/api/client/cluster/listers/cluster/v1" 18 | "sigs.k8s.io/apiserver-runtime/pkg/util/loopback" 19 | "sigs.k8s.io/controller-runtime/pkg/client" 20 | controllerruntimeconfig "sigs.k8s.io/controller-runtime/pkg/client/config" 21 | 22 | "github.com/oam-dev/cluster-gateway/pkg/config" 23 | "github.com/oam-dev/cluster-gateway/pkg/featuregates" 24 | "github.com/oam-dev/cluster-gateway/pkg/util/cert" 25 | clusterutil "github.com/oam-dev/cluster-gateway/pkg/util/cluster" 26 | "github.com/oam-dev/cluster-gateway/pkg/util/scheme" 27 | ) 28 | 29 | var kubeClient kubernetes.Interface 30 | var ocmClient ocmclient.Interface 31 | var ctrlClient client.Client 32 | 33 | var secretInformer cache.SharedIndexInformer 34 | var secretLister corev1lister.SecretLister 35 | 36 | var secretControl cert.SecretControl 37 | 38 | var clusterInformer cache.SharedIndexInformer 39 | var clusterLister clusterv1Lister.ManagedClusterLister 40 | var clusterControl clusterutil.OCMClusterControl 41 | 42 | func GetSecretControl() cert.SecretControl { 43 | return secretControl 44 | } 45 | 46 | func GetOCMClient() ocmclient.Interface { 47 | return ocmClient 48 | } 49 | 50 | func GetKubeClient() kubernetes.Interface { 51 | return kubeClient 52 | } 53 | 54 | func GetCtrlClient() client.Client { 55 | return ctrlClient 56 | } 57 | 58 | func SetCtrlClient(cli client.Client) { 59 | ctrlClient = cli 60 | } 61 | 62 | func InitLoopbackClient(ctx server.PostStartHookContext) error { 63 | var err error 64 | cfg := loopback.GetLoopbackMasterClientConfig() 65 | if cfg == nil { 66 | if cfg, err = controllerruntimeconfig.GetConfig(); err != nil { 67 | return err 68 | } 69 | } 70 | copiedCfg := clientgorest.CopyConfig(cfg) 71 | copiedCfg.RateLimiter = nil 72 | kubeClient, err = kubernetes.NewForConfig(copiedCfg) 73 | if err != nil { 74 | return err 75 | } 76 | ocmClient, err = ocmclient.NewForConfig(copiedCfg) 77 | if err != nil { 78 | return err 79 | } 80 | ctrlClient, err = client.New(copiedCfg, client.Options{Scheme: scheme.Scheme}) 81 | if err != nil { 82 | return err 83 | } 84 | if utilfeature.DefaultMutableFeatureGate.Enabled(featuregates.SecretCache) { 85 | if err := setInformer(kubeClient, ctx.StopCh); err != nil { 86 | return err 87 | } 88 | secretControl = cert.NewCachedSecretControl(config.SecretNamespace, secretLister) 89 | } 90 | if secretControl == nil { 91 | secretControl = cert.NewDirectApiSecretControl(config.SecretNamespace, kubeClient) 92 | } 93 | 94 | if utilfeature.DefaultMutableFeatureGate.Enabled(featuregates.OCMClusterCache) { 95 | installed, err := clusterutil.IsOCMManagedClusterInstalled(ocmClient) 96 | if err != nil { 97 | klog.Error(err) 98 | } else if !installed { 99 | klog.Infof("OCM ManagedCluster CRD not installed, skip bootstrapping informer for OCM ManagedCluster") 100 | } else if err := setOCMClusterInformer(ocmClient, ctx.StopCh); err != nil { 101 | return err 102 | } 103 | clusterControl = clusterutil.NewCacheOCMClusterControl(clusterLister) 104 | } 105 | if clusterControl == nil { 106 | clusterControl = clusterutil.NewDirectOCMClusterControl(ocmClient) 107 | } 108 | 109 | return nil 110 | } 111 | 112 | func setInformer(k kubernetes.Interface, stopCh <-chan struct{}) error { 113 | secretInformer = corev1informer.NewSecretInformer(k, config.SecretNamespace, 0, cache.Indexers{ 114 | cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, 115 | }) 116 | secretLister = corev1lister.NewSecretLister(secretInformer.GetIndexer()) 117 | go secretInformer.Run(stopCh) 118 | return wait.PollImmediateUntil(time.Second, func() (done bool, err error) { 119 | return secretInformer.HasSynced(), nil 120 | }, stopCh) 121 | } 122 | 123 | // SetSecretControl is for test only 124 | func SetSecretControl(ctrl cert.SecretControl) { 125 | secretControl = ctrl 126 | } 127 | 128 | // SetOCMClient is for test only 129 | func SetOCMClient(c ocmclient.Interface) { 130 | ocmClient = c 131 | } 132 | 133 | // SetKubeClient is for test only 134 | func SetKubeClient(k kubernetes.Interface) { 135 | kubeClient = k 136 | } 137 | 138 | func setOCMClusterInformer(c ocmclient.Interface, stopCh <-chan struct{}) error { 139 | ocmClusterInformers := clusterinformers.NewSharedInformerFactory(c, 0) 140 | clusterInformer = ocmClusterInformers.Cluster().V1().ManagedClusters().Informer() 141 | clusterLister = ocmClusterInformers.Cluster().V1().ManagedClusters().Lister() 142 | go clusterInformer.Run(stopCh) 143 | return wait.PollImmediateUntil(time.Second, func() (done bool, err error) { 144 | return clusterInformer.HasSynced(), nil 145 | }, stopCh) 146 | } 147 | 148 | func GetClusterControl() clusterutil.OCMClusterControl { 149 | return clusterControl 150 | } 151 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Image URL to use all building/pushing image targets 3 | IMG ?= controller:latest 4 | IMG_TAG ?= latest 5 | # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) 6 | CRD_OPTIONS ?= "crd" 7 | 8 | OS?=linux 9 | ARCH?=amd64 10 | 11 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 12 | ifeq (,$(shell go env GOBIN)) 13 | GOBIN=$(shell go env GOPATH)/bin 14 | else 15 | GOBIN=$(shell go env GOBIN) 16 | endif 17 | 18 | VERSION=v0.0.21 19 | 20 | all: manager 21 | 22 | # Run tests 23 | test: generate generate-openapi fmt vet manifests 24 | go test ./pkg/... -coverprofile cover.out 25 | 26 | # Build manager binary 27 | manager: generate generate-openapi fmt vet 28 | go build -o bin/manager ./cmd/apiserver/main.go 29 | 30 | # Run against the configured Kubernetes cluster in ~/.kube/config 31 | run: generate generate-openapi fmt vet manifests 32 | go run ./cmd/apiserver/main.go 33 | 34 | local-run: 35 | go run ./cmd/apiserver/main.go \ 36 | --standalone-debug-mode=true \ 37 | --bind-address=127.0.0.1 \ 38 | --etcd-servers=127.0.0.1:2379 \ 39 | --secure-port=9443 40 | 41 | # Install CRDs into a cluster 42 | install: manifests kustomize 43 | $(KUSTOMIZE) build config/crd | kubectl apply -f - 44 | 45 | # Uninstall CRDs from a cluster 46 | uninstall: manifests kustomize 47 | $(KUSTOMIZE) build config/crd | kubectl delete -f - 48 | 49 | # Deploy controller in the configured Kubernetes cluster in ~/.kube/config 50 | deploy: manifests kustomize 51 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} 52 | $(KUSTOMIZE) build config/default | kubectl apply -f - 53 | 54 | # Run go fmt against code 55 | fmt: 56 | go fmt ./... 57 | 58 | # Run go vet against code 59 | vet: 60 | go vet ./... 61 | 62 | # Build the docker image 63 | docker-build: test 64 | docker build . -t ${IMG} 65 | 66 | # Push the docker image 67 | docker-push: 68 | docker push ${IMG} 69 | 70 | # find or download controller-gen 71 | # download controller-gen if necessary 72 | controller-gen: 73 | ifeq (, $(shell which controller-gen)) 74 | @{ \ 75 | set -e ;\ 76 | CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ 77 | cd $$CONTROLLER_GEN_TMP_DIR ;\ 78 | go mod init tmp ;\ 79 | go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 ;\ 80 | rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ 81 | } 82 | CONTROLLER_GEN=$(GOBIN)/controller-gen 83 | else 84 | CONTROLLER_GEN=$(shell which controller-gen) 85 | endif 86 | 87 | openapi-gen: 88 | ifeq (, $(shell which openapi-gen)) 89 | @{ \ 90 | set -e ;\ 91 | CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ 92 | cd $$CONTROLLER_GEN_TMP_DIR ;\ 93 | go mod init tmp ;\ 94 | go install k8s.io/kube-openapi/cmd/openapi-gen@v0.0.0-20240228011516-70dd3763d340 ;\ 95 | rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ 96 | } 97 | OPENAPI_GEN=$(GOBIN)/openapi-gen 98 | else 99 | OPENAPI_GEN=$(shell which openapi-gen) 100 | endif 101 | 102 | kustomize: 103 | ifeq (, $(shell which kustomize)) 104 | @{ \ 105 | set -e ;\ 106 | KUSTOMIZE_GEN_TMP_DIR=$$(mktemp -d) ;\ 107 | cd $$KUSTOMIZE_GEN_TMP_DIR ;\ 108 | go mod init tmp ;\ 109 | go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4 ;\ 110 | rm -rf $$KUSTOMIZE_GEN_TMP_DIR ;\ 111 | }chore: update event handling to use typed events and bump dependencies 112 | KUSTOMIZE=$(GOBIN)/kustomize 113 | else 114 | KUSTOMIZE=$(shell which kustomize) 115 | endif 116 | 117 | 118 | client-gen: 119 | go install k8s.io/code-generator/cmd/client-gen@v0.31.1 120 | apiserver-runtime-gen \ 121 | --module github.com/oam-dev/cluster-gateway \ 122 | -g client-gen \ 123 | --versions=github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1 \ 124 | --install-generators=false 125 | 126 | 127 | generate: controller-gen 128 | ${CONTROLLER_GEN} object:headerFile="hack/boilerplate.go.txt" paths="./pkg/apis/proxy/..." 129 | 130 | .PHONY: generate-openapi 131 | generate-openapi: openapi-gen 132 | ${OPENAPI_GEN} \ 133 | --output-pkg github.com/oam-dev/cluster-gateway/pkg/apis \ 134 | --output-file zz_generated.openapi.go \ 135 | --output-dir ./pkg/apis/generated \ 136 | --output-pkg "generated" \ 137 | --go-header-file ./hack/boilerplate.go.txt \ 138 | ./pkg/apis/proxy/v1alpha1 \ 139 | ./pkg/apis/cluster/v1alpha1 \ 140 | k8s.io/apimachinery/pkg/api/resource \ 141 | k8s.io/apimachinery/pkg/apis/meta/v1 \ 142 | k8s.io/apimachinery/pkg/runtime \ 143 | k8s.io/apimachinery/pkg/version 144 | 145 | manifests: controller-gen 146 | ${CONTROLLER_GEN} $(CRD_OPTIONS) \ 147 | paths="./pkg/apis/proxy/..." \ 148 | rbac:roleName=manager-role \ 149 | output:crd:artifacts:config=hack/crd/bases 150 | 151 | gateway: 152 | docker build -t oamdev/cluster-gateway:${IMG_TAG} \ 153 | --build-arg OS=${OS} \ 154 | --build-arg ARCH=${ARCH} \ 155 | -f cmd/apiserver/Dockerfile \ 156 | . 157 | 158 | ocm-addon-manager: 159 | docker build -t oamdev/cluster-gateway-addon-manager:${IMG_TAG} \ 160 | --build-arg OS=${OS} \ 161 | --build-arg ARCH=${ARCH} \ 162 | -f cmd/addon-manager/Dockerfile \ 163 | . 164 | 165 | image: gateway ocm-addon-manager 166 | 167 | e2e-binary: 168 | mkdir -p bin 169 | go test -o bin/e2e -c ./e2e/ 170 | 171 | e2e-binary-ocm: 172 | mkdir -p bin 173 | go test -o bin/e2e.ocm -c ./e2e/ocm/ 174 | 175 | e2e-bench-binary: 176 | go test -c ./e2e/benchmark/ 177 | 178 | test-e2e: e2e-binary 179 | ./bin/e2e --test-cluster=loopback 180 | 181 | test-e2e-ocm: e2e-binary-ocm 182 | ./bin/e2e.ocm --test-cluster=loopback -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/cluster/v1alpha1/clustergateway_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | package v1alpha1 16 | 17 | import ( 18 | "context" 19 | "net/http" 20 | "strings" 21 | 22 | "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" 23 | "github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned/scheme" 24 | contextutil "github.com/oam-dev/cluster-gateway/pkg/util/context" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/kubernetes" 27 | "k8s.io/client-go/rest" 28 | "k8s.io/client-go/transport" 29 | "sigs.k8s.io/controller-runtime/pkg/client" 30 | ) 31 | 32 | type ClusterGatewayExpansion interface { 33 | RESTClient(clusterName string) rest.Interface 34 | 35 | GetKubernetesClient(clusterName string) kubernetes.Interface 36 | GetControllerRuntimeClient(clusterName string, options client.Options) (client.Client, error) 37 | 38 | RoundTripperForCluster(clusterName string) http.RoundTripper 39 | RoundTripperForClusterFromContext() http.RoundTripper 40 | RoundTripperForClusterFromContextWrapper(http.RoundTripper) http.RoundTripper 41 | 42 | GetHealthiness(ctx context.Context, name string, options metav1.GetOptions) (*v1alpha1.ClusterGateway, error) 43 | UpdateHealthiness(ctx context.Context, clusterGateway *v1alpha1.ClusterGateway, options metav1.UpdateOptions) (*v1alpha1.ClusterGateway, error) 44 | } 45 | 46 | func (c *clusterGateways) RESTClient(clusterName string) rest.Interface { 47 | restClient := c.client.(*rest.RESTClient) 48 | shallowCopiedClient := *restClient 49 | shallowCopiedHTTPClient := *(restClient.Client) 50 | shallowCopiedClient.Client = &shallowCopiedHTTPClient 51 | shallowCopiedClient.Client.Transport = c.RoundTripperForCluster(clusterName) 52 | return &shallowCopiedClient 53 | } 54 | 55 | func (c *clusterGateways) RoundTripperForCluster(clusterName string) http.RoundTripper { 56 | return c.getRoundTripper(func(ctx context.Context) string { 57 | return clusterName 58 | }) 59 | } 60 | 61 | func (c *clusterGateways) GetKubernetesClient(clusterName string) kubernetes.Interface { 62 | return kubernetes.New(c.RESTClient(clusterName)) 63 | } 64 | 65 | func (c *clusterGateways) GetControllerRuntimeClient(clusterName string, options client.Options) (client.Client, error) { 66 | return client.New(&rest.Config{ 67 | Host: c.client.Verb("").URL().String(), 68 | WrapTransport: c.RoundTripperForClusterWrapperGenerator(clusterName), 69 | }, options) 70 | } 71 | 72 | func (c *clusterGateways) RoundTripperForClusterFromContext() http.RoundTripper { 73 | return c.getRoundTripper(contextutil.GetClusterName) 74 | } 75 | 76 | func (c *clusterGateways) RoundTripperForClusterFromContextWrapper(http.RoundTripper) http.RoundTripper { 77 | return c.RoundTripperForClusterFromContext() 78 | } 79 | 80 | func (c *clusterGateways) RoundTripperForClusterWrapperGenerator(clusterName string) transport.WrapperFunc { 81 | return func(rt http.RoundTripper) http.RoundTripper { 82 | return c.getRoundTripper(func(_ context.Context) string { return clusterName }) 83 | } 84 | } 85 | 86 | func (c *clusterGateways) getRoundTripper(clusterNameGetter func(ctx context.Context) string) http.RoundTripper { 87 | restClient := c.client.(*rest.RESTClient) 88 | return gatewayAPIPrefixPrepender{ 89 | clusterNameGetter: clusterNameGetter, 90 | delegate: restClient.Client.Transport, 91 | } 92 | } 93 | 94 | var _ http.RoundTripper = &gatewayAPIPrefixPrepender{} 95 | 96 | type gatewayAPIPrefixPrepender struct { 97 | clusterNameGetter func(ctx context.Context) string 98 | delegate http.RoundTripper 99 | } 100 | 101 | func (p gatewayAPIPrefixPrepender) RoundTrip(req *http.Request) (*http.Response, error) { 102 | originalPath := req.URL.Path 103 | prefix := "/apis/" + 104 | v1alpha1.SchemeGroupVersion.Group + 105 | "/" + 106 | v1alpha1.SchemeGroupVersion.Version + 107 | "/clustergateways/" 108 | if !strings.HasPrefix(originalPath, "/") { 109 | originalPath = "/" + originalPath 110 | } 111 | fullPath := prefix + p.clusterNameGetter(req.Context()) + "/proxy" + originalPath 112 | req.URL.Path = fullPath 113 | return p.delegate.RoundTrip(req) 114 | } 115 | 116 | func (c *clusterGateways) GetHealthiness(ctx context.Context, name string, options metav1.GetOptions) (*v1alpha1.ClusterGateway, error) { 117 | result := &v1alpha1.ClusterGateway{} 118 | err := c.client.Get(). 119 | Resource("clustergateways"). 120 | Name(name). 121 | VersionedParams(&options, scheme.ParameterCodec). 122 | SubResource("health"). 123 | Do(ctx). 124 | Into(result) 125 | return result, err 126 | } 127 | 128 | func (c *clusterGateways) UpdateHealthiness(ctx context.Context, clusterGateway *v1alpha1.ClusterGateway, options metav1.UpdateOptions) (*v1alpha1.ClusterGateway, error) { 129 | result := &v1alpha1.ClusterGateway{} 130 | err := c.client.Put(). 131 | Resource("clustergateways"). 132 | Name(clusterGateway.Name). 133 | VersionedParams(&options, scheme.ParameterCodec). 134 | Body(clusterGateway). 135 | SubResource("health"). 136 | Do(ctx). 137 | Into(result) 138 | return result, err 139 | } 140 | -------------------------------------------------------------------------------- /docs/local-run.md: -------------------------------------------------------------------------------- 1 | # Running Non-Etcd Apiserver Locally 2 | 3 | ### Setting Up Environment 4 | 5 | 1. Build the container: 6 | 7 | ```shell 8 | docker build \ 9 | -t "cluster-gateway:v0.0.0-non-etcd" \ 10 | -f cmd/apiserver/Dockerfile . 11 | ``` 12 | 13 | 2. Spawn a local KinD cluster: 14 | 15 | ```shell 16 | kind create cluster --name hub 17 | kind export kubeconfig --kubeconfig /tmp/hub.kubeconfig --name hub 18 | kind load docker-image "cluster-gateway:v0.0.0-non-etcd" --name hub 19 | ``` 20 | 21 | 3. Apply the manifests below: 22 | 23 | ```yaml 24 | apiVersion: apps/v1 25 | kind: Deployment 26 | metadata: 27 | name: gateway-deployment 28 | labels: 29 | app: gateway 30 | spec: 31 | replicas: 3 32 | selector: 33 | matchLabels: 34 | app: gateway 35 | template: 36 | metadata: 37 | labels: 38 | app: gateway 39 | spec: 40 | containers: 41 | - name: gateway 42 | image: "cluster-gateway:v0.0.0-non-etcd" 43 | command: 44 | - ./apiserver 45 | - --secure-port=9443 46 | - --secret-namespace=default 47 | - --feature-gates=APIPriorityAndFairness=false 48 | ports: 49 | - containerPort: 9443 50 | --- 51 | apiVersion: v1 52 | kind: Service 53 | metadata: 54 | name: gateway-service 55 | spec: 56 | selector: 57 | app: gateway 58 | ports: 59 | - protocol: TCP 60 | port: 9443 61 | targetPort: 9443 62 | --- 63 | apiVersion: apiregistration.k8s.io/v1 64 | kind: APIService 65 | metadata: 66 | name: v1alpha1.cluster.core.oam.dev 67 | labels: 68 | api: cluster-extension-apiserver 69 | apiserver: "true" 70 | spec: 71 | version: v1alpha1 72 | group: cluster.core.oam.dev 73 | groupPriorityMinimum: 2000 74 | service: 75 | name: gateway-service 76 | namespace: default 77 | port: 9443 78 | versionPriority: 10 79 | insecureSkipTLSVerify: true 80 | --- 81 | apiVersion: rbac.authorization.k8s.io/v1 82 | kind: RoleBinding 83 | metadata: 84 | name: system::extension-apiserver-authentication-reader:cluster-gateway 85 | namespace: kube-system 86 | roleRef: 87 | apiGroup: rbac.authorization.k8s.io 88 | kind: Role 89 | name: extension-apiserver-authentication-reader 90 | subjects: 91 | - kind: ServiceAccount 92 | name: default 93 | namespace: default 94 | --- 95 | apiVersion: rbac.authorization.k8s.io/v1 96 | kind: Role 97 | metadata: 98 | namespace: default 99 | name: cluster-gateway-secret-reader 100 | rules: 101 | - apiGroups: 102 | - "" 103 | resources: 104 | - "secrets" 105 | verbs: 106 | - get 107 | - list 108 | - watch 109 | --- 110 | apiVersion: rbac.authorization.k8s.io/v1 111 | kind: RoleBinding 112 | metadata: 113 | name: cluster-gateway-secret-reader 114 | namespace: default 115 | roleRef: 116 | apiGroup: rbac.authorization.k8s.io 117 | kind: Role 118 | name: cluster-gateway-secret-reader 119 | subjects: 120 | - kind: ServiceAccount 121 | name: default 122 | namespace: default 123 | --- 124 | ``` 125 | 126 | 4. Check if apiserver aggregation working properly: 127 | 128 | ```shell 129 | $ KUBECONFIG=/tmp/hub.kubeconfig kubectl api-resources | grep clustergateway 130 | $ KUBECONFIG=/tmp/hub.kubeconfig kubectl get clustergateway # A 404 error is expected 131 | ``` 132 | 133 | ### Proxying Multi-Cluster 134 | 135 | 1. Prepare a second cluster `managed1` that accessible from `hub`'s network. 136 | 137 | 2.1. Creates a secret containing X509 certificate/key to the hub cluster: 138 | 139 | ```yaml 140 | apiVersion: v1 141 | kind: Secret 142 | metadata: 143 | name: managed1 144 | labels: 145 | cluster.core.oam.dev/cluster-credential-type: X509 146 | type: Opaque # <--- Has to be opaque 147 | data: 148 | endpoint: "..." # Should NOT be 127.0.0.1 149 | ca.crt: "..." # ca cert for cluster "managed1" 150 | tls.crt: "..." # x509 cert for cluster "managed1" 151 | tls.key: "..." # private key for cluster "managed1" 152 | ``` 153 | 154 | 2.2. (Alternatively) Create a secret containing service-account token to the hub cluster: 155 | 156 | ```yaml 157 | apiVersion: v1 158 | kind: Secret 159 | metadata: 160 | name: managed1 161 | labels: 162 | cluster.core.oam.dev/cluster-credential-type: ServiceAccountToken 163 | type: Opaque # <--- Has to be opaque 164 | data: 165 | endpoint: "..." # ditto 166 | ca.crt: "..." # ditto 167 | token: "..." # working jwt token 168 | ``` 169 | 170 | 2.3. (Alternatively) Create a secret containing an exec config to dynamically fetch the cluster credential from an external command: 171 | 172 | ```yaml 173 | apiVersion: v1 174 | kind: Secret 175 | metadata: 176 | name: managed1 177 | labels: 178 | cluster.core.oam.dev/cluster-credential-type: Dynamic 179 | type: Opaque # <--- Has to be opaque 180 | data: 181 | endpoint: "..." # ditto 182 | exec: "..." # an exec config in JSON format; see ExecConfig (https://github.com/kubernetes/kubernetes/blob/2016fab3085562b4132e6d3774b6ded5ba9939fd/staging/src/k8s.io/client-go/tools/clientcmd/api/types.go#L206, https://kubernetes.io/docs/reference/access-authn-authz/authentication/#configuration) 183 | ``` 184 | 185 | 3. Proxy to cluster `managed1`'s `/healthz` endpoint 186 | 187 | ```shell 188 | $ KUBECONFIG=/tmp/hub.kubeconfig kubectl get \ 189 | --raw="/apis/cluster.core.oam.dev/v1alpha1/clustergateways/managed1/proxy/healthz" 190 | ``` 191 | 192 | 4. Craft a dedicated kubeconfig for proxying `managed1` from `hub` cluster: 193 | 194 | ```shell 195 | $ cat /tmp/hub.kubeconfig \ 196 | | sed 's/\(server: .*\)/\1\/apis\/cluster.core.oam.dev\/v1alpha1\/clustergateways\/managed1\/proxy\//' \ 197 | > /tmp/hub-managed1.kubeconfig 198 | ``` 199 | 200 | try the tweaked kubeconfig: 201 | 202 | ```shell 203 | # list namespaces under cluster managed1 204 | KUBECONFIG=/tmp/hub-managed1.kubeconfig kubectl get ns 205 | ``` 206 | 207 | ### Clean up 208 | 209 | 1. Deletes the sandbox clusters: 210 | 211 | ```shell 212 | $ kind delete cluster --name tmp 213 | ``` 214 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/clustergateway_proxy_configuration.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "regexp" 23 | 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apimachinery/pkg/util/yaml" 26 | "k8s.io/apiserver/pkg/authentication/user" 27 | "k8s.io/client-go/rest" 28 | "k8s.io/utils/strings/slices" 29 | 30 | "github.com/oam-dev/cluster-gateway/pkg/config" 31 | ) 32 | 33 | const ( 34 | AnnotationClusterGatewayProxyConfiguration = "cluster.core.oam.dev/cluster-gateway-proxy-configuration" 35 | ) 36 | 37 | type ClusterGatewayProxyConfiguration struct { 38 | metav1.TypeMeta `json:",inline"` 39 | Spec ClusterGatewayProxyConfigurationSpec `json:"spec"` 40 | } 41 | 42 | type ClusterGatewayProxyConfigurationSpec struct { 43 | ClientIdentityExchanger `json:"clientIdentityExchanger"` 44 | } 45 | 46 | type ClientIdentityExchanger struct { 47 | Rules []ClientIdentityExchangeRule `json:"rules,omitempty"` 48 | } 49 | 50 | type ClientIdentityExchangeType string 51 | 52 | const ( 53 | PrivilegedIdentityExchanger ClientIdentityExchangeType = "PrivilegedIdentityExchanger" 54 | StaticMappingIdentityExchanger ClientIdentityExchangeType = "StaticMappingIdentityExchanger" 55 | ExternalIdentityExchanger ClientIdentityExchangeType = "ExternalIdentityExchanger" 56 | ) 57 | 58 | type ClientIdentityExchangeRule struct { 59 | Name string `json:"name"` 60 | Type ClientIdentityExchangeType `json:"type"` 61 | Source *IdentityExchangerSource `json:"source"` 62 | 63 | Target *IdentityExchangerTarget `json:"target,omitempty"` 64 | URL *string `json:"url,omitempty"` 65 | } 66 | 67 | type IdentityExchangerTarget struct { 68 | User string `json:"user,omitempty"` 69 | Groups []string `json:"groups,omitempty"` 70 | UID string `json:"uid,omitempty"` 71 | } 72 | 73 | type IdentityExchangerSource struct { 74 | User *string `json:"user,omitempty"` 75 | Group *string `json:"group,omitempty"` 76 | UID *string `json:"uid,omitempty"` 77 | Cluster *string `json:"cluster,omitempty"` 78 | 79 | UserPattern *string `json:"userPattern,omitempty"` 80 | GroupPattern *string `json:"groupPattern,omitempty"` 81 | ClusterPattern *string `json:"clusterPattern,omitempty"` 82 | } 83 | 84 | var GlobalClusterGatewayProxyConfiguration = &ClusterGatewayProxyConfiguration{} 85 | 86 | func LoadGlobalClusterGatewayProxyConfig() error { 87 | if config.ClusterGatewayProxyConfigPath == "" { 88 | return nil 89 | } 90 | bs, err := os.ReadFile(config.ClusterGatewayProxyConfigPath) 91 | if err != nil { 92 | return err 93 | } 94 | return yaml.Unmarshal(bs, GlobalClusterGatewayProxyConfiguration) 95 | } 96 | 97 | func ExchangeIdentity(exchanger *ClientIdentityExchanger, userInfo user.Info, cluster string) (matched bool, ruleName string, projected *rest.ImpersonationConfig, err error) { 98 | for _, rule := range exchanger.Rules { 99 | if matched, projected, err = exchangeIdentity(&rule, userInfo, cluster); matched { 100 | return matched, rule.Name, projected, err 101 | } 102 | } 103 | return false, "", nil, nil 104 | } 105 | 106 | func exchangeIdentity(rule *ClientIdentityExchangeRule, userInfo user.Info, cluster string) (matched bool, projected *rest.ImpersonationConfig, err error) { 107 | if !matchIdentity(rule.Source, userInfo, cluster) { 108 | return false, nil, nil 109 | } 110 | switch rule.Type { 111 | case PrivilegedIdentityExchanger: 112 | return true, &rest.ImpersonationConfig{}, nil 113 | case StaticMappingIdentityExchanger: 114 | return true, &rest.ImpersonationConfig{ 115 | UserName: rule.Target.User, 116 | Groups: rule.Target.Groups, 117 | UID: rule.Target.UID, 118 | }, nil 119 | case ExternalIdentityExchanger: 120 | return true, nil, fmt.Errorf("ExternalIdentityExchanger is not implemented") 121 | } 122 | return true, nil, fmt.Errorf("unknown exchanger type: %s", rule.Type) 123 | } 124 | 125 | // denyQuery return true when the pattern is valid and could be used as regular expression, 126 | // and the given query does not match the pattern, otherwise return false 127 | func (in *IdentityExchangerSource) denyQuery(pattern *string, query string) bool { 128 | if pattern == nil { 129 | return false 130 | } 131 | matched, err := regexp.Match(*pattern, []byte(query)) 132 | if err != nil { 133 | return false 134 | } 135 | return !matched 136 | } 137 | 138 | // denyGroups return true if none of the group matches the given pattern 139 | func (in *IdentityExchangerSource) denyGroups(groupPattern *string, groups []string) bool { 140 | if groupPattern == nil { 141 | return false 142 | } 143 | for _, group := range groups { 144 | if !in.denyQuery(groupPattern, group) { 145 | return false 146 | } 147 | } 148 | return true 149 | } 150 | 151 | func matchIdentity(in *IdentityExchangerSource, userInfo user.Info, cluster string) bool { 152 | if in == nil { 153 | return false 154 | } 155 | switch { 156 | case in.User != nil && userInfo.GetName() != *in.User: 157 | return false 158 | case in.Group != nil && !slices.Contains(userInfo.GetGroups(), *in.Group): 159 | return false 160 | case in.UID != nil && userInfo.GetUID() != *in.UID: 161 | return false 162 | case in.Cluster != nil && cluster != *in.Cluster: 163 | return false 164 | case in.denyQuery(in.UserPattern, userInfo.GetName()): 165 | return false 166 | case in.denyGroups(in.GroupPattern, userInfo.GetGroups()): 167 | return false 168 | case in.denyQuery(in.ClusterPattern, cluster): 169 | return false 170 | } 171 | return true 172 | } 173 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | workflow_dispatch: {} 8 | pull_request: 9 | branches: 10 | - master 11 | 12 | env: 13 | # Common versions 14 | GO_VERSION: '1.23' 15 | GOLANGCI_VERSION: 'v1.56' 16 | KIND_VERSION: 'v0.29.0' 17 | 18 | jobs: 19 | 20 | detect-noop: 21 | runs-on: ubuntu-22.04 22 | outputs: 23 | noop: ${{ steps.noop.outputs.should_skip }} 24 | steps: 25 | - name: Detect No-op Changes 26 | id: noop 27 | uses: fkirc/skip-duplicate-actions@v3.3.0 28 | with: 29 | github_token: ${{ secrets.GITHUB_TOKEN }} 30 | paths_ignore: '["**.md", "**.mdx", "**.png", "**.jpg"]' 31 | do_not_skip: '["workflow_dispatch", "schedule", "push"]' 32 | concurrent_skipping: false 33 | 34 | test: 35 | runs-on: ubuntu-22.04 36 | needs: detect-noop 37 | if: needs.detect-noop.outputs.noop != 'true' 38 | steps: 39 | - name: Set up Go 40 | uses: actions/setup-go@v5 41 | with: 42 | go-version: ${{ env.GO_VERSION }} 43 | id: go 44 | - name: Checkout 45 | uses: actions/checkout@v4 46 | with: 47 | submodules: true 48 | - name: Cache Go Dependencies 49 | uses: actions/cache@v4 50 | with: 51 | path: .work/pkg 52 | key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} 53 | restore-keys: ${{ runner.os }}-pkg- 54 | - name: install Kubebuilder 55 | run: | 56 | # Install setup-envtest for managing test binaries 57 | go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest 58 | 59 | # Setup the test environment with the correct Kubernetes version 60 | export KUBEBUILDER_ASSETS=$(setup-envtest use 1.31.x --print path --bin-dir ~/.local/bin) 61 | echo "KUBEBUILDER_ASSETS=$KUBEBUILDER_ASSETS" >> $GITHUB_ENV 62 | 63 | # Verify the setup 64 | ls -la $KUBEBUILDER_ASSETS 65 | - name: Run Make test 66 | run: make test 67 | - name: Upload coverage report 68 | uses: codecov/codecov-action@v5 69 | with: 70 | token: ${{ secrets.CODECOV_TOKEN }} 71 | file: cover.out 72 | flags: unit-test 73 | name: codecov-umbrella 74 | fail_ci_if_error: true 75 | verbose: true 76 | - name: Run Make 77 | run: make 78 | 79 | e2e-cluster-gateway: 80 | runs-on: ubuntu-22.04 81 | needs: detect-noop 82 | if: needs.detect-noop.outputs.noop != 'true' 83 | steps: 84 | - name: Set up Go 85 | uses: actions/setup-go@v5 86 | with: 87 | go-version: ${{ env.GO_VERSION }} 88 | id: go 89 | - name: Checkout 90 | uses: actions/checkout@v4 91 | with: 92 | submodules: true 93 | - name: Cache Go Dependencies 94 | uses: actions/cache@v4 95 | with: 96 | path: .work/pkg 97 | key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} 98 | restore-keys: ${{ runner.os }}-pkg- 99 | - name: Create k8s Kind Cluster 100 | uses: helm/kind-action@v1.2.0 101 | with: 102 | version: v0.29.0 103 | node_image: kindest/node:v1.31.9 104 | - name: Build Image 105 | run: | 106 | make image 107 | kind load docker-image oamdev/cluster-gateway:latest --name chart-testing 108 | - name: Prepare ClusterGateway E2E Environment 109 | run: | 110 | helm install --create-namespace -n vela-system \ 111 | cluster-gateway ./charts/cluster-gateway \ 112 | --set featureGate.healthiness=true \ 113 | --set featureGate.secretCache=true \ 114 | --set tag=latest 115 | kubectl wait --for=condition=Available apiservice/v1alpha1.cluster.core.oam.dev 116 | go run ./e2e/env/prepare | kubectl apply -f - 117 | - name: Run Make test 118 | run: | 119 | kubectl get clustergateway 120 | make test-e2e 121 | 122 | e2e-ocm-addon-cluster-gateway: 123 | runs-on: ubuntu-22.04 124 | needs: detect-noop 125 | if: needs.detect-noop.outputs.noop != 'true' 126 | steps: 127 | - name: Set up Go 128 | uses: actions/setup-go@v5 129 | with: 130 | go-version: ${{ env.GO_VERSION }} 131 | id: go 132 | - name: Checkout 133 | uses: actions/checkout@v4 134 | with: 135 | submodules: true 136 | - name: Cache Go Dependencies 137 | uses: actions/cache@v4 138 | with: 139 | path: .work/pkg 140 | key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} 141 | restore-keys: ${{ runner.os }}-pkg- 142 | - name: Install clusteradm 143 | run: curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash 144 | - name: Create k8s Kind Cluster 145 | uses: helm/kind-action@v1.2.0 146 | with: 147 | version: v0.29.0 148 | node_image: kindest/node:v1.31.9 149 | - name: Prepare OCM testing environment 150 | run: | 151 | clusteradm init --output-join-command-file join.sh --wait 152 | sh -c "$(cat join.sh) loopback --force-internal-endpoint-lookup" 153 | clusteradm accept --clusters loopback --wait 30 154 | kubectl wait --for=condition=ManagedClusterConditionAvailable managedcluster/loopback 155 | - name: Build image 156 | run: | 157 | make image 158 | kind load docker-image oamdev/cluster-gateway:latest --name chart-testing 159 | kind load docker-image oamdev/cluster-gateway-addon-manager:latest --name chart-testing 160 | - name: Install latest cluster-gateway 161 | run: | 162 | helm install --create-namespace -n open-cluster-management-addon \ 163 | cluster-gateway ./charts/addon-manager \ 164 | --set tag=latest 165 | go run ./e2e/env/prepare | kubectl apply -f - 166 | kubectl rollout status deployment -n vela-system gateway-deployment --timeout 1m 167 | kubectl wait --for=condition=Available apiservice/v1alpha1.cluster.core.oam.dev 168 | - name: Run e2e test 169 | run: | 170 | make test-e2e-ocm 171 | -------------------------------------------------------------------------------- /pkg/apis/cluster/v1alpha1/virtualcluster_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The KubeVela Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | "context" 21 | "sort" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | apierrors "k8s.io/apimachinery/pkg/api/errors" 25 | "k8s.io/apimachinery/pkg/api/meta" 26 | "k8s.io/apimachinery/pkg/labels" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/apimachinery/pkg/runtime/schema" 29 | "k8s.io/apimachinery/pkg/selection" 30 | "k8s.io/apimachinery/pkg/types" 31 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 32 | "k8s.io/utils/strings/slices" 33 | ocmclusterv1 "open-cluster-management.io/api/cluster/v1" 34 | "sigs.k8s.io/controller-runtime/pkg/client" 35 | 36 | "github.com/oam-dev/cluster-gateway/pkg/common" 37 | "github.com/oam-dev/cluster-gateway/pkg/config" 38 | ) 39 | 40 | // VirtualClusterClient client for reading cluster information 41 | // +kubebuilder:object:generate=false 42 | type VirtualClusterClient interface { 43 | Get(ctx context.Context, name string) (*VirtualCluster, error) 44 | List(ctx context.Context, options ...client.ListOption) (*VirtualClusterList, error) 45 | } 46 | 47 | type virtualClusterClient struct { 48 | client.Client 49 | namespace string 50 | withControlPlane bool 51 | } 52 | 53 | // NewVirtualClusterClient create a client for accessing cluster 54 | func NewVirtualClusterClient(cli client.Client, namespace string, withControlPlane bool) VirtualClusterClient { 55 | return &virtualClusterClient{Client: cli, namespace: namespace, withControlPlane: withControlPlane} 56 | } 57 | 58 | func (c *virtualClusterClient) Get(ctx context.Context, name string) (*VirtualCluster, error) { 59 | if name == ClusterLocalName { 60 | return NewLocalCluster(), nil 61 | } 62 | key := types.NamespacedName{Name: name, Namespace: c.namespace} 63 | var cluster *VirtualCluster 64 | secret := &corev1.Secret{} 65 | err := c.Client.Get(ctx, key, secret) 66 | var secretErr error 67 | if err == nil { 68 | if cluster, secretErr = NewClusterFromSecret(secret); secretErr == nil { 69 | return cluster, nil 70 | } 71 | } 72 | if err != nil && !apierrors.IsNotFound(err) { 73 | secretErr = err 74 | } 75 | 76 | managedCluster := &ocmclusterv1.ManagedCluster{} 77 | err = c.Client.Get(ctx, key, managedCluster) 78 | var managedClusterErr error 79 | if err == nil { 80 | if cluster, managedClusterErr = NewClusterFromManagedCluster(managedCluster); managedClusterErr == nil { 81 | return cluster, nil 82 | } 83 | } 84 | 85 | if err != nil && !apierrors.IsNotFound(err) && !meta.IsNoMatchError(err) && !runtime.IsNotRegisteredError(err) { 86 | managedClusterErr = err 87 | } 88 | 89 | errs := utilerrors.NewAggregate([]error{secretErr, managedClusterErr}) 90 | if errs == nil { 91 | return nil, apierrors.NewNotFound(schema.GroupResource{ 92 | Group: config.MetaApiGroupName, 93 | Resource: "virtualclusters", 94 | }, name) 95 | } else if len(errs.Errors()) == 1 { 96 | return nil, errs.Errors()[0] 97 | } else { 98 | return nil, errs 99 | } 100 | } 101 | 102 | func (c *virtualClusterClient) List(ctx context.Context, options ...client.ListOption) (*VirtualClusterList, error) { 103 | opts := &client.ListOptions{} 104 | for _, opt := range options { 105 | opt.ApplyToList(opts) 106 | } 107 | local := NewLocalCluster() 108 | clusters := &VirtualClusterList{Items: []VirtualCluster{*local}} 109 | 110 | secrets := &corev1.SecretList{} 111 | err := c.Client.List(ctx, secrets, virtualClusterSelector{selector: opts.LabelSelector, requireCredentialType: true, namespace: c.namespace}) 112 | if err != nil { 113 | return nil, err 114 | } 115 | for _, secret := range secrets.Items { 116 | if cluster, err := NewClusterFromSecret(secret.DeepCopy()); err == nil { 117 | clusters.Items = append(clusters.Items, *cluster) 118 | } 119 | } 120 | 121 | managedClusters := &ocmclusterv1.ManagedClusterList{} 122 | err = c.Client.List(ctx, managedClusters, virtualClusterSelector{selector: opts.LabelSelector, requireCredentialType: false}) 123 | if err != nil && !meta.IsNoMatchError(err) && !runtime.IsNotRegisteredError(err) { 124 | return nil, err 125 | } 126 | for _, managedCluster := range managedClusters.Items { 127 | if !clusters.HasCluster(managedCluster.Name) { 128 | if cluster, err := NewClusterFromManagedCluster(managedCluster.DeepCopy()); err == nil { 129 | clusters.Items = append(clusters.Items, *cluster) 130 | } 131 | } 132 | } 133 | 134 | // filter clusters 135 | var items []VirtualCluster 136 | for _, cluster := range clusters.Items { 137 | if opts.LabelSelector == nil || opts.LabelSelector.Matches(labels.Set(cluster.GetLabels())) { 138 | items = append(items, cluster) 139 | } 140 | } 141 | clusters.Items = items 142 | 143 | // sort clusters 144 | sort.Slice(clusters.Items, func(i, j int) bool { 145 | if clusters.Items[i].Name == ClusterLocalName { 146 | return true 147 | } else if clusters.Items[j].Name == ClusterLocalName { 148 | return false 149 | } else { 150 | return clusters.Items[i].CreationTimestamp.After(clusters.Items[j].CreationTimestamp.Time) 151 | } 152 | }) 153 | return clusters, nil 154 | } 155 | 156 | // virtualClusterSelector filters the list/delete operation of cluster list 157 | type virtualClusterSelector struct { 158 | selector labels.Selector 159 | requireCredentialType bool 160 | namespace string 161 | } 162 | 163 | // ApplyToList applies this configuration to the given list options. 164 | func (m virtualClusterSelector) ApplyToList(opts *client.ListOptions) { 165 | opts.LabelSelector = labels.NewSelector() 166 | if m.selector != nil { 167 | requirements, _ := m.selector.Requirements() 168 | for _, r := range requirements { 169 | if !slices.Contains([]string{LabelClusterControlPlane}, r.Key()) { 170 | opts.LabelSelector = opts.LabelSelector.Add(r) 171 | } 172 | } 173 | } 174 | if m.requireCredentialType { 175 | r, _ := labels.NewRequirement(common.LabelKeyClusterCredentialType, selection.Exists, nil) 176 | opts.LabelSelector = opts.LabelSelector.Add(*r) 177 | } 178 | opts.Namespace = m.namespace 179 | } 180 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/oam-dev/cluster-gateway 2 | 3 | go 1.23.8 4 | 5 | require ( 6 | github.com/ghodss/yaml v1.0.0 7 | github.com/onsi/ginkgo/v2 v2.20.1 8 | github.com/onsi/gomega v1.34.2 9 | github.com/openshift/library-go v0.0.0-20230327085348-8477ec72b725 10 | github.com/pkg/errors v0.9.1 11 | github.com/spf13/cobra v1.8.1 12 | github.com/spf13/pflag v1.0.5 13 | github.com/stretchr/testify v1.9.0 14 | google.golang.org/grpc v1.67.1 15 | k8s.io/api v0.31.10 16 | k8s.io/apimachinery v0.31.10 17 | k8s.io/apiserver v0.31.10 18 | k8s.io/client-go v0.31.10 19 | k8s.io/component-base v0.31.10 20 | k8s.io/klog v1.0.0 21 | k8s.io/klog/v2 v2.130.1 22 | k8s.io/kube-aggregator v0.31.10 23 | k8s.io/kube-openapi v0.0.0-20250610211856-8b98d1ed966a 24 | k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 25 | open-cluster-management.io/addon-framework v0.7.0 26 | open-cluster-management.io/api v0.11.0 27 | open-cluster-management.io/managed-serviceaccount v0.2.0 28 | sigs.k8s.io/apiserver-network-proxy v0.31.4 29 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.3 30 | sigs.k8s.io/apiserver-runtime v1.1.2-0.20221102045245-fb656940062f 31 | sigs.k8s.io/controller-runtime v0.19.7 32 | sigs.k8s.io/controller-tools v0.16.5 33 | ) 34 | 35 | require ( 36 | github.com/NYTimes/gziphandler v1.1.1 // indirect 37 | github.com/antlr4-go/antlr/v4 v4.13.0 // indirect 38 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect 39 | github.com/beorn7/perks v1.0.1 // indirect 40 | github.com/blang/semver/v4 v4.0.0 // indirect 41 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 42 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 43 | github.com/coreos/go-semver v0.3.1 // indirect 44 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect 45 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 46 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 47 | github.com/evanphx/json-patch v5.6.0+incompatible // indirect 48 | github.com/evanphx/json-patch/v5 v5.9.0 // indirect 49 | github.com/felixge/httpsnoop v1.0.4 // indirect 50 | github.com/fsnotify/fsnotify v1.7.0 // indirect 51 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 52 | github.com/go-logr/logr v1.4.2 // indirect 53 | github.com/go-logr/stdr v1.2.2 // indirect 54 | github.com/go-logr/zapr v1.3.0 // indirect 55 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 56 | github.com/go-openapi/jsonreference v0.20.2 // indirect 57 | github.com/go-openapi/swag v0.23.0 // indirect 58 | github.com/go-task/slim-sprig/v3 v3.0.0 // indirect 59 | github.com/gogo/protobuf v1.3.2 // indirect 60 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 61 | github.com/golang/protobuf v1.5.4 // indirect 62 | github.com/google/cel-go v0.20.1 // indirect 63 | github.com/google/gnostic-models v0.6.9 // indirect 64 | github.com/google/go-cmp v0.6.0 // indirect 65 | github.com/google/gofuzz v1.2.0 // indirect 66 | github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect 67 | github.com/google/uuid v1.6.0 // indirect 68 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 69 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect 70 | github.com/imdario/mergo v0.3.15 // indirect 71 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 72 | github.com/josharian/intern v1.0.0 // indirect 73 | github.com/json-iterator/go v1.1.12 // indirect 74 | github.com/klauspost/compress v1.17.10 // indirect 75 | github.com/kylelemons/godebug v1.1.0 // indirect 76 | github.com/mailru/easyjson v0.7.7 // indirect 77 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 78 | github.com/modern-go/reflect2 v1.0.2 // indirect 79 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 80 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect 81 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 82 | github.com/prometheus/client_golang v1.20.5 // indirect 83 | github.com/prometheus/client_model v0.6.1 // indirect 84 | github.com/prometheus/common v0.55.0 // indirect 85 | github.com/prometheus/procfs v0.15.1 // indirect 86 | github.com/stoewer/go-strcase v1.2.0 // indirect 87 | github.com/x448/float16 v0.8.4 // indirect 88 | go.etcd.io/etcd/api/v3 v3.5.16 // indirect 89 | go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect 90 | go.etcd.io/etcd/client/v3 v3.5.16 // indirect 91 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect 92 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect 93 | go.opentelemetry.io/otel v1.28.0 // indirect 94 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect 95 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect 96 | go.opentelemetry.io/otel/metric v1.28.0 // indirect 97 | go.opentelemetry.io/otel/sdk v1.28.0 // indirect 98 | go.opentelemetry.io/otel/trace v1.28.0 // indirect 99 | go.opentelemetry.io/proto/otlp v1.3.1 // indirect 100 | go.uber.org/multierr v1.11.0 // indirect 101 | go.uber.org/zap v1.26.0 // indirect 102 | golang.org/x/crypto v0.31.0 // indirect 103 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect 104 | golang.org/x/net v0.33.0 // indirect 105 | golang.org/x/oauth2 v0.22.0 // indirect 106 | golang.org/x/sync v0.10.0 // indirect 107 | golang.org/x/sys v0.28.0 // indirect 108 | golang.org/x/term v0.27.0 // indirect 109 | golang.org/x/text v0.21.0 // indirect 110 | golang.org/x/time v0.5.0 // indirect 111 | golang.org/x/tools v0.26.0 // indirect 112 | gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect 113 | google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect 114 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect 115 | google.golang.org/protobuf v1.35.1 // indirect 116 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 117 | gopkg.in/inf.v0 v0.9.1 // indirect 118 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect 119 | gopkg.in/yaml.v2 v2.4.0 // indirect 120 | gopkg.in/yaml.v3 v3.0.1 // indirect 121 | k8s.io/apiextensions-apiserver v0.31.2 // indirect 122 | k8s.io/kms v0.31.10 // indirect 123 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 124 | sigs.k8s.io/randfill v1.0.0 // indirect 125 | sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect 126 | sigs.k8s.io/yaml v1.4.0 // indirect 127 | ) 128 | 129 | replace ( 130 | cloud.google.com/go => cloud.google.com/go v0.100.2 131 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.36 132 | sigs.k8s.io/apiserver-runtime => github.com/kmodules/apiserver-runtime v1.1.2-0.20250422194347-c5ac4abaf2ae 133 | ) 134 | --------------------------------------------------------------------------------