├── .ci └── yamllint.yml ├── .gitignore ├── .licenserc.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── cmd └── lazyxds │ ├── app │ ├── app.go │ ├── config │ │ ├── config.go │ │ └── const.go │ ├── options │ │ └── options.go │ └── run.go │ └── main.go ├── docker └── Dockerfile ├── docs └── images │ ├── arch.png │ ├── performance-test-arch.png │ ├── performance-test-mem.png │ ├── performance-test-xds.png │ ├── productpage-accesslog-1.png │ ├── productpage-accesslog-2.png │ └── sotw-xds.png ├── go.mod ├── go.sum ├── install ├── lazyxds-controller.yaml └── lazyxds-egress.yaml ├── pkg ├── accesslog │ ├── handler.go │ └── server.go ├── bootstrap │ └── egress_gateway.go ├── controller │ ├── aggregation.go │ ├── cluster.go │ ├── discovery_selector.go │ ├── discoveryselector │ │ └── controller.go │ ├── endpoints.go │ ├── endpoints │ │ └── controller.go │ ├── handle_crd.go │ ├── lazy_source.go │ ├── lazyservice │ │ └── controller.go │ ├── namespace.go │ ├── namespace │ │ └── controller.go │ ├── placeholder_service.go │ ├── service.go │ ├── service │ │ └── controller.go │ ├── serviceentry.go │ ├── serviceentry │ │ └── controller.go │ ├── sidecar.go │ ├── sidecar │ │ └── controller.go │ ├── virtual_service.go │ └── virtualservice │ │ └── controller.go ├── manager │ └── manager.go ├── model │ ├── endpoints.go │ ├── namespace.go │ └── service.go └── utils │ ├── app │ ├── app.go │ ├── cmd.go │ ├── config.go │ ├── flag.go │ ├── help.go │ ├── options.go │ └── version │ │ ├── flag.go │ │ └── version.go │ ├── crd.go │ ├── k8s.go │ ├── leaderelectionconfig │ └── config.go │ ├── log │ ├── context.go │ └── log.go │ ├── protocal.go │ ├── service.go │ └── signal │ ├── signal.go │ ├── signal_posix.go │ └── signal_windows.go └── test └── e2e └── lazyxds ├── data ├── services │ ├── data-svc1.yaml │ ├── headless-svc.yaml │ ├── mix-svc.yaml │ ├── serviceentry.yaml │ ├── web-svc-multi-version.yaml │ ├── web-svc-related.yaml │ ├── web-svc1.yaml │ └── web-svc2.yaml └── source │ ├── external-name-svc.yaml │ ├── lazy_source.yaml │ └── normal_source.yaml ├── lazyxds ├── lazyxds_suite_test.go └── lazyxds_test.go └── utils ├── cmd.go ├── kube_runner.go └── request_id.go /.ci/yamllint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | ignore: | 5 | customresourcedefinitions.gen.yaml 6 | manifests/charts/aeraki/templates/deployment.yaml 7 | manifests/charts/aeraki/templates/service.yaml 8 | manifests/charts/aeraki/templates/serviceaccount.yaml 9 | k8s/crd.yaml 10 | 11 | yaml-files: 12 | - '*.yaml' 13 | - '*.yml' 14 | 15 | rules: 16 | truthy: disable 17 | # 80 chars should be enough, but don't fail if a line is longer 18 | line-length: disable 19 | comments-indentation: disable 20 | indentation: 21 | spaces: consistent 22 | indent-sequences: whatever 23 | check-multi-line-strings: false 24 | braces: 25 | level: warning 26 | max-spaces-inside: 1 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/.DS_Store 2 | debug 3 | out 4 | .idea 5 | tmp -------------------------------------------------------------------------------- /.licenserc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright Aeraki Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | header: 17 | license: 18 | spdx-id: Apache-2.0 19 | copyright-owner: Aeraki 20 | 21 | paths-ignore: 22 | - '.gitignore' 23 | - 'LICENSE' 24 | - 'NOTICE' 25 | - '**/*.json' 26 | - '.github/' 27 | - 'go.mod' 28 | - 'go.sum' 29 | - '**/*.gen.go' 30 | - '**/*.pb.go' 31 | - '**/*.pb.html' 32 | - '**/*.proto' 33 | - '**/*.dic' 34 | - '**/*.aff' 35 | - 'test/' 36 | - 'crd/' 37 | - '.ci/' 38 | - 'k8s/crd.yaml' 39 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Contributor Covenant Code of Conduct 18 | 19 | ## Our Pledge 20 | 21 | We as members, contributors, and leaders pledge to make participation in our 22 | community a harassment-free experience for everyone, regardless of age, body 23 | size, visible or invisible disability, ethnicity, sex characteristics, gender 24 | identity and expression, level of experience, education, socio-economic status, 25 | nationality, personal appearance, race, religion, or sexual identity 26 | and orientation. 27 | 28 | We pledge to act and interact in ways that contribute to an open, welcoming, 29 | diverse, inclusive, and healthy community. 30 | 31 | ## Our Standards 32 | 33 | Examples of behavior that contributes to a positive environment for our 34 | community include: 35 | 36 | * Demonstrating empathy and kindness toward other people 37 | * Being respectful of differing opinions, viewpoints, and experiences 38 | * Giving and gracefully accepting constructive feedback 39 | * Accepting responsibility and apologizing to those affected by our mistakes, 40 | and learning from the experience 41 | * Focusing on what is best not just for us as individuals, but for the 42 | overall community 43 | 44 | Examples of unacceptable behavior include: 45 | 46 | * The use of sexualized language or imagery, and sexual attention or 47 | advances of any kind 48 | * Trolling, insulting or derogatory comments, and personal or political attacks 49 | * Public or private harassment 50 | * Publishing others' private information, such as a physical or email 51 | address, without their explicit permission 52 | * Other conduct which could reasonably be considered inappropriate in a 53 | professional setting 54 | 55 | ## Enforcement Responsibilities 56 | 57 | Community leaders are responsible for clarifying and enforcing our standards of 58 | acceptable behavior and will take appropriate and fair corrective action in 59 | response to any behavior that they deem inappropriate, threatening, offensive, 60 | or harmful. 61 | 62 | Community leaders have the right and responsibility to remove, edit, or reject 63 | comments, commits, code, wiki edits, issues, and other contributions that are 64 | not aligned to this Code of Conduct, and will communicate reasons for moderation 65 | decisions when appropriate. 66 | 67 | ## Scope 68 | 69 | This Code of Conduct applies within all community spaces, and also applies when 70 | an individual is officially representing the community in public spaces. 71 | Examples of representing our community include using an official e-mail address, 72 | posting via an official social media account, or acting as an appointed 73 | representative at an online or offline event. 74 | 75 | ## Enforcement 76 | 77 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 78 | reported to the community leaders responsible for enforcement at 79 | zhaohuabing@gmail.com. 80 | All complaints will be reviewed and investigated promptly and fairly. 81 | 82 | All community leaders are obligated to respect the privacy and security of the 83 | reporter of any incident. 84 | 85 | ## Enforcement Guidelines 86 | 87 | Community leaders will follow these Community Impact Guidelines in determining 88 | the consequences for any action they deem in violation of this Code of Conduct: 89 | 90 | ### 1. Correction 91 | 92 | **Community Impact**: Use of inappropriate language or other behavior deemed 93 | unprofessional or unwelcome in the community. 94 | 95 | **Consequence**: A private, written warning from community leaders, providing 96 | clarity around the nature of the violation and an explanation of why the 97 | behavior was inappropriate. A public apology may be requested. 98 | 99 | ### 2. Warning 100 | 101 | **Community Impact**: A violation through a single incident or series 102 | of actions. 103 | 104 | **Consequence**: A warning with consequences for continued behavior. No 105 | interaction with the people involved, including unsolicited interaction with 106 | those enforcing the Code of Conduct, for a specified period of time. This 107 | includes avoiding interactions in community spaces as well as external channels 108 | like social media. Violating these terms may lead to a temporary or 109 | permanent ban. 110 | 111 | ### 3. Temporary Ban 112 | 113 | **Community Impact**: A serious violation of community standards, including 114 | sustained inappropriate behavior. 115 | 116 | **Consequence**: A temporary ban from any sort of interaction or public 117 | communication with the community for a specified period of time. No public or 118 | private interaction with the people involved, including unsolicited interaction 119 | with those enforcing the Code of Conduct, is allowed during this period. 120 | Violating these terms may lead to a permanent ban. 121 | 122 | ### 4. Permanent Ban 123 | 124 | **Community Impact**: Demonstrating a pattern of violation of community 125 | standards, including sustained inappropriate behavior, harassment of an 126 | individual, or aggression toward or disparagement of classes of individuals. 127 | 128 | **Consequence**: A permanent ban from any sort of public interaction within 129 | the community. 130 | 131 | ## Attribution 132 | 133 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 134 | version 2.0, available at 135 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 136 | 137 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 138 | enforcement ladder](https://github.com/mozilla/diversity). 139 | 140 | [homepage]: https://www.contributor-covenant.org 141 | 142 | For answers to common questions about this code of conduct, see the FAQ at 143 | https://www.contributor-covenant.org/faq. Translations are available at 144 | https://www.contributor-covenant.org/translations. 145 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Contributing to Aeraki 18 | 19 | Welcome to Aeraki! You are welcome to [report Issues](https://github.com/aeraki-mesh/aeraki/issues/new/choose) or [pull requests](https://github.com/aeraki-mesh/aeraki/compare). It's recommended to read the following Contributing Guide first before contributing. 20 | 21 | ## Issues 22 | 23 | We use Github issues to track public bugs and feature requests. 24 | 25 | ### Search Known Issues First 26 | 27 | Please search the existing issues to see if any similar issue or feature request has already been filed. You should make sure your issue isn't redundant. 28 | 29 | ### Reporting New Issues 30 | If you open an issue, the more information the better. Such as detailed description, screenshot or video of your problem, logcat or code blocks for your crash. 31 | 32 | ## Pull Requests 33 | 34 | We strongly welcome your pull request to make Aeraki better. 35 | 36 | ### Make Pull Requests 37 | 38 | The code team will monitor all pull request, we run some code check and test on it. After all tests passed, we will accept this PR. But it won't merge to master branch at once, which have some delay. 39 | 40 | Before submitting a pull request, please make sure the followings are done: 41 | 42 | 1. Fork the repo and create your branch from master. 43 | 2. Update code or documentation if you have changed APIs. 44 | 3. Add the copyright notice to the top of any new files you've added. 45 | 4. Check your code lints and checkstyles. 46 | 5. Test and test again your code. 47 | 6. Now, you can submit your pull request on dev. 48 | 49 | # Sign the CLA 50 | 51 | You must sign the Contributor License Agreement in order to contribute. 52 | 53 | # License 54 | 55 | By contributing to Aeraki, you agree that your contributions will be licensed under its Apache v2 License. 56 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Aeraki Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Go parameters 16 | GOCMD?=go 17 | GOBUILD?=$(GOCMD) build 18 | GOTEST?=$(GOCMD) test 19 | GOCLEAN?=$(GOCMD) clean 20 | GOTEST?=$(GOCMD) test 21 | GOGET?=$(GOCMD) get 22 | GOBIN?=$(GOPATH)/bin 23 | GOMOD?=$(GOCMD) mod 24 | 25 | # Build parameters 26 | IMAGE_TAG := $(tag) 27 | 28 | ifeq ($(IMAGE_TAG),) 29 | IMAGE_TAG := latest 30 | endif 31 | 32 | OUT?=./out 33 | DOCKER_TMP?=$(OUT)/docker_temp/ 34 | 35 | # lazyxds 36 | LAZYXDS_DOCKER_TAG?=aeraki/lazyxds:$(IMAGE_TAG) 37 | LAZYXDS_DOCKER_TAG_E2E?=aeraki/lazyxds:`git log --format="%H" -n 1` 38 | LAZYXDS_BINARY_NAME?=$(OUT)/lazyxds 39 | LAZYXDS_BINARY_NAME_DARWIN?=$(LAZYXDS_BINARY_NAME)-darwin 40 | LAZYXDS_MAIN?=./cmd/lazyxds/main.go 41 | 42 | test: style-check 43 | $(GOMOD) tidy 44 | $(GOTEST) -race `go list ./... | grep -v e2e` 45 | clean: 46 | rm -rf $(OUT) 47 | style-check: 48 | gofmt -l -d ./ 49 | goimports -l -d ./ 50 | lint: 51 | golint ./... 52 | golangci-lint run --tests="false" 53 | build.lazyxds: test 54 | CGO_ENABLED=0 GOOS=linux $(GOBUILD) -o $(LAZYXDS_BINARY_NAME) $(LAZYXDS_MAIN) 55 | build-mac.lazyxds: test 56 | CGO_ENABLED=0 GOOS=darwin $(GOBUILD) -o $(LAZYXDS_BINARY_NAME_DARWIN) $(LAZYXDS_MAIN) 57 | docker-build.lazyxds: build.lazyxds 58 | rm -rf $(DOCKER_TMP) 59 | mkdir $(DOCKER_TMP) 60 | cp ./lazyxds/docker/Dockerfile $(DOCKER_TMP) 61 | cp $(LAZYXDS_BINARY_NAME) $(DOCKER_TMP) 62 | docker build -t $(LAZYXDS_DOCKER_TAG) $(DOCKER_TMP) 63 | rm -rf $(DOCKER_TMP) 64 | docker-push.lazyxds: docker-build.lazyxds 65 | docker push $(LAZYXDS_DOCKER_TAG) 66 | docker-build-e2e.lazyxds: build.lazyxds 67 | rm -rf $(DOCKER_TMP) 68 | mkdir $(DOCKER_TMP) 69 | cp ./lazyxds/docker/Dockerfile $(DOCKER_TMP) 70 | cp $(LAZYXDS_BINARY_NAME) $(DOCKER_TMP) 71 | docker build -t $(LAZYXDS_DOCKER_TAG_E2E) $(DOCKER_TMP) 72 | rm -rf $(DOCKER_TMP) 73 | e2e-lazyxds: 74 | ginkgo -v ./test/e2e/lazyxds/lazyxds/ 75 | .DEFAULT_GOAL := docker-build 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # LazyXds 18 | 19 | LazyXds enables Istio only push needed xDS to sidecars to reduce resource consumption and speed up xDS configuration propagation. 20 | 21 | Note: LazyXds is an experimental project, please don't use it in production. 22 | 23 | ## Problems to solve 24 | 25 | ![SotW xDS](docs/images/sotw-xds.png) 26 | 27 | ## Architecture 28 | 29 | ![SotW xDS](docs/images/arch.png) 30 | 31 | ## Build 32 | 33 | ```bash 34 | # build lazyxds binary on linux 35 | make build.lazyxds 36 | 37 | # build lazyxds binary on darwin 38 | make build-mac.lazyxds 39 | ``` 40 | 41 | ### Build Image 42 | 43 | ```bash 44 | # build lazyxds docker image with the default latest tag 45 | make docker-build.lazyxds 46 | 47 | # build lazyxds docker image with xxx tag 48 | make docker-build.lazyxds tag=xxx 49 | 50 | # build lazyxds e2e docker image 51 | make docker-build-e2e.lazyxds 52 | ``` 53 | 54 | ## Install 55 | 56 | ### Pre-requirements: 57 | 58 | * A running Kubernetes cluster, and istio(version >= 1.10.0) installed 59 | * Kubectl installed, and the `~/.kube/conf` points to the cluster in the first step 60 | 61 | ### Install Lazyxds Egress and Controller 62 | 63 | ``` 64 | kubectl apply -f https://raw.githubusercontent.com/aeraki-mesh/lazyxds/master/install/lazyxds-egress.yaml 65 | kubectl apply -f https://raw.githubusercontent.com/aeraki-mesh/lazyxds/master/install/lazyxds-controller.yaml 66 | ``` 67 | 68 | The above commands install the lazyxds egress and controller into the istio-system namespace. 69 | 70 | ## How to enable LazyXDS 71 | 72 | You can choose to enable lazyXDS on some particular services or enable it namespace wide. To enable lazyXDS on a service or a namespace, you just need to add an annotation `lazy-xds: "true"` to the target service or namespace. 73 | 74 | ### Enable on a Service 75 | 76 | ``` 77 | apiVersion: v1 78 | kind: Service 79 | metadata: 80 | name: my-service 81 | annotations: 82 | lazy-xds: "true" 83 | spec: 84 | ``` 85 | 86 | or use kubectl: 87 | 88 | `kubectl annotate service my-service lazy-xds=true --overwrite` 89 | 90 | ### Enable on a Namespace 91 | 92 | ``` 93 | apiVersion: v1 94 | kind: Namespace 95 | metadata: 96 | name: my-namespace 97 | annotations: 98 | lazy-xds: "true" 99 | spec: 100 | ``` 101 | 102 | or use kubectl: 103 | 104 | `kubectl annotate namespace my-namespace lazy-xds=true --overwrite` 105 | 106 | ## Bookinfo Demo 107 | 108 | 1. Install istio(version >= 1.10.0), and enable access log for debug purpose. 109 | 110 | ``` 111 | istioctl install -y --set meshConfig.accessLogFile=/dev/stdout 112 | ``` 113 | 114 | 2. Install lazyXds by following the instructions in [Install Lazyxds egress and controller](https://github.com/aeraki-mesh/lazyxds/blob/master/README.md#install-lazyxds-egress-and-controller). 115 | 116 | 3. Install bookinfo application: 117 | 118 | ``` 119 | kubectl label namespace default istio-injection=enabled 120 | kubectl apply -f https://raw.githubusercontent.com/istio/istio/release-1.10/samples/bookinfo/platform/kube/bookinfo.yaml 121 | kubectl apply -f https://raw.githubusercontent.com/istio/istio/release-1.10/samples/bookinfo/networking/bookinfo-gateway.yaml 122 | ``` 123 | 124 | Determine the ingress IP, and we use 80 as the ingress port by default. 125 | ``` 126 | export INGRESS_HOST=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 127 | ``` 128 | 129 | Save product page pod name to env for later use. 130 | ``` 131 | export PRODUCT_PAGE_POD=$(kubectl get pod -l app=productpage -o jsonpath="{.items[0].metadata.name}") 132 | ``` 133 | 134 | Check the eds of product page pod, we can see product page gets all eds of bookinfo, though it does not need all of them: 135 | ``` 136 | istioctl pc endpoints $PRODUCT_PAGE_POD | grep '9080' 137 | 172.22.0.10:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local 138 | 172.22.0.11:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local 139 | 172.22.0.12:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local 140 | 172.22.0.13:9080 HEALTHY OK outbound|9080||productpage.default.svc.cluster.local 141 | 172.22.0.8:9080 HEALTHY OK outbound|9080||details.default.svc.cluster.local 142 | 172.22.0.9:9080 HEALTHY OK outbound|9080||ratings.default.svc.cluster.local 143 | ``` 144 | 145 | 4. Enable lazyXds for the productpage service: 146 | 147 | ``` 148 | kubectl annotate service productpage lazy-xds=true --overwrite 149 | ``` 150 | 151 | Check the eds of product page: 152 | ``` 153 | istioctl pc endpoints $PRODUCT_PAGE_POD | grep '9080' 154 | // no eds show 155 | ``` 156 | Once enabling lazyXds, product page pod won't get any endpoints of bookinfo. 157 | 158 | 5. Access bookinfo the first time: 159 | 160 | ``` 161 | curl -I "http://${INGRESS_HOST}/productpage" 162 | ``` 163 | 164 | check the access log of product page pod: 165 | 166 | ``` 167 | kubectl logs -c istio-proxy -f $PRODUCT_PAGE_POD 168 | ``` 169 | 170 | ![access to egress](docs/images/productpage-accesslog-1.png) 171 | 172 | We can see the first request form product page to details and reviews has been redirected to `istio-egressgateway-lazyxds` 173 | 174 | Check the eds of product page again: 175 | 176 | ``` 177 | 172.22.0.10:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local 178 | 172.22.0.11:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local 179 | 172.22.0.12:9080 HEALTHY OK outbound|9080||reviews.default.svc.cluster.local 180 | 172.22.0.8:9080 HEALTHY OK outbound|9080||details.default.svc.cluster.local 181 | ``` 182 | 183 | Only reviews and details endpoints are in the eds, which are the exact endpoints product page needs. 184 | 185 | 6. Access bookinfo again: 186 | 187 | ``` 188 | curl -I "http://${INGRESS_HOST}/productpage" 189 | ``` 190 | 191 | Check the access log of product page pod: 192 | 193 | ``` 194 | kubectl logs -c istio-proxy -f $PRODUCT_PAGE_POD 195 | ``` 196 | 197 | ![access to egress](docs/images/productpage-accesslog-2.png) 198 | 199 | Now the traffic goes directly to the target services since the sidecar proxy already has all the endpoints it needs. 200 | 201 | ## Uninstall 202 | 203 | ``` 204 | kubectl delete -f https://raw.githubusercontent.com/aeraki-mesh/aeraki/master/lazyxds/install/lazyxds-controller.yaml 205 | kubectl delete -f https://raw.githubusercontent.com/aeraki-mesh/aeraki/master/lazyxds/install/lazyxds-egress.yaml 206 | ``` 207 | 208 | ## Performance 209 | 210 | We have set up two bookinfo applications in an istio mesh with lazyxds installed, the product page in `lazy-on` namespace has lazyXds enabled, and the other one hasn't. 211 | Then we use [istio load testing](https://github.com/istio/tools/tree/master/perf/load) to increasingly create a large number of services, 212 | each load test namespace contains 19 services, each service contains 5 pods. The following is the test result for your reference: 213 | 214 | ![performance-test-arch](docs/images/performance-test-arch.png) 215 | 216 | Memory compare: 217 | 218 | ![performance-test-mem](docs/images/performance-test-mem.png) 219 | 220 | EDS and CDS compare: 221 | 222 | ![performance-test-xds](docs/images/performance-test-xds.png) 223 | -------------------------------------------------------------------------------- /cmd/lazyxds/app/app.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "github.com/aeraki-mesh/lazyxds/pkg/utils/app" 19 | "github.com/aeraki-mesh/lazyxds/pkg/utils/signal" 20 | "k8s.io/klog/v2" 21 | 22 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 23 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/options" 24 | ) 25 | 26 | const commandDesc = `lazyxds enables istio only push needed xds to sidecars` 27 | 28 | // New creates a App object with default parameters. 29 | func New(basename string) *app.App { 30 | opts := options.New(basename) 31 | application := app.NewApp("lazyxds", 32 | basename, 33 | app.WithOptions(opts), 34 | app.WithDescription(commandDesc), 35 | app.WithRunFunc(run(opts)), 36 | ) 37 | return application 38 | } 39 | 40 | func run(opts *options.Options) app.RunFunc { 41 | return func(basename string) error { 42 | defer klog.Flush() 43 | 44 | cfg, err := config.New(opts) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | stopCh := signal.SetupSignalHandler() 50 | return Run(cfg, stopCh) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cmd/lazyxds/app/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "os" 19 | "strings" 20 | 21 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/options" 22 | componentbaseconfig "k8s.io/component-base/config" 23 | ) 24 | 25 | const ( 26 | // AutoCreateEgress ... 27 | AutoCreateEgress = "LAZYXDS_AUTO_CREATE_EGRESS" 28 | ) 29 | 30 | // New creates a new Config from Options 31 | func New(options *options.Options) (*Config, error) { 32 | config := &Config{ 33 | LeaderElection: options.LeaderElection, 34 | } 35 | 36 | config.KubeConfig = options.KubeConfig 37 | config.AutoCreateEgress = strings.ToLower(os.Getenv(AutoCreateEgress)) == "true" 38 | config.IstiodAddress = options.IstiodAddress 39 | config.ProxyImage = options.ProxyImage 40 | 41 | return config, nil 42 | } 43 | 44 | // Config is the lazyxds manager configuration 45 | type Config struct { 46 | KubeConfig string 47 | AutoCreateEgress bool 48 | IstiodAddress string 49 | ProxyImage string 50 | LeaderElection *componentbaseconfig.LeaderElectionConfiguration 51 | } 52 | -------------------------------------------------------------------------------- /cmd/lazyxds/app/config/const.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import "fmt" 18 | 19 | const ( 20 | // IstioNamespace is the default root namespace of istio 21 | IstioNamespace = "istio-system" 22 | // LazyXdsManager is the controller name which will put into fieldManager 23 | LazyXdsManager = "lazyxds-manager" 24 | // EgressName is name of egress deployment and service 25 | EgressName = "istio-egressgateway-lazyxds" 26 | // EgressServicePort is default port of egress service 27 | EgressServicePort = 8080 28 | // EgressGatewayName is the istio gateway name of lazyxds egress 29 | EgressGatewayName = "lazyxds-egress" 30 | // AccessLogServicePort is the default port of access log service 31 | AccessLogServicePort = 8080 32 | // EgressVirtualServiceName the vs name of lazyxds egress 33 | EgressVirtualServiceName = "lazyxds-egress" 34 | // LazyLoadingAnnotation is the annotation name which use to enable/disable lazy xds feature 35 | LazyLoadingAnnotation = "lazy-xds" 36 | // ManagedByLabel is the common label indicate the component is managed by which controller 37 | ManagedByLabel = "app.kubernetes.io/managed-by" 38 | ) 39 | 40 | // GetEgressCluster returns the egress xds cluster string 41 | // default is "outbound|8080||istio-egressgateway-lazyxds.istio-system.svc.cluster.local" 42 | func GetEgressCluster() string { 43 | return fmt.Sprintf("outbound|%d||%s.%s.svc.cluster.local", 44 | EgressServicePort, 45 | EgressName, 46 | IstioNamespace, 47 | ) 48 | } 49 | -------------------------------------------------------------------------------- /cmd/lazyxds/app/options/options.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package options 16 | 17 | import ( 18 | "github.com/aeraki-mesh/lazyxds/pkg/utils/leaderelectionconfig" 19 | "github.com/spf13/pflag" 20 | "k8s.io/component-base/config" 21 | ) 22 | 23 | const ( 24 | // DefaultIstiodAddress is the default istiod address 25 | DefaultIstiodAddress = "istiod.istio-system.svc:15012" 26 | // DefaultProxyImage is the default sidecar image of istio 27 | DefaultProxyImage = "docker.io/istio/proxyv2:1.10.0" 28 | ) 29 | 30 | // Options for lazyxds 31 | type Options struct { 32 | KubeConfig string 33 | IstiodAddress string 34 | ProxyImage string 35 | LeaderElection *config.LeaderElectionConfiguration 36 | } 37 | 38 | // New creates an Options 39 | func New(basename string) *Options { 40 | return &Options{ 41 | LeaderElection: leaderelectionconfig.New(basename), 42 | } 43 | } 44 | 45 | // AddFlags add several flags of lazyxds 46 | func (o *Options) AddFlags(fs *pflag.FlagSet) { 47 | fs.StringVar(&o.KubeConfig, "kube-config", "", 48 | "service discovery kube config file") 49 | fs.StringVar(&o.IstiodAddress, "istiod-address", DefaultIstiodAddress, 50 | "istiod address, use to create lazyxds egress") 51 | fs.StringVar(&o.ProxyImage, "proxy-image", DefaultProxyImage, 52 | "proxy image, use to create lazyxds egress") 53 | leaderelectionconfig.AddFlags(o.LeaderElection, fs) 54 | } 55 | 56 | // Validate will check the requirements of options 57 | func (o *Options) Validate() []error { 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /cmd/lazyxds/app/run.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "context" 19 | "os" 20 | "time" 21 | 22 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 23 | "github.com/aeraki-mesh/lazyxds/pkg/manager" 24 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 25 | "k8s.io/client-go/tools/leaderelection" 26 | "k8s.io/client-go/tools/leaderelection/resourcelock" 27 | "k8s.io/klog/v2" 28 | ) 29 | 30 | // Run start leader election and run main process 31 | func Run(conf *config.Config, stopCh <-chan struct{}) error { 32 | ctx, cancel := context.WithCancel(context.Background()) 33 | go func() { 34 | <-stopCh 35 | cancel() 36 | }() 37 | 38 | if !conf.LeaderElection.LeaderElect { 39 | doRun(ctx, conf) 40 | return nil 41 | } 42 | 43 | client, err := utils.NewKubeClient(conf.KubeConfig) 44 | if err != nil { 45 | klog.Fatalf("build kube client failed: %v", err) 46 | } 47 | 48 | id, err := os.Hostname() 49 | if err != nil { 50 | return err 51 | } 52 | 53 | rl, err := resourcelock.New(conf.LeaderElection.ResourceLock, 54 | conf.LeaderElection.ResourceNamespace, 55 | conf.LeaderElection.ResourceName, 56 | client.CoreV1(), 57 | client.CoordinationV1(), 58 | resourcelock.ResourceLockConfig{ 59 | Identity: id, 60 | }) 61 | if err != nil { 62 | klog.Fatalf("error creating lock: %v", err) 63 | } 64 | 65 | leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ 66 | Lock: rl, 67 | ReleaseOnCancel: true, 68 | LeaseDuration: conf.LeaderElection.LeaseDuration.Duration, 69 | RenewDeadline: conf.LeaderElection.RenewDeadline.Duration, 70 | RetryPeriod: conf.LeaderElection.RetryPeriod.Duration, 71 | Callbacks: leaderelection.LeaderCallbacks{ 72 | OnStartedLeading: func(c context.Context) { 73 | klog.Infof("%s: leading", id) 74 | doRun(c, conf) 75 | }, 76 | OnStoppedLeading: func() { 77 | klog.Infof("%s: stop leading", id) 78 | time.Sleep(3 * time.Second) 79 | klog.Infof("%s: stopped leading", id) 80 | }, 81 | OnNewLeader: func(identity string) { 82 | if identity == id { 83 | return 84 | } 85 | klog.Infof("new leader elected: %v", identity) 86 | }, 87 | }, 88 | }) 89 | 90 | klog.Info("app exiting") 91 | return nil 92 | } 93 | 94 | func doRun(ctx context.Context, conf *config.Config) { 95 | var err error 96 | var m manager.LazyXdsManager 97 | m, err = manager.NewManager(conf, ctx.Done()) 98 | if err != nil { 99 | klog.Fatalf("new lazyxds manager failed: %v", err) 100 | } 101 | if err = m.Run(); err != nil { 102 | klog.Fatalf("run lazyxds manager failed: %v", err) 103 | } 104 | 105 | <-ctx.Done() 106 | } 107 | -------------------------------------------------------------------------------- /cmd/lazyxds/main.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app" 19 | ) 20 | 21 | func main() { 22 | app.New("lazyxds").Run() 23 | } 24 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright Aeraki Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM alpine:3.10 16 | 17 | COPY lazyxds /usr/local/bin/ 18 | 19 | ENTRYPOINT ["/usr/local/bin/lazyxds", "-v", "4"] 20 | -------------------------------------------------------------------------------- /docs/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aeraki-mesh/lazyxds/f33f640c2a4da68c7d0ae5b69f8ccd8906f90063/docs/images/arch.png -------------------------------------------------------------------------------- /docs/images/performance-test-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aeraki-mesh/lazyxds/f33f640c2a4da68c7d0ae5b69f8ccd8906f90063/docs/images/performance-test-arch.png -------------------------------------------------------------------------------- /docs/images/performance-test-mem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aeraki-mesh/lazyxds/f33f640c2a4da68c7d0ae5b69f8ccd8906f90063/docs/images/performance-test-mem.png -------------------------------------------------------------------------------- /docs/images/performance-test-xds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aeraki-mesh/lazyxds/f33f640c2a4da68c7d0ae5b69f8ccd8906f90063/docs/images/performance-test-xds.png -------------------------------------------------------------------------------- /docs/images/productpage-accesslog-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aeraki-mesh/lazyxds/f33f640c2a4da68c7d0ae5b69f8ccd8906f90063/docs/images/productpage-accesslog-1.png -------------------------------------------------------------------------------- /docs/images/productpage-accesslog-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aeraki-mesh/lazyxds/f33f640c2a4da68c7d0ae5b69f8ccd8906f90063/docs/images/productpage-accesslog-2.png -------------------------------------------------------------------------------- /docs/images/sotw-xds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aeraki-mesh/lazyxds/f33f640c2a4da68c7d0ae5b69f8ccd8906f90063/docs/images/sotw-xds.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/aeraki-mesh/lazyxds 2 | 3 | go 1.16 4 | 5 | replace github.com/spf13/viper => github.com/istio/viper v1.3.3-0.20190515210538-2789fed3109c 6 | 7 | // Old version had no license 8 | replace github.com/chzyer/logex => github.com/chzyer/logex v1.1.11-0.20170329064859-445be9e134b2 9 | 10 | // Avoid pulling in incompatible libraries 11 | replace github.com/docker/distribution => github.com/docker/distribution v0.0.0-20191216044856-a8371794149d 12 | 13 | replace github.com/docker/docker => github.com/moby/moby v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible 14 | 15 | // Client-go does not handle different versions of mergo due to some breaking changes - use the matching version 16 | replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.5 17 | 18 | //replace github.com/envoyproxy/go-control-plane => /Users/huabingzhao/workspace/go-control-plane 19 | 20 | //replace github.com/aeraki-mesh/meta-protocol-control-plane-api => github.com/aeraki-mesh/meta-protocol-control-plane-api v0.0.0-20220325074604-63adf119a7bc 21 | 22 | require ( 23 | github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158 // indirect 24 | github.com/envoyproxy/go-control-plane v0.9.10-0.20210804155723-c55ac1656905 25 | github.com/envoyproxy/protoc-gen-validate v0.6.1 // indirect 26 | github.com/fatih/color v1.12.0 27 | github.com/go-logr/logr v0.4.0 28 | github.com/gogo/protobuf v1.3.2 29 | github.com/google/go-cmp v0.5.6 // indirect 30 | github.com/google/gofuzz v1.2.0 // indirect 31 | github.com/google/uuid v1.3.0 // indirect 32 | github.com/gosuri/uitable v0.0.4 33 | github.com/imdario/mergo v0.3.12 // indirect 34 | github.com/magiconair/properties v1.8.1 // indirect 35 | github.com/mattn/go-isatty v0.0.13 // indirect 36 | github.com/mattn/go-runewidth v0.0.13 // indirect 37 | github.com/onsi/ginkgo v1.16.4 38 | github.com/onsi/gomega v1.15.0 39 | github.com/pelletier/go-toml v1.8.1 // indirect 40 | github.com/spf13/cast v1.3.1 // indirect 41 | github.com/spf13/cobra v1.2.1 42 | github.com/spf13/pflag v1.0.5 43 | github.com/spf13/viper v1.8.1 44 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect 45 | golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a // indirect 46 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect 47 | golang.org/x/text v0.3.7 // indirect 48 | google.golang.org/appengine v1.6.7 // indirect 49 | google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f // indirect 50 | google.golang.org/grpc v1.40.0 51 | istio.io/api v0.0.0-20210819145325-4e216752748c 52 | istio.io/client-go v1.11.0 53 | istio.io/gogo-genproto v0.0.0-20210806192525-32ebb2f9006c // indirect 54 | k8s.io/api v0.22.0 55 | k8s.io/apimachinery v0.22.0 56 | k8s.io/client-go v0.22.0 57 | k8s.io/component-base v0.22.0 58 | k8s.io/klog/v2 v2.9.0 59 | k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d // indirect 60 | k8s.io/utils v0.0.0-20210802155522-efc7438f0176 // indirect 61 | sigs.k8s.io/yaml v1.2.0 62 | ) 63 | -------------------------------------------------------------------------------- /install/lazyxds-controller.yaml: -------------------------------------------------------------------------------- 1 | # Copyright Aeraki Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | apiVersion: v1 17 | kind: ServiceAccount 18 | metadata: 19 | name: lazyxds 20 | namespace: istio-system 21 | labels: 22 | app: lazyxds 23 | --- 24 | apiVersion: rbac.authorization.k8s.io/v1 25 | kind: ClusterRole 26 | metadata: 27 | name: lazyxds 28 | labels: 29 | app: lazyxds 30 | rules: 31 | - apiGroups: ["*"] 32 | resources: ["pods", "nodes", "namespaces", "endpoints", "secrets"] 33 | verbs: ["get", "list", "watch"] 34 | - apiGroups: ["*"] 35 | resources: ["configmaps", "deployments", "services", "roles", "rolebindings", "serviceaccounts"] 36 | verbs: ["get", "list", "watch", "create", "update", "delete"] 37 | - apiGroups: ["networking.istio.io"] 38 | resources: ["*"] 39 | verbs: ["*"] 40 | --- 41 | apiVersion: rbac.authorization.k8s.io/v1 42 | kind: ClusterRoleBinding 43 | metadata: 44 | name: lazyxds 45 | labels: 46 | app: lazyxds 47 | roleRef: 48 | apiGroup: rbac.authorization.k8s.io 49 | kind: ClusterRole 50 | name: lazyxds 51 | subjects: 52 | - kind: ServiceAccount 53 | namespace: istio-system 54 | name: lazyxds 55 | --- 56 | apiVersion: apps/v1 57 | kind: Deployment 58 | metadata: 59 | labels: 60 | app: lazyxds 61 | name: lazyxds 62 | namespace: istio-system 63 | spec: 64 | replicas: 1 65 | selector: 66 | matchLabels: 67 | app: lazyxds 68 | template: 69 | metadata: 70 | labels: 71 | app: lazyxds 72 | spec: 73 | serviceAccountName: lazyxds 74 | containers: 75 | - image: aeraki/lazyxds:latest 76 | imagePullPolicy: Always 77 | name: app 78 | ports: 79 | - containerPort: 8080 80 | protocol: TCP 81 | --- 82 | apiVersion: v1 83 | kind: Service 84 | metadata: 85 | labels: 86 | app: lazyxds 87 | name: lazyxds 88 | namespace: istio-system 89 | spec: 90 | ports: 91 | - name: grpc-als 92 | port: 8080 93 | protocol: TCP 94 | selector: 95 | app: lazyxds 96 | type: ClusterIP 97 | --- 98 | -------------------------------------------------------------------------------- /pkg/accesslog/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package accesslog 16 | 17 | import ( 18 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 19 | envoycore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" 20 | al "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" 21 | als "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3" 22 | ) 23 | 24 | // AccessHandler ... 25 | type AccessHandler interface { 26 | HandleAccess(fromIP, svcID, toIP string) error 27 | } 28 | 29 | // StreamAccessLogs accept access log from lazy xds egress gateway 30 | func (server *Server) StreamAccessLogs(logStream als.AccessLogService_StreamAccessLogsServer) error { 31 | for { 32 | data, err := logStream.Recv() 33 | if err != nil { 34 | return err 35 | } 36 | 37 | httpLog := data.GetHttpLogs() 38 | if httpLog != nil { 39 | for _, entry := range httpLog.LogEntry { 40 | server.log.V(4).Info("http log entry", "entry", entry) 41 | fromIP := getDownstreamIP(entry) 42 | if fromIP == "" { 43 | continue 44 | } 45 | 46 | upstreamCluster := entry.CommonProperties.UpstreamCluster 47 | svcID := utils.UpstreamCluster2ServiceID(upstreamCluster) 48 | 49 | toIP := getUpstreamIP(entry) 50 | 51 | if err := server.handler.HandleAccess(fromIP, svcID, toIP); err != nil { 52 | server.log.Error(err, "handle access error") 53 | } 54 | } 55 | } 56 | } 57 | } 58 | 59 | func getDownstreamIP(entry *al.HTTPAccessLogEntry) string { 60 | if entry.CommonProperties.DownstreamRemoteAddress == nil { 61 | return "" 62 | } 63 | downstreamSock, ok := entry.CommonProperties.DownstreamRemoteAddress.Address.(*envoycore.Address_SocketAddress) 64 | if !ok { 65 | return "" 66 | } 67 | if downstreamSock == nil || downstreamSock.SocketAddress == nil { 68 | return "" 69 | } 70 | return downstreamSock.SocketAddress.Address 71 | } 72 | 73 | func getUpstreamIP(entry *al.HTTPAccessLogEntry) string { 74 | if entry.CommonProperties.UpstreamRemoteAddress == nil { 75 | return "" 76 | } 77 | upstreamSock, ok := entry.CommonProperties.UpstreamRemoteAddress.Address.(*envoycore.Address_SocketAddress) 78 | if !ok { 79 | return "" 80 | } 81 | if upstreamSock == nil || upstreamSock.SocketAddress == nil { 82 | return "" 83 | } 84 | 85 | return upstreamSock.SocketAddress.Address 86 | } 87 | -------------------------------------------------------------------------------- /pkg/accesslog/server.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package accesslog 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "net" 21 | 22 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 23 | envoy_service_accesslog_v3 "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v3" 24 | "github.com/go-logr/logr" 25 | "google.golang.org/grpc" 26 | "k8s.io/klog/v2/klogr" 27 | ) 28 | 29 | // Server ... 30 | type Server struct { 31 | log logr.Logger 32 | handler AccessHandler 33 | } 34 | 35 | // NewAccessLogServer ... 36 | func NewAccessLogServer(handler AccessHandler) *Server { 37 | return &Server{ 38 | log: klogr.New().WithName("access-log-server"), 39 | handler: handler, 40 | } 41 | } 42 | 43 | // Serve ... 44 | func (server *Server) Serve() error { 45 | port := fmt.Sprintf(":%d", config.AccessLogServicePort) 46 | lis, err := net.Listen("tcp", port) 47 | if err != nil { 48 | return errors.New("failed to listen") 49 | } 50 | svc := grpc.NewServer() 51 | envoy_service_accesslog_v3.RegisterAccessLogServiceServer(svc, server) 52 | 53 | go func() { 54 | if err := svc.Serve(lis); err != nil { 55 | server.log.Error(err, "serve error") 56 | } 57 | }() 58 | // todo maybe graceful shutdown 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /pkg/controller/aggregation.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sync" 21 | 22 | "github.com/aeraki-mesh/lazyxds/pkg/controller/discoveryselector" 23 | "k8s.io/apimachinery/pkg/labels" 24 | 25 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 26 | "github.com/aeraki-mesh/lazyxds/pkg/controller/endpoints" 27 | "github.com/aeraki-mesh/lazyxds/pkg/controller/lazyservice" 28 | "github.com/aeraki-mesh/lazyxds/pkg/controller/namespace" 29 | "github.com/aeraki-mesh/lazyxds/pkg/controller/service" 30 | "github.com/aeraki-mesh/lazyxds/pkg/controller/serviceentry" 31 | "github.com/aeraki-mesh/lazyxds/pkg/controller/sidecar" 32 | "github.com/aeraki-mesh/lazyxds/pkg/controller/virtualservice" 33 | "github.com/aeraki-mesh/lazyxds/pkg/model" 34 | "github.com/go-logr/logr" 35 | istioclient "istio.io/client-go/pkg/clientset/versioned" 36 | istioinformer "istio.io/client-go/pkg/informers/externalversions" 37 | "k8s.io/client-go/kubernetes" 38 | "k8s.io/klog/v2" 39 | "k8s.io/klog/v2/klogr" 40 | ) 41 | 42 | const ( 43 | // ResourcePrefix ... 44 | ResourcePrefix = "lazyxds-" 45 | // EgressGatewayFullName ... 46 | EgressGatewayFullName = config.IstioNamespace + "/" + config.EgressGatewayName 47 | // ServiceAddressKey ... 48 | ServiceAddressKey = "lazyxds-service-address" 49 | ) 50 | 51 | // AggregationController ... 52 | type AggregationController struct { 53 | log logr.Logger 54 | stop <-chan struct{} 55 | 56 | KubeClient *kubernetes.Clientset 57 | istioClient *istioclient.Clientset 58 | istioInformer istioinformer.SharedInformerFactory 59 | multiCluster map[string]*Cluster 60 | 61 | namespaceController *namespace.Controller 62 | serviceController *service.Controller 63 | endpointsController *endpoints.Controller 64 | 65 | virtualServiceController *virtualservice.Controller 66 | sidecarController *sidecar.Controller 67 | serviceEntryController *serviceentry.Controller 68 | lazyServiceController *lazyservice.Controller 69 | configMapController *discoveryselector.Controller 70 | 71 | // all services of all k8s clusters 72 | services sync.Map // format: {svcID: *model.svc} 73 | // all lazy services 74 | lazyServices map[string]*model.Service 75 | // istio discovery namespace selector 76 | selectors []labels.Selector 77 | 78 | namespaces sync.Map 79 | endpoints sync.Map 80 | 81 | serviceEntries sync.Map 82 | httpServicesBinding sync.Map // {vsID: {svcID set}} 83 | tcpServicesBinding sync.Map // {vsID: {svcID set}} 84 | } 85 | 86 | // NewController ... 87 | func NewController(istioClient *istioclient.Clientset, kubeClient *kubernetes.Clientset, stop <-chan struct{}) *AggregationController { 88 | c := &AggregationController{ 89 | log: klogr.New().WithName("AggregationController"), 90 | istioClient: istioClient, 91 | KubeClient: kubeClient, 92 | istioInformer: istioinformer.NewSharedInformerFactory(istioClient, 0), 93 | stop: stop, 94 | multiCluster: make(map[string]*Cluster), 95 | lazyServices: make(map[string]*model.Service), 96 | } 97 | 98 | c.virtualServiceController = virtualservice.NewController( 99 | c.istioInformer.Networking().V1alpha3().VirtualServices(), 100 | c.syncVirtualService, 101 | c.deleteVirtualService, 102 | ) 103 | 104 | c.sidecarController = sidecar.NewController( 105 | c.istioInformer.Networking().V1alpha3().Sidecars(), 106 | c.syncSidecar, 107 | c.deleteSidecar, 108 | ) 109 | 110 | c.serviceEntryController = serviceentry.NewController( 111 | c.istioInformer.Networking().V1alpha3().ServiceEntries(), 112 | c.syncServiceEntry, 113 | c.deleteServiceEntry, 114 | ) 115 | 116 | c.lazyServiceController = lazyservice.NewController( 117 | c.syncLazyService, 118 | ) 119 | 120 | c.configMapController = discoveryselector.NewController( 121 | c.KubeClient, 122 | c.updateDiscoverySelector, 123 | c.reconcileAllNamespaces, 124 | ) 125 | 126 | return c 127 | } 128 | 129 | // AddCluster ... 130 | func (c *AggregationController) AddCluster(name string, client *kubernetes.Clientset) error { 131 | if _, ok := c.multiCluster[name]; ok { 132 | return fmt.Errorf("cluster %s already exists", name) 133 | } 134 | cluster := NewCluster(name, client) 135 | c.multiCluster[name] = cluster 136 | 137 | c.namespaceController = namespace.NewController( 138 | name, 139 | client.CoreV1(), 140 | cluster.Informer.Core().V1().Namespaces(), 141 | c.syncNamespace, 142 | c.deleteNamespace, 143 | ) 144 | 145 | c.serviceController = service.NewController( 146 | name, 147 | client.CoreV1(), 148 | cluster.Informer.Core().V1().Services(), 149 | c.syncService, 150 | c.deleteService, 151 | ) 152 | 153 | c.endpointsController = endpoints.NewController( 154 | name, 155 | client.CoreV1(), 156 | cluster.Informer.Core().V1().Endpoints(), 157 | c.syncEndpoints, 158 | c.deleteEndpoints, 159 | ) 160 | 161 | klog.Info("Starting Namespace controller", "cluster", name) 162 | go c.namespaceController.Run(2, c.stop) 163 | 164 | klog.Info("Starting Service controller", "cluster", name) 165 | go c.serviceController.Run(4, c.stop) 166 | 167 | klog.Infof("Starting Endpoints controller", "cluster", name) 168 | go c.endpointsController.Run(4, c.stop) 169 | 170 | cluster.Informer.Start(c.stop) 171 | return nil 172 | } 173 | 174 | // Run ... 175 | func (c *AggregationController) Run() { 176 | go c.virtualServiceController.Run(2, c.stop) 177 | go c.sidecarController.Run(2, c.stop) 178 | go c.serviceEntryController.Run(2, c.stop) 179 | go c.lazyServiceController.Run(4, c.stop) 180 | go c.configMapController.Run(c.stop) 181 | 182 | c.istioInformer.Start(c.stop) 183 | } 184 | 185 | // ClusterClient ... 186 | func (c *AggregationController) ClusterClient(name string) *kubernetes.Clientset { 187 | cluster := c.multiCluster[name] 188 | if cluster == nil { 189 | return nil 190 | } 191 | return cluster.Client 192 | } 193 | 194 | // HandleAccess ... 195 | func (c *AggregationController) HandleAccess(fromIP, svcID, toIP string) error { 196 | c.log.Info("HandleAccess", "fromIP", fromIP, "svcID", svcID, "toIP", toIP) 197 | 198 | fromSvcID := c.IP2ServiceID(fromIP) 199 | if fromSvcID == "" { 200 | return nil 201 | } 202 | 203 | lazySvc, ok := c.lazyServices[fromSvcID] 204 | if !ok { 205 | return nil 206 | } 207 | 208 | if svcID == "" { 209 | svcID := c.IP2ServiceID(toIP) 210 | if svcID == "" { 211 | return nil 212 | } 213 | } 214 | 215 | c.log.Info("Add service to egress of lazyservice", "fromService", fromSvcID, "toService", svcID) 216 | lazySvc.EgressService[svcID] = struct{}{} 217 | 218 | return c.reconcileLazyService(context.TODO(), lazySvc) 219 | } 220 | -------------------------------------------------------------------------------- /pkg/controller/cluster.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "k8s.io/client-go/informers" 19 | "k8s.io/client-go/kubernetes" 20 | ) 21 | 22 | // Cluster ... 23 | type Cluster struct { 24 | name string 25 | Client *kubernetes.Clientset 26 | Informer informers.SharedInformerFactory 27 | } 28 | 29 | // NewCluster ... 30 | func NewCluster(name string, client *kubernetes.Clientset) *Cluster { 31 | return &Cluster{ 32 | name: name, 33 | Client: client, 34 | Informer: informers.NewSharedInformerFactory(client, 0), 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/controller/discovery_selector.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 19 | "k8s.io/apimachinery/pkg/labels" 20 | ) 21 | 22 | func (c *AggregationController) updateDiscoverySelector(discoverySelector []*metav1.LabelSelector) error { 23 | var selectors []labels.Selector 24 | // convert LabelSelectors to Selectors 25 | for _, selector := range discoverySelector { 26 | ls, err := metav1.LabelSelectorAsSelector(selector) 27 | if err != nil { 28 | c.log.Error(err, "error initializing discovery namespaces filter, invalid discovery selector: %v") 29 | return err 30 | } 31 | selectors = append(selectors, ls) 32 | } 33 | 34 | c.selectors = selectors 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /pkg/controller/discoveryselector/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package discoveryselector 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 22 | "github.com/go-logr/logr" 23 | meshv1alpha1 "istio.io/api/mesh/v1alpha1" 24 | corev1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/kubernetes" 27 | "k8s.io/klog/v2/klogr" 28 | "sigs.k8s.io/yaml" 29 | ) 30 | 31 | // Controller is responsible for watching discoverySelectors 32 | type Controller struct { 33 | log logr.Logger 34 | clientset *kubernetes.Clientset 35 | updateDiscoverySelector func([]*metav1.LabelSelector) error 36 | reconcileAllNamespaces func(context.Context) error 37 | } 38 | 39 | // NewController creates a new service controller 40 | func NewController( 41 | clientset *kubernetes.Clientset, 42 | updateDiscoverySelector func([]*metav1.LabelSelector) error, 43 | reconcileAllNamespaces func(context.Context) error, 44 | ) *Controller { 45 | logger := klogr.New().WithName("ConfigMapController") 46 | c := &Controller{ 47 | log: logger, 48 | clientset: clientset, 49 | updateDiscoverySelector: updateDiscoverySelector, 50 | reconcileAllNamespaces: reconcileAllNamespaces, 51 | } 52 | 53 | return c 54 | } 55 | 56 | // Run begins watching and syncing. 57 | func (c *Controller) Run(stopCh <-chan struct{}) { 58 | c.log.Info("starting discoveryselector controller...") 59 | ctx := log.WithContext(context.Background(), c.log) 60 | configMapWatcher, err := c.clientset.CoreV1().ConfigMaps("istio-system").Watch(ctx, metav1.ListOptions{ 61 | LabelSelector: "release=istio", 62 | }) 63 | if err != nil { 64 | c.log.Error(err, "watch configMap failed") 65 | return 66 | } 67 | for { 68 | select { 69 | case e, ok := <-configMapWatcher.ResultChan(): 70 | if !ok { 71 | c.log.Info("configMapWatcher chan has been close!") 72 | c.log.Info("clean chan over!") 73 | time.Sleep(time.Second * 5) 74 | } 75 | if e.Object != nil { 76 | c.log.Info("configMapWatcher chan is ok") 77 | dataMap := e.Object.DeepCopyObject().(*corev1.ConfigMap).Data 78 | if _, ok := dataMap["mesh"]; !ok { 79 | break 80 | } 81 | meshconfig := meshv1alpha1.MeshConfig{} 82 | err := yaml.Unmarshal([]byte(dataMap["mesh"]), &meshconfig) 83 | if err != nil { 84 | c.log.Error(err, "deserialize meshconfig failed") 85 | break 86 | } 87 | c.log.Info("meshconfig.DiscoverySelectors modified", "matchLabels", meshconfig.DiscoverySelectors) 88 | err = c.updateDiscoverySelector(meshconfig.DiscoverySelectors) 89 | if err != nil { 90 | c.log.Error(err, "update DiscoverySelector error") 91 | break 92 | } 93 | err = c.reconcileAllNamespaces(ctx) 94 | if err != nil { 95 | c.log.Error(err, "reconcileAllNamespaces error") 96 | } 97 | } 98 | case <-stopCh: 99 | c.log.Info("close configMapWatcher") 100 | return 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /pkg/controller/endpoints.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/aeraki-mesh/lazyxds/pkg/model" 21 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 22 | corev1 "k8s.io/api/core/v1" 23 | ) 24 | 25 | func (c *AggregationController) syncEndpoints(ctx context.Context, endpoints *corev1.Endpoints) error { 26 | ep := model.NewEndpoints(endpoints) 27 | c.endpoints.Store(ep.ID(), ep) 28 | 29 | return nil 30 | } 31 | 32 | func (c *AggregationController) deleteEndpoints(ctx context.Context, name, ns string) error { 33 | id := utils.FQDN(name, ns) 34 | c.endpoints.Delete(id) 35 | return nil 36 | } 37 | 38 | // IP2ServiceID ... 39 | func (c *AggregationController) IP2ServiceID(targetIP string) string { 40 | var svcID string 41 | c.endpoints.Range(func(key, value interface{}) bool { 42 | ep := value.(*model.Endpoints) 43 | for _, ip := range ep.IPList { 44 | if targetIP == ip { 45 | svcID = ep.ID() 46 | return false 47 | } 48 | } 49 | return true 50 | }) 51 | 52 | return svcID 53 | } 54 | -------------------------------------------------------------------------------- /pkg/controller/endpoints/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package endpoints 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "reflect" 21 | "time" 22 | 23 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 24 | "github.com/go-logr/logr" 25 | corev1 "k8s.io/api/core/v1" 26 | apierrors "k8s.io/apimachinery/pkg/api/errors" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | "k8s.io/apimachinery/pkg/util/wait" 29 | coreinformers "k8s.io/client-go/informers/core/v1" 30 | v1 "k8s.io/client-go/kubernetes/typed/core/v1" 31 | corelisters "k8s.io/client-go/listers/core/v1" 32 | "k8s.io/client-go/tools/cache" 33 | queue "k8s.io/client-go/util/workqueue" 34 | "k8s.io/klog/v2/klogr" 35 | ) 36 | 37 | // Controller is responsible for synchronizing endpoints objects. 38 | type Controller struct { 39 | clusterName string 40 | log logr.Logger 41 | getter v1.EndpointsGetter 42 | lister corelisters.EndpointsLister 43 | listerSynced cache.InformerSynced 44 | queue queue.RateLimitingInterface 45 | syncEndpoints func(context.Context, *corev1.Endpoints) error 46 | deleteEndpoints func(context.Context, string, string) error 47 | } 48 | 49 | // NewController creates a new endpoints controller 50 | func NewController( 51 | clusterName string, 52 | getter v1.EndpointsGetter, 53 | informer coreinformers.EndpointsInformer, 54 | sync func(context.Context, *corev1.Endpoints) error, 55 | delete func(context.Context, string, string) error, 56 | ) *Controller { 57 | logger := klogr.New().WithName("EndpointsController") 58 | c := &Controller{ 59 | clusterName: clusterName, 60 | log: logger, 61 | getter: getter, 62 | lister: informer.Lister(), 63 | listerSynced: informer.Informer().HasSynced, 64 | queue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "endpoints"), 65 | syncEndpoints: sync, 66 | deleteEndpoints: delete, 67 | } 68 | 69 | informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 70 | AddFunc: c.add, 71 | UpdateFunc: c.update, 72 | DeleteFunc: c.delete, 73 | }) 74 | 75 | return c 76 | } 77 | 78 | func (c *Controller) add(obj interface{}) { 79 | endpoints, _ := obj.(*corev1.Endpoints) 80 | c.log.V(4).Info("Adding Endpoints", "name", endpoints.Name) 81 | c.enqueue(endpoints) 82 | } 83 | 84 | func (c *Controller) update(oldObj, curObj interface{}) { 85 | old, _ := oldObj.(*corev1.Endpoints) 86 | current, _ := curObj.(*corev1.Endpoints) 87 | if !c.needsUpdate(old, current) { 88 | return 89 | } 90 | 91 | c.log.V(4).Info("Updating Endpoints", "name", current.Name) 92 | c.enqueue(current) 93 | } 94 | 95 | func (c *Controller) needsUpdate(old *corev1.Endpoints, new *corev1.Endpoints) bool { 96 | return !reflect.DeepEqual(old.Subsets, new.Subsets) || new.GetDeletionTimestamp() != nil 97 | } 98 | 99 | func (c *Controller) delete(obj interface{}) { 100 | endpoints, ok := obj.(*corev1.Endpoints) 101 | if !ok { 102 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 103 | if !ok { 104 | c.log.Info("Couldn't get object from tombstone", "obj", obj) 105 | return 106 | } 107 | endpoints, ok = tombstone.Obj.(*corev1.Endpoints) 108 | if !ok { 109 | c.log.Info("Tombstone contained object that is not a Endpoints", "obj", obj) 110 | return 111 | } 112 | } 113 | c.log.V(4).Info("Deleting Endpoints", "name", endpoints.Name) 114 | c.enqueue(obj) 115 | } 116 | 117 | func (c *Controller) enqueue(obj interface{}) { 118 | key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) 119 | if err != nil { 120 | utilruntime.HandleError(err) 121 | return 122 | } 123 | c.queue.Add(key) 124 | } 125 | 126 | // Run begins watching and syncing. 127 | func (c *Controller) Run(workers int, stopCh <-chan struct{}) { 128 | defer utilruntime.HandleCrash() 129 | defer c.queue.ShutDown() 130 | 131 | c.log.Info("Starting Endpoints controller") 132 | defer c.log.Info("Shutting down Endpoints controller") 133 | 134 | if !cache.WaitForNamedCacheSync("Endpoints", stopCh, c.listerSynced) { 135 | return 136 | } 137 | 138 | for i := 0; i < workers; i++ { 139 | go wait.Until(c.worker, time.Second, stopCh) 140 | } 141 | 142 | <-stopCh 143 | } 144 | 145 | func (c *Controller) worker() { 146 | for c.processNextWorkItem() { 147 | } 148 | } 149 | 150 | func (c *Controller) processNextWorkItem() bool { 151 | key, quit := c.queue.Get() 152 | if quit { 153 | return false 154 | } 155 | defer c.queue.Done(key) 156 | 157 | logger := c.log.WithValues("key", key) 158 | ctx := log.WithContext(context.Background(), logger) 159 | err := c.syncFromKey(ctx, key.(string)) 160 | if err != nil { 161 | c.queue.AddRateLimited(key) 162 | logger.Error(err, "Sync error") 163 | return true 164 | } 165 | 166 | c.queue.Forget(key) 167 | logger.Info("Successfully synced") 168 | 169 | return true 170 | } 171 | 172 | func (c *Controller) syncFromKey(ctx context.Context, key string) error { 173 | startTime := time.Now() 174 | logger := log.FromContext(ctx) 175 | logger.V(4).Info("Starting sync") 176 | defer func() { 177 | logger.V(4).Info("Finished sync endpoints", "duration", time.Since(startTime).String()) 178 | }() 179 | 180 | ns, name, err := cache.SplitMetaNamespaceKey(key) 181 | if err != nil { 182 | return err 183 | } 184 | 185 | endpoints, err := c.lister.Endpoints(ns).Get(name) 186 | if err != nil && apierrors.IsNotFound(err) { 187 | logger.V(4).Info("Endpoints has been deleted") 188 | // todo may need delay for a while, because access log may be late 189 | return c.deleteEndpoints(ctx, name, ns) 190 | } 191 | if err != nil { 192 | return fmt.Errorf("unable to retrieve endpoints from store: error %w", err) 193 | } 194 | 195 | return c.syncEndpoints(ctx, endpoints) 196 | } 197 | -------------------------------------------------------------------------------- /pkg/controller/lazy_source.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sort" 21 | 22 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 23 | "github.com/aeraki-mesh/lazyxds/pkg/model" 24 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 25 | ) 26 | 27 | func (c *AggregationController) syncLazyService(ctx context.Context, id string) (err error) { 28 | // todo if something wrong, need put back to queue 29 | lazySvc := c.lazyServices[id] 30 | if lazySvc == nil { 31 | return c.deleteLazyService(ctx, id) 32 | } 33 | return c.reconcileLazyService(ctx, lazySvc) 34 | } 35 | 36 | func (c *AggregationController) tryReconcileLazyService(ctx context.Context, svc *model.Service) (err error) { 37 | id := svc.ID() 38 | v, ok := c.namespaces.Load(svc.Namespace) 39 | if !ok { 40 | return fmt.Errorf("namespace %s not found", svc.Namespace) 41 | } 42 | ns := v.(*model.Namespace) 43 | svc.UpdateNSLazy(ns.LazyStatus) 44 | 45 | if !svc.Status.LazyEnabled && svc.Spec.LazyEnabled { 46 | c.lazyServices[id] = svc 47 | c.lazyServiceController.Add(id) 48 | } else if svc.Status.LazyEnabled && !svc.Spec.LazyEnabled { 49 | delete(c.lazyServices, id) 50 | c.lazyServiceController.Add(id) 51 | } 52 | 53 | return nil 54 | } 55 | 56 | func (c *AggregationController) reconcileLazyService(ctx context.Context, lazySvc *model.Service) (err error) { 57 | defer func() { 58 | if err == nil { 59 | lazySvc.FinishReconcileLazy() 60 | } 61 | }() 62 | if err := c.syncEnvoyFilterOfLazySource(ctx, lazySvc); err != nil { 63 | return err 64 | } 65 | if err := c.syncSidecarOfLazySource(ctx, lazySvc); err != nil { 66 | return err 67 | } 68 | return nil 69 | } 70 | 71 | func (c *AggregationController) reconcileAllLazyServices(ctx context.Context) error { 72 | var err error 73 | for _, ls := range c.lazyServices { 74 | c.lazyServiceController.Add(ls.ID()) 75 | } 76 | 77 | return err 78 | } 79 | 80 | func (c *AggregationController) deleteLazyService(ctx context.Context, id string) error { 81 | defer func() { 82 | v, ok := c.services.Load(id) 83 | if ok { 84 | svc := v.(*model.Service) 85 | svc.FinishReconcileLazy() 86 | } 87 | }() 88 | name, namespace := utils.ParseID(id) 89 | 90 | if err := c.removeEnvoyFilter(ctx, name, namespace); err != nil { 91 | return err 92 | } 93 | if err := c.removeSidecar(ctx, name, namespace); err != nil { 94 | return err 95 | } 96 | return nil 97 | } 98 | 99 | func (c *AggregationController) visibleServiceOfLazyService(lazySvc *model.Service) map[string]struct{} { 100 | egress := make(map[string]struct{}) 101 | 102 | for svcID := range lazySvc.EgressService { 103 | binding := c.getHTTPServiceBinding(svcID) 104 | for id := range binding { 105 | egress[id] = struct{}{} 106 | } 107 | } 108 | // currently all tcp service should be visible 109 | c.services.Range(func(key, value interface{}) bool { 110 | svc := value.(*model.Service) 111 | if svc.Namespace == config.IstioNamespace { // istio-system always exported 112 | return true 113 | } 114 | 115 | if len(svc.Spec.TCPPorts) > 0 { 116 | egress[key.(string)] = struct{}{} 117 | } 118 | return true 119 | }) 120 | 121 | return egress 122 | } 123 | 124 | func (c *AggregationController) egressListOfLazySource(lazySvc *model.Service) []string { 125 | var list []string 126 | 127 | for id := range c.visibleServiceOfLazyService(lazySvc) { 128 | list = append(list, utils.ServiceID2EgressString(id)) 129 | } 130 | 131 | c.serviceEntries.Range(func(key, value interface{}) bool { 132 | _, ns := utils.ParseID(key.(string)) 133 | hosts := value.([]string) 134 | for _, host := range hosts { 135 | list = append(list, fmt.Sprintf("%s/%s", ns, host)) 136 | } 137 | return true 138 | }) 139 | 140 | sort.Slice(list, func(i, j int) bool { 141 | return list[i] < list[j] 142 | }) 143 | 144 | list = append([]string{"istio-system/*"}, list...) 145 | return list 146 | } 147 | -------------------------------------------------------------------------------- /pkg/controller/lazyservice/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lazyservice 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | "github.com/go-logr/logr" 22 | "k8s.io/klog/v2/klogr" 23 | 24 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 25 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 26 | "k8s.io/apimachinery/pkg/util/wait" 27 | queue "k8s.io/client-go/util/workqueue" 28 | ) 29 | 30 | // Controller is responsible for synchronizing lazy service. 31 | type Controller struct { 32 | log logr.Logger 33 | queue queue.RateLimitingInterface 34 | syncService func(context.Context, string) error 35 | } 36 | 37 | // NewController creates a new lazy service controller 38 | func NewController(syncService func(context.Context, string) error) *Controller { 39 | logger := klogr.New().WithName("LazyServiceController") 40 | c := &Controller{ 41 | log: logger, 42 | queue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "lazyservice"), 43 | syncService: syncService, 44 | } 45 | 46 | return c 47 | } 48 | 49 | // Add ... 50 | func (c *Controller) Add(id string) { 51 | c.log.V(4).Info("Adding LazyService", "name", id) 52 | c.queue.Add(id) 53 | } 54 | 55 | // Run begins watching and syncing. 56 | func (c *Controller) Run(workers int, stopCh <-chan struct{}) { 57 | defer utilruntime.HandleCrash() 58 | defer c.queue.ShutDown() 59 | 60 | c.log.Info("Starting LazyService controller") 61 | defer c.log.Info("Shutting down LazyService controller") 62 | 63 | for i := 0; i < workers; i++ { 64 | go wait.Until(c.worker, time.Second, stopCh) 65 | } 66 | 67 | <-stopCh 68 | } 69 | 70 | func (c *Controller) worker() { 71 | for c.processNextWorkItem() { 72 | } 73 | } 74 | 75 | func (c *Controller) processNextWorkItem() bool { 76 | key, quit := c.queue.Get() 77 | if quit { 78 | return false 79 | } 80 | defer c.queue.Done(key) 81 | 82 | logger := c.log.WithValues("key", key) 83 | ctx := log.WithContext(context.Background(), logger) 84 | err := c.syncFromKey(ctx, key.(string)) 85 | if err != nil { 86 | c.queue.AddRateLimited(key) 87 | logger.Error(err, "Sync error") 88 | return true 89 | } 90 | 91 | c.queue.Forget(key) 92 | logger.Info("Successfully synced") 93 | 94 | return true 95 | } 96 | 97 | func (c *Controller) syncFromKey(ctx context.Context, key string) error { 98 | startTime := time.Now() 99 | logger := log.FromContext(ctx) 100 | logger.V(4).Info("Starting sync") 101 | defer func() { 102 | logger.V(4).Info("Finished sync LazyService", "duration", time.Since(startTime).String()) 103 | }() 104 | 105 | return c.syncService(ctx, key) 106 | } 107 | -------------------------------------------------------------------------------- /pkg/controller/namespace.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/aeraki-mesh/lazyxds/pkg/model" 21 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 22 | corev1 "k8s.io/api/core/v1" 23 | ) 24 | 25 | func (c *AggregationController) syncNamespace(ctx context.Context, clusterName string, namespace *corev1.Namespace) (err error) { 26 | id := namespace.Name 27 | v, _ := c.namespaces.LoadOrStore(id, model.NewNamespace(namespace)) 28 | ns := v.(*model.Namespace) 29 | 30 | ns.Update(clusterName, namespace) 31 | 32 | return c.reconcileNamespace(ctx, ns) 33 | } 34 | 35 | func (c *AggregationController) deleteNamespace(ctx context.Context, clusterName string, name string) (err error) { 36 | logger := log.FromContext(ctx) 37 | 38 | logger.Info("Namespace has been deleted") 39 | id := name 40 | 41 | v, ok := c.namespaces.Load(id) 42 | if !ok { 43 | return nil 44 | } 45 | ns := v.(*model.Namespace) 46 | 47 | ns.Delete(clusterName) 48 | if len(ns.Distribution) == 0 { 49 | c.namespaces.Delete(name) 50 | return nil 51 | } 52 | 53 | return c.reconcileNamespace(ctx, ns) 54 | } 55 | 56 | // reconcileAllNamespace do this when namespace labels updated 57 | func (c *AggregationController) reconcileNamespace(ctx context.Context, ns *model.Namespace) (err error) { 58 | c.serviceController.ReconcileServices(ns) 59 | return c.reconcileAllLazyServices(ctx) 60 | } 61 | 62 | // reconcileAllNamespace do this when discoverySelector updated 63 | func (c *AggregationController) reconcileAllNamespaces(ctx context.Context) (err error) { 64 | c.serviceController.ReconcileAllServices() 65 | return c.reconcileAllLazyServices(ctx) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/controller/namespace/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package namespace 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "reflect" 21 | "time" 22 | 23 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 24 | "github.com/go-logr/logr" 25 | corev1 "k8s.io/api/core/v1" 26 | apierrors "k8s.io/apimachinery/pkg/api/errors" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | "k8s.io/apimachinery/pkg/util/wait" 29 | coreinformers "k8s.io/client-go/informers/core/v1" 30 | v1 "k8s.io/client-go/kubernetes/typed/core/v1" 31 | corelisters "k8s.io/client-go/listers/core/v1" 32 | "k8s.io/client-go/tools/cache" 33 | queue "k8s.io/client-go/util/workqueue" 34 | "k8s.io/klog/v2/klogr" 35 | ) 36 | 37 | // Controller is responsible for synchronizing namespace objects. 38 | type Controller struct { 39 | clusterName string 40 | log logr.Logger 41 | getter v1.NamespacesGetter 42 | lister corelisters.NamespaceLister 43 | listerSynced cache.InformerSynced 44 | queue queue.RateLimitingInterface 45 | syncNamespace func(context.Context, string, *corev1.Namespace) error 46 | deleteNamespace func(context.Context, string, string) error 47 | } 48 | 49 | // NewController creates a new namespace controller 50 | func NewController( 51 | clusterName string, 52 | getter v1.NamespacesGetter, 53 | informer coreinformers.NamespaceInformer, 54 | sync func(context.Context, string, *corev1.Namespace) error, 55 | delete func(context.Context, string, string) error, 56 | ) *Controller { 57 | logger := klogr.New().WithName("NamespaceController") 58 | c := &Controller{ 59 | clusterName: clusterName, 60 | log: logger, 61 | getter: getter, 62 | lister: informer.Lister(), 63 | listerSynced: informer.Informer().HasSynced, 64 | queue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "namespace"), 65 | syncNamespace: sync, 66 | deleteNamespace: delete, 67 | } 68 | 69 | informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 70 | AddFunc: c.add, 71 | UpdateFunc: c.update, 72 | DeleteFunc: c.delete, 73 | }) 74 | 75 | return c 76 | } 77 | 78 | func (c *Controller) add(obj interface{}) { 79 | namespace, _ := obj.(*corev1.Namespace) 80 | c.log.V(4).Info("Adding Namespace", "name", namespace.Name) 81 | c.enqueue(namespace) 82 | } 83 | 84 | func (c *Controller) update(oldObj, curObj interface{}) { 85 | old, _ := oldObj.(*corev1.Namespace) 86 | current, _ := curObj.(*corev1.Namespace) 87 | if !c.needsUpdate(old, current) { 88 | return 89 | } 90 | 91 | c.log.V(4).Info("Updating Namespace", "name", current.Name) 92 | c.enqueue(current) 93 | } 94 | 95 | func (c *Controller) needsUpdate(old *corev1.Namespace, new *corev1.Namespace) bool { 96 | return !reflect.DeepEqual(old.Annotations, new.Annotations) || !reflect.DeepEqual(old.Labels, new.Labels) || new.GetDeletionTimestamp() != nil 97 | } 98 | 99 | func (c *Controller) delete(obj interface{}) { 100 | namespace, ok := obj.(*corev1.Namespace) 101 | if !ok { 102 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 103 | if !ok { 104 | c.log.Info("Couldn't get object from tombstone", "obj", obj) 105 | return 106 | } 107 | namespace, ok = tombstone.Obj.(*corev1.Namespace) 108 | if !ok { 109 | c.log.Info("Tombstone contained object that is not a Namespace", "obj", obj) 110 | return 111 | } 112 | } 113 | c.log.V(4).Info("Deleting Namespace", "name", namespace.Name) 114 | c.enqueue(obj) 115 | } 116 | 117 | func (c *Controller) enqueue(obj interface{}) { 118 | key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) 119 | if err != nil { 120 | utilruntime.HandleError(err) 121 | return 122 | } 123 | c.queue.Add(key) 124 | } 125 | 126 | // Run begins watching and syncing. 127 | func (c *Controller) Run(workers int, stopCh <-chan struct{}) { 128 | defer utilruntime.HandleCrash() 129 | defer c.queue.ShutDown() 130 | 131 | c.log.Info("Starting Namespace controller") 132 | defer c.log.Info("Shutting down Namespace controller") 133 | 134 | if !cache.WaitForNamedCacheSync("Namespace", stopCh, c.listerSynced) { 135 | return 136 | } 137 | 138 | for i := 0; i < workers; i++ { 139 | go wait.Until(c.worker, time.Second, stopCh) 140 | } 141 | 142 | <-stopCh 143 | } 144 | 145 | func (c *Controller) worker() { 146 | for c.processNextWorkItem() { 147 | } 148 | } 149 | 150 | func (c *Controller) processNextWorkItem() bool { 151 | key, quit := c.queue.Get() 152 | if quit { 153 | return false 154 | } 155 | defer c.queue.Done(key) 156 | 157 | logger := c.log.WithValues("key", key) 158 | ctx := log.WithContext(context.Background(), logger) 159 | err := c.syncFromKey(ctx, key.(string)) 160 | if err != nil { 161 | c.queue.AddRateLimited(key) 162 | logger.Error(err, "Sync error") 163 | return true 164 | } 165 | 166 | c.queue.Forget(key) 167 | logger.Info("Successfully synced") 168 | 169 | return true 170 | } 171 | 172 | func (c *Controller) syncFromKey(ctx context.Context, key string) error { 173 | startTime := time.Now() 174 | logger := log.FromContext(ctx) 175 | logger.V(4).Info("Starting sync") 176 | defer func() { 177 | logger.V(4).Info("Finished sync namespace", "duration", time.Since(startTime).String()) 178 | }() 179 | name := key 180 | 181 | namespace, err := c.lister.Get(name) 182 | if err != nil && apierrors.IsNotFound(err) { 183 | logger.V(4).Info("Namespace has been deleted") 184 | return c.deleteNamespace(ctx, c.clusterName, name) 185 | } 186 | if err != nil { 187 | return fmt.Errorf("unable to retrieve namespace from store: error %w", err) 188 | } 189 | if !namespace.DeletionTimestamp.IsZero() { 190 | return c.deleteNamespace(ctx, c.clusterName, name) 191 | } 192 | 193 | return c.syncNamespace(ctx, c.clusterName, namespace) 194 | } 195 | -------------------------------------------------------------------------------- /pkg/controller/placeholder_service.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License.s 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "strconv" 21 | 22 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 23 | 24 | "github.com/aeraki-mesh/lazyxds/pkg/model" 25 | coreV1 "k8s.io/api/core/v1" 26 | "k8s.io/apimachinery/pkg/api/errors" 27 | metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | ) 29 | 30 | // PlaceHolderService ... 31 | const PlaceHolderService = "lazyxds-placeholder-service" 32 | 33 | // buildPlaceHolderService creates a fake service to make Istio create all the needed HTTP listeners at the envoy 34 | // proxy. A route will be added to all the created HTTP listeners to redirect traffic to the lazyxds egress gateway. 35 | func (c *AggregationController) buildPlaceHolderService(ctx context.Context) error { 36 | existingGlobalService, err := c.KubeClient.CoreV1().Services("istio-system").Get(ctx, 37 | PlaceHolderService, metaV1.GetOptions{}) 38 | firstCreate := false 39 | if err != nil { 40 | if errors.IsNotFound(err) { 41 | firstCreate = true 42 | } else { 43 | return err 44 | } 45 | } 46 | 47 | currentHTTPPorts := make(map[string]struct{}) 48 | handler := func(key, value interface{}) bool { 49 | svc := value.(*model.Service) 50 | for servicePort := range svc.Spec.HTTPPorts { 51 | currentHTTPPorts[servicePort] = struct{}{} 52 | } 53 | return true 54 | } 55 | c.services.Range(handler) 56 | if len(currentHTTPPorts) == 0 { 57 | // todo delete empty svc 58 | return nil 59 | } 60 | 61 | var servicePorts []coreV1.ServicePort 62 | for port := range currentHTTPPorts { 63 | portNum, _ := strconv.Atoi(port) 64 | servicePorts = append(servicePorts, coreV1.ServicePort{ 65 | Name: "http-" + port, 66 | Port: int32(portNum), 67 | }) 68 | } 69 | 70 | if firstCreate { 71 | if err := c.createPlaceholderService(ctx, servicePorts); err != nil { 72 | return err 73 | } 74 | } 75 | 76 | if !firstCreate && c.isServicePortsChanged(existingGlobalService, currentHTTPPorts) { 77 | if err := c.updatePlaceholderService(ctx, existingGlobalService, servicePorts); err != nil { 78 | return err 79 | } 80 | } 81 | 82 | return nil 83 | } 84 | 85 | func (c *AggregationController) isServicePortsChanged(existingGlobalService *coreV1.Service, 86 | currentHTTPPorts map[string]struct{}) bool { 87 | globalServicePorts := make(map[string]struct{}) 88 | for port := range existingGlobalService.Spec.Ports { 89 | portStr := fmt.Sprint(port) 90 | globalServicePorts[portStr] = struct{}{} 91 | } 92 | 93 | for httpPort := range currentHTTPPorts { 94 | if _, ok := globalServicePorts[httpPort]; !ok { 95 | return true 96 | } 97 | } 98 | 99 | for httpPort := range globalServicePorts { 100 | if _, ok := currentHTTPPorts[httpPort]; !ok { 101 | return true 102 | } 103 | } 104 | return false 105 | } 106 | 107 | func (c *AggregationController) updatePlaceholderService(ctx context.Context, existingGlobalService *coreV1.Service, 108 | servicePorts []coreV1.ServicePort) error { 109 | existingGlobalService.Spec.Ports = servicePorts 110 | _, err := c.KubeClient.CoreV1().Services("istio-system").Update(ctx, existingGlobalService, 111 | metaV1.UpdateOptions{ 112 | FieldManager: config.LazyXdsManager, 113 | }) 114 | if err != nil { 115 | return err 116 | } 117 | return nil 118 | } 119 | 120 | func (c *AggregationController) createPlaceholderService(ctx context.Context, servicePorts []coreV1.ServicePort) error { 121 | newGlobalService := &coreV1.Service{ 122 | ObjectMeta: metaV1.ObjectMeta{ 123 | Name: PlaceHolderService, 124 | Labels: map[string]string{ 125 | config.ManagedByLabel: config.LazyXdsManager, 126 | }, 127 | }, 128 | Spec: coreV1.ServiceSpec{ 129 | Selector: map[string]string{ 130 | "app": PlaceHolderService, 131 | }, 132 | Ports: servicePorts, 133 | }, 134 | } 135 | 136 | _, err := c.KubeClient.CoreV1().Services("istio-system").Create(ctx, newGlobalService, 137 | metaV1.CreateOptions{ 138 | FieldManager: config.LazyXdsManager, 139 | }) 140 | return err 141 | } 142 | -------------------------------------------------------------------------------- /pkg/controller/service.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/aeraki-mesh/lazyxds/pkg/model" 22 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 23 | corev1 "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/labels" 25 | ) 26 | 27 | func (c *AggregationController) syncService(ctx context.Context, clusterName string, service *corev1.Service) (err error) { 28 | selectors := c.selectors 29 | id := utils.FQDN(service.Name, service.Namespace) 30 | 31 | matched, err := c.matchDiscoverySelector(selectors, service) 32 | if err != nil { 33 | return err 34 | } 35 | if !matched { 36 | c.services.Delete(id) 37 | c.log.Info("Namespace label of service not match DiscoverySelector, delete it", "service", service.Name, "namespace", service.Namespace) 38 | return nil 39 | } 40 | 41 | v, _ := c.services.LoadOrStore(id, model.NewService(service)) 42 | svc := v.(*model.Service) 43 | svc.UpdateClusterService(clusterName, service) 44 | 45 | return c.doSyncService(ctx, svc) 46 | } 47 | 48 | func (c *AggregationController) matchDiscoverySelector(selectors []labels.Selector, service *corev1.Service) (bool, error) { 49 | isNsLabelsMatch := false 50 | if len(selectors) > 0 { 51 | v, ok := c.namespaces.Load(service.Namespace) 52 | if !ok { 53 | return false, fmt.Errorf("namespace %s not found", service.Namespace) 54 | } 55 | ns := v.(*model.Namespace) 56 | for _, selector := range selectors { 57 | if selector.Matches(labels.Set(ns.Labels)) { 58 | isNsLabelsMatch = true 59 | } 60 | } 61 | } else { 62 | // omitting discoverySelectors indicates discovering all namespaces 63 | isNsLabelsMatch = true 64 | } 65 | return isNsLabelsMatch, nil 66 | } 67 | 68 | func (c *AggregationController) deleteService(ctx context.Context, clusterName, svcID string) (err error) { 69 | v, ok := c.services.Load(svcID) 70 | if !ok { 71 | return nil 72 | } 73 | svc := v.(*model.Service) 74 | svc.DeleteFromCluster(clusterName) 75 | 76 | return c.doSyncService(ctx, svc) 77 | } 78 | 79 | func (c *AggregationController) doSyncService(ctx context.Context, svc *model.Service) error { 80 | // todo 可以并发 81 | if err := c.reconcileService(ctx, svc); err != nil { 82 | return err 83 | } 84 | 85 | if err := c.tryReconcileLazyService(ctx, svc); err != nil { 86 | return err 87 | } 88 | 89 | return nil 90 | } 91 | 92 | func (c *AggregationController) reconcileService(ctx context.Context, svc *model.Service) (err error) { 93 | defer func() { 94 | if len(svc.Distribution) == 0 { 95 | c.services.Delete(svc.ID()) 96 | } 97 | }() 98 | if !svc.NeedReconcileService() { 99 | return nil 100 | } 101 | 102 | defer func() { 103 | if err == nil { 104 | svc.FinishReconcileService() 105 | } 106 | }() 107 | 108 | if err = c.syncServiceRuleOfEgress(ctx, svc); err != nil { 109 | return err 110 | } 111 | 112 | // todo we haven't consider tcp 113 | //if len(svc.TCPPorts) > 0 { 114 | //} else { 115 | //} 116 | 117 | // UpdateClusterService global service 118 | if err = c.buildPlaceHolderService(ctx); err != nil { 119 | return err 120 | } 121 | 122 | if err = c.reconcileAllLazyServices(ctx); err != nil { // todo 检查必要性 123 | return err 124 | } 125 | 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /pkg/controller/service/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package service 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "reflect" 21 | "time" 22 | 23 | "github.com/aeraki-mesh/lazyxds/pkg/model" 24 | "k8s.io/apimachinery/pkg/labels" 25 | 26 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 27 | "github.com/go-logr/logr" 28 | corelisters "k8s.io/client-go/listers/core/v1" 29 | "k8s.io/klog/v2/klogr" 30 | 31 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 32 | corev1 "k8s.io/api/core/v1" 33 | apierrors "k8s.io/apimachinery/pkg/api/errors" 34 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 35 | "k8s.io/apimachinery/pkg/util/wait" 36 | coreinformers "k8s.io/client-go/informers/core/v1" 37 | v1 "k8s.io/client-go/kubernetes/typed/core/v1" 38 | "k8s.io/client-go/tools/cache" 39 | queue "k8s.io/client-go/util/workqueue" 40 | ) 41 | 42 | // Controller is responsible for synchronizing service objects. 43 | type Controller struct { 44 | clusterName string 45 | log logr.Logger 46 | getter v1.ServicesGetter 47 | lister corelisters.ServiceLister 48 | listerSynced cache.InformerSynced 49 | queue queue.RateLimitingInterface 50 | 51 | syncService func(context.Context, string, *corev1.Service) error 52 | deleteService func(context.Context, string, string) error 53 | } 54 | 55 | // NewController creates a new service controller 56 | func NewController( 57 | clusterName string, 58 | getter v1.ServicesGetter, 59 | informer coreinformers.ServiceInformer, 60 | syncService func(context.Context, string, *corev1.Service) error, 61 | deleteService func(context.Context, string, string) error, 62 | ) *Controller { 63 | logger := klogr.New().WithName("ServiceController") 64 | c := &Controller{ 65 | clusterName: clusterName, 66 | log: logger, 67 | getter: getter, 68 | lister: informer.Lister(), 69 | listerSynced: informer.Informer().HasSynced, 70 | queue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "service"), 71 | 72 | syncService: syncService, 73 | deleteService: deleteService, 74 | } 75 | 76 | informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 77 | AddFunc: c.add, 78 | UpdateFunc: c.update, 79 | DeleteFunc: c.delete, 80 | }) 81 | 82 | return c 83 | } 84 | 85 | // ReconcileAllServices reconcile all the services 86 | func (c *Controller) ReconcileAllServices() { 87 | services, _ := c.lister.List(labels.Everything()) 88 | for _, service := range services { 89 | c.log.V(4).Info("Adding Service", "name", service.Name) 90 | c.enqueue(service) 91 | } 92 | } 93 | 94 | // ReconcileServices reconcile services in a certain namespace 95 | func (c *Controller) ReconcileServices(ns *model.Namespace) { 96 | services, _ := c.lister.List(labels.Everything()) 97 | for _, service := range services { 98 | if service.Namespace == ns.Name { 99 | c.log.V(4).Info("Adding Service", "name", service.Name) 100 | c.enqueue(service) 101 | } 102 | } 103 | } 104 | 105 | func (c *Controller) add(obj interface{}) { 106 | service, _ := obj.(*corev1.Service) 107 | c.log.V(4).Info("Adding Service", "name", service.Name) 108 | c.enqueue(service) 109 | } 110 | 111 | func (c *Controller) update(oldObj, curObj interface{}) { 112 | old, _ := oldObj.(*corev1.Service) 113 | current, _ := curObj.(*corev1.Service) 114 | if !c.needsUpdate(old, current) { 115 | return 116 | } 117 | 118 | c.log.V(4).Info("Updating Service", "name", current.Name) 119 | c.enqueue(current) 120 | } 121 | 122 | func (c *Controller) needsUpdate(old *corev1.Service, new *corev1.Service) bool { 123 | return !reflect.DeepEqual(old.Annotations, new.Annotations) || 124 | !reflect.DeepEqual(old.Spec, new.Spec) || new.GetDeletionTimestamp() != nil 125 | } 126 | 127 | func (c *Controller) delete(obj interface{}) { 128 | service, ok := obj.(*corev1.Service) 129 | if !ok { 130 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 131 | if !ok { 132 | c.log.Info("Couldn't get object from tombstone", "obj", obj) 133 | return 134 | } 135 | service, ok = tombstone.Obj.(*corev1.Service) 136 | if !ok { 137 | c.log.Info("Tombstone contained object that is not a Service", "obj", obj) 138 | return 139 | } 140 | } 141 | c.log.V(4).Info("Deleting Service", "name", service.Name) 142 | c.enqueue(obj) 143 | } 144 | 145 | func (c *Controller) enqueue(obj interface{}) { 146 | key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) 147 | if err != nil { 148 | utilruntime.HandleError(err) 149 | return 150 | } 151 | c.queue.Add(key) 152 | } 153 | 154 | // Run begins watching and syncing. 155 | func (c *Controller) Run(workers int, stopCh <-chan struct{}) { 156 | defer utilruntime.HandleCrash() 157 | defer c.queue.ShutDown() 158 | 159 | c.log.Info("Starting Service controller") 160 | defer c.log.Info("Shutting down Service controller") 161 | 162 | if !cache.WaitForNamedCacheSync("Service", stopCh, c.listerSynced) { 163 | return 164 | } 165 | 166 | for i := 0; i < workers; i++ { 167 | go wait.Until(c.worker, time.Second, stopCh) 168 | } 169 | 170 | <-stopCh 171 | } 172 | 173 | func (c *Controller) worker() { 174 | for c.processNextWorkItem() { 175 | } 176 | } 177 | 178 | func (c *Controller) processNextWorkItem() bool { 179 | key, quit := c.queue.Get() 180 | if quit { 181 | return false 182 | } 183 | defer c.queue.Done(key) 184 | 185 | logger := c.log.WithValues("key", key) 186 | ctx := log.WithContext(context.Background(), logger) 187 | err := c.syncFromKey(ctx, key.(string)) 188 | if err != nil { 189 | c.queue.AddRateLimited(key) 190 | logger.Error(err, "Sync error") 191 | return true 192 | } 193 | 194 | c.queue.Forget(key) 195 | logger.Info("Successfully synced") 196 | 197 | return true 198 | } 199 | 200 | func (c *Controller) syncFromKey(ctx context.Context, key string) error { 201 | startTime := time.Now() 202 | logger := log.FromContext(ctx) 203 | logger.V(4).Info("Starting sync") 204 | defer func() { 205 | logger.V(4).Info("Finished sync service", "duration", time.Since(startTime).String()) 206 | }() 207 | 208 | ns, name, err := cache.SplitMetaNamespaceKey(key) 209 | if err != nil { 210 | return err 211 | } 212 | 213 | service, err := c.lister.Services(ns).Get(name) 214 | if err != nil && apierrors.IsNotFound(err) { 215 | logger.V(4).Info("Service has been deleted") 216 | return c.deleteService(ctx, c.clusterName, utils.FQDN(name, ns)) 217 | } 218 | if err != nil { 219 | return fmt.Errorf("unable to retrieve service from store: error %w", err) 220 | } 221 | 222 | if !service.DeletionTimestamp.IsZero() { 223 | return c.deleteService(ctx, c.clusterName, utils.FQDN(name, ns)) 224 | } 225 | 226 | return c.syncService(ctx, c.clusterName, service) 227 | } 228 | -------------------------------------------------------------------------------- /pkg/controller/serviceentry.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | "reflect" 20 | 21 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 22 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 23 | istio "istio.io/client-go/pkg/apis/networking/v1alpha3" 24 | ) 25 | 26 | func (c *AggregationController) syncServiceEntry(ctx context.Context, serviceEntry *istio.ServiceEntry) error { 27 | if serviceEntry.Namespace == config.IstioNamespace { // ignore istio system, it's always exported 28 | return nil 29 | } 30 | 31 | id := utils.ObjectID(serviceEntry.Name, serviceEntry.Namespace) 32 | 33 | var oldHosts, newHosts []string 34 | if value, ok := c.serviceEntries.Load(id); ok { 35 | oldHosts = value.([]string) 36 | } 37 | newHosts = append(newHosts, serviceEntry.Spec.Hosts...) 38 | 39 | if reflect.DeepEqual(oldHosts, newHosts) { 40 | return nil 41 | } 42 | 43 | c.serviceEntries.Store(id, newHosts) 44 | 45 | return c.reconcileAllLazyServices(ctx) 46 | } 47 | 48 | func (c *AggregationController) deleteServiceEntry(ctx context.Context, name, ns string) error { 49 | if ns == config.IstioNamespace { // ignore istio system, it's always exported 50 | return nil 51 | } 52 | 53 | id := utils.ObjectID(name, ns) 54 | 55 | if _, found := c.serviceEntries.LoadAndDelete(id); found { 56 | return c.reconcileAllLazyServices(ctx) 57 | } 58 | 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /pkg/controller/serviceentry/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package serviceentry 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "reflect" 21 | "time" 22 | 23 | istio "istio.io/client-go/pkg/apis/networking/v1alpha3" 24 | apierrors "k8s.io/apimachinery/pkg/api/errors" 25 | 26 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 27 | "github.com/go-logr/logr" 28 | istioinformers "istio.io/client-go/pkg/informers/externalversions/networking/v1alpha3" 29 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 30 | "k8s.io/apimachinery/pkg/util/wait" 31 | "k8s.io/client-go/tools/cache" 32 | queue "k8s.io/client-go/util/workqueue" 33 | "k8s.io/klog/v2/klogr" 34 | 35 | networklister "istio.io/client-go/pkg/listers/networking/v1alpha3" 36 | ) 37 | 38 | // LazyXdsManager ... 39 | const LazyXdsManager = "lazyxds" 40 | 41 | // Controller is responsible for synchronizing Istio serviceEntry objects. 42 | type Controller struct { 43 | log logr.Logger 44 | lister networklister.ServiceEntryLister 45 | listerSynced cache.InformerSynced 46 | queue queue.RateLimitingInterface 47 | syncServiceEntry func(context.Context, *istio.ServiceEntry) error 48 | deleteServiceEntry func(context.Context, string, string) error 49 | } 50 | 51 | // NewController creates a new serviceEntry controller 52 | func NewController( 53 | informer istioinformers.ServiceEntryInformer, 54 | syncServiceEntryConfig func(context.Context, *istio.ServiceEntry) error, 55 | deleteServiceEntryConfig func(context.Context, string, string) error, 56 | ) *Controller { 57 | logger := klogr.New().WithName("ServiceEntryController") 58 | c := &Controller{ 59 | log: logger, 60 | lister: informer.Lister(), 61 | listerSynced: informer.Informer().HasSynced, 62 | queue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "ServiceEntry"), 63 | syncServiceEntry: syncServiceEntryConfig, 64 | deleteServiceEntry: deleteServiceEntryConfig, 65 | } 66 | 67 | informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 68 | AddFunc: c.add, 69 | UpdateFunc: c.update, 70 | DeleteFunc: c.delete, 71 | }) 72 | 73 | return c 74 | } 75 | 76 | func (c *Controller) add(obj interface{}) { 77 | serviceEntry, _ := obj.(*istio.ServiceEntry) 78 | c.log.V(4).Info("Adding ServiceEntry", "name", serviceEntry.Name) 79 | c.enqueue(serviceEntry) 80 | } 81 | 82 | func (c *Controller) update(oldObj, curObj interface{}) { 83 | old, _ := oldObj.(*istio.ServiceEntry) 84 | current, _ := curObj.(*istio.ServiceEntry) 85 | if !c.needsUpdate(old, current) { 86 | return 87 | } 88 | 89 | c.log.V(4).Info("Updating ServiceEntry", "name", current.Name) 90 | c.enqueue(current) 91 | } 92 | 93 | func (c *Controller) needsUpdate(old *istio.ServiceEntry, new *istio.ServiceEntry) bool { 94 | return !reflect.DeepEqual(old.Spec, new.Spec) || new.GetDeletionTimestamp() != nil 95 | } 96 | 97 | func (c *Controller) delete(obj interface{}) { 98 | serviceEntry, ok := obj.(*istio.ServiceEntry) 99 | if !ok { 100 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 101 | if !ok { 102 | c.log.Info("Couldn't get object from tombstone", "obj", obj) 103 | return 104 | } 105 | serviceEntry, ok = tombstone.Obj.(*istio.ServiceEntry) 106 | if !ok { 107 | c.log.Info("Tombstone contained object that is not a Service", "obj", obj) 108 | return 109 | } 110 | } 111 | c.log.V(4).Info("Deleting ServiceEntry", "name", serviceEntry.Name) 112 | c.enqueue(obj) 113 | } 114 | 115 | func (c *Controller) enqueue(obj interface{}) { 116 | key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) 117 | if err != nil { 118 | utilruntime.HandleError(err) 119 | return 120 | } 121 | c.queue.Add(key) 122 | } 123 | 124 | // Run begins watching and syncing. 125 | func (c *Controller) Run(workers int, stopCh <-chan struct{}) { 126 | defer utilruntime.HandleCrash() 127 | defer c.queue.ShutDown() 128 | 129 | c.log.Info("Starting ServiceEntry controller") 130 | defer c.log.Info("Shutting down ServiceEntry controller") 131 | 132 | if !cache.WaitForNamedCacheSync("ServiceEntry", stopCh, c.listerSynced) { 133 | return 134 | } 135 | 136 | for i := 0; i < workers; i++ { 137 | go wait.Until(c.worker, time.Second, stopCh) 138 | } 139 | 140 | <-stopCh 141 | } 142 | 143 | func (c *Controller) worker() { 144 | for c.processNextWorkItem() { 145 | } 146 | } 147 | 148 | func (c *Controller) processNextWorkItem() bool { 149 | key, quit := c.queue.Get() 150 | if quit { 151 | return false 152 | } 153 | defer c.queue.Done(key) 154 | 155 | logger := c.log.WithValues("key", key) 156 | ctx := log.WithContext(context.Background(), logger) 157 | err := c.syncFromKey(ctx, key.(string)) 158 | if err != nil { 159 | c.queue.AddRateLimited(key) 160 | logger.Error(err, "Sync error") 161 | return true 162 | } 163 | 164 | c.queue.Forget(key) 165 | logger.Info("Successfully synced") 166 | 167 | return true 168 | } 169 | 170 | func (c *Controller) syncFromKey(ctx context.Context, key string) error { 171 | startTime := time.Now() 172 | logger := log.FromContext(ctx) 173 | logger.V(4).Info("Starting sync") 174 | defer func() { 175 | logger.V(4).Info("Finished sync ServiceEntry", "duration", time.Since(startTime).String()) 176 | }() 177 | 178 | ns, name, err := cache.SplitMetaNamespaceKey(key) 179 | if err != nil { 180 | return err 181 | } 182 | 183 | serviceEntry, err := c.lister.ServiceEntries(ns).Get(name) 184 | if err != nil && apierrors.IsNotFound(err) { 185 | logger.V(4).Info("ServiceEntry has been deleted") 186 | return c.deleteServiceEntry(ctx, name, ns) 187 | } 188 | if err != nil { 189 | return fmt.Errorf("unable to retrieve serviceentry from store: error %w", err) 190 | } 191 | 192 | if !serviceEntry.DeletionTimestamp.IsZero() { 193 | logger.V(4).Info("ServiceEntry has been deleted, should not be here") 194 | return c.deleteServiceEntry(ctx, name, ns) 195 | } 196 | 197 | return c.syncServiceEntry(ctx, serviceEntry) 198 | } 199 | -------------------------------------------------------------------------------- /pkg/controller/sidecar.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/aeraki-mesh/lazyxds/pkg/model" 22 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 23 | istio "istio.io/client-go/pkg/apis/networking/v1alpha3" 24 | ) 25 | 26 | func (c *AggregationController) syncSidecar(ctx context.Context, sidecar *istio.Sidecar) (err error) { 27 | v, ok := c.namespaces.Load(sidecar.Namespace) 28 | if !ok { 29 | return fmt.Errorf("namespace of sidecar(%s/%s) is not exist", sidecar.Namespace, sidecar.Name) 30 | } 31 | ns := v.(*model.Namespace) 32 | ns.AddSidecar(sidecar.Name) 33 | 34 | return c.reconcileNamespace(ctx, ns) 35 | } 36 | 37 | func (c *AggregationController) deleteSidecar(ctx context.Context, id string) (err error) { 38 | name, namespace := utils.ParseID(id) 39 | 40 | v, ok := c.namespaces.Load(namespace) 41 | if !ok { 42 | return fmt.Errorf("namespace of sidecar(%s/%s) is not exist", namespace, name) 43 | } 44 | ns := v.(*model.Namespace) 45 | ns.DeleteSidecar(name) 46 | 47 | return c.reconcileNamespace(ctx, ns) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/controller/sidecar/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sidecar 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 23 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 24 | istio "istio.io/client-go/pkg/apis/networking/v1alpha3" 25 | apierrors "k8s.io/apimachinery/pkg/api/errors" 26 | 27 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 28 | "github.com/go-logr/logr" 29 | istioinformers "istio.io/client-go/pkg/informers/externalversions/networking/v1alpha3" 30 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 31 | "k8s.io/apimachinery/pkg/util/wait" 32 | "k8s.io/client-go/tools/cache" 33 | queue "k8s.io/client-go/util/workqueue" 34 | "k8s.io/klog/v2/klogr" 35 | 36 | networklister "istio.io/client-go/pkg/listers/networking/v1alpha3" 37 | ) 38 | 39 | // Controller is responsible for synchronizing Istio sidecar objects. 40 | type Controller struct { 41 | log logr.Logger 42 | lister networklister.SidecarLister 43 | listerSynced cache.InformerSynced 44 | queue queue.RateLimitingInterface 45 | syncSidecarConfig func(context.Context, *istio.Sidecar) error 46 | deleteSidecarConfig func(context.Context, string) error 47 | } 48 | 49 | // NewController creates a new sidecar controller 50 | func NewController( 51 | informer istioinformers.SidecarInformer, 52 | syncSidecarConfig func(context.Context, *istio.Sidecar) error, 53 | deleteSidecarConfig func(context.Context, string) error, 54 | ) *Controller { 55 | logger := klogr.New().WithName("SidecarController") 56 | c := &Controller{ 57 | log: logger, 58 | lister: informer.Lister(), 59 | listerSynced: informer.Informer().HasSynced, 60 | queue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "Sidecar"), 61 | syncSidecarConfig: syncSidecarConfig, 62 | deleteSidecarConfig: deleteSidecarConfig, 63 | } 64 | 65 | // only handle AddFunc, we don't automatically turn on lazyxds for namespaces or services 66 | informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 67 | AddFunc: c.add, 68 | DeleteFunc: c.delete, 69 | }) 70 | 71 | return c 72 | } 73 | 74 | func (c *Controller) add(obj interface{}) { 75 | sidecar, _ := obj.(*istio.Sidecar) 76 | 77 | // We only handle Sidecar configs which are not managed by LazyXds controller 78 | if c.isSidecarConfigManagedByLazyXds(sidecar) { 79 | return 80 | } 81 | 82 | c.log.V(4).Info("Adding Sidecar", "name", sidecar.Name) 83 | c.enqueue(sidecar) 84 | } 85 | 86 | func (c *Controller) delete(obj interface{}) { 87 | sidecar, ok := obj.(*istio.Sidecar) 88 | 89 | if c.isSidecarConfigManagedByLazyXds(sidecar) { 90 | return 91 | } 92 | 93 | if !ok { 94 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 95 | if !ok { 96 | c.log.Info("Couldn't get object from tombstone", "obj", obj) 97 | return 98 | } 99 | sidecar, ok = tombstone.Obj.(*istio.Sidecar) 100 | if !ok { 101 | c.log.Info("Tombstone contained object that is not a Service", "obj", obj) 102 | return 103 | } 104 | } 105 | c.log.V(4).Info("Deleting Sidecar", "name", sidecar.Name) 106 | c.enqueue(obj) 107 | } 108 | 109 | func (c *Controller) isSidecarConfigManagedByLazyXds(sidecar *istio.Sidecar) bool { 110 | // old kubernetes versions not support ManagedFields, so need use label 111 | // todo: first create a sidecar without the label(so the ns is disabled), then add the label to the sidecar, 112 | // now we need make the ns enabled, but seems it's not a normal process, ignore now. 113 | if sidecar.Labels[config.ManagedByLabel] == config.LazyXdsManager { 114 | return true 115 | } 116 | for _, mangedFiled := range sidecar.ManagedFields { 117 | if mangedFiled.Manager == config.LazyXdsManager { 118 | return true 119 | } 120 | } 121 | return false 122 | } 123 | 124 | func (c *Controller) enqueue(obj interface{}) { 125 | key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) 126 | if err != nil { 127 | utilruntime.HandleError(err) 128 | return 129 | } 130 | c.queue.Add(key) 131 | } 132 | 133 | // Run begins watching and syncing. 134 | func (c *Controller) Run(workers int, stopCh <-chan struct{}) { 135 | defer utilruntime.HandleCrash() 136 | defer c.queue.ShutDown() 137 | 138 | c.log.Info("Starting Sidecar controller") 139 | defer c.log.Info("Shutting down Sidecar controller") 140 | 141 | if !cache.WaitForNamedCacheSync("Sidecar", stopCh, c.listerSynced) { 142 | return 143 | } 144 | 145 | for i := 0; i < workers; i++ { 146 | go wait.Until(c.worker, time.Second, stopCh) 147 | } 148 | 149 | <-stopCh 150 | } 151 | 152 | func (c *Controller) worker() { 153 | for c.processNextWorkItem() { 154 | } 155 | } 156 | 157 | func (c *Controller) processNextWorkItem() bool { 158 | key, quit := c.queue.Get() 159 | if quit { 160 | return false 161 | } 162 | defer c.queue.Done(key) 163 | 164 | logger := c.log.WithValues("key", key) 165 | ctx := log.WithContext(context.Background(), logger) 166 | err := c.syncFromKey(ctx, key.(string)) 167 | if err != nil { 168 | c.queue.AddRateLimited(key) 169 | logger.Error(err, "Sync error") 170 | return true 171 | } 172 | 173 | c.queue.Forget(key) 174 | logger.Info("Successfully synced") 175 | 176 | return true 177 | } 178 | 179 | func (c *Controller) syncFromKey(ctx context.Context, key string) error { 180 | startTime := time.Now() 181 | logger := log.FromContext(ctx) 182 | logger.V(4).Info("Starting sync") 183 | defer func() { 184 | logger.V(4).Info("Finished sync Sidecar", "duration", time.Since(startTime).String()) 185 | }() 186 | 187 | ns, name, err := cache.SplitMetaNamespaceKey(key) 188 | if err != nil { 189 | return err 190 | } 191 | 192 | sidecar, err := c.lister.Sidecars(ns).Get(name) 193 | if err != nil && apierrors.IsNotFound(err) { 194 | logger.V(4).Info("Sidecar has been deleted") 195 | return c.deleteSidecarConfig(ctx, utils.ObjectID(name, ns)) 196 | } 197 | if err != nil { 198 | return fmt.Errorf("unable to retrieve sidecar from store: error %w", err) 199 | } 200 | 201 | if !sidecar.DeletionTimestamp.IsZero() { 202 | return c.deleteSidecarConfig(ctx, utils.ObjectID(name, ns)) 203 | } 204 | 205 | return c.syncSidecarConfig(ctx, sidecar) 206 | } 207 | -------------------------------------------------------------------------------- /pkg/controller/virtual_service.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controller 16 | 17 | import ( 18 | "context" 19 | "reflect" 20 | "strings" 21 | 22 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 23 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 24 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 25 | istio "istio.io/client-go/pkg/apis/networking/v1alpha3" 26 | ) 27 | 28 | // todo miss tls 29 | func (c *AggregationController) syncVirtualService(ctx context.Context, vs *istio.VirtualService) error { 30 | logger := log.FromContext(ctx) 31 | 32 | // k8s.AddFinalizer(endpoints, meshv1alpha1.ClusterFinalizer) 33 | if !vs.DeletionTimestamp.IsZero() { 34 | logger.Info("todo endpoints deleted, 需要 Finalizer") 35 | } 36 | 37 | if utils.FQDN(vs.Name, vs.Namespace) == utils.FQDN(config.EgressVirtualServiceName, config.IstioNamespace) { 38 | return nil 39 | } 40 | 41 | httpChanged := c.updateHTTPServiceBinding(vs) 42 | tcpChanged := c.updateTCPServiceBinding(vs) 43 | 44 | if httpChanged || tcpChanged { 45 | return c.reconcileAllLazyServices(ctx) 46 | } 47 | 48 | return nil 49 | 50 | } 51 | 52 | // return whether the value changed. 53 | func (c *AggregationController) updateHTTPServiceBinding(vs *istio.VirtualService) bool { 54 | id := utils.FQDN(vs.Name, vs.Namespace) 55 | binding := make(map[string]struct{}) 56 | 57 | // todo consider delegate VirtualService and sort domain 58 | // Note for Kubernetes users: When short names are used 59 | // (e.g. “reviews” instead of “reviews.default.svc.cluster.local”), 60 | // Istio will interpret the short name based on the namespace of the rule, not the service. 61 | // A rule in the “default” namespace containing a host “reviews” will be interpreted as 62 | // “reviews.default.svc.cluster.local”, irrespective of the actual namespace associated with the reviews service. 63 | // To avoid potential misconfigurations, it is recommended to always use fully qualified domain names 64 | for _, host := range vs.Spec.Hosts { 65 | if !strings.HasSuffix(host, "svc.cluster.local") { // hard code 66 | continue 67 | } 68 | if strings.Contains(host, "*") { 69 | continue 70 | } 71 | binding[host] = struct{}{} 72 | } 73 | 74 | if len(binding) == 0 { 75 | _, ok := c.httpServicesBinding.LoadAndDelete(id) 76 | return ok 77 | } 78 | 79 | for _, hr := range vs.Spec.Http { 80 | for _, r := range hr.Route { 81 | host := r.Destination.Host 82 | if !strings.HasSuffix(host, "svc.cluster.local") { 83 | continue 84 | } 85 | binding[host] = struct{}{} 86 | } 87 | } 88 | if len(binding) <= 1 { 89 | _, ok := c.httpServicesBinding.LoadAndDelete(id) 90 | return ok 91 | } 92 | 93 | val, ok := c.httpServicesBinding.Load(id) 94 | if !ok { 95 | c.httpServicesBinding.Store(id, binding) 96 | return true 97 | } 98 | old := val.(map[string]struct{}) 99 | 100 | if reflect.DeepEqual(binding, old) { 101 | return false 102 | } 103 | c.httpServicesBinding.Store(id, binding) 104 | return true 105 | } 106 | 107 | // return whether the value changed. 108 | func (c *AggregationController) updateTCPServiceBinding(vs *istio.VirtualService) bool { 109 | id := utils.FQDN(vs.Name, vs.Namespace) 110 | binding := make(map[string]struct{}) 111 | 112 | for _, host := range vs.Spec.Hosts { 113 | if !strings.HasSuffix(host, "svc.cluster.local") { // hard code 114 | continue 115 | } 116 | if strings.Contains(host, "*") { 117 | continue 118 | } 119 | binding[host] = struct{}{} 120 | } 121 | 122 | if len(binding) == 0 { 123 | _, ok := c.tcpServicesBinding.LoadAndDelete(id) 124 | return ok 125 | } 126 | 127 | for _, tr := range vs.Spec.Tcp { 128 | for _, r := range tr.Route { 129 | host := r.Destination.Host 130 | if !strings.HasSuffix(host, "svc.cluster.local") { 131 | continue 132 | } 133 | binding[host] = struct{}{} 134 | } 135 | } 136 | if len(binding) <= 1 { 137 | _, ok := c.tcpServicesBinding.LoadAndDelete(id) 138 | return ok 139 | } 140 | 141 | val, ok := c.tcpServicesBinding.Load(id) 142 | if !ok { 143 | c.tcpServicesBinding.Store(id, binding) 144 | return true 145 | } 146 | 147 | old := val.(map[string]struct{}) 148 | if reflect.DeepEqual(binding, old) { 149 | return false 150 | } 151 | c.tcpServicesBinding.Store(id, binding) 152 | return true 153 | } 154 | 155 | func (c *AggregationController) deleteVirtualService(ctx context.Context, id string) error { 156 | if id == utils.FQDN(config.EgressVirtualServiceName, config.IstioNamespace) { 157 | return nil 158 | } 159 | logger := log.FromContext(ctx) 160 | 161 | logger.V(2).Info("VirtualService has been deleted") 162 | _, httpDeleted := c.httpServicesBinding.LoadAndDelete(id) 163 | _, tcpDeleted := c.tcpServicesBinding.LoadAndDelete(id) 164 | 165 | if httpDeleted || tcpDeleted { 166 | return c.reconcileAllLazyServices(ctx) 167 | } 168 | return nil 169 | } 170 | 171 | // this is only one level binding 172 | func (c *AggregationController) getHTTPServiceBinding(svcID string) map[string]struct{} { 173 | binding := make(map[string]struct{}) 174 | binding[svcID] = struct{}{} 175 | 176 | c.httpServicesBinding.Range(func(key, value interface{}) bool { 177 | b := value.(map[string]struct{}) 178 | if _, ok := b[svcID]; ok { 179 | for id := range b { 180 | if _, ok := c.services.Load(id); ok { // todo service in vs may not exist, do we need check if it's http 181 | binding[id] = struct{}{} 182 | } 183 | } 184 | } 185 | return true 186 | }) 187 | 188 | return binding 189 | } 190 | -------------------------------------------------------------------------------- /pkg/controller/virtualservice/controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package virtualservice 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "reflect" 21 | "time" 22 | 23 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 24 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 25 | istio "istio.io/client-go/pkg/apis/networking/v1alpha3" 26 | 27 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 28 | "github.com/go-logr/logr" 29 | istioinformers "istio.io/client-go/pkg/informers/externalversions/networking/v1alpha3" 30 | apierrors "k8s.io/apimachinery/pkg/api/errors" 31 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 32 | "k8s.io/apimachinery/pkg/util/wait" 33 | "k8s.io/client-go/tools/cache" 34 | queue "k8s.io/client-go/util/workqueue" 35 | "k8s.io/klog/v2/klogr" 36 | 37 | networklister "istio.io/client-go/pkg/listers/networking/v1alpha3" 38 | ) 39 | 40 | // Controller is responsible for synchronizing virtualService objects. 41 | type Controller struct { 42 | log logr.Logger 43 | lister networklister.VirtualServiceLister 44 | listerSynced cache.InformerSynced 45 | queue queue.RateLimitingInterface 46 | syncVirtualService func(context.Context, *istio.VirtualService) error 47 | deleteVirtualService func(context.Context, string) error 48 | } 49 | 50 | // NewController creates a new virtualService controller 51 | func NewController( 52 | informer istioinformers.VirtualServiceInformer, 53 | syncVirtualService func(context.Context, *istio.VirtualService) error, 54 | deleteVirtualService func(context.Context, string) error, 55 | ) *Controller { 56 | logger := klogr.New().WithName("VirtualServiceController") 57 | c := &Controller{ 58 | log: logger, 59 | lister: informer.Lister(), 60 | listerSynced: informer.Informer().HasSynced, 61 | queue: queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "virtualService"), 62 | syncVirtualService: syncVirtualService, 63 | deleteVirtualService: deleteVirtualService, 64 | } 65 | 66 | informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 67 | AddFunc: c.add, 68 | UpdateFunc: c.update, 69 | DeleteFunc: c.delete, 70 | }) 71 | 72 | return c 73 | } 74 | 75 | func (c *Controller) add(obj interface{}) { 76 | virtualService, _ := obj.(*istio.VirtualService) 77 | c.log.V(4).Info("Adding VirtualService", "name", virtualService.Name) 78 | c.enqueue(virtualService) 79 | } 80 | 81 | func (c *Controller) update(oldObj, curObj interface{}) { 82 | old, _ := oldObj.(*istio.VirtualService) 83 | current, _ := curObj.(*istio.VirtualService) 84 | if !c.needsUpdate(old, current) { 85 | return 86 | } 87 | 88 | c.log.V(4).Info("Updating VirtualService", "name", current.Name) 89 | c.enqueue(current) 90 | } 91 | 92 | func (c *Controller) needsUpdate(old *istio.VirtualService, new *istio.VirtualService) bool { 93 | return !reflect.DeepEqual(old.Spec, new.Spec) || new.GetDeletionTimestamp() != nil 94 | } 95 | 96 | func (c *Controller) delete(obj interface{}) { 97 | virtualService, ok := obj.(*istio.VirtualService) 98 | if !ok { 99 | tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 100 | if !ok { 101 | c.log.Info("Couldn't get object from tombstone", "obj", obj) 102 | return 103 | } 104 | virtualService, ok = tombstone.Obj.(*istio.VirtualService) 105 | if !ok { 106 | c.log.Info("Tombstone contained object that is not a VirtualService", "obj", obj) 107 | return 108 | } 109 | } 110 | c.log.V(4).Info("Deleting VirtualService", "name", virtualService.Name) 111 | c.enqueue(obj) 112 | } 113 | 114 | func (c *Controller) enqueue(obj interface{}) { 115 | key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) 116 | if err != nil { 117 | utilruntime.HandleError(err) 118 | return 119 | } 120 | c.queue.Add(key) 121 | } 122 | 123 | // Run begins watching and syncing. 124 | func (c *Controller) Run(workers int, stopCh <-chan struct{}) { 125 | defer utilruntime.HandleCrash() 126 | defer c.queue.ShutDown() 127 | 128 | c.log.Info("Starting VirtualService controller") 129 | defer c.log.Info("Shutting down VirtualService controller") 130 | 131 | if !cache.WaitForNamedCacheSync("VirtualService", stopCh, c.listerSynced) { 132 | return 133 | } 134 | 135 | for i := 0; i < workers; i++ { 136 | go wait.Until(c.worker, time.Second, stopCh) 137 | } 138 | 139 | <-stopCh 140 | } 141 | 142 | func (c *Controller) worker() { 143 | for c.processNextWorkItem() { 144 | } 145 | } 146 | 147 | func (c *Controller) processNextWorkItem() bool { 148 | key, quit := c.queue.Get() 149 | if quit { 150 | return false 151 | } 152 | defer c.queue.Done(key) 153 | 154 | logger := c.log.WithValues("key", key) 155 | ctx := log.WithContext(context.Background(), logger) 156 | err := c.syncFromKey(ctx, key.(string)) 157 | if err != nil { 158 | c.queue.AddRateLimited(key) 159 | logger.Error(err, "Sync error") 160 | return true 161 | } 162 | 163 | c.queue.Forget(key) 164 | logger.Info("Successfully synced") 165 | 166 | return true 167 | } 168 | 169 | func (c *Controller) syncFromKey(ctx context.Context, key string) error { 170 | startTime := time.Now() 171 | logger := log.FromContext(ctx) 172 | logger.V(4).Info("Starting sync") 173 | defer func() { 174 | logger.V(4).Info("Finished sync virtualService", "duration", time.Since(startTime).String()) 175 | }() 176 | 177 | ns, name, err := cache.SplitMetaNamespaceKey(key) 178 | if ns == config.IstioNamespace && name == config.EgressVirtualServiceName { 179 | return nil 180 | } 181 | if err != nil { 182 | return err 183 | } 184 | 185 | virtualService, err := c.lister.VirtualServices(ns).Get(name) 186 | if err != nil && apierrors.IsNotFound(err) { 187 | logger.V(4).Info("VirtualService has been deleted") 188 | return c.deleteVirtualService(ctx, utils.FQDN(name, ns)) 189 | } 190 | if err != nil { 191 | return fmt.Errorf("unable to retrieve virtualService from store: error %w", err) 192 | } 193 | if !virtualService.DeletionTimestamp.IsZero() { 194 | return c.deleteVirtualService(ctx, utils.FQDN(name, ns)) 195 | } 196 | 197 | return c.syncVirtualService(ctx, virtualService) 198 | } 199 | -------------------------------------------------------------------------------- /pkg/manager/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package manager 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 22 | "github.com/aeraki-mesh/lazyxds/pkg/accesslog" 23 | "github.com/aeraki-mesh/lazyxds/pkg/bootstrap" 24 | "github.com/aeraki-mesh/lazyxds/pkg/controller" 25 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 26 | istioclient "istio.io/client-go/pkg/clientset/versioned" 27 | "k8s.io/client-go/kubernetes" 28 | "k8s.io/klog/v2" 29 | ) 30 | 31 | // LazyXdsManager defines the high-level interface of multiCluster management. 32 | type LazyXdsManager interface { 33 | AddCluster(name string, client *kubernetes.Clientset) error 34 | DeleteCluster(name string) error 35 | Run() error 36 | //Run(stopCh <-chan struct{}) error 37 | } 38 | 39 | // Manager contains the main lazy xds controller 40 | type Manager struct { 41 | conf *config.Config 42 | stop <-chan struct{} // todo move to args of run function 43 | masterClient *kubernetes.Clientset 44 | istioClient *istioclient.Clientset 45 | controller *controller.AggregationController 46 | accessLogServer *accesslog.Server 47 | } 48 | 49 | var singleton *Manager 50 | 51 | // NewManager ... 52 | func NewManager(conf *config.Config, stop <-chan struct{}) (*Manager, error) { 53 | if singleton != nil { 54 | klog.Error("LazyXds Manager already exist") 55 | return singleton, nil 56 | } 57 | 58 | kubeClient, err := utils.NewKubeClient(conf.KubeConfig) 59 | if err != nil { 60 | return nil, err 61 | } 62 | istioClient, err := utils.NewIstioClient(conf.KubeConfig) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | singleton = &Manager{ 68 | conf: conf, 69 | stop: stop, 70 | masterClient: kubeClient, 71 | istioClient: istioClient, 72 | controller: controller.NewController(istioClient, kubeClient, stop), 73 | } 74 | singleton.accessLogServer = accesslog.NewAccessLogServer(singleton.controller) 75 | 76 | return singleton, nil 77 | } 78 | 79 | // Run ... 80 | func (m *Manager) Run() error { 81 | klog.Info("Starting access log server...") 82 | if err := m.accessLogServer.Serve(); err != nil { 83 | return fmt.Errorf("start access log server failed: %w", err) 84 | } 85 | 86 | m.controller.Run() 87 | 88 | // todo we need support multiple cluster 89 | if err := m.AddCluster("Kubernetes", m.masterClient); err != nil { 90 | return err 91 | } 92 | m.controller.KubeClient = m.masterClient 93 | 94 | return nil 95 | } 96 | 97 | // AddCluster add new cluster to the mesh 98 | func (m *Manager) AddCluster(name string, client *kubernetes.Clientset) error { 99 | if m.conf.AutoCreateEgress { 100 | klog.Info("Starting create lazyxds egress", "cluster", name) 101 | if err := bootstrap.InitEgress(context.TODO(), name, client, m.istioClient, m.conf.IstiodAddress, m.conf.ProxyImage); err != nil { 102 | return fmt.Errorf("init egress of cluster %s failed: %w", name, err) 103 | } 104 | } 105 | 106 | return m.controller.AddCluster(name, client) 107 | } 108 | 109 | // DeleteCluster remove cluster from the mesh 110 | // todo need implement 111 | func (m *Manager) DeleteCluster(name string) error { 112 | return nil 113 | } 114 | -------------------------------------------------------------------------------- /pkg/model/endpoints.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package model 16 | 17 | import ( 18 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 19 | corev1 "k8s.io/api/core/v1" 20 | ) 21 | 22 | // Endpoints contains the k8s endpoints info which lazyxds just need 23 | type Endpoints struct { 24 | Name string 25 | Namespace string 26 | 27 | IPList []string 28 | } 29 | 30 | // NewEndpoints creates new Endpoints from k8s endpoints 31 | func NewEndpoints(endpoints *corev1.Endpoints) *Endpoints { 32 | ep := &Endpoints{ 33 | Name: endpoints.Name, 34 | Namespace: endpoints.Namespace, 35 | } 36 | 37 | for _, subset := range endpoints.Subsets { 38 | for _, address := range subset.Addresses { 39 | ep.IPList = append(ep.IPList, address.IP) 40 | } 41 | } 42 | 43 | return ep 44 | } 45 | 46 | // ID use service fqdn as endpoints id 47 | func (ep *Endpoints) ID() string { 48 | return utils.FQDN(ep.Name, ep.Namespace) 49 | } 50 | -------------------------------------------------------------------------------- /pkg/model/namespace.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package model 16 | 17 | import ( 18 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 19 | corev1 "k8s.io/api/core/v1" 20 | ) 21 | 22 | // NSLazyStatus represents the status of lazy xds 23 | type NSLazyStatus int 24 | 25 | const ( 26 | // NSLazyStatusNone ... 27 | NSLazyStatusNone NSLazyStatus = 0 28 | // NSLazyStatusEnabled ... 29 | NSLazyStatusEnabled NSLazyStatus = 1 30 | // NSLazyStatusDisabled ... 31 | NSLazyStatusDisabled NSLazyStatus = 2 32 | ) 33 | 34 | // Namespace represent the namespace which contains multi-cluster info 35 | type Namespace struct { 36 | Name string 37 | 38 | Distribution map[string]bool 39 | UserSidecar map[string]struct{} 40 | 41 | Labels map[string]string 42 | 43 | LazyStatus NSLazyStatus 44 | } 45 | 46 | // NewNamespace creates new Namespace struct from kubernetes namespace 47 | func NewNamespace(namespace *corev1.Namespace) *Namespace { 48 | return &Namespace{ 49 | Name: namespace.Name, 50 | Distribution: make(map[string]bool), 51 | UserSidecar: make(map[string]struct{}), 52 | Labels: namespace.Labels, 53 | } 54 | } 55 | 56 | // ID use name of namespace as id 57 | func (ns *Namespace) ID() string { 58 | return ns.Name 59 | } 60 | 61 | // LazyEnabled check if the namespace enable lazyxds 62 | // Currently, if there is any user created sidecar CRD, the lazyxds will be disabled 63 | func (ns *Namespace) LazyEnabled(clusterName string) bool { 64 | if len(ns.UserSidecar) > 0 { 65 | return false 66 | } 67 | 68 | return ns.Distribution[clusterName] 69 | } 70 | 71 | // Update update one namespace of the multiCluster 72 | func (ns *Namespace) Update(clusterName string, namespace *corev1.Namespace) { 73 | ns.Distribution[clusterName] = utils.IsLazyEnabled(namespace.Annotations) 74 | ns.Labels = namespace.Labels 75 | ns.updateLazyStatus() 76 | } 77 | 78 | // AddSidecar record user-created sidecar crd 79 | func (ns *Namespace) AddSidecar(sidecarName string) { 80 | ns.UserSidecar[sidecarName] = struct{}{} 81 | ns.updateLazyStatus() 82 | } 83 | 84 | // DeleteSidecar delete use-created sidecar crd 85 | func (ns *Namespace) DeleteSidecar(sidecarName string) { 86 | delete(ns.UserSidecar, sidecarName) 87 | ns.updateLazyStatus() 88 | } 89 | 90 | // Delete delete a namespace of one cluster 91 | func (ns *Namespace) Delete(clusterName string) { 92 | delete(ns.Distribution, clusterName) 93 | ns.updateLazyStatus() 94 | } 95 | 96 | func (ns *Namespace) updateLazyStatus() { 97 | lazyStatus := NSLazyStatusNone 98 | 99 | if len(ns.UserSidecar) > 0 { 100 | lazyStatus = NSLazyStatusDisabled 101 | } else { 102 | for _, lazy := range ns.Distribution { 103 | if lazy { // if one is true, then true 104 | lazyStatus = NSLazyStatusEnabled 105 | break 106 | } 107 | } 108 | } 109 | 110 | ns.LazyStatus = lazyStatus 111 | } 112 | -------------------------------------------------------------------------------- /pkg/model/service.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package model 16 | 17 | import ( 18 | "fmt" 19 | "reflect" 20 | "sort" 21 | "sync" 22 | 23 | "github.com/aeraki-mesh/lazyxds/pkg/utils" 24 | corev1 "k8s.io/api/core/v1" 25 | ) 26 | 27 | // Service represent one service cross multi-cluster 28 | type Service struct { 29 | mu sync.Mutex // todo 30 | Name string 31 | Namespace string 32 | Distribution map[string]*clusterServiceStatus 33 | EgressService map[string]struct{} // http service which reported from als 34 | NSLazy NSLazyStatus 35 | 36 | Spec serviceStatus // desired status 37 | Status serviceStatus 38 | } 39 | 40 | // serviceStatus is the status of lazyxds service 41 | type serviceStatus struct { 42 | ClusterIPs []string 43 | HTTPPorts map[string]struct{} 44 | TCPPorts map[string]struct{} 45 | 46 | LazyEnabled bool 47 | LazySelector map[string]string 48 | } 49 | 50 | // clusterServiceStatus is the service status of one cluster 51 | type clusterServiceStatus struct { 52 | ClusterIP string 53 | HTTPPorts map[string]struct{} 54 | TCPPorts map[string]struct{} 55 | 56 | LazyEnabled bool 57 | Selector map[string]string 58 | } 59 | 60 | // NewService creates new Service 61 | func NewService(service *corev1.Service) *Service { 62 | return &Service{ 63 | Name: service.Name, 64 | Namespace: service.Namespace, 65 | Distribution: make(map[string]*clusterServiceStatus), 66 | EgressService: make(map[string]struct{}), 67 | } 68 | } 69 | 70 | // ID use FQDN as service id 71 | func (svc *Service) ID() string { 72 | return utils.FQDN(svc.Name, svc.Namespace) 73 | } 74 | 75 | // NeedReconcileService check if service need reconcile 76 | func (svc *Service) NeedReconcileService() bool { 77 | return !reflect.DeepEqual(svc.Status.HTTPPorts, svc.Spec.HTTPPorts) || 78 | !reflect.DeepEqual(svc.Status.TCPPorts, svc.Spec.TCPPorts) || 79 | !reflect.DeepEqual(svc.Status.ClusterIPs, svc.Spec.ClusterIPs) 80 | } 81 | 82 | // FinishReconcileService update status using spec info 83 | func (svc *Service) FinishReconcileService() { 84 | svc.Status.HTTPPorts = svc.Spec.HTTPPorts 85 | svc.Status.TCPPorts = svc.Spec.TCPPorts 86 | svc.Status.ClusterIPs = svc.Spec.ClusterIPs 87 | } 88 | 89 | // NeedReconcileLazy check if lazy info is equal in status and spec 90 | func (svc *Service) NeedReconcileLazy() bool { 91 | return !reflect.DeepEqual(svc.Status.LazyEnabled, svc.Spec.LazyEnabled) || 92 | !reflect.DeepEqual(svc.Status.LazySelector, svc.Spec.LazySelector) 93 | } 94 | 95 | // FinishReconcileLazy update lazy status using spec info 96 | func (svc *Service) FinishReconcileLazy() { 97 | svc.Status.LazyEnabled = svc.Spec.LazyEnabled 98 | svc.Status.LazySelector = svc.Spec.LazySelector 99 | } 100 | 101 | // UpdateClusterService update the service of one cluster 102 | func (svc *Service) UpdateClusterService(clusterName string, service *corev1.Service) { 103 | svc.mu.Lock() 104 | defer svc.mu.Unlock() 105 | cs := &clusterServiceStatus{ 106 | ClusterIP: service.Spec.ClusterIP, 107 | HTTPPorts: make(map[string]struct{}), 108 | TCPPorts: make(map[string]struct{}), 109 | 110 | LazyEnabled: utils.IsLazyEnabled(service.Annotations), 111 | Selector: service.Spec.Selector, 112 | } 113 | 114 | // https://github.com/aeraki-mesh/aeraki/issues/83 115 | // if a service without selector, the lazy loading is disabled 116 | // we always disable lazy loading feature on a service with ExternalName 117 | if service.Spec.Type == corev1.ServiceTypeExternalName { 118 | cs.Selector = nil 119 | } 120 | 121 | for _, servicePort := range service.Spec.Ports { 122 | if utils.IsHTTP(servicePort) { 123 | cs.HTTPPorts[fmt.Sprint(servicePort.Port)] = struct{}{} 124 | } else { 125 | cs.TCPPorts[fmt.Sprint(servicePort.Port)] = struct{}{} 126 | } 127 | } 128 | svc.Distribution[clusterName] = cs 129 | 130 | svc.updateSpec() 131 | } 132 | 133 | // UpdateNSLazy update the enabled status of the namespace 134 | // If the ns lazy status changed, we need update service spec 135 | func (svc *Service) UpdateNSLazy(status NSLazyStatus) { 136 | svc.mu.Lock() 137 | defer svc.mu.Unlock() 138 | if svc.NSLazy != status { 139 | svc.NSLazy = status 140 | svc.updateSpec() 141 | } 142 | } 143 | 144 | // DeleteFromCluster delete the service of one cluster 145 | func (svc *Service) DeleteFromCluster(clusterName string) { 146 | svc.mu.Lock() 147 | defer svc.mu.Unlock() 148 | delete(svc.Distribution, clusterName) 149 | svc.updateSpec() 150 | } 151 | 152 | func (svc *Service) updateSpec() { 153 | spec := serviceStatus{ 154 | HTTPPorts: make(map[string]struct{}), 155 | TCPPorts: make(map[string]struct{}), 156 | LazySelector: make(map[string]string), 157 | } 158 | 159 | ports := make(map[string]bool) 160 | clusterIPSet := make(map[string]bool) 161 | svcMustDisableLazy := false 162 | for _, cs := range svc.Distribution { 163 | for p := range cs.HTTPPorts { 164 | if old, ok := ports[p]; ok { 165 | ports[p] = old // only if the port of all clusters are http, it's http 166 | } else { 167 | ports[p] = true 168 | } 169 | } 170 | for p := range cs.TCPPorts { 171 | ports[p] = false 172 | } 173 | 174 | if len(cs.Selector) == 0 { 175 | svcMustDisableLazy = true 176 | } 177 | 178 | if svcMustDisableLazy { 179 | spec.LazyEnabled = false 180 | } else { 181 | if svc.NSLazy == NSLazyStatusDisabled { 182 | spec.LazyEnabled = false 183 | } else if svc.NSLazy == NSLazyStatusEnabled { 184 | spec.LazyEnabled = true 185 | } 186 | spec.LazyEnabled = spec.LazyEnabled || cs.LazyEnabled 187 | } 188 | 189 | // if cs.LazyEnabled { 190 | spec.LazySelector = cs.Selector // random now, need doc this 191 | // } 192 | 193 | ip := cs.ClusterIP 194 | if clusterIPSet[ip] { 195 | continue 196 | } 197 | clusterIPSet[ip] = true 198 | spec.ClusterIPs = append(spec.ClusterIPs, ip) 199 | } 200 | sort.Slice(spec.ClusterIPs, func(i, j int) bool { 201 | return spec.ClusterIPs[i] > spec.ClusterIPs[j] 202 | }) 203 | 204 | for p, isHTTP := range ports { 205 | if isHTTP { 206 | spec.HTTPPorts[p] = struct{}{} 207 | } else { 208 | spec.TCPPorts[p] = struct{}{} 209 | } 210 | } 211 | 212 | svc.Spec = spec 213 | } 214 | 215 | // DomainListOfPort return the whole list of domains related with this port 216 | func (svc *Service) DomainListOfPort(num, sourceNS string) []string { 217 | fqdn := svc.ID() 218 | list := []string{ 219 | fqdn, 220 | fmt.Sprintf("%s:%s", fqdn, num), 221 | 222 | fmt.Sprintf("%s.%s.%s", svc.Name, svc.Namespace, "svc.cluster"), 223 | fmt.Sprintf("%s.%s.%s:%s", svc.Name, svc.Namespace, "svc.cluster", num), 224 | 225 | fmt.Sprintf("%s.%s.%s", svc.Name, svc.Namespace, "svc"), 226 | fmt.Sprintf("%s.%s.%s:%s", svc.Name, svc.Namespace, "svc", num), 227 | 228 | fmt.Sprintf("%s.%s", svc.Name, svc.Namespace), 229 | fmt.Sprintf("%s.%s:%s", svc.Name, svc.Namespace, num), 230 | } 231 | if svc.Namespace == sourceNS { // in case 2 services with same name are in 2 different ns 232 | list = append(list, svc.Name) 233 | list = append(list, fmt.Sprintf("%s:%s", svc.Name, num)) 234 | } 235 | 236 | for _, ip := range svc.Spec.ClusterIPs { 237 | if ip == "None" { // todo: for headless service, but the sub domain is miss in outbound of egreee, need fix 238 | l := len(list) 239 | for i := 0; i < l; i++ { 240 | list = append(list, fmt.Sprintf("*.%s", list[i])) 241 | } 242 | } else if ip != "" { 243 | list = append(list, ip) 244 | list = append(list, fmt.Sprintf("%s:%s", ip, num)) 245 | } 246 | } 247 | 248 | return list 249 | } 250 | -------------------------------------------------------------------------------- /pkg/utils/app/app.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "flag" 19 | "fmt" 20 | "math/rand" 21 | "os" 22 | "time" 23 | 24 | "github.com/fatih/color" 25 | "github.com/spf13/cobra" 26 | 27 | "github.com/aeraki-mesh/lazyxds/pkg/utils/app/version" 28 | ) 29 | 30 | var ( 31 | progressMessage = color.GreenString("==>") 32 | usageTemplate = fmt.Sprintf(`%s{{if .Runnable}} 33 | %s{{end}}{{if .HasAvailableSubCommands}} 34 | %s{{end}}{{if gt (len .Aliases) 0}} 35 | 36 | %s 37 | {{.NameAndAliases}}{{end}}{{if .HasExample}} 38 | 39 | %s 40 | {{.Example}}{{end}}{{if .HasAvailableSubCommands}} 41 | 42 | %s{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} 43 | %s {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} 44 | 45 | %s 46 | {{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} 47 | 48 | %s 49 | {{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} 50 | 51 | %s{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} 52 | {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} 53 | 54 | Use "%s --help" for more information about a command.{{end}} 55 | `, 56 | color.CyanString("Usage:"), 57 | color.GreenString("{{.UseLine}}"), 58 | color.GreenString("{{.CommandPath}} [command]"), 59 | color.CyanString("Aliases:"), 60 | color.CyanString("Examples:"), 61 | color.CyanString("Available Commands:"), 62 | color.GreenString("{{rpad .Name .NamePadding }}"), 63 | color.CyanString("Flags:"), 64 | color.CyanString("Global Flags:"), 65 | color.CyanString("Additional help topics:"), 66 | color.GreenString("{{.CommandPath}} [command]"), 67 | ) 68 | ) 69 | 70 | // App is the main structure of a cli application. 71 | // It is recommended that an app be created with the app.NewApp() function. 72 | type App struct { 73 | basename string 74 | name string 75 | description string 76 | options CliOptions 77 | runFunc RunFunc 78 | silence bool 79 | noVersion bool 80 | commands []*Command 81 | } 82 | 83 | // Option defines optional parameters for initializing the application 84 | // structure. 85 | type Option func(*App) 86 | 87 | // WithOptions to open the application's function to read from the command line 88 | // or read parameters from the configuration file. 89 | func WithOptions(opt CliOptions) Option { 90 | return func(a *App) { 91 | a.options = opt 92 | } 93 | } 94 | 95 | // RunFunc defines the application's startup callback function. 96 | type RunFunc func(basename string) error 97 | 98 | // WithRunFunc is used to set the application startup callback function option. 99 | func WithRunFunc(run RunFunc) Option { 100 | return func(a *App) { 101 | a.runFunc = run 102 | } 103 | } 104 | 105 | // WithDescription is used to set the description of the application. 106 | func WithDescription(desc string) Option { 107 | return func(a *App) { 108 | a.description = desc 109 | } 110 | } 111 | 112 | // WithSilence sets the application to silent mode, in which the program startup 113 | // information, configuration information, and version information are not 114 | // printed in the console. 115 | func WithSilence() Option { 116 | return func(a *App) { 117 | a.silence = true 118 | } 119 | } 120 | 121 | // WithNoVersion set the application does not provide version flag. 122 | func WithNoVersion() Option { 123 | return func(a *App) { 124 | a.noVersion = true 125 | } 126 | } 127 | 128 | // NewApp creates a new application instance based on the given application name, 129 | // binary name, and other options. 130 | func NewApp(name string, basename string, opts ...Option) *App { 131 | a := &App{ 132 | name: name, 133 | basename: basename, 134 | } 135 | 136 | for _, o := range opts { 137 | o(a) 138 | } 139 | 140 | return a 141 | } 142 | 143 | // Run is used to launch the application. 144 | func (a *App) Run() { 145 | rand.Seed(time.Now().UTC().UnixNano()) 146 | 147 | initFlag() 148 | 149 | cmd := cobra.Command{ 150 | Use: FormatBaseName(a.basename), 151 | Long: a.description, 152 | SilenceUsage: true, 153 | SilenceErrors: true, 154 | } 155 | cmd.SetUsageTemplate(usageTemplate) 156 | cmd.Flags().SortFlags = false 157 | if len(a.commands) > 0 { 158 | for _, command := range a.commands { 159 | cmd.AddCommand(command.cobraCommand()) 160 | } 161 | cmd.SetHelpCommand(helpCommand(a.name)) 162 | } 163 | if a.runFunc != nil { 164 | cmd.Run = a.runCommand 165 | } 166 | 167 | cmd.Flags().AddGoFlagSet(flag.CommandLine) 168 | if a.options != nil { 169 | if _, ok := a.options.(ConfigurableOptions); ok { 170 | addConfigFlag(a.basename, cmd.Flags()) 171 | } 172 | a.options.AddFlags(cmd.Flags()) 173 | } 174 | 175 | if !a.noVersion { 176 | version.AddFlags(cmd.Flags()) 177 | } 178 | addHelpFlag(a.name, cmd.Flags()) 179 | 180 | if err := cmd.Execute(); err != nil { 181 | fmt.Printf("%v %v\n", color.RedString("Error:"), err) 182 | os.Exit(1) 183 | } 184 | } 185 | 186 | func (a *App) runCommand(cmd *cobra.Command, args []string) { 187 | if !a.noVersion { 188 | version.PrintAndExitIfRequested(a.name) 189 | } 190 | if !a.silence { 191 | fmt.Printf("%v Starting %s...\n", progressMessage, a.name) 192 | wd, _ := os.Getwd() 193 | fmt.Printf("%v WorkingDir: %s\n", progressMessage, wd) 194 | fmt.Printf("%v Args: %v\n", progressMessage, os.Args) 195 | } 196 | 197 | // merge configuration and print it 198 | if a.options != nil { 199 | if configurableOptions, ok := a.options.(ConfigurableOptions); ok { 200 | if errs := configurableOptions.ApplyFlags(); len(errs) > 0 { 201 | for _, err := range errs { 202 | fmt.Printf("%v %v\n", color.RedString("Error:"), err) 203 | } 204 | os.Exit(1) 205 | } 206 | if !a.silence { 207 | printConfig() 208 | } 209 | } 210 | 211 | if validater, ok := a.options.(OptionValidater); ok { 212 | if errs := validater.Validate(); len(errs) > 0 { 213 | for _, err := range errs { 214 | fmt.Printf("%v %v\n", color.RedString("Error:"), err) 215 | } 216 | os.Exit(1) 217 | } 218 | if !a.silence { 219 | printConfig() 220 | } 221 | } 222 | } 223 | 224 | if !a.silence && !a.noVersion { 225 | fmt.Printf("%v Version:\n", progressMessage) 226 | fmt.Printf("%s\n", version.Get()) 227 | } 228 | 229 | if a.runFunc != nil { 230 | if !a.silence { 231 | fmt.Printf("%v Log data will now stream in as it occurs:\n", progressMessage) 232 | } 233 | if err := a.runFunc(a.basename); err != nil { 234 | fmt.Printf("%v %v\n", color.RedString("Error:"), err) 235 | os.Exit(1) 236 | } 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /pkg/utils/app/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "runtime" 21 | "strings" 22 | 23 | "github.com/fatih/color" 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | // Command is a sub command structure of a cli application. 28 | // It is recommended that a command be created with the app.NewCommand() 29 | // function. 30 | type Command struct { 31 | usage string 32 | desc string 33 | options CliOptions 34 | commands []*Command 35 | runFunc RunCommandFunc 36 | } 37 | 38 | // CommandOption defines optional parameters for initializing the command 39 | // structure. 40 | type CommandOption func(*Command) 41 | 42 | // WithCommandOptions to open the application's function to read from the 43 | // command line. 44 | func WithCommandOptions(opt CliOptions) CommandOption { 45 | return func(c *Command) { 46 | c.options = opt 47 | } 48 | } 49 | 50 | // RunCommandFunc defines the application's command startup callback function. 51 | type RunCommandFunc func(args []string) error 52 | 53 | // WithCommandRunFunc is used to set the application's command startup callback 54 | // function option. 55 | func WithCommandRunFunc(run RunCommandFunc) CommandOption { 56 | return func(c *Command) { 57 | c.runFunc = run 58 | } 59 | } 60 | 61 | // NewCommand creates a new sub command instance based on the given command name 62 | // and other options. 63 | func NewCommand(usage string, desc string, opts ...CommandOption) *Command { 64 | c := &Command{ 65 | usage: usage, 66 | desc: desc, 67 | } 68 | 69 | for _, o := range opts { 70 | o(c) 71 | } 72 | 73 | return c 74 | } 75 | 76 | // AddCommand adds sub command to the current command. 77 | func (c *Command) AddCommand(cmd *Command) { 78 | c.commands = append(c.commands, cmd) 79 | } 80 | 81 | // AddCommands adds multiple sub commands to the current command. 82 | func (c *Command) AddCommands(cmds ...*Command) { 83 | c.commands = append(c.commands, cmds...) 84 | } 85 | 86 | func (c *Command) cobraCommand() *cobra.Command { 87 | cmd := &cobra.Command{ 88 | Use: c.usage, 89 | Short: c.desc, 90 | } 91 | cmd.SetOutput(os.Stdout) 92 | cmd.Flags().SortFlags = false 93 | if len(c.commands) > 0 { 94 | for _, command := range c.commands { 95 | cmd.AddCommand(command.cobraCommand()) 96 | } 97 | } 98 | if c.runFunc != nil { 99 | cmd.Run = c.runCommand 100 | } 101 | if c.options != nil { 102 | c.options.AddFlags(cmd.Flags()) 103 | } 104 | addHelpCommandFlag(c.usage, cmd.Flags()) 105 | return cmd 106 | } 107 | 108 | func (c *Command) runCommand(cmd *cobra.Command, args []string) { 109 | if c.runFunc != nil { 110 | if err := c.runFunc(args); err != nil { 111 | fmt.Printf("%v %v\n", color.RedString("Error:"), err) 112 | os.Exit(1) 113 | } 114 | } 115 | } 116 | 117 | // AddCommand adds sub command to the application. 118 | func (a *App) AddCommand(cmd *Command) { 119 | a.commands = append(a.commands, cmd) 120 | } 121 | 122 | // AddCommands adds multiple sub commands to the application. 123 | func (a *App) AddCommands(cmds ...*Command) { 124 | a.commands = append(a.commands, cmds...) 125 | } 126 | 127 | // FormatBaseName is formatted as an executable file name under different 128 | // operating systems according to the given name. 129 | func FormatBaseName(basename string) string { 130 | // Make case-insensitive and strip executable suffix if present 131 | if runtime.GOOS == "windows" { 132 | basename = strings.ToLower(basename) 133 | basename = strings.TrimSuffix(basename, ".exe") 134 | } 135 | return basename 136 | } 137 | -------------------------------------------------------------------------------- /pkg/utils/app/config.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "strings" 21 | 22 | "github.com/gosuri/uitable" 23 | "github.com/spf13/cobra" 24 | "github.com/spf13/pflag" 25 | "github.com/spf13/viper" 26 | ) 27 | 28 | var cfgFile string 29 | 30 | // addConfigFlag adds flags for a specific server to the specified FlagSet 31 | // object. 32 | func addConfigFlag(basename string, fs *pflag.FlagSet) { 33 | viper.SetEnvPrefix(strings.Replace(strings.ToUpper(basename), "-", "_", -1)) 34 | viper.AutomaticEnv() 35 | viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) 36 | cobra.OnInitialize(initCobra) 37 | fs.StringVarP(&cfgFile, "config", "C", cfgFile, 38 | "Read configuration from specified `FILE`, support JSON, TOML, YAML, HCL, or Java properties formats.") 39 | } 40 | 41 | func initCobra() { 42 | if cfgFile != "" { 43 | viper.SetConfigFile(cfgFile) 44 | 45 | if err := viper.ReadInConfig(); err != nil { 46 | _, _ = fmt.Fprintf(os.Stderr, "Error: failed to read configuration file(%s): %v\n", cfgFile, err) 47 | os.Exit(1) 48 | } 49 | } 50 | } 51 | 52 | func printConfig() { 53 | keys := viper.AllKeys() 54 | if len(keys) > 0 { 55 | fmt.Printf("%v Configuration items:\n", progressMessage) 56 | table := uitable.New() 57 | table.Separator = " " 58 | table.MaxColWidth = 80 59 | table.RightAlign(0) 60 | for _, k := range keys { 61 | table.AddRow(fmt.Sprintf("%s:", k), viper.Get(k)) 62 | } 63 | fmt.Println(table) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/utils/app/flag.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "strings" 19 | 20 | "github.com/spf13/pflag" 21 | ) 22 | 23 | func initFlag() { 24 | pflag.CommandLine.SetNormalizeFunc(WordSepNormalizeFunc) 25 | } 26 | 27 | // WordSepNormalizeFunc changes all flags that contain "_" separators. 28 | func WordSepNormalizeFunc(_ *pflag.FlagSet, name string) pflag.NormalizedName { 29 | if strings.Contains(name, "_") { 30 | return pflag.NormalizedName(strings.Replace(name, "_", "-", -1)) 31 | } 32 | return pflag.NormalizedName(name) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/utils/app/help.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | "github.com/fatih/color" 22 | "github.com/spf13/cobra" 23 | "github.com/spf13/pflag" 24 | ) 25 | 26 | const ( 27 | flagHelp = "help" 28 | flagHelpShorthand = "H" 29 | ) 30 | 31 | func helpCommand(name string) *cobra.Command { 32 | return &cobra.Command{ 33 | Use: "help [command]", 34 | Short: "Help about any command.", 35 | Long: `Help provides help for any command in the application. 36 | Simply type ` + name + ` help [path to command] for full details.`, 37 | 38 | Run: func(c *cobra.Command, args []string) { 39 | cmd, _, e := c.Root().Find(args) 40 | if cmd == nil || e != nil { 41 | c.Printf("Unknown help topic %#q\n", args) 42 | _ = c.Root().Usage() 43 | } else { 44 | cmd.InitDefaultHelpFlag() // make possible 'help' flag to be shown 45 | _ = cmd.Help() 46 | } 47 | }, 48 | } 49 | } 50 | 51 | // addHelpFlag adds flags for a specific application to the specified FlagSet 52 | // object. 53 | func addHelpFlag(name string, fs *pflag.FlagSet) { 54 | fs.BoolP(flagHelp, flagHelpShorthand, false, fmt.Sprintf("Help for %s.", name)) 55 | } 56 | 57 | // addHelpCommandFlag adds flags for a specific command of application to the 58 | // specified FlagSet object. 59 | func addHelpCommandFlag(usage string, fs *pflag.FlagSet) { 60 | fs.BoolP(flagHelp, flagHelpShorthand, false, 61 | fmt.Sprintf("Help for the %s command.", color.GreenString(strings.Split(usage, " ")[0]))) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/utils/app/options.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "github.com/spf13/pflag" 19 | ) 20 | 21 | // CliOptions abstracts configuration options for reading parameters from the 22 | // command line. 23 | type CliOptions interface { 24 | // AddFlags adds flags to the specified FlagSet object. 25 | AddFlags(fs *pflag.FlagSet) 26 | } 27 | 28 | // ConfigurableOptions abstracts configuration options for reading parameters 29 | // from a configuration file. 30 | type ConfigurableOptions interface { 31 | // ApplyFlags parsing parameters from the command line or configuration file 32 | // to the options instance. 33 | ApplyFlags() []error 34 | } 35 | 36 | // OptionValidater abstracts option validater. 37 | type OptionValidater interface { 38 | Validate() []error 39 | } 40 | -------------------------------------------------------------------------------- /pkg/utils/app/version/flag.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package version 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "strconv" 21 | 22 | "github.com/spf13/pflag" 23 | ) 24 | 25 | type value int 26 | 27 | const ( 28 | boolFalse value = 0 29 | boolTrue value = 1 30 | raw value = 2 31 | ) 32 | 33 | const strRawVersion string = "raw" 34 | 35 | func (v *value) IsBoolFlag() bool { 36 | return true 37 | } 38 | 39 | func (v *value) Get() interface{} { 40 | return *v 41 | } 42 | 43 | func (v *value) Set(s string) error { 44 | if s == strRawVersion { 45 | *v = raw 46 | return nil 47 | } 48 | boolVal, err := strconv.ParseBool(s) 49 | if boolVal { 50 | *v = boolTrue 51 | } else { 52 | *v = boolFalse 53 | } 54 | return err 55 | } 56 | 57 | func (v *value) String() string { 58 | if *v == raw { 59 | return strRawVersion 60 | } 61 | return fmt.Sprintf("%v", bool(*v == boolTrue)) 62 | } 63 | 64 | // The type of the flag as required by the pflag.value interface 65 | func (v *value) Type() string { 66 | return "version" 67 | } 68 | 69 | const flagName = "version" 70 | const flagShortHand = "V" 71 | 72 | var ( 73 | v = boolFalse 74 | ) 75 | 76 | // AddFlags registers this package's flags on arbitrary FlagSets, such that they 77 | // point to the same value as the global flags. 78 | func AddFlags(fs *pflag.FlagSet) { 79 | fs.VarP(&v, flagName, flagShortHand, "Print version information and quit.") 80 | // "--version" will be treated as "--version=true" 81 | fs.Lookup(flagName).NoOptDefVal = "true" 82 | } 83 | 84 | // PrintAndExitIfRequested will check if the -version flag was passed and, if so, 85 | // print the version and exit. 86 | func PrintAndExitIfRequested(appName string) { 87 | if v == raw { 88 | fmt.Printf("%s\n", Get()) 89 | os.Exit(0) 90 | } else if v == boolTrue { 91 | fmt.Printf("%s %s\n", appName, Get().GitVersion) 92 | os.Exit(0) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pkg/utils/app/version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package version 16 | 17 | import ( 18 | "fmt" 19 | "runtime" 20 | 21 | "github.com/gosuri/uitable" 22 | ) 23 | 24 | var ( 25 | // GitVersion is semantic version. 26 | GitVersion = "v0.0.0-master+$Format:%h$" 27 | // BuildDate in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') 28 | BuildDate = "1970-01-01T00:00:00Z" 29 | // GitCommit sha1 from git, output of $(git rev-parse HEAD) 30 | GitCommit = "$Format:%H$" 31 | // GitTreeState state of git tree, either "clean" or "dirty" 32 | GitTreeState = "" 33 | ) 34 | 35 | // Info contains versioning information. 36 | type Info struct { 37 | GitVersion string `json:"gitVersion"` 38 | GitCommit string `json:"gitCommit"` 39 | GitTreeState string `json:"gitTreeState"` 40 | BuildDate string `json:"buildDate"` 41 | GoVersion string `json:"goVersion"` 42 | Compiler string `json:"compiler"` 43 | Platform string `json:"platform"` 44 | } 45 | 46 | // BaseName returns info as a human-friendly version string. 47 | func (info Info) String() string { 48 | if s, err := info.Text(); err == nil { 49 | return string(s) 50 | } 51 | return info.GitVersion 52 | } 53 | 54 | // Text encodes the version information into UTF-8-encoded text and 55 | // returns the result. 56 | func (info Info) Text() ([]byte, error) { 57 | table := uitable.New() 58 | table.RightAlign(0) 59 | table.MaxColWidth = 80 60 | table.Separator = " " 61 | table.AddRow("gitVersion:", info.GitVersion) 62 | table.AddRow("gitCommit:", info.GitCommit) 63 | table.AddRow("gitTreeState:", info.GitTreeState) 64 | table.AddRow("buildDate:", info.BuildDate) 65 | table.AddRow("goVersion:", info.GoVersion) 66 | table.AddRow("compiler:", info.Compiler) 67 | table.AddRow("platform:", info.Platform) 68 | return table.Bytes(), nil 69 | } 70 | 71 | // Get returns the overall codebase version. It's for detecting 72 | // what code a binary was built from. 73 | func Get() Info { 74 | // These variables typically come from -ldflags settings and in 75 | // their absence fallback to the settings in pkg/version/base.go 76 | return Info{ 77 | GitVersion: GitVersion, 78 | GitCommit: GitCommit, 79 | GitTreeState: GitTreeState, 80 | BuildDate: BuildDate, 81 | GoVersion: runtime.Version(), 82 | Compiler: runtime.Compiler, 83 | Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pkg/utils/crd.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "bytes" 19 | 20 | "github.com/gogo/protobuf/jsonpb" 21 | "github.com/gogo/protobuf/types" 22 | ) 23 | 24 | // BuildPatchStruct creates types.Struct from istio EnvoyFilter slice string 25 | func BuildPatchStruct(config string) (*types.Struct, error) { 26 | m := jsonpb.Unmarshaler{} 27 | 28 | out := &types.Struct{} 29 | err := m.Unmarshal(bytes.NewReader([]byte(config)), out) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return out, nil 35 | } 36 | -------------------------------------------------------------------------------- /pkg/utils/k8s.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "github.com/aeraki-mesh/lazyxds/pkg/utils/log" 23 | istioclient "istio.io/client-go/pkg/clientset/versioned" 24 | "k8s.io/apimachinery/pkg/api/errors" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/kubernetes" 27 | "k8s.io/client-go/rest" 28 | "k8s.io/client-go/tools/clientcmd" 29 | ) 30 | 31 | // NewKubeClient creates new kube client 32 | func NewKubeClient(kubeconfigPath string) (*kubernetes.Clientset, error) { 33 | var err error 34 | var kubeConf *rest.Config 35 | 36 | if kubeconfigPath == "" { 37 | // creates the in-cluster config 38 | kubeConf, err = rest.InClusterConfig() 39 | if err != nil { 40 | return nil, fmt.Errorf("build default in cluster kube config failed: %w", err) 41 | } 42 | } else { 43 | kubeConf, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath) 44 | if err != nil { 45 | return nil, fmt.Errorf("build kube client config from config file failed: %w", err) 46 | } 47 | } 48 | return kubernetes.NewForConfig(kubeConf) 49 | } 50 | 51 | // NewIstioClient creates new istio client which use to handle istio CRD 52 | func NewIstioClient(kubeconfigPath string) (*istioclient.Clientset, error) { 53 | var err error 54 | var istioConf *rest.Config 55 | 56 | if kubeconfigPath == "" { 57 | istioConf, err = rest.InClusterConfig() 58 | if err != nil { 59 | return nil, fmt.Errorf("build default in cluster istio config failed: %w", err) 60 | } 61 | } else { 62 | istioConf, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath) 63 | if err != nil { 64 | return nil, fmt.Errorf("build istio client config from config file failed: %w", err) 65 | } 66 | } 67 | 68 | return istioclient.NewForConfig(istioConf) 69 | } 70 | 71 | // WaitDeployment wait the deployment ready 72 | func WaitDeployment(ctx context.Context, client *kubernetes.Clientset, ns, name string) (err error) { 73 | logger := log.FromContext(ctx) 74 | for { 75 | select { 76 | case <-ctx.Done(): 77 | err = ctx.Err() 78 | return 79 | default: 80 | logger.Info(fmt.Sprintf("Waiting deployment: %s/%s", ns, name)) 81 | time.Sleep(5 * time.Second) 82 | deployment, e := client.AppsV1().Deployments(ns).Get(ctx, name, metav1.GetOptions{}) 83 | if e != nil { 84 | logger.Error(e, fmt.Sprintf("failed to get deployment %s/%s", ns, name)) 85 | if errors.IsNotFound(e) { 86 | continue 87 | } else { 88 | err = e 89 | return 90 | } 91 | } 92 | desired := *deployment.Spec.Replicas 93 | current := deployment.Status.Replicas 94 | available := deployment.Status.AvailableReplicas 95 | updated := deployment.Status.UpdatedReplicas 96 | if current != desired { 97 | logger.Info("replicas of currently pods is not in line with the desire", 98 | "current", current, "desired", desired) 99 | continue 100 | } 101 | if available != desired { 102 | logger.Info("replicas of available pods is not in line with the desire", 103 | "available", available, "desired", desired) 104 | continue 105 | } 106 | if updated != desired { 107 | logger.Info("replicas of updated pods if not in line with the desire", 108 | "updated", updated, "desired", desired) 109 | continue 110 | } 111 | 112 | return 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /pkg/utils/leaderelectionconfig/config.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package leaderelectionconfig 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/spf13/pflag" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/client-go/tools/leaderelection/resourcelock" 23 | componentbaseconfig "k8s.io/component-base/config" 24 | ) 25 | 26 | // New creates a LeaderElectionConfiguration 27 | func New(name string) *componentbaseconfig.LeaderElectionConfiguration { 28 | return &componentbaseconfig.LeaderElectionConfiguration{ 29 | LeaderElect: true, 30 | ResourceName: name, 31 | ResourceNamespace: metav1.NamespaceSystem, 32 | ResourceLock: resourcelock.ConfigMapsResourceLock, 33 | LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, 34 | RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, 35 | RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, 36 | } 37 | } 38 | 39 | // AddFlags binds the LeaderElectionConfiguration struct fields to a flagset 40 | func AddFlags(l *componentbaseconfig.LeaderElectionConfiguration, fs *pflag.FlagSet) { 41 | fs.BoolVar(&l.LeaderElect, "leader-elect", l.LeaderElect, ""+ 42 | "Start a leader election client and gain leadership before "+ 43 | "executing the main loop. Enable this when running replicated "+ 44 | "components for high availability.") 45 | fs.DurationVar(&l.LeaseDuration.Duration, "leader-elect-lease-duration", l.LeaseDuration.Duration, ""+ 46 | "The duration that non-leader candidates will wait after observing a leadership "+ 47 | "renewal until attempting to acquire leadership of a led but unrenewed leader "+ 48 | "slot. This is effectively the maximum duration that a leader can be stopped "+ 49 | "before it is replaced by another candidate. This is only applicable if leader "+ 50 | "election is enabled.") 51 | fs.DurationVar(&l.RenewDeadline.Duration, "leader-elect-renew-deadline", l.RenewDeadline.Duration, ""+ 52 | "The interval between attempts by the acting master to renew a leadership slot "+ 53 | "before it stops leading. This must be less than or equal to the lease duration. "+ 54 | "This is only applicable if leader election is enabled.") 55 | fs.DurationVar(&l.RetryPeriod.Duration, "leader-elect-retry-period", l.RetryPeriod.Duration, ""+ 56 | "The duration the clients should wait between attempting acquisition and renewal "+ 57 | "of a leadership. This is only applicable if leader election is enabled.") 58 | fs.StringVar(&l.ResourceLock, "leader-elect-resource-lock", l.ResourceLock, ""+ 59 | "The type of resource object that is used for locking during "+ 60 | "leader election. Supported options are `endpoints` (default) and `configmaps`.") 61 | fs.StringVar(&l.ResourceName, "leader-elect-resource-name", l.ResourceName, ""+ 62 | "The name of resource object that is used for locking during "+ 63 | "leader election.") 64 | fs.StringVar(&l.ResourceNamespace, "leader-elect-resource-namespace", l.ResourceNamespace, ""+ 65 | "The namespace of resource object that is used for locking during "+ 66 | "leader election.") 67 | } 68 | -------------------------------------------------------------------------------- /pkg/utils/log/context.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package log 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/go-logr/logr" 21 | ) 22 | 23 | type key int 24 | 25 | const ( 26 | logKey key = iota 27 | ) 28 | 29 | // WithContext returns a copy of parent in which the logger value is set 30 | func WithContext(parent context.Context, logger logr.Logger) context.Context { 31 | return context.WithValue(parent, logKey, logger) 32 | } 33 | 34 | // Ctx returns the logger from context. 35 | func Ctx(ctx context.Context) logr.Logger { 36 | return FromContext(ctx) 37 | } 38 | 39 | // FromContext returns the logger from context. 40 | func FromContext(ctx context.Context) logr.Logger { 41 | log, ok := ctx.Value(logKey).(logr.Logger) 42 | if !ok { 43 | return Logger() 44 | } 45 | 46 | return log 47 | } 48 | -------------------------------------------------------------------------------- /pkg/utils/log/log.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package log 16 | 17 | import ( 18 | "github.com/go-logr/logr" 19 | "k8s.io/klog/v2" 20 | "k8s.io/klog/v2/klogr" 21 | ) 22 | 23 | var logger = klogr.New() 24 | 25 | func init() { 26 | klog.InitFlags(nil) 27 | } 28 | 29 | // Logger return the default logger. 30 | func Logger() logr.Logger { 31 | return logger 32 | } 33 | 34 | // Info logs a non-error message with the given key/value pairs as context. 35 | // 36 | // The msg argument should be used to add some constant description to 37 | // the log line. The key/value pairs can then be used to add additional 38 | // variable information. The key/value pairs should alternate string 39 | // keys and arbitrary values. 40 | func Info(msg string, keysAndValues ...interface{}) { 41 | logger.Info(msg, keysAndValues...) 42 | } 43 | 44 | // Error logs an error, with the given message and key/value pairs as context. 45 | // It functions similarly to calling Info with the "error" named value, but may 46 | // have unique behavior, and should be preferred for logging errors (see the 47 | // package documentations for more information). 48 | // 49 | // The msg field should be used to add context to any underlying error, 50 | // while the err field should be used to attach the actual error that 51 | // triggered this log line, if present. 52 | func Error(err error, msg string, keysAndValues ...interface{}) { 53 | logger.Error(err, msg, keysAndValues...) 54 | } 55 | 56 | // V returns an Logger value for a specific verbosity level, relative to 57 | // this Logger. In other words, V values are additive. V higher verbosity 58 | // level means a log message is less important. It's illegal to pass a log 59 | // level less than zero. 60 | func V(level int) logr.Logger { 61 | return logger.V(level) 62 | } 63 | 64 | // WithValues adds some key-value pairs of context to a logger. 65 | // See Info for documentation on how key/value pairs work. 66 | func WithValues(keysAndValues ...interface{}) logr.Logger { 67 | return logger.WithValues(keysAndValues...) 68 | } 69 | 70 | // WithName adds a new element to the logger's name. 71 | // Successive calls with WithName continue to append 72 | // suffixes to the logger's name. It's strongly recommended 73 | // that name segments contain only letters, digits, and hyphens 74 | // (see the package documentation for more information). 75 | func WithName(name string) logr.Logger { 76 | return logger.WithName(name) 77 | } 78 | -------------------------------------------------------------------------------- /pkg/utils/protocal.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "strings" 19 | 20 | corev1 "k8s.io/api/core/v1" 21 | ) 22 | 23 | // IsHTTP check if the port is using http protocol 24 | func IsHTTP(port corev1.ServicePort) bool { 25 | // If application protocol is set, we will use that 26 | // If not, use the port name 27 | var name string 28 | if port.AppProtocol != nil { 29 | name = *port.AppProtocol 30 | } 31 | if name == "" { 32 | name = port.Name 33 | } 34 | name = strings.ToLower(name) 35 | 36 | if strings.HasPrefix(name, "https") { 37 | return false 38 | } 39 | 40 | return strings.HasPrefix(name, "http") || 41 | strings.HasPrefix(name, "http2") || 42 | strings.HasPrefix(name, "grpc") 43 | } 44 | -------------------------------------------------------------------------------- /pkg/utils/service.go: -------------------------------------------------------------------------------- 1 | // Copyright Aeraki Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | "github.com/aeraki-mesh/lazyxds/cmd/lazyxds/app/config" 22 | ) 23 | 24 | const ( 25 | // DefaultDNS is the suffix of default fqdn in kubernetes 26 | DefaultDNS = "svc.cluster.local" 27 | ) 28 | 29 | // FQDN returns the fully qualified domain name of service 30 | func FQDN(name, namespace string) string { 31 | // return fmt.Sprintf("%s/%s", name, namespace) 32 | return fmt.Sprintf("%s.%s.%s", name, namespace, DefaultDNS) 33 | } 34 | 35 | // ObjectID use name.namespace as object id 36 | func ObjectID(name, namespace string) string { 37 | return fmt.Sprintf("%s.%s", name, namespace) 38 | } 39 | 40 | // PortID use svcID:port as port id 41 | func PortID(svcID, port string) string { 42 | return fmt.Sprintf("%s:%s", svcID, port) 43 | } 44 | 45 | // ParseID parse the object id to name and namespace 46 | func ParseID(id string) (string, string) { 47 | parts := strings.Split(id, ".") 48 | if len(parts) < 2 { 49 | // todo panic 50 | return "", "" 51 | } 52 | name := parts[0] 53 | namespace := parts[1] 54 | return name, namespace 55 | } 56 | 57 | // ServiceID2EgressString turn the service id to egress string of sidecar crd 58 | func ServiceID2EgressString(id string) string { 59 | _, namespace := ParseID(id) 60 | 61 | return fmt.Sprintf("%s/%s", namespace, id) 62 | } 63 | 64 | // UpstreamCluster2ServiceID extract the service id from xds cluster id 65 | func UpstreamCluster2ServiceID(cluster string) string { 66 | parts := strings.Split(cluster, "|") 67 | if len(parts) != 4 { 68 | return "" 69 | } 70 | if parts[0] != "outbound" { 71 | return "" 72 | } 73 | return parts[3] 74 | } 75 | 76 | // IsLazyEnabled check if lazyxds is enabled 77 | func IsLazyEnabled(annotations map[string]string) bool { 78 | return strings.ToLower(annotations[config.LazyLoadingAnnotation]) == "true" 79 | } 80 | -------------------------------------------------------------------------------- /pkg/utils/signal/signal.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package signal 18 | 19 | import ( 20 | "os" 21 | "os/signal" 22 | ) 23 | 24 | var onlyOneSignalHandler = make(chan struct{}) 25 | 26 | // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned 27 | // which is closed on one of these signals. If a second signal is caught, the program 28 | // is terminated with exit code 1. 29 | func SetupSignalHandler() (stopCh <-chan struct{}) { 30 | close(onlyOneSignalHandler) // panics when called twice 31 | 32 | stop := make(chan struct{}) 33 | c := make(chan os.Signal, 2) 34 | signal.Notify(c, shutdownSignals...) 35 | go func() { 36 | <-c 37 | close(stop) 38 | <-c 39 | os.Exit(1) // second signal. Exit directly. 40 | }() 41 | 42 | return stop 43 | } 44 | -------------------------------------------------------------------------------- /pkg/utils/signal/signal_posix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | /* 4 | Copyright 2017 The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package signal 20 | 21 | import ( 22 | "os" 23 | "syscall" 24 | ) 25 | 26 | var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} 27 | -------------------------------------------------------------------------------- /pkg/utils/signal/signal_windows.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package signal 18 | 19 | import ( 20 | "os" 21 | ) 22 | 23 | var shutdownSignals = []os.Signal{os.Interrupt} 24 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/services/data-svc1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: data-svc1 7 | name: data-svc1 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: data-svc1 13 | template: 14 | metadata: 15 | labels: 16 | app: data-svc1 17 | spec: 18 | containers: 19 | - image: zhongfox/anyserver:v1 20 | imagePullPolicy: Always 21 | name: app 22 | ports: 23 | - containerPort: 4000 24 | protocol: TCP 25 | env: 26 | - name: SERVICE 27 | value: data-svc1 28 | - name: TCP_PORTS 29 | value: "4000" 30 | - name: POD_IP 31 | valueFrom: 32 | fieldRef: 33 | fieldPath: status.podIP 34 | - name: POD_NAME 35 | valueFrom: 36 | fieldRef: 37 | fieldPath: metadata.name 38 | --- 39 | 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | labels: 44 | app: data-svc1 45 | name: data-svc1 46 | spec: 47 | ports: 48 | - name: tcp-1 49 | port: 4000 50 | protocol: TCP 51 | selector: 52 | app: data-svc1 53 | type: ClusterIP 54 | --- 55 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/services/headless-svc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: headless-svc 7 | name: headless-svc 8 | spec: 9 | replicas: 0 10 | selector: 11 | matchLabels: 12 | app: headless-svc 13 | template: 14 | metadata: 15 | labels: 16 | app: headless-svc 17 | spec: 18 | containers: 19 | - image: zhongfox/anyserver:v1 20 | imagePullPolicy: Always 21 | name: app 22 | ports: 23 | - containerPort: 7000 24 | protocol: TCP 25 | env: 26 | - name: SERVICE 27 | value: headless-svc 28 | - name: HTTP_PORTS 29 | value: "7000" 30 | - name: POD_IP 31 | valueFrom: 32 | fieldRef: 33 | fieldPath: status.podIP 34 | - name: POD_NAME 35 | valueFrom: 36 | fieldRef: 37 | fieldPath: metadata.name 38 | --- 39 | 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | labels: 44 | app: headless-svc 45 | name: headless-svc 46 | spec: 47 | clusterIP: None 48 | ports: 49 | - name: http-1 50 | port: 7000 51 | protocol: TCP 52 | selector: 53 | app: headless-svc 54 | type: ClusterIP 55 | --- 56 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/services/mix-svc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: mix-svc 7 | name: mix-svc 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: mix-svc 13 | template: 14 | metadata: 15 | labels: 16 | app: mix-svc 17 | spec: 18 | containers: 19 | - image: zhongfox/anyserver:v1 20 | imagePullPolicy: Always 21 | name: app 22 | ports: 23 | - containerPort: 7001 24 | protocol: TCP 25 | - containerPort: 7002 26 | protocol: TCP 27 | - containerPort: 4001 28 | protocol: TCP 29 | - containerPort: 4002 30 | protocol: TCP 31 | env: 32 | - name: SERVICE 33 | value: mix-svc 34 | - name: HTTP_PORTS 35 | value: "7001,7002" 36 | - name: TCP_PORTS 37 | value: "4001,4002" 38 | - name: POD_IP 39 | valueFrom: 40 | fieldRef: 41 | fieldPath: status.podIP 42 | - name: POD_NAME 43 | valueFrom: 44 | fieldRef: 45 | fieldPath: metadata.name 46 | --- 47 | 48 | apiVersion: v1 49 | kind: Service 50 | metadata: 51 | labels: 52 | app: mix-svc 53 | name: mix-svc 54 | spec: 55 | ports: 56 | - name: http-1 57 | port: 7001 58 | protocol: TCP 59 | - name: http-2 60 | port: 7002 61 | protocol: TCP 62 | - name: tcp-1 63 | port: 4001 64 | protocol: TCP 65 | - name: tcp-2 66 | port: 4002 67 | protocol: TCP 68 | selector: 69 | app: mix-svc 70 | type: ClusterIP 71 | --- 72 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/services/serviceentry.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: networking.istio.io/v1alpha3 3 | kind: ServiceEntry 4 | metadata: 5 | name: web-svc1-se 6 | spec: 7 | hosts: 8 | - qq.com 9 | location: MESH_INTERNAL 10 | ports: 11 | - number: 7000 12 | name: http 13 | protocol: HTTP 14 | resolution: STATIC 15 | workloadSelector: 16 | labels: 17 | app: web-svc1 18 | --- 19 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/services/web-svc-multi-version.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: web-svc-multi-version 7 | version: v1 8 | name: web-svc-multi-version-v1 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: web-svc-multi-version 14 | version: v1 15 | template: 16 | metadata: 17 | labels: 18 | app: web-svc-multi-version 19 | version: v1 20 | spec: 21 | containers: 22 | - image: zhongfox/anyserver:v1 23 | imagePullPolicy: Always 24 | name: app 25 | ports: 26 | - containerPort: 7001 27 | protocol: TCP 28 | env: 29 | - name: SERVICE 30 | value: web-svc-multi-version 31 | - name: HTTP_PORTS 32 | value: "7001" 33 | - name: POD_IP 34 | valueFrom: 35 | fieldRef: 36 | fieldPath: status.podIP 37 | - name: POD_NAME 38 | valueFrom: 39 | fieldRef: 40 | fieldPath: metadata.name 41 | --- 42 | 43 | apiVersion: apps/v1 44 | kind: Deployment 45 | metadata: 46 | labels: 47 | app: web-svc-multi-version 48 | version: v2 49 | name: web-svc-multi-version-v2 50 | spec: 51 | replicas: 1 52 | selector: 53 | matchLabels: 54 | app: web-svc-multi-version 55 | version: v2 56 | template: 57 | metadata: 58 | labels: 59 | app: web-svc-multi-version 60 | version: v2 61 | spec: 62 | containers: 63 | - image: zhongfox/anyserver:v1 64 | imagePullPolicy: Always 65 | name: app 66 | ports: 67 | - containerPort: 7001 68 | protocol: TCP 69 | env: 70 | - name: SERVICE 71 | value: web-svc-multi-version 72 | - name: HTTP_PORTS 73 | value: "7001" 74 | - name: POD_IP 75 | valueFrom: 76 | fieldRef: 77 | fieldPath: status.podIP 78 | - name: POD_NAME 79 | valueFrom: 80 | fieldRef: 81 | fieldPath: metadata.name 82 | --- 83 | 84 | apiVersion: v1 85 | kind: Service 86 | metadata: 87 | labels: 88 | app: web-svc-multi-version 89 | name: web-svc-multi-version 90 | spec: 91 | ports: 92 | - name: http-1 93 | port: 7001 94 | protocol: TCP 95 | selector: 96 | app: web-svc-multi-version 97 | type: ClusterIP 98 | --- 99 | apiVersion: networking.istio.io/v1alpha3 100 | kind: VirtualService 101 | metadata: 102 | name: web-svc-multi-version 103 | spec: 104 | hosts: 105 | - web-svc-multi-version.lazyxds-example.svc.cluster.local 106 | gateways: 107 | - mesh 108 | - istio-system/lazyxds-egress 109 | http: 110 | - match: 111 | - headers: 112 | user: 113 | exact: admin 114 | route: 115 | - destination: 116 | host: web-svc-multi-version.lazyxds-example.svc.cluster.local 117 | subset: v2 118 | port: 119 | number: 7001 120 | - route: 121 | - destination: 122 | host: web-svc-multi-version.lazyxds-example.svc.cluster.local 123 | subset: v1 124 | port: 125 | number: 7001 126 | --- 127 | apiVersion: networking.istio.io/v1alpha3 128 | kind: DestinationRule 129 | metadata: 130 | name: web-svc-multi-version 131 | spec: 132 | host: web-svc-multi-version 133 | subsets: 134 | - name: v1 135 | labels: 136 | version: v1 137 | - name: v2 138 | labels: 139 | version: v2 140 | --- 141 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/services/web-svc-related.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: web-svc-related1 7 | name: web-svc-related1 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: web-svc-related1 13 | template: 14 | metadata: 15 | labels: 16 | app: web-svc-related1 17 | spec: 18 | containers: 19 | - image: zhongfox/anyserver:v1 20 | imagePullPolicy: Always 21 | name: app 22 | ports: 23 | - containerPort: 7011 24 | protocol: TCP 25 | env: 26 | - name: SERVICE 27 | value: web-svc-related1 28 | - name: HTTP_PORTS 29 | value: "7011" 30 | - name: POD_IP 31 | valueFrom: 32 | fieldRef: 33 | fieldPath: status.podIP 34 | - name: POD_NAME 35 | valueFrom: 36 | fieldRef: 37 | fieldPath: metadata.name 38 | --- 39 | 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | labels: 44 | app: web-svc-related1 45 | name: web-svc-related1 46 | spec: 47 | ports: 48 | - name: http-1 49 | port: 7011 50 | protocol: TCP 51 | selector: 52 | app: web-svc-related1 53 | type: ClusterIP 54 | --- 55 | 56 | apiVersion: apps/v1 57 | kind: Deployment 58 | metadata: 59 | labels: 60 | app: web-svc-related2 61 | name: web-svc-related2 62 | spec: 63 | replicas: 1 64 | selector: 65 | matchLabels: 66 | app: web-svc-related2 67 | template: 68 | metadata: 69 | labels: 70 | app: web-svc-related2 71 | spec: 72 | containers: 73 | - image: zhongfox/anyserver:v1 74 | imagePullPolicy: Always 75 | name: app 76 | ports: 77 | - containerPort: 7012 78 | protocol: TCP 79 | env: 80 | - name: SERVICE 81 | value: web-svc-related2 82 | - name: HTTP_PORTS 83 | value: "7012" 84 | - name: POD_IP 85 | valueFrom: 86 | fieldRef: 87 | fieldPath: status.podIP 88 | - name: POD_NAME 89 | valueFrom: 90 | fieldRef: 91 | fieldPath: metadata.name 92 | --- 93 | 94 | apiVersion: v1 95 | kind: Service 96 | metadata: 97 | labels: 98 | app: web-svc-related2 99 | name: web-svc-related2 100 | spec: 101 | ports: 102 | - name: http-1 103 | port: 7012 104 | protocol: TCP 105 | selector: 106 | app: web-svc-related2 107 | type: ClusterIP 108 | --- 109 | 110 | apiVersion: apps/v1 111 | kind: Deployment 112 | metadata: 113 | labels: 114 | app: web-svc-related3 115 | name: web-svc-related3 116 | spec: 117 | replicas: 1 118 | selector: 119 | matchLabels: 120 | app: web-svc-related3 121 | template: 122 | metadata: 123 | labels: 124 | app: web-svc-related3 125 | spec: 126 | containers: 127 | - image: zhongfox/anyserver:v1 128 | imagePullPolicy: Always 129 | name: app 130 | ports: 131 | - containerPort: 7000 132 | protocol: TCP 133 | env: 134 | - name: SERVICE 135 | value: web-svc-related3 136 | - name: HTTP_PORTS 137 | value: "7013" 138 | - name: POD_IP 139 | valueFrom: 140 | fieldRef: 141 | fieldPath: status.podIP 142 | - name: POD_NAME 143 | valueFrom: 144 | fieldRef: 145 | fieldPath: metadata.name 146 | --- 147 | 148 | apiVersion: v1 149 | kind: Service 150 | metadata: 151 | labels: 152 | app: web-svc-related3 153 | name: web-svc-related3 154 | spec: 155 | ports: 156 | - name: http-1 157 | port: 7013 158 | protocol: TCP 159 | selector: 160 | app: web-svc-related3 161 | type: ClusterIP 162 | --- 163 | 164 | apiVersion: networking.istio.io/v1alpha3 165 | kind: VirtualService 166 | metadata: 167 | name: web-svc-related 168 | spec: 169 | hosts: 170 | - web-svc-related1.lazyxds-example.svc.cluster.local 171 | gateways: 172 | - mesh 173 | - istio-system/lazyxds-egress 174 | http: 175 | - match: 176 | - queryParams: 177 | user: 178 | exact: "user2" 179 | route: 180 | - destination: 181 | host: web-svc-related2.lazyxds-example.svc.cluster.local 182 | port: 183 | number: 7012 184 | - match: 185 | - queryParams: 186 | user: 187 | exact: "user3" 188 | route: 189 | - destination: 190 | host: web-svc-related3.lazyxds-example.svc.cluster.local 191 | port: 192 | number: 7013 193 | - route: 194 | - destination: 195 | host: web-svc-related1.lazyxds-example.svc.cluster.local 196 | port: 197 | number: 7011 198 | --- 199 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/services/web-svc1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: web-svc1 7 | name: web-svc1 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: web-svc1 13 | template: 14 | metadata: 15 | labels: 16 | app: web-svc1 17 | spec: 18 | containers: 19 | - image: zhongfox/anyserver:v1 20 | imagePullPolicy: Always 21 | name: app 22 | ports: 23 | - containerPort: 7000 24 | protocol: TCP 25 | env: 26 | - name: SERVICE 27 | value: web-svc1 28 | - name: HTTP_PORTS 29 | value: "7000" 30 | - name: POD_IP 31 | valueFrom: 32 | fieldRef: 33 | fieldPath: status.podIP 34 | - name: POD_NAME 35 | valueFrom: 36 | fieldRef: 37 | fieldPath: metadata.name 38 | --- 39 | 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | labels: 44 | app: web-svc1 45 | name: web-svc1 46 | spec: 47 | ports: 48 | - name: http-1 49 | port: 7000 50 | protocol: TCP 51 | selector: 52 | app: web-svc1 53 | type: ClusterIP 54 | --- 55 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/services/web-svc2.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: web-svc2 7 | name: web-svc2 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: web-svc2 13 | template: 14 | metadata: 15 | labels: 16 | app: web-svc2 17 | spec: 18 | containers: 19 | - image: zhongfox/anyserver:v1 20 | imagePullPolicy: Always 21 | name: app 22 | ports: 23 | - containerPort: 7000 24 | protocol: TCP 25 | env: 26 | - name: SERVICE 27 | value: web-svc2 28 | - name: HTTP_PORTS 29 | value: "7000" 30 | - name: POD_IP 31 | valueFrom: 32 | fieldRef: 33 | fieldPath: status.podIP 34 | - name: POD_NAME 35 | valueFrom: 36 | fieldRef: 37 | fieldPath: metadata.name 38 | --- 39 | 40 | apiVersion: v1 41 | kind: Service 42 | metadata: 43 | labels: 44 | app: web-svc2 45 | name: web-svc2 46 | spec: 47 | ports: 48 | - name: http-1 49 | port: 7000 50 | protocol: TCP 51 | selector: 52 | app: web-svc2 53 | type: ClusterIP 54 | --- 55 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/source/external-name-svc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: external-name-svc-without-selector 6 | labels: 7 | app: external-name-svc-without-selector 8 | annotations: 9 | lazy-xds: "true" 10 | spec: 11 | externalName: www.baidu.com 12 | ports: 13 | - name: http-1 14 | port: 80 15 | protocol: TCP 16 | type: ExternalName 17 | --- 18 | apiVersion: v1 19 | kind: Service 20 | metadata: 21 | name: external-name-svc-with-selector 22 | labels: 23 | app: external-name-svc-with-selector 24 | annotations: 25 | lazy-xds: "true" 26 | spec: 27 | externalName: www.google.com 28 | selector: 29 | app: external-name-svc-with-selector 30 | ports: 31 | - name: http-1 32 | port: 80 33 | protocol: TCP 34 | type: ExternalName 35 | 36 | --- 37 | apiVersion: v1 38 | kind: Service 39 | metadata: 40 | name: selector-less-svc 41 | labels: 42 | app: selector-less-svc 43 | annotations: 44 | lazy-xds: "true" 45 | spec: 46 | ports: 47 | - name: http-1 48 | port: 7000 49 | protocol: TCP 50 | type: ClusterIP 51 | --- 52 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/source/lazy_source.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: lazy-source 7 | name: lazy-source 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: lazy-source 13 | template: 14 | metadata: 15 | labels: 16 | app: lazy-source 17 | spec: 18 | containers: 19 | - image: zhongfox/anyserver:v1 20 | imagePullPolicy: Always 21 | name: app 22 | ports: 23 | - containerPort: 7123 24 | protocol: TCP 25 | env: 26 | - name: SERVICE 27 | value: lazy-source 28 | - name: HTTP_PORTS 29 | value: "7123" 30 | - name: POD_IP 31 | valueFrom: 32 | fieldRef: 33 | fieldPath: status.podIP 34 | - name: POD_NAME 35 | valueFrom: 36 | fieldRef: 37 | fieldPath: metadata.name 38 | --- 39 | apiVersion: v1 40 | kind: Service 41 | metadata: 42 | labels: 43 | app: lazy-source 44 | annotations: 45 | lazy-xds: "true" 46 | name: lazy-source 47 | spec: 48 | ports: 49 | - name: http 50 | port: 7123 51 | protocol: TCP 52 | selector: 53 | app: lazy-source 54 | type: ClusterIP 55 | --- 56 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/data/source/normal_source.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: normal-source 7 | name: normal-source 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: normal-source 13 | template: 14 | metadata: 15 | labels: 16 | app: normal-source 17 | spec: 18 | containers: 19 | - image: zhongfox/anyserver:v1 20 | imagePullPolicy: Always 21 | name: app 22 | ports: 23 | - containerPort: 7123 24 | protocol: TCP 25 | env: 26 | - name: SERVICE 27 | value: normal-source 28 | - name: HTTP_PORTS 29 | value: "7123" 30 | - name: POD_IP 31 | valueFrom: 32 | fieldRef: 33 | fieldPath: status.podIP 34 | - name: POD_NAME 35 | valueFrom: 36 | fieldRef: 37 | fieldPath: metadata.name 38 | --- 39 | apiVersion: v1 40 | kind: Service 41 | metadata: 42 | labels: 43 | app: normal-source 44 | name: normal-source 45 | spec: 46 | ports: 47 | - name: http 48 | port: 7123 49 | protocol: TCP 50 | selector: 51 | app: normal-source 52 | type: ClusterIP 53 | --- 54 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/lazyxds/lazyxds_suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * // Copyright Aeraki Authors 3 | * // 4 | * // Licensed under the Apache License, Version 2.0 (the "License"); 5 | * // you may not use this file except in compliance with the License. 6 | * // You may obtain a copy of the License at 7 | * // 8 | * // http://www.apache.org/licenses/LICENSE-2.0 9 | * // 10 | * // Unless required by applicable law or agreed to in writing, software 11 | * // distributed under the License is distributed on an "AS IS" BASIS, 12 | * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package lazyxds_test 18 | 19 | import ( 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | "testing" 23 | ) 24 | 25 | func TestLazyxds(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecs(t, "Lazyxds Suite") 28 | } 29 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/utils/cmd.go: -------------------------------------------------------------------------------- 1 | /* 2 | * // Copyright Aeraki Authors 3 | * // 4 | * // Licensed under the Apache License, Version 2.0 (the "License"); 5 | * // you may not use this file except in compliance with the License. 6 | * // You may obtain a copy of the License at 7 | * // 8 | * // http://www.apache.org/licenses/LICENSE-2.0 9 | * // 10 | * // Unless required by applicable law or agreed to in writing, software 11 | * // distributed under the License is distributed on an "AS IS" BASIS, 12 | * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import ( 20 | "bytes" 21 | "log" 22 | "os/exec" 23 | "strings" 24 | ) 25 | 26 | // RunCMD run the shell command 27 | func RunCMD(s string) string { 28 | // log.Printf("run cmd: %s\n", s) 29 | parts := strings.Split(s, " ") 30 | cmd := exec.Command(parts[0], parts[1:]...) 31 | 32 | var out bytes.Buffer 33 | var stderr bytes.Buffer 34 | cmd.Stdout = &out 35 | cmd.Stderr = &stderr 36 | err := cmd.Run() 37 | 38 | if err != nil { 39 | log.Fatalf("cmd failed: %v\n%s\n", err, stderr.String()) 40 | } 41 | return out.String() 42 | } 43 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/utils/kube_runner.go: -------------------------------------------------------------------------------- 1 | /* 2 | * // Copyright Aeraki Authors 3 | * // 4 | * // Licensed under the Apache License, Version 2.0 (the "License"); 5 | * // you may not use this file except in compliance with the License. 6 | * // You may obtain a copy of the License at 7 | * // 8 | * // http://www.apache.org/licenses/LICENSE-2.0 9 | * // 10 | * // Unless required by applicable law or agreed to in writing, software 11 | * // distributed under the License is distributed on an "AS IS" BASIS, 12 | * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import ( 20 | "bufio" 21 | "bytes" 22 | "context" 23 | "fmt" 24 | corev1 "k8s.io/api/core/v1" 25 | v1 "k8s.io/api/core/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | v12 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/client-go/kubernetes" 29 | "k8s.io/client-go/kubernetes/scheme" 30 | "k8s.io/client-go/rest" 31 | "k8s.io/client-go/tools/clientcmd" 32 | "k8s.io/client-go/tools/remotecommand" 33 | "log" 34 | "os" 35 | "strings" 36 | "time" 37 | ) 38 | 39 | // KubeRunner represent the command running in the container 40 | type KubeRunner struct { 41 | client *kubernetes.Clientset 42 | config *rest.Config 43 | } 44 | 45 | // NewKubeRunnerFromENV ... 46 | func NewKubeRunnerFromENV() (*KubeRunner, error) { 47 | runner := &KubeRunner{} 48 | 49 | kubeConfFile := os.Getenv("KUBECONFIG") 50 | if kubeConfFile == "" { 51 | dirname, err := os.UserHomeDir() 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | kubeConfFile = fmt.Sprintf("%s/.kube/config", dirname) // todo check exists 56 | //return nil, fmt.Errorf("miss env KUBECONFIG") 57 | } 58 | 59 | if kubeConf, err := clientcmd.BuildConfigFromFlags("", kubeConfFile); err == nil { 60 | runner.config = kubeConf 61 | } else { 62 | return nil, fmt.Errorf("build kube client config from config file failed: %w", err) 63 | } 64 | 65 | if kubeClient, err := kubernetes.NewForConfig(runner.config); err == nil { 66 | runner.client = kubeClient 67 | } else { 68 | return nil, fmt.Errorf("create kube client failed: %w", err) 69 | } 70 | 71 | return runner, nil 72 | } 73 | 74 | // ExecPod execute the command in the specified pod 75 | func (r *KubeRunner) ExecPod(container, podName, namespace, command string) (string, error) { 76 | log.Printf("pod %s exce cmd: %s\n", podName, command) 77 | req := r.client.CoreV1().RESTClient().Post().Resource("pods").Name(podName).Namespace(namespace).SubResource("exec") 78 | option := &v1.PodExecOptions{ 79 | Container: container, 80 | Command: []string{"sh", "-c", command}, // strings.Fields(command), 81 | Stdin: false, 82 | Stdout: true, 83 | Stderr: true, 84 | TTY: true, 85 | } 86 | 87 | req.VersionedParams( 88 | option, 89 | scheme.ParameterCodec, 90 | ) 91 | 92 | var stdout, stderr bytes.Buffer 93 | executor, err := remotecommand.NewSPDYExecutor(r.config, "POST", req.URL()) 94 | if err != nil { 95 | log.Printf("NewSPDYExecutor error: %v\n", err) 96 | return "", err 97 | } 98 | err = executor.Stream(remotecommand.StreamOptions{ 99 | Stdin: nil, 100 | Stdout: &stdout, 101 | Stderr: &stderr, 102 | Tty: true, 103 | }) 104 | if err != nil { 105 | log.Printf("exce stdout: %s\n", stderr.String()) 106 | return "", err 107 | } 108 | 109 | return strings.TrimSpace(stdout.String()), err 110 | } 111 | 112 | // XDSStatistics get the cds and eds statistics of pod 113 | func (r *KubeRunner) XDSStatistics(name, ns string) string { 114 | return fmt.Sprintf("XDS statistics of %s/%s: CDS count: %d EDS count: %d", 115 | ns, name, 116 | r.CountOfCDS(name, ns), 117 | r.CountOfEDS(name, ns)) 118 | } 119 | 120 | // CountOfCDS get cds count of pod 121 | func (r *KubeRunner) CountOfCDS(name, ns string) int { 122 | s := RunCMD(fmt.Sprintf("istioctl pc cluster -n %s %s", ns, name)) 123 | cds := strings.Split(s, "\n") 124 | 125 | return len(cds) - 1 // remove header 126 | } 127 | 128 | // CountOfEDS get eds count of pod 129 | func (r *KubeRunner) CountOfEDS(name, ns string) int { 130 | s := RunCMD(fmt.Sprintf("istioctl pc endpoints -n %s %s", ns, name)) 131 | cds := strings.Split(s, "\n") 132 | 133 | return len(cds) - 1 // remove header 134 | } 135 | 136 | // GetAccessLog get the access log of container 137 | func (r *KubeRunner) GetAccessLog(container, podName, ns string, since time.Time, match string) (string, error) { 138 | rs, err := r.client.CoreV1().Pods(ns).GetLogs(podName, &v1.PodLogOptions{ 139 | Follow: true, 140 | SinceTime: &metav1.Time{Time: since}, 141 | Container: container, 142 | }).Stream(context.TODO()) 143 | 144 | if err != nil { 145 | return "", err 146 | } 147 | defer rs.Close() 148 | 149 | // todo may need timeout control 150 | sc := bufio.NewScanner(rs) 151 | 152 | for sc.Scan() { 153 | line := sc.Text() 154 | if strings.Contains(line, match) { 155 | return line, nil 156 | } 157 | } 158 | return "", fmt.Errorf("log not found") 159 | } 160 | 161 | // GetServiceIP get service ip by service name and namespace 162 | func (r *KubeRunner) GetServiceIP(name, ns string) string { 163 | svc, _ := r.client.CoreV1().Services(ns).Get(context.TODO(), name, metav1.GetOptions{}) 164 | 165 | return svc.Spec.ClusterIP 166 | } 167 | 168 | // GetFirstPodByLabels get the first pod by labels 169 | func (r *KubeRunner) GetFirstPodByLabels(namespace, labels string) (*corev1.Pod, error) { 170 | pods, err := r.client.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ 171 | LabelSelector: labels, 172 | }) 173 | if err != nil { 174 | return nil, err 175 | } 176 | if len(pods.Items) == 0 { 177 | return nil, fmt.Errorf("no pod found with label %s", labels) 178 | } 179 | return &pods.Items[0], nil 180 | } 181 | 182 | // CreateNamespace creates namespace, with the control of istio-injection label 183 | func (r *KubeRunner) CreateNamespace(namespace string, inject bool) error { 184 | ns := &v1.Namespace{ 185 | ObjectMeta: v12.ObjectMeta{ 186 | Name: namespace, 187 | Labels: map[string]string{}, 188 | }, 189 | } 190 | 191 | if inject { 192 | // ns.Labels["istio.io/rev"] = "1-8-1" 193 | ns.Labels["istio-injection"] = "enabled" 194 | } 195 | 196 | _, err := r.client.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) 197 | return err 198 | } 199 | -------------------------------------------------------------------------------- /test/e2e/lazyxds/utils/request_id.go: -------------------------------------------------------------------------------- 1 | /* 2 | * // Copyright Aeraki Authors 3 | * // 4 | * // Licensed under the Apache License, Version 2.0 (the "License"); 5 | * // you may not use this file except in compliance with the License. 6 | * // You may obtain a copy of the License at 7 | * // 8 | * // http://www.apache.org/licenses/LICENSE-2.0 9 | * // 10 | * // Unless required by applicable law or agreed to in writing, software 11 | * // distributed under the License is distributed on an "AS IS" BASIS, 12 | * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import "sync" 20 | 21 | var requestID int 22 | var mu sync.Mutex 23 | 24 | // GetRequestID return an auto-increase id 25 | func GetRequestID() int { 26 | mu.Lock() 27 | defer mu.Unlock() 28 | requestID = requestID + 1 29 | return requestID 30 | } 31 | --------------------------------------------------------------------------------