├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── enhancement-request.md │ └── feature_request.md ├── dependabot.yml ├── labeler.yml └── workflows │ └── presubmit.yaml ├── .gitignore ├── .go-version ├── .ko.yaml ├── ATTRIBUTION.md ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DEVELOPER_GUIDE.md ├── Dockerfile ├── LICENSE ├── Makefile ├── NOTICE ├── PROJECT ├── README.md ├── apis └── vpcresources │ ├── v1alpha1 │ ├── cninode_types.go │ ├── groupversion_info.go │ └── zz_generated.deepcopy.go │ └── v1beta1 │ ├── groupversion_info.go │ ├── securitygrouppolicy_types.go │ └── zz_generated.deepcopy.go ├── config ├── certmanager │ ├── certificate.yaml │ ├── kustomization.yaml │ └── kustomizeconfig.yaml ├── controller │ ├── controller.yaml │ └── kustomization.yaml ├── crd │ ├── bases │ │ ├── vpcresources.k8s.aws_cninodes.yaml │ │ └── vpcresources.k8s.aws_securitygrouppolicies.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_securitygrouppolicies.yaml │ │ └── webhook_in_securitygrouppolicies.yaml ├── default │ ├── controller_auth_proxy_patch.yaml │ ├── controller_webhook_patch.yaml │ ├── kustomization.yaml │ └── webhookcainjection_patch.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── role.yaml │ ├── role_binding.yaml │ ├── securitygrouppolicy_editor_role.yaml │ └── securitygrouppolicy_viewer_role.yaml ├── sa │ ├── kustomization.yaml │ └── sa.yaml ├── samples │ ├── vpcresources_v1alpha1_cninode.yaml │ └── vpcresources_v1beta1_securitygrouppolicy.yaml └── webhook │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ ├── manifests.yaml │ └── service.yaml ├── controllers ├── apps │ ├── deployment_controller.go │ └── deployment_controller_test.go ├── core │ ├── configmap_controller.go │ ├── configmap_controller_test.go │ ├── node_controller.go │ ├── node_controller_test.go │ ├── pod_controller.go │ ├── pod_controller_test.go │ └── suite_test.go ├── crds │ ├── cninode_controller.go │ ├── cninode_controller_test.go │ └── suite_test.go └── custom │ ├── builder.go │ └── custom_controller.go ├── docs ├── images │ ├── README.md │ ├── sgp-node-create.png │ ├── sgp-pod-create.png │ ├── windows-node-create-event.png │ ├── windows-pod-create-event.png │ ├── windows-prefix-delegation-node-create-events.jpg │ └── windows-prefix-delegation-pod-create-events.jpg ├── sgp │ ├── sgp_config_options.md │ └── workflow.md ├── troubleshooting.md └── windows │ ├── prefix_delegation_config_options.md │ ├── prefix_delegation_hld_workflow.md │ ├── secondary_ip_mode_config_options.md │ └── secondary_ip_mode_workflow.md ├── go.mod ├── go.sum ├── hack └── toolchain.sh ├── main.go ├── mocks └── amazon-vcp-resource-controller-k8s │ ├── controllers │ └── custom │ │ └── custom_controller.go │ └── pkg │ ├── aws │ └── ec2 │ │ ├── api │ │ ├── cleanup │ │ │ └── mock_resource_cleaner.go │ │ ├── mock_ec2_apihelper.go │ │ └── mock_ec2_wrapper.go │ │ └── mock_instance.go │ ├── condition │ └── mock_condition.go │ ├── handler │ └── mock_handler.go │ ├── k8s │ ├── mock_finalizer.go │ ├── mock_k8swrapper.go │ └── pod │ │ └── mock_podapiwrapper.go │ ├── node │ ├── manager │ │ └── mock_manager.go │ └── mock_node.go │ ├── pool │ └── mock_pool.go │ ├── provider │ ├── branch │ │ ├── cooldown │ │ │ └── mock_cooldown.go │ │ └── trunk │ │ │ └── mock_trunk.go │ ├── ip │ │ └── eni │ │ │ └── mock_eni.go │ └── mock_provider.go │ ├── resource │ └── mock_resources.go │ ├── utils │ └── mock_k8shelper.go │ └── worker │ └── mock_worker.go ├── pkg ├── api │ └── wrapper.go ├── aws │ ├── ec2 │ │ ├── api │ │ │ ├── cleanup │ │ │ │ ├── eni_cleanup.go │ │ │ │ ├── eni_cleanup_test.go │ │ │ │ ├── node_cleanup.go │ │ │ │ └── resource_cleaner.go │ │ │ ├── helper.go │ │ │ ├── helper_test.go │ │ │ ├── wrapper.go │ │ │ └── wrapper_test.go │ │ ├── instance.go │ │ └── instance_test.go │ ├── errors │ │ └── ec2_errors.go │ └── vpc │ │ ├── README.md │ │ ├── limits.go │ │ └── limits_test.go ├── condition │ ├── conditions.go │ └── conditions_test.go ├── config │ ├── loader.go │ ├── loader_test.go │ └── type.go ├── handler │ ├── handler.go │ ├── on_demand.go │ ├── on_demand_test.go │ ├── warm.go │ └── warm_test.go ├── healthz │ ├── healthz.go │ └── healthz_test.go ├── k8s │ ├── finalizer.go │ ├── pod │ │ ├── client_wrapper.go │ │ ├── client_wrapper_test.go │ │ └── converter.go │ ├── wrapper.go │ └── wrapper_test.go ├── node │ ├── manager │ │ ├── manager.go │ │ └── manager_test.go │ ├── node.go │ └── node_test.go ├── pool │ ├── pool.go │ ├── pool_test.go │ ├── utils.go │ └── utils_test.go ├── provider │ ├── branch │ │ ├── cooldown │ │ │ ├── cooldown.go │ │ │ └── cooldown_test.go │ │ ├── provider.go │ │ ├── provider_test.go │ │ └── trunk │ │ │ ├── trunk.go │ │ │ └── trunk_test.go │ ├── ip │ │ ├── eni │ │ │ ├── eni.go │ │ │ └── eni_test.go │ │ ├── provider.go │ │ └── provider_test.go │ ├── prefix │ │ ├── provider.go │ │ └── provider_test.go │ └── provider.go ├── resource │ ├── introspect.go │ ├── introspect_test.go │ ├── manager.go │ └── manager_test.go ├── utils │ ├── errors.go │ ├── events.go │ ├── helper.go │ ├── helper_test.go │ ├── httpClient.go │ ├── httpClient_test.go │ ├── map.go │ ├── math.go │ ├── math_test.go │ ├── set.go │ ├── set_test.go │ └── setup_test.go ├── version │ └── version.go └── worker │ ├── jobs.go │ ├── jobs_test.go │ ├── worker.go │ └── worker_test.go ├── scripts ├── gen_mocks.sh ├── templates │ ├── boilerplate.go.txt │ └── copyright.txt └── test │ ├── README.md │ ├── create-cluster-karpenter.sh │ ├── create-cluster.sh │ ├── delete-cluster-karpenter.sh │ ├── delete-cluster.sh │ ├── iam-resources.sh │ ├── install-cert-manager.sh │ ├── install-vpc-cni.sh │ ├── lib │ ├── aws.sh │ ├── cluster.sh │ ├── common.sh │ ├── config.sh │ └── k8s.sh │ ├── run-canary-test.sh │ ├── run-integration-tests.sh │ ├── template │ ├── eksctl │ │ └── eks-cluster.yaml │ ├── iam │ │ ├── associate-trunk-policy.json │ │ ├── aws-trust-relationship.json │ │ └── vpc-resource-controller-policy.json │ └── rbac │ │ └── cp-vpc-leader-election-role-patch.yaml │ └── test-with-eksctl.sh ├── test ├── README.md ├── framework │ ├── framework.go │ ├── manifest │ │ ├── configmap.go │ │ ├── container.go │ │ ├── deployment.go │ │ ├── eniconfig.go │ │ ├── jobs.go │ │ ├── pod.go │ │ ├── service.go │ │ ├── serviceaccount.go │ │ └── sgp.go │ ├── options.go │ ├── resource │ │ ├── aws │ │ │ ├── autoscaling │ │ │ │ └── manager.go │ │ │ └── ec2 │ │ │ │ └── manager.go │ │ └── k8s │ │ │ ├── configmap │ │ │ ├── manager.go │ │ │ └── wrapper.go │ │ │ ├── controller │ │ │ ├── manager.go │ │ │ └── wrapper.go │ │ │ ├── deployment │ │ │ ├── manager.go │ │ │ └── wrapper.go │ │ │ ├── jobs │ │ │ └── manager.go │ │ │ ├── namespace │ │ │ └── manager.go │ │ │ ├── node │ │ │ ├── manager.go │ │ │ └── wrapper.go │ │ │ ├── pod │ │ │ ├── manager.go │ │ │ └── wrapper.go │ │ │ ├── rbac │ │ │ └── manager.go │ │ │ ├── service │ │ │ └── manager.go │ │ │ ├── serviceaccount │ │ │ └── manager.go │ │ │ └── sgp │ │ │ ├── manager.go │ │ │ └── wrapper.go │ ├── utils │ │ ├── metrics_helper.go │ │ ├── poll.go │ │ ├── resource.go │ │ └── utils.go │ └── verify │ │ └── pod.go └── integration │ ├── cninode │ ├── cninode_suite_test.go │ └── cninode_test.go │ ├── ec2api │ ├── ec2api_suite_test.go │ └── ec2api_test.go │ ├── metrics │ ├── metrics_suite_test.go │ ├── metrics_test.go │ └── metrics_test_config.go │ ├── perpodsg │ ├── job_test.go │ ├── perpodsg_suite_test.go │ └── perpodsg_test.go │ ├── scale │ ├── pod_scale_test.go │ └── scale_suite_test.go │ ├── webhook │ ├── validating_webhook_suite_test.go │ └── validating_webhook_test.go │ └── windows │ ├── stress_test.go │ ├── windows_suite_test.go │ └── windows_test.go └── webhooks ├── core ├── annotation_validation_webhook.go ├── annotation_validation_webhook_test.go ├── node_update_webhook.go ├── node_update_webhook_test.go ├── pod_webhook.go └── pod_webhook_test.go └── idle └── webhooks.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | 15 | 16 | **Describe the Bug**: 17 | 23 | 24 | **Observed Behavior**: 25 | 26 | **Expected Behavior**: 27 | 28 | **How to reproduce it (as minimally and precisely as possible)**: 29 | 30 | **Additional Context**: 31 | 32 | **Environment**: 33 | - Kubernetes version (use `kubectl version`): 34 | - CNI Version 35 | - OS (Linux/Windows): 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Enhancement Request 3 | about: Create a request to enhance an existing feature 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | 16 | 17 | **What would you like to be enhanced**: 18 | 19 | **Why is the change needed and what use case will it solve**: 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: feature request 6 | assignees: '' 7 | 8 | --- 9 | 10 | 16 | 17 | **Is your feature request related to a problem? Please describe.** 18 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 19 | 20 | **Describe the solution you'd like** 21 | A clear and concise description of what you want to happen. 22 | 23 | **Describe alternatives you've considered** 24 | A clear and concise description of any alternative solutions or features you've considered. 25 | 26 | **Additional context** 27 | Add any other context or screenshots about the feature request here. 28 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # See https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#package-ecosystem 2 | version: 2 3 | updates: 4 | - package-ecosystem: "gomod" 5 | directory: "/" 6 | schedule: 7 | interval: "monthly" -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | test-disabled: 2 | - '*' 3 | -------------------------------------------------------------------------------- /.github/workflows/presubmit.yaml: -------------------------------------------------------------------------------- 1 | name: Presubmit 2 | on: 3 | push: 4 | branches: [master] 5 | pull_request: 6 | workflow_dispatch: 7 | permissions: 8 | contents: read 9 | jobs: 10 | presubmit: 11 | name: Presubmit 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: actions/setup-go@v4 16 | with: 17 | go-version-file: go.mod 18 | check-latest: true 19 | cache-dependency-path: "**/go.sum" 20 | - uses: actions/cache@v3 21 | with: 22 | path: | 23 | ~/.kubebuilder/bin 24 | ~/go/bin 25 | key: ${{ runner.os }}-toolchain-cache-${{ hashFiles('hack/toolchain.sh') }} 26 | - run: make toolchain 27 | - run: make presubmit 28 | deprecated-apigroups: 29 | name: Detect deprecated apiGroups 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v3 33 | - run: | 34 | version=$(curl -sL https://api.github.com/repos/FairwindsOps/pluto/releases/latest | jq -r ".tag_name") 35 | number=${version:1} 36 | wget https://github.com/FairwindsOps/pluto/releases/download/${version}/pluto_${number}_linux_amd64.tar.gz 37 | sudo tar -C /usr/local -xzf pluto_${number}_linux_amd64.tar.gz 38 | - run: | 39 | /usr/local/pluto detect-files -d . 40 | vuln_check: 41 | runs-on: ubuntu-latest 42 | timeout-minutes: 5 43 | steps: 44 | - name: Checkout 45 | uses: actions/checkout@v3 46 | - name: Setup Go Version 47 | run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV 48 | - uses: actions/setup-go@v4 49 | with: 50 | go-version: ${{ env.GO_VERSION }} 51 | cache-dependency-path: "**/go.sum" 52 | - name: Install `govulncheck` 53 | run: go install golang.org/x/vuln/cmd/govulncheck@latest 54 | - name: Run `govulncheck` 55 | run: ~/go/bin/govulncheck ./... 56 | static-security-analysis: 57 | runs-on: ubuntu-latest 58 | steps: 59 | - name: Checkout 60 | uses: actions/checkout@v3 61 | - name: Setup Go Version 62 | run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV 63 | - uses: actions/setup-go@v4 64 | with: 65 | go-version: ${{ env.GO_VERSION }} 66 | cache-dependency-path: "**/go.sum" 67 | - name: Install `gosec` 68 | run: go install github.com/securego/gosec/v2/cmd/gosec@latest 69 | - name: Run Gosec Security Scanner 70 | run: ~/go/bin/gosec -exclude-dir test -exclude-generated -severity medium ./... 71 | 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | 10 | # Test build files 11 | scripts/test/build/ 12 | 13 | # Internal only file 14 | scripts/gen_vpc_limits.go 15 | 16 | # Test binary, build with `go test -c` 17 | *.test 18 | 19 | # Output of the go coverage tool, specifically when used with LiteIDE 20 | *.out 21 | 22 | # Kubernetes Generated files - skip generated files, except for vendored files 23 | !vendor/**/zz_generated.* 24 | # editor and IDE paraphernalia 25 | .idea 26 | *.swp 27 | *.swo 28 | *~ 29 | /AWSWesleyVpcResourceControllers.iml 30 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.24 -------------------------------------------------------------------------------- /.ko.yaml: -------------------------------------------------------------------------------- 1 | defaultBaseImage: public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-nonroot:latest.2 2 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @aws/eks-networking 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /DEVELOPER_GUIDE.md: -------------------------------------------------------------------------------- 1 | # Developer Guide 2 | 3 | ## Setup 4 | 5 | ```sh 6 | make toolchain # Install required to develop the project 7 | ``` 8 | 9 | ## Testing a code change 10 | 11 | Deploy your changes to a local development cluster and run the tests against it. You will need to allowlist your account for ENI trunking before the deployment. 12 | 13 | If you are testing on EKS beta cluster, set 14 | ```sh 15 | BETA_CLUSTER=true 16 | ``` 17 | 18 | ```sh 19 | make apply-dependencies # install the cert manager and certificate 20 | make apply # Apply your changes 21 | make test-e2e # Run the integration test suite 22 | ``` 23 | 24 | In another terminal, you can tail the logs with stern 25 | ```sh 26 | stern -l app=vpc-resource-controller -n kube-system 27 | ``` 28 | 29 | ## Submitting a PR 30 | Run the presubmit target and check in all generated code before submitting a PR. 31 | 32 | ```sh 33 | make presubmit 34 | ``` 35 | 36 | ## Troubleshooting 37 | 38 | ### Invalid value 'trunk' for InterfaceType 39 | 40 | The following error means that must be allowlisted for EC2 Networking 41 | ``` 42 | {"level":"error","timestamp":"2023-06-09T21:53:00.705Z","logger":"branch eni provider","msg":"failed to create trunk interface","node name":"ip-192-168-60-153.us-west-2.compute.internal","request":"initialize","instance ID":"i-0d892c7fa08bf7bbd","error":"InvalidParameterValue: Invalid value 'trunk' for InterfaceType. Allowed values are ('EFA')\n\tstatus code: 400, request id: 7b94401f-686f-46a4-a5e9-3cfda8e12cd6","stacktrace":"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/trunk.(*trunkENI).InitTrunk\n\tgithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/trunk/trunk.go:194\ngithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch.(*branchENIProvider).InitResource\n\tgithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/provider.go:154\ngithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/node.(*node).InitResources\n\tgithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/node.go:156\ngithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager.(*manager).performAsyncOperation\n\tgithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager/manager.go:316\ngithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker.(*worker).processNextItem\n\tgithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker/worker.go:162\ngithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker.(*worker).runWorker\n\tgithub.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker/worker.go:147"} 43 | ``` -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE 2 | ARG BUILD_IMAGE 3 | ARG ARCH 4 | # Build the controller binary 5 | FROM $BUILD_IMAGE as builder 6 | 7 | WORKDIR /workspace 8 | ENV GOPROXY direct 9 | 10 | # Copy the Go Modules manifests 11 | COPY go.mod go.mod 12 | COPY go.sum go.sum 13 | # cache deps before building and copying source so that we don't need to re-download as much 14 | # and so that source changes don't invalidate our downloaded layer 15 | RUN go mod download 16 | 17 | # Copy the go source 18 | COPY .git/ .git/ 19 | COPY main.go main.go 20 | COPY apis/ apis/ 21 | COPY pkg/ pkg/ 22 | COPY controllers/ controllers/ 23 | COPY webhooks/ webhooks/ 24 | 25 | # Version package for passing the ldflags 26 | ENV VERSION_PKG=github.com/aws/amazon-vpc-resource-controller-k8s/pkg/version 27 | ENV GOARCH $ARCH 28 | # Build 29 | RUN GIT_VERSION=$(git describe --tags --always) && \ 30 | GIT_COMMIT=$(git rev-parse HEAD) && \ 31 | BUILD_DATE=$(date +%Y-%m-%dT%H:%M:%S%z) && \ 32 | CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build \ 33 | -ldflags="-X ${VERSION_PKG}.GitVersion=${GIT_VERSION} -X ${VERSION_PKG}.GitCommit=${GIT_COMMIT} -X ${VERSION_PKG}.BuildDate=${BUILD_DATE}" -a -o controller main.go 34 | 35 | FROM $BASE_IMAGE 36 | 37 | WORKDIR / 38 | COPY --from=public.ecr.aws/eks-distro/kubernetes/go-runner:v0.18.0-eks-1-32-11 /go-runner /usr/local/bin/go-runner 39 | COPY --from=builder /workspace/controller . 40 | 41 | ENTRYPOINT ["/controller"] 42 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | # Code generated by tool. DO NOT EDIT. 2 | # This file is used to track the info used to scaffold your project 3 | # and allow the plugins properly work. 4 | # More info: https://book.kubebuilder.io/reference/project-config.html 5 | domain: k8s.aws 6 | layout: 7 | - go.kubebuilder.io/v3 8 | multigroup: true 9 | projectName: amazon-vpc-resource-controller-k8s 10 | repo: github.com/aws/amazon-vpc-resource-controller-k8s 11 | resources: 12 | - api: 13 | crdVersion: v1 14 | namespaced: true 15 | domain: k8s.aws 16 | group: vpcresources 17 | kind: SecurityGroupPolicy 18 | path: github.com/aws/amazon-vpc-resource-controller-k8s/apis/v1beta1 19 | version: v1beta1 20 | - api: 21 | crdVersion: v1 22 | controller: true 23 | domain: k8s.aws 24 | group: vpcresources 25 | kind: CNINode 26 | path: github.com/aws/amazon-vpc-resource-controller-k8s/apis/v1alpha1 27 | version: v1alpha1 28 | version: "3" 29 | -------------------------------------------------------------------------------- /apis/vpcresources/v1alpha1/cninode_types.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package v1alpha1 15 | 16 | import ( 17 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 18 | ) 19 | 20 | // FeatureName is a type of feature name supported by AWS VPC CNI. It can be Security Group for Pods, custom networking, or others 21 | type FeatureName string 22 | 23 | const ( 24 | SecurityGroupsForPods FeatureName = "SecurityGroupsForPods" 25 | CustomNetworking FeatureName = "CustomNetworking" 26 | ) 27 | 28 | // Feature is a type of feature being supported by VPC resource controller and other AWS Services 29 | type Feature struct { 30 | Name FeatureName `json:"name,omitempty"` 31 | Value string `json:"value,omitempty"` 32 | } 33 | 34 | // Important: Run "make" to regenerate code after modifying this file 35 | // CNINodeSpec defines the desired state of CNINode 36 | type CNINodeSpec struct { 37 | Features []Feature `json:"features,omitempty"` 38 | // Additional tag key/value added to all network interfaces provisioned by the vpc-resource-controller and VPC-CNI 39 | Tags map[string]string `json:"tags,omitempty"` 40 | } 41 | 42 | // CNINodeStatus defines the managed VPC resources. 43 | type CNINodeStatus struct { 44 | //TODO: add VPC resources which will be managed by this CRD and its finalizer 45 | } 46 | 47 | // +kubebuilder:object:root=true 48 | // +kubebuilder:printcolumn:name="Features",type=string,JSONPath=`.spec.features`,description="The features delegated to VPC resource controller" 49 | // +kubebuilder:resource:shortName=cnd,scope=Cluster 50 | 51 | // +kubebuilder:object:root=true 52 | type CNINode struct { 53 | metav1.TypeMeta `json:",inline"` 54 | metav1.ObjectMeta `json:"metadata,omitempty"` 55 | Spec CNINodeSpec `json:"spec,omitempty"` 56 | Status CNINodeStatus `json:"status,omitempty"` 57 | } 58 | 59 | // +kubebuilder:object:root=true 60 | // CNINodeList contains a list of CNINodeList 61 | type CNINodeList struct { 62 | metav1.TypeMeta `json:",inline"` 63 | metav1.ListMeta `json:"metadata,omitempty"` 64 | Items []CNINode `json:"items"` 65 | } 66 | 67 | func init() { 68 | SchemeBuilder.Register(&CNINode{}, &CNINodeList{}) 69 | } 70 | -------------------------------------------------------------------------------- /apis/vpcresources/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Package v1beta1 contains API Schema definitions for the vpcresources v1beta1 API group 15 | // +kubebuilder:object:generate=true 16 | // +groupName=vpcresources.k8s.aws 17 | package v1alpha1 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime/schema" 21 | "sigs.k8s.io/controller-runtime/pkg/scheme" 22 | ) 23 | 24 | var ( 25 | // GroupVersion is group version used to register these objects 26 | GroupVersion = schema.GroupVersion{Group: "vpcresources.k8s.aws", Version: "v1alpha1"} 27 | 28 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 29 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 30 | 31 | // AddToScheme adds the types in this group-version to the given scheme. 32 | AddToScheme = SchemeBuilder.AddToScheme 33 | ) 34 | -------------------------------------------------------------------------------- /apis/vpcresources/v1beta1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Package v1beta1 contains API Schema definitions for the vpcresources v1beta1 API group 15 | // +kubebuilder:object:generate=true 16 | // +groupName=vpcresources.k8s.aws 17 | package v1beta1 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime/schema" 21 | "sigs.k8s.io/controller-runtime/pkg/scheme" 22 | ) 23 | 24 | var ( 25 | // GroupVersion is group version used to register these objects 26 | GroupVersion = schema.GroupVersion{Group: "vpcresources.k8s.aws", Version: "v1beta1"} 27 | 28 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 29 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 30 | 31 | // AddToScheme adds the types in this group-version to the given scheme. 32 | AddToScheme = SchemeBuilder.AddToScheme 33 | ) 34 | -------------------------------------------------------------------------------- /apis/vpcresources/v1beta1/securitygrouppolicy_types.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package v1beta1 15 | 16 | import ( 17 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 18 | ) 19 | 20 | // Important: Run "make" to regenerate code after modifying this file 21 | 22 | // SecurityGroupPolicySpec defines the desired state of SecurityGroupPolicy 23 | type SecurityGroupPolicySpec struct { 24 | PodSelector *metav1.LabelSelector `json:"podSelector,omitempty"` 25 | ServiceAccountSelector *metav1.LabelSelector `json:"serviceAccountSelector,omitempty"` 26 | SecurityGroups GroupIds `json:"securityGroups,omitempty"` 27 | } 28 | 29 | // GroupIds contains the list of security groups that will be applied to the network interface of the pod matching the criteria. 30 | type GroupIds struct { 31 | // Groups is the list of EC2 Security Groups Ids that need to be applied to the ENI of a Pod. 32 | // +kubebuilder:validation:MinItems=1 33 | // +kubebuilder:validation:MaxItems=5 34 | Groups []string `json:"groupIds,omitempty"` 35 | } 36 | 37 | // ServiceAccountSelector contains the selection criteria for matching pod with service account that matches the label selector 38 | // requirement and the exact name of the service account. 39 | type ServiceAccountSelector struct { 40 | *metav1.LabelSelector `json:",omitempty"` 41 | // matchNames is the list of service account names. The requirements are ANDed 42 | // +kubebuilder:validation:MinItems=1 43 | MatchNames []string `json:"matchNames,omitempty"` 44 | } 45 | 46 | // +kubebuilder:object:root=true 47 | // +kubebuilder:printcolumn:name="Security-Group-Ids",type=string,JSONPath=`.spec.securityGroups.groupIds`,description="The security group IDs to apply to the elastic network interface of pods that match this policy" 48 | // +kubebuilder:resource:shortName=sgp 49 | 50 | // Custom Resource Definition for applying security groups to pods 51 | type SecurityGroupPolicy struct { 52 | metav1.TypeMeta `json:",inline"` 53 | metav1.ObjectMeta `json:"metadata,omitempty"` 54 | 55 | Spec SecurityGroupPolicySpec `json:"spec,omitempty"` 56 | } 57 | 58 | // +kubebuilder:object:root=true 59 | 60 | // SecurityGroupPolicyList contains a list of SecurityGroupPolicy 61 | type SecurityGroupPolicyList struct { 62 | metav1.TypeMeta `json:",inline"` 63 | metav1.ListMeta `json:"metadata,omitempty"` 64 | Items []SecurityGroupPolicy `json:"items"` 65 | } 66 | 67 | func init() { 68 | SchemeBuilder.Register(&SecurityGroupPolicy{}, &SecurityGroupPolicyList{}) 69 | } 70 | -------------------------------------------------------------------------------- /config/certmanager/certificate.yaml: -------------------------------------------------------------------------------- 1 | # The following manifests contain a self-signed issuer CR and a certificate CR. 2 | # More document can be found at https://docs.cert-manager.io 3 | # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for 4 | # breaking changes 5 | apiVersion: cert-manager.io/v1 6 | kind: Issuer 7 | metadata: 8 | name: selfsigned-issuer 9 | namespace: system 10 | spec: 11 | selfSigned: {} 12 | --- 13 | apiVersion: cert-manager.io/v1 14 | kind: Certificate 15 | metadata: 16 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml 17 | namespace: system 18 | spec: 19 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize 20 | dnsNames: 21 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc 22 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local 23 | issuerRef: 24 | kind: Issuer 25 | name: selfsigned-issuer 26 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize 27 | -------------------------------------------------------------------------------- /config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - certificate.yaml 3 | 4 | configurations: 5 | - kustomizeconfig.yaml 6 | -------------------------------------------------------------------------------- /config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration is for teaching kustomize how to update name ref and var substitution 2 | nameReference: 3 | - kind: Issuer 4 | group: cert-manager.io 5 | fieldSpecs: 6 | - kind: Certificate 7 | group: cert-manager.io 8 | path: spec/issuerRef/name 9 | 10 | varReference: 11 | - kind: Certificate 12 | group: cert-manager.io 13 | path: spec/commonName 14 | - kind: Certificate 15 | group: cert-manager.io 16 | path: spec/dnsNames 17 | -------------------------------------------------------------------------------- /config/controller/controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller 5 | namespace: system 6 | labels: 7 | control-plane: controller 8 | app: vpc-resource-controller 9 | spec: 10 | selector: 11 | matchLabels: 12 | control-plane: controller 13 | replicas: 2 14 | template: 15 | metadata: 16 | labels: 17 | control-plane: controller 18 | app: vpc-resource-controller 19 | spec: 20 | volumes: 21 | - name: aws-iam-token 22 | projected: 23 | defaultMode: 420 24 | sources: 25 | - serviceAccountToken: 26 | audience: sts.amazonaws.com 27 | expirationSeconds: 86400 28 | path: token 29 | serviceAccountName: vpc-resource-controller 30 | containers: 31 | - args: 32 | - --cluster-name=CLUSTER_NAME 33 | - --role-arn=USER_ROLE_ARN 34 | - --leader-elect 35 | - --metrics-bind-address=:8443 36 | - --introspect-bind-addr=:22775 37 | - --vpc-id=VPC_ID 38 | image: controller:latest 39 | name: controller 40 | resources: 41 | limits: 42 | cpu: 0.5 43 | memory: 1Gi 44 | requests: 45 | cpu: 0.3 46 | memory: 400Mi 47 | livenessProbe: 48 | failureThreshold: 5 49 | httpGet: 50 | path: /healthz 51 | port: 61779 52 | scheme: HTTP 53 | initialDelaySeconds: 30 54 | timeoutSeconds: 30 55 | periodSeconds: 30 56 | ports: 57 | - containerPort: 9443 58 | name: webhook-server 59 | protocol: TCP 60 | - containerPort: 8443 61 | name: metrics 62 | protocol: TCP 63 | terminationGracePeriodSeconds: 10 64 | nodeSelector: 65 | kubernetes.io/os: linux -------------------------------------------------------------------------------- /config/controller/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - controller.yaml 3 | apiVersion: kustomize.config.k8s.io/v1beta1 4 | kind: Kustomization 5 | namePrefix: local- 6 | # Adding additional prefix as a workaround, because the controller will fail running locally 7 | # for Windows test, as it checks the same deployment name should not be deployed to enable Windows 8 | # IPAM 9 | images: 10 | - digest: latest 11 | name: controller 12 | newName: controller 13 | -------------------------------------------------------------------------------- /config/crd/bases/vpcresources.k8s.aws_cninodes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.14.0 7 | name: cninodes.vpcresources.k8s.aws 8 | spec: 9 | group: vpcresources.k8s.aws 10 | names: 11 | kind: CNINode 12 | listKind: CNINodeList 13 | plural: cninodes 14 | shortNames: 15 | - cnd 16 | singular: cninode 17 | scope: Cluster 18 | versions: 19 | - additionalPrinterColumns: 20 | - description: The features delegated to VPC resource controller 21 | jsonPath: .spec.features 22 | name: Features 23 | type: string 24 | name: v1alpha1 25 | schema: 26 | openAPIV3Schema: 27 | properties: 28 | apiVersion: 29 | description: |- 30 | APIVersion defines the versioned schema of this representation of an object. 31 | Servers should convert recognized schemas to the latest internal value, and 32 | may reject unrecognized values. 33 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources 34 | type: string 35 | kind: 36 | description: |- 37 | Kind is a string value representing the REST resource this object represents. 38 | Servers may infer this from the endpoint the client submits requests to. 39 | Cannot be updated. 40 | In CamelCase. 41 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds 42 | type: string 43 | metadata: 44 | type: object 45 | spec: 46 | description: |- 47 | Important: Run "make" to regenerate code after modifying this file 48 | CNINodeSpec defines the desired state of CNINode 49 | properties: 50 | features: 51 | items: 52 | description: Feature is a type of feature being supported by VPC 53 | resource controller and other AWS Services 54 | properties: 55 | name: 56 | description: FeatureName is a type of feature name supported 57 | by AWS VPC CNI. It can be Security Group for Pods, custom 58 | networking, or others 59 | type: string 60 | value: 61 | type: string 62 | type: object 63 | type: array 64 | tags: 65 | additionalProperties: 66 | type: string 67 | description: Additional tag key/value added to all network interfaces 68 | provisioned by the vpc-resource-controller and VPC-CNI 69 | type: object 70 | type: object 71 | status: 72 | description: CNINodeStatus defines the managed VPC resources. 73 | type: object 74 | type: object 75 | served: true 76 | storage: true 77 | subresources: {} 78 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/vpcresources.k8s.aws_cninodes.yaml 6 | - bases/vpcresources.k8s.aws_securitygrouppolicies.yaml 7 | # +kubebuilder:scaffold:crdkustomizeresource 8 | 9 | patchesStrategicMerge: 10 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 11 | # patches here are for enabling the conversion webhook for each CRD 12 | #- patches/webhook_in_securitygrouppolicies.yaml 13 | # +kubebuilder:scaffold:crdkustomizewebhookpatch 14 | 15 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 16 | # patches here are for enabling the CA injection for each CRD 17 | #- patches/cainjection_in_securitygrouppolicies.yaml 18 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch 19 | 20 | # the following config is for teaching kustomize how to do kustomization for CRDs. 21 | configurations: 22 | - kustomizeconfig.yaml 23 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | group: apiextensions.k8s.io 8 | path: spec/conversion/webhookClientConfig/service/name 9 | 10 | namespace: 11 | - kind: CustomResourceDefinition 12 | group: apiextensions.k8s.io 13 | path: spec/conversion/webhookClientConfig/service/namespace 14 | create: false 15 | 16 | varReference: 17 | - path: metadata/annotations 18 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_securitygrouppolicies.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: securitygrouppolicies.vpcresources.k8s.aws 9 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_securitygrouppolicies.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: securitygrouppolicies.vpcresources.k8s.aws 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /config/default/controller_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kube-rbac-proxy 13 | image: registry.k8s.io/kubebuilder/kube-rbac-proxy:v0.5.0 14 | args: 15 | - "--secure-listen-address=0.0.0.0:8443" 16 | - "--upstream=http://127.0.0.1:8080/" 17 | - "--logtostderr=true" 18 | - "--v=10" 19 | ports: 20 | - containerPort: 8443 21 | name: https 22 | - name: controller 23 | args: 24 | - "--metrics-bind-address=127.0.0.1:8080" 25 | - "--leader-elect" 26 | -------------------------------------------------------------------------------- /config/default/controller_webhook_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: controller 11 | ports: 12 | - containerPort: 9443 13 | name: webhook-server 14 | protocol: TCP 15 | volumeMounts: 16 | - mountPath: /tmp/k8s-webhook-server/serving-certs 17 | name: cert 18 | readOnly: true 19 | volumes: 20 | - name: cert 21 | secret: 22 | defaultMode: 420 23 | secretName: webhook-server-cert 24 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: kube-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: vpc-resource- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../sa 19 | - ../controller 20 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 21 | # crd/kustomization.yaml 22 | - ../webhook 23 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 24 | - ../certmanager 25 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 26 | #- ../prometheus 27 | 28 | patchesStrategicMerge: 29 | # Protect the /metrics endpoint by putting it behind auth. 30 | # If you want your controller to expose the /metrics 31 | # endpoint w/o any authn/z, please comment the following line. 32 | #- controller_auth_proxy_patch.yaml 33 | 34 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 35 | # crd/kustomization.yaml 36 | - controller_webhook_patch.yaml 37 | 38 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 39 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 40 | # 'CERTMANAGER' needs to be enabled to use ca injection 41 | - webhookcainjection_patch.yaml 42 | 43 | # the following config is for teaching kustomize how to do var substitution 44 | vars: 45 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 46 | - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 47 | objref: 48 | kind: Certificate 49 | group: cert-manager.io 50 | version: v1 51 | name: serving-cert # this name should match the one in certificate.yaml 52 | fieldref: 53 | fieldpath: metadata.namespace 54 | - name: CERTIFICATE_NAME 55 | objref: 56 | kind: Certificate 57 | group: cert-manager.io 58 | version: v1 59 | name: serving-cert # this name should match the one in certificate.yaml 60 | - name: SERVICE_NAMESPACE # namespace of the service 61 | objref: 62 | kind: Service 63 | version: v1 64 | name: webhook-service 65 | fieldref: 66 | fieldpath: metadata.namespace 67 | - name: SERVICE_NAME 68 | objref: 69 | kind: Service 70 | version: v1 71 | name: webhook-service 72 | -------------------------------------------------------------------------------- /config/default/webhookcainjection_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch add annotation to admission webhook config and 2 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. 3 | # admissionregistration.k8s.io/v1 was added in Kubernetes 1.16 4 | apiVersion: admissionregistration.k8s.io/v1 5 | kind: MutatingWebhookConfiguration 6 | metadata: 7 | name: mutating-webhook-configuration 8 | annotations: 9 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 10 | --- 11 | apiVersion: admissionregistration.k8s.io/v1 12 | kind: ValidatingWebhookConfiguration 13 | metadata: 14 | name: validating-webhook-configuration 15 | annotations: 16 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 17 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller 8 | name: controller-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | selector: 15 | matchLabels: 16 | control-plane: controller 17 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: ["/metrics"] 7 | verbs: ["get"] 8 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: ["authentication.k8s.io"] 7 | resources: 8 | - tokenreviews 9 | verbs: ["create"] 10 | - apiGroups: ["authorization.k8s.io"] 11 | resources: 12 | - subjectaccessreviews 13 | verbs: ["create"] 14 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller 6 | name: controller-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller 15 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | 2 | resources: 3 | - role.yaml 4 | - role_binding.yaml 5 | - leader_election_role.yaml 6 | - leader_election_role_binding.yaml 7 | # Comment the following 4 lines if you want to disable 8 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 9 | # which protects your /metrics endpoint. 10 | #- auth_proxy_service.yaml 11 | #- auth_proxy_role.yaml 12 | #- auth_proxy_role_binding.yaml 13 | #- auth_proxy_client_clusterrole.yaml 14 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - apiGroups: 19 | - "" 20 | resources: 21 | - configmaps/status 22 | verbs: 23 | - get 24 | - update 25 | - patch 26 | - apiGroups: 27 | - "" 28 | resources: 29 | - events 30 | verbs: 31 | - create 32 | - apiGroups: 33 | - coordination.k8s.io 34 | resources: 35 | - leases 36 | verbs: 37 | - create 38 | - apiGroups: 39 | - coordination.k8s.io 40 | resourceNames: 41 | - cp-vpc-resource-controller 42 | resources: 43 | - leases 44 | verbs: 45 | - get 46 | - update 47 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: controller-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - events 11 | verbs: 12 | - create 13 | - patch 14 | - update 15 | - apiGroups: 16 | - "" 17 | resources: 18 | - pods 19 | verbs: 20 | - get 21 | - list 22 | - patch 23 | - watch 24 | - apiGroups: 25 | - "" 26 | resources: 27 | - nodes 28 | verbs: 29 | - get 30 | - list 31 | - watch 32 | - apiGroups: 33 | - "" 34 | resources: 35 | - nodes/status 36 | verbs: 37 | - get 38 | - patch 39 | - apiGroups: 40 | - "" 41 | resources: 42 | - serviceaccounts 43 | verbs: 44 | - get 45 | - list 46 | - watch 47 | - apiGroups: 48 | - crd.k8s.amazonaws.com 49 | resources: 50 | - eniconfigs 51 | verbs: 52 | - get 53 | - list 54 | - watch 55 | - apiGroups: 56 | - vpcresources.k8s.aws 57 | resources: 58 | - cninodes 59 | verbs: 60 | - create 61 | - delete 62 | - get 63 | - list 64 | - patch 65 | - update 66 | - watch 67 | - apiGroups: 68 | - vpcresources.k8s.aws 69 | resources: 70 | - securitygrouppolicies 71 | verbs: 72 | - get 73 | - list 74 | - watch 75 | --- 76 | apiVersion: rbac.authorization.k8s.io/v1 77 | kind: Role 78 | metadata: 79 | name: controller-role 80 | namespace: kube-system 81 | rules: 82 | - apiGroups: 83 | - apps 84 | resourceNames: 85 | - vpc-resource-controller 86 | resources: 87 | - deployments 88 | verbs: 89 | - get 90 | - list 91 | - watch 92 | - apiGroups: 93 | - "" 94 | resourceNames: 95 | - amazon-vpc-cni 96 | resources: 97 | - configmaps 98 | verbs: 99 | - get 100 | - list 101 | - watch 102 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: controller-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: controller-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller 12 | namespace: system 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | kind: RoleBinding 16 | metadata: 17 | name: controller-rolebinding 18 | roleRef: 19 | apiGroup: rbac.authorization.k8s.io 20 | kind: Role 21 | name: controller-role 22 | subjects: 23 | - kind: ServiceAccount 24 | name: controller 25 | namespace: system 26 | 27 | -------------------------------------------------------------------------------- /config/rbac/securitygrouppolicy_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit securitygrouppolicies. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: securitygrouppolicy-editor-role 6 | rules: 7 | - apiGroups: 8 | - vpcresources.k8s.aws 9 | resources: 10 | - securitygrouppolicies 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - vpcresources.k8s.aws 21 | resources: 22 | - securitygrouppolicies/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/securitygrouppolicy_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view securitygrouppolicies. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: securitygrouppolicy-viewer-role 6 | rules: 7 | - apiGroups: 8 | - vpcresources.k8s.aws 9 | resources: 10 | - securitygrouppolicies 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - vpcresources.k8s.aws 17 | resources: 18 | - securitygrouppolicies/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/sa/kustomization.yaml: -------------------------------------------------------------------------------- 1 | 2 | resources: 3 | - sa.yaml -------------------------------------------------------------------------------- /config/sa/sa.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: controller 6 | namespace: system -------------------------------------------------------------------------------- /config/samples/vpcresources_v1alpha1_cninode.yaml: -------------------------------------------------------------------------------- 1 | # Example of a CNINode 2 | apiVersion: vpcresources.k8s.aws/v1alpha1 3 | kind: CNINode 4 | metadata: 5 | name: cninode-example 6 | spec: 7 | features: 8 | - SecurityGroupsForPods 9 | - CustomNetworking 10 | -------------------------------------------------------------------------------- /config/samples/vpcresources_v1beta1_securitygrouppolicy.yaml: -------------------------------------------------------------------------------- 1 | # Example of SecurityGroupPolicy that uses Pod labels to determine if a new Pod get a ENI with the specified Security Groups. 2 | apiVersion: vpcresources.k8s.aws/v1beta1 3 | kind: SecurityGroupPolicy 4 | metadata: 5 | name: securitygrouppolicy-sample-podselector 6 | spec: 7 | podSelector: # Select eligible Pod using the Pod's label. The result from each item will be ANDed to retrieve final result. 8 | matchLabels: # Select Pod with label that exactly match the key and value given below. 9 | role: db 10 | matchExpressions: # Select Pod with label that matches the pattern In/NotIn/Exists/DoesNotExist for given key-value pair. 11 | - key: environment 12 | operator: In 13 | values: 14 | - production 15 | - qa 16 | securityGroups: 17 | groupIds: # List of security groups to be applied to the ENI and assigned to a Pod. 18 | - sg-07b9fafd9006da7d5 19 | - sg-f7990sfng33684fad 20 | --- 21 | # Example of SecurityGroupPolicy that uses Service Account labels to determine if a new Pod get a ENI with the specified Security Groups. 22 | apiVersion: vpcresources.k8s.aws/v1beta1 23 | kind: SecurityGroupPolicy 24 | metadata: 25 | name: securitygrouppolicy-sample-serviceaccountselector 26 | spec: 27 | serviceAccountSelector: # Select eligible Pod using the Service Account labels. The result from each item will be ANDed to retrieve final result. 28 | matchLabels: # Select Service Account with label that exactly match the key and value given below. 29 | role: db 30 | matchExpressions: # Select Service Account with label that matches the pattern In/NotIn/Exists/DoesNotExist for given key-value pair. 31 | - key: environment 32 | operator: In 33 | values: 34 | - production 35 | - qa 36 | securityGroups: 37 | groupIds: # List of security groups to be applied to the ENI and assigned to a Pod. 38 | - sg-07b9fafd9006da7d5 39 | - sg-f7990sfng33684fad -------------------------------------------------------------------------------- /config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manifests.yaml 3 | - service.yaml 4 | 5 | configurations: 6 | - kustomizeconfig.yaml 7 | -------------------------------------------------------------------------------- /config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /config/webhook/manifests.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: admissionregistration.k8s.io/v1 3 | kind: MutatingWebhookConfiguration 4 | metadata: 5 | name: mutating-webhook-configuration 6 | webhooks: 7 | - admissionReviewVersions: 8 | - v1 9 | clientConfig: 10 | service: 11 | name: webhook-service 12 | namespace: system 13 | path: /mutate-v1-pod 14 | failurePolicy: Ignore 15 | matchPolicy: Equivalent 16 | name: mpod.vpc.k8s.aws 17 | rules: 18 | - apiGroups: 19 | - "" 20 | apiVersions: 21 | - v1 22 | operations: 23 | - CREATE 24 | resources: 25 | - pods 26 | sideEffects: None 27 | --- 28 | apiVersion: admissionregistration.k8s.io/v1 29 | kind: ValidatingWebhookConfiguration 30 | metadata: 31 | name: validating-webhook-configuration 32 | webhooks: 33 | - admissionReviewVersions: 34 | - v1 35 | clientConfig: 36 | service: 37 | name: webhook-service 38 | namespace: system 39 | path: /validate-v1-node 40 | failurePolicy: Ignore 41 | matchPolicy: Equivalent 42 | name: vnode.vpc.k8s.aws 43 | rules: 44 | - apiGroups: 45 | - "" 46 | apiVersions: 47 | - v1 48 | operations: 49 | - UPDATE 50 | resources: 51 | - nodes 52 | sideEffects: None 53 | - admissionReviewVersions: 54 | - v1 55 | clientConfig: 56 | service: 57 | name: webhook-service 58 | namespace: system 59 | path: /validate-v1-pod 60 | failurePolicy: Ignore 61 | matchPolicy: Equivalent 62 | name: vpod.vpc.k8s.aws 63 | rules: 64 | - apiGroups: 65 | - "" 66 | apiVersions: 67 | - v1 68 | operations: 69 | - CREATE 70 | - UPDATE 71 | resources: 72 | - pods 73 | sideEffects: None 74 | -------------------------------------------------------------------------------- /config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: webhook-service 6 | namespace: system 7 | spec: 8 | ports: 9 | - port: 443 10 | targetPort: 9443 11 | selector: 12 | control-plane: controller 13 | -------------------------------------------------------------------------------- /controllers/apps/deployment_controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package apps 15 | 16 | import ( 17 | "context" 18 | 19 | controllers "github.com/aws/amazon-vpc-resource-controller-k8s/controllers/core" 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition" 21 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" 22 | rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" 23 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" 24 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" 25 | 26 | "github.com/go-logr/logr" 27 | appV1 "k8s.io/api/apps/v1" 28 | ctrl "sigs.k8s.io/controller-runtime" 29 | "sigs.k8s.io/controller-runtime/pkg/healthz" 30 | ) 31 | 32 | type DeploymentReconciler struct { 33 | Log logr.Logger 34 | NodeManager manager.Manager 35 | K8sAPI k8s.K8sWrapper 36 | Condition condition.Conditions 37 | wasOldControllerDeployed bool 38 | } 39 | 40 | func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 41 | var isOldControllerDeployed bool 42 | // Only process old controller deployment events 43 | if req.Name != config.OldVPCControllerDeploymentName || 44 | req.Namespace != config.KubeSystemNamespace { 45 | return ctrl.Result{}, nil 46 | } 47 | 48 | isOldControllerDeployed = r.Condition.IsOldVPCControllerDeploymentPresent() 49 | 50 | // State didn't change, no ops 51 | if isOldControllerDeployed == r.wasOldControllerDeployed { 52 | return ctrl.Result{}, nil 53 | } 54 | 55 | r.wasOldControllerDeployed = isOldControllerDeployed 56 | 57 | r.Log.Info("condition changed", "was deployment present before", 58 | r.wasOldControllerDeployed, "is deployment present now", isOldControllerDeployed) 59 | 60 | err := controllers.UpdateNodesOnConfigMapChanges(r.K8sAPI, r.NodeManager) 61 | if err != nil { 62 | return ctrl.Result{}, err 63 | } 64 | 65 | return ctrl.Result{}, nil 66 | } 67 | 68 | func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager, healthzHandler *rcHealthz.HealthzHandler) error { 69 | // add health check on subpath for deployment controller 70 | // TODO: this is a simple controller and unlikely hit blocking issue but we can revisit this after subpaths are released for a while 71 | healthzHandler.AddControllersHealthCheckers( 72 | map[string]healthz.Checker{"health-deploy-controller": rcHealthz.SimplePing("deployment controller", r.Log)}, 73 | ) 74 | 75 | return ctrl.NewControllerManagedBy(mgr). 76 | For(&appV1.Deployment{}). 77 | Complete(r) 78 | } 79 | -------------------------------------------------------------------------------- /controllers/core/suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package controllers 15 | 16 | import ( 17 | "path/filepath" 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo/v2" 21 | . "github.com/onsi/gomega" 22 | corev1 "k8s.io/api/core/v1" 23 | "k8s.io/client-go/kubernetes/scheme" 24 | "k8s.io/client-go/rest" 25 | "sigs.k8s.io/controller-runtime/pkg/client" 26 | "sigs.k8s.io/controller-runtime/pkg/envtest" 27 | logf "sigs.k8s.io/controller-runtime/pkg/log" 28 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 29 | // +kubebuilder:scaffold:imports 30 | ) 31 | 32 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 33 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 34 | 35 | var cfg *rest.Config 36 | var k8sClient client.Client 37 | var testEnv *envtest.Environment 38 | 39 | func TestAPIs(t *testing.T) { 40 | RegisterFailHandler(Fail) 41 | 42 | RunSpecs(t, "Controller Suite") 43 | } 44 | 45 | var _ = BeforeSuite(func() { 46 | done := make(chan interface{}) 47 | 48 | go func() { 49 | // user test code to run asynchronously 50 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter))) 51 | 52 | By("bootstrapping test environment") 53 | testEnv = &envtest.Environment{ 54 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 55 | } 56 | 57 | var err error 58 | cfg, err = testEnv.Start() 59 | Expect(err).ToNot(HaveOccurred()) 60 | Expect(cfg).ToNot(BeNil()) 61 | 62 | err = corev1.AddToScheme(scheme.Scheme) 63 | Expect(err).NotTo(HaveOccurred()) 64 | 65 | // +kubebuilder:scaffold:scheme 66 | 67 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) 68 | Expect(err).ToNot(HaveOccurred()) 69 | Expect(k8sClient).ToNot(BeNil()) 70 | close(done) //signifies the code is done 71 | }() 72 | Eventually(done, 60).Should(BeClosed()) 73 | }) 74 | 75 | var _ = AfterSuite(func() { 76 | By("tearing down the test environment") 77 | err := testEnv.Stop() 78 | Expect(err).ToNot(HaveOccurred()) 79 | }) 80 | -------------------------------------------------------------------------------- /controllers/crds/suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package crds 15 | 16 | import ( 17 | "path/filepath" 18 | "testing" 19 | 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1" 21 | . "github.com/onsi/ginkgo/v2" 22 | . "github.com/onsi/gomega" 23 | 24 | "k8s.io/client-go/kubernetes/scheme" 25 | "k8s.io/client-go/rest" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | "sigs.k8s.io/controller-runtime/pkg/envtest" 28 | logf "sigs.k8s.io/controller-runtime/pkg/log" 29 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 30 | //+kubebuilder:scaffold:imports 31 | ) 32 | 33 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 34 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 35 | 36 | var cfg *rest.Config 37 | var k8sClient client.Client 38 | var testEnv *envtest.Environment 39 | 40 | func TestAPIs(t *testing.T) { 41 | RegisterFailHandler(Fail) 42 | 43 | RunSpecs(t, "Controller Suite") 44 | } 45 | 46 | var _ = BeforeSuite(func() { 47 | done := make(chan interface{}) 48 | go func() { 49 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 50 | 51 | By("bootstrapping test environment") 52 | testEnv = &envtest.Environment{ 53 | CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, 54 | ErrorIfCRDPathMissing: false, 55 | } 56 | 57 | var err error 58 | // cfg is defined in this file globally. 59 | cfg, err = testEnv.Start() 60 | Expect(err).NotTo(HaveOccurred()) 61 | Expect(cfg).NotTo(BeNil()) 62 | 63 | err = v1alpha1.AddToScheme(scheme.Scheme) 64 | Expect(err).NotTo(HaveOccurred()) 65 | //+kubebuilder:scaffold:scheme 66 | 67 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) 68 | Expect(err).NotTo(HaveOccurred()) 69 | Expect(k8sClient).NotTo(BeNil()) 70 | }() 71 | Eventually(done, 60).Should(BeClosed()) 72 | 73 | }) 74 | 75 | var _ = AfterSuite(func() { 76 | By("tearing down the test environment") 77 | err := testEnv.Stop() 78 | Expect(err).NotTo(HaveOccurred()) 79 | }) 80 | -------------------------------------------------------------------------------- /docs/images/README.md: -------------------------------------------------------------------------------- 1 | ### Updating the Workflow Images 2 | The images have embedded data and can be imported to draw.io to for updates. -------------------------------------------------------------------------------- /docs/images/sgp-node-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/366cfc1cf51265bf0dc045959348a93ab9f687dc/docs/images/sgp-node-create.png -------------------------------------------------------------------------------- /docs/images/sgp-pod-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/366cfc1cf51265bf0dc045959348a93ab9f687dc/docs/images/sgp-pod-create.png -------------------------------------------------------------------------------- /docs/images/windows-node-create-event.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/366cfc1cf51265bf0dc045959348a93ab9f687dc/docs/images/windows-node-create-event.png -------------------------------------------------------------------------------- /docs/images/windows-pod-create-event.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/366cfc1cf51265bf0dc045959348a93ab9f687dc/docs/images/windows-pod-create-event.png -------------------------------------------------------------------------------- /docs/images/windows-prefix-delegation-node-create-events.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/366cfc1cf51265bf0dc045959348a93ab9f687dc/docs/images/windows-prefix-delegation-node-create-events.jpg -------------------------------------------------------------------------------- /docs/images/windows-prefix-delegation-pod-create-events.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/366cfc1cf51265bf0dc045959348a93ab9f687dc/docs/images/windows-prefix-delegation-pod-create-events.jpg -------------------------------------------------------------------------------- /docs/sgp/sgp_config_options.md: -------------------------------------------------------------------------------- 1 | # Configuration options for Security groups for pods 2 | 3 | Users are able to configure the controller functionality related to security group for pods by updating the `data` fields in EKS-managed configmap `amazon-vpc-cni`. 4 | 5 | * **branch-eni-cooldown**: Cooldown period for the branch ENIs, the period of time to wait before deleting the branch ENI for propagation of iptables rules for the deleted pod. The default cooldown period is 60s, and the minimum value for the cool period is 30s. If user updates configmap to a lower value than 30s, this will be overridden and set to 30s. 6 | 7 | Add `branch-eni-cooldown` field in the configmap to set the cooldown period, example: 8 | ``` 9 | apiVersion: v1 10 | data: 11 | branch-eni-cooldown: "60" 12 | kind: ConfigMap 13 | metadata: 14 | name: amazon-vpc-cni 15 | namespace: kube-system 16 | ``` 17 | 18 | After changing the value of `branch-eni-cooldown`, you can verify if the change has been applied by the controller. You need describe any node in your cluster and check node events in Events list. Note: this value is applied to the cluster instead of only certain nodes. 19 | 20 | For example, after setting the value to `90`, the change will be reflected immediately in node events: 21 | ``` 22 | Events: 23 | Type Reason Age From Message 24 | ---- ------ ---- ---- ------- 25 | Normal BranchENICoolDownPeriodUpdated 18s vpc-resource-controller Branch ENI cool down period has been updated to 1m30s 26 | ``` -------------------------------------------------------------------------------- /docs/sgp/workflow.md: -------------------------------------------------------------------------------- 1 | # Security Group for Pods Event Workflows 2 | This document presents high level workflow diagram for Events associated with Nodes and Pods using Security Group for Pods feature. 3 | 4 | ## Adding a supported Node to Cluster 5 | 6 | Security Group for Pods is supported only on Nitro Based Instances. 7 | 8 | ![New Nitro Based Node Create Event Diagram](../images/sgp-node-create.png) 9 | 10 | 1. User adds a new supported node or enables ENI Trunking with existing nodes present in the cluster. 11 | 2. VPC CNI Plugin updates EKS-managed CRD `CNINode ` to add feature `SecurityGroupsForPods` if the node has capacity to create 1 additional ENI. 12 | 3. Controller watches for node events and acts on node if the feature is added in `CNINode` CRD by creating a Trunk ENI. 13 | 4. Controller updates the resource capacity on this node to `vpc.amazonaws.com/pod-eni: # Supported Branch ENI`. Controller also publishes an event on the node upon successful trunk ENI creation. 14 | 15 | ## Creating a Pod using Security Groups 16 | 17 | ![New Security Group for Pod Create Event Diagram](../images/sgp-pod-create.png) 18 | 19 | 1. User creates a Pod with labels/service account that matches at-least one Security Group Policy. 20 | 2. Webhook mutates the Create Pod request by adding the following resource limit and capacity `vpc.amazonaws.com/pod-eni: 1`. 21 | 3. The Pod is scheduled on a Node which has capacity to provide 1 Branch ENI. 22 | 4. Controller creates a Branch ENI with Security Group from the matching Security Group Policy. This Branch ENI is then associate with the Trunk ENI of the Node. 23 | 5. Controller annotates the Pod with the Branch ENI details. 24 | 6. VPC CNI reads the Annotation and sets up the Networking for the Pod. 25 | -------------------------------------------------------------------------------- /docs/windows/prefix_delegation_hld_workflow.md: -------------------------------------------------------------------------------- 1 | # Windows Event Workflows in IPv4 Prefix Delegation mode 2 | This document presents high level workflow diagram for Events associated with Windows Nodes and Pods when using the IPv4 prefix delegation mode. 3 | 4 | ## Adding a Windows Node to the Cluster 5 | 6 | New Windows Node Create Event Diagram 7 | 8 | 1. Controller watches for Node Event from the Kube API server. 9 | 2. User Adds a Windows Node to the Cluster with the label `kubernetes.io/os: windows`. 10 | 3. Resource controller would start managing an IP address warm pool for the Windows node. It would invoke EC2 APIs on behalf of the customer to allocate /28 prefixes to the primary ENI. Internally, it would deconstruct the prefix into IP addresses and the pods would later be assigned one of the IP address from the prefix range. 11 | 12 | In order to reduce latency after pod creation, controller would warm up a prefix beforehand. Customer can control the pre-scaling/warm settings using configuration options as specified [here](prefix_delegation_config_options.md). 13 | 4. Controller updates the resource capacity on this node to `vpc.amazonaws.com/PrivateIPv4Address: # (Secondary IP per interface -1)*16`. This limits the Number of Windows Pod that can be scheduled on Windows Node based on the number of available IPv4 addresses. 14 | 15 | ## Creating a new Windows Pod 16 | 17 | New Windows Pod Create Event Diagram 18 | 19 | 1. User Creates a new Windows Pod with the nodeSelector `kubernetes.io/os: windows`. 20 | 2. Webhook mutates the Create Pod request by adding the following resource limit and capacity `vpc.amazonaws.com/PrivateIPv4Address: 1`. This tells the scheduler that the Pod has to be scheduled on a Node with 1 available IPv4 Address. 21 | 3. Controller receives the Pod Create event and allocates a IPv4 address from the Prefix Warm Pool. The IP address assigned to the pod would be from the range of one of the prefixes assigned to the primary ENI on the node. 22 | 23 | It is worthwhile to note that the controller would assign the IP address to the pods such that the prefix with the fewest remaining IP addresses would be consumed first. This means that if there are 2 prefixes on the node such that 10 IP addresses from the second prefix are yet to be allocated and 5 from the first, then newer pods will be allocated the IP addresses from the first prefix while it has unassigned IP addresses. 24 | 25 | 4. Controller annotates the Pod with `vpc.amazonaws.com/PrivateIPv4Address: IPv4 Address`. 26 | 5. VPC CNI Plugin Binary on the Windows host reads the IPv4 address present in the annotation from API Server and sets up the Networking for the Pod 27 | 28 | 29 | ## Delete events 30 | 31 | When the pods are terminated, the IP addresses are released back into the warmpool. If the available IP addresses in the warmpool are greater than the required number, then controller will release the free prefixes. This essentially means that a prefix is released back only if all the IP addresses from its range are unallocated from the pods. -------------------------------------------------------------------------------- /docs/windows/secondary_ip_mode_workflow.md: -------------------------------------------------------------------------------- 1 | # Windows Event Workflows in Secondary IPv4 address mode 2 | This document presents high level workflow diagram for Events associated with Windows Nodes and Pods when using the secondary IPv4 address mode. 3 | 4 | ## Adding a Windows Node to the Cluster 5 | ![New Windows Node Create Event Diagram](../images/windows-node-create-event.png) 6 | 7 | 1. Controller watches for Node Event from the kube-apiserver. 8 | 2. User Adds a Windows Node to the Cluster with the label `kubernetes.io/os: windows`. 9 | 3. Controller starts managing a Warm Pool for this Node. It maintains a pool of secondary IPv4 Address in this pool by using EC2 API on behalf of the user. 10 | 4. Controller updates the resource capacity on this node to `vpc.amazonaws.com/PrivateIPv4Address: # Secondary IP on single ENI`. This limits the Number of Windows Pod that can be scheduled on Windows Node based on the number of available IPv4 addresses. 11 | 12 | ## Creating a new Windows Pod 13 | 14 | ![New Windows Pod Create Event Diagram](../images/windows-pod-create-event.png) 15 | 16 | 1. User Creates a new Windows Pod with the nodeSelector `kubernetes.io/os: windows`. 17 | 2. Webhook mutates the Create Pod request by adding the following resource limit and capacity `vpc.amazonaws.com/PrivateIPv4Address: 1`. This tells the scheduler that the Pod has to be scheduled on a Node with 1 available Secondary IPv4 Address. 18 | 3. Controller receives the Pod Create event and allocates a Secondary IPv4 address from the Warm Pool. 19 | 4. Controller annotates the Pod with `vpc.amazonaws.com/PrivateIPv4Address: IPv4 Address`. 20 | 5. VPC CNI Plugin Binary on the Windows host reads the IPv4 address present in the annotation from API Server and sets up the Networking for the Pod 21 | 22 | For Delete Events the resource is added back to the Warm Pool and the controller maintains the warm pool size to a fixed count. 23 | -------------------------------------------------------------------------------- /hack/toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | K8S_VERSION="${K8S_VERSION:="1.22.x"}" 5 | KUBEBUILDER_ASSETS="${KUBEBUILDER_ASSETS:="${HOME}/.kubebuilder/bin"}" 6 | 7 | main() { 8 | tools 9 | kubebuilder 10 | } 11 | 12 | tools() { 13 | go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20230216140739-c98506dc3b8e 14 | go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 15 | go install github.com/google/ko@latest 16 | 17 | if ! echo "$PATH" | grep -q "${GOPATH:-undefined}/bin\|$HOME/go/bin"; then 18 | echo "Go workspace's \"bin\" directory is not in PATH. Run 'export PATH=\"\$PATH:\${GOPATH:-\$HOME/go}/bin\"'." 19 | fi 20 | } 21 | 22 | kubebuilder() { 23 | mkdir -p $KUBEBUILDER_ASSETS 24 | arch=$(go env GOARCH) 25 | ## Kubebuilder does not support darwin/arm64, so use amd64 through Rosetta instead 26 | if [[ $(go env GOOS) == "darwin" ]] && [[ $(go env GOARCH) == "arm64" ]]; then 27 | arch="amd64" 28 | fi 29 | ln -sf $(setup-envtest use -p path "${K8S_VERSION}" --arch="${arch}" --bin-dir="${KUBEBUILDER_ASSETS}")/* ${KUBEBUILDER_ASSETS} 30 | find $KUBEBUILDER_ASSETS 31 | } 32 | 33 | main "$@" 34 | -------------------------------------------------------------------------------- /mocks/amazon-vcp-resource-controller-k8s/controllers/custom/custom_controller.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Code generated by MockGen. DO NOT EDIT. 15 | // Source: github.com/aws/amazon-vpc-resource-controller-k8s/controllers/custom (interfaces: Controller) 16 | 17 | // Package mock_custom is a generated GoMock package. 18 | package mock_custom 19 | 20 | import ( 21 | gomock "github.com/golang/mock/gomock" 22 | cache "k8s.io/client-go/tools/cache" 23 | reflect "reflect" 24 | ) 25 | 26 | // MockController is a mock of Controller interface 27 | type MockController struct { 28 | ctrl *gomock.Controller 29 | recorder *MockControllerMockRecorder 30 | } 31 | 32 | // MockControllerMockRecorder is the mock recorder for MockController 33 | type MockControllerMockRecorder struct { 34 | mock *MockController 35 | } 36 | 37 | // NewMockController creates a new mock instance 38 | func NewMockController(ctrl *gomock.Controller) *MockController { 39 | mock := &MockController{ctrl: ctrl} 40 | mock.recorder = &MockControllerMockRecorder{mock} 41 | return mock 42 | } 43 | 44 | // EXPECT returns an object that allows the caller to indicate expected use 45 | func (m *MockController) EXPECT() *MockControllerMockRecorder { 46 | return m.recorder 47 | } 48 | 49 | // GetDataStore mocks base method 50 | func (m *MockController) GetDataStore() cache.Indexer { 51 | m.ctrl.T.Helper() 52 | ret := m.ctrl.Call(m, "GetDataStore") 53 | ret0, _ := ret[0].(cache.Indexer) 54 | return ret0 55 | } 56 | 57 | // GetDataStore indicates an expected call of GetDataStore 58 | func (mr *MockControllerMockRecorder) GetDataStore() *gomock.Call { 59 | mr.mock.ctrl.T.Helper() 60 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDataStore", reflect.TypeOf((*MockController)(nil).GetDataStore)) 61 | } 62 | 63 | // StartController mocks base method 64 | func (m *MockController) StartController(arg0 cache.Indexer, arg1 chan struct{}) { 65 | m.ctrl.T.Helper() 66 | m.ctrl.Call(m, "StartController", arg0, arg1) 67 | } 68 | 69 | // StartController indicates an expected call of StartController 70 | func (mr *MockControllerMockRecorder) StartController(arg0, arg1 interface{}) *gomock.Call { 71 | mr.mock.ctrl.T.Helper() 72 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartController", reflect.TypeOf((*MockController)(nil).StartController), arg0, arg1) 73 | } 74 | -------------------------------------------------------------------------------- /mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api/cleanup/mock_resource_cleaner.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Code generated by MockGen. DO NOT EDIT. 15 | // Source: github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api/cleanup (interfaces: ResourceCleaner) 16 | 17 | // Package mock_cleanup is a generated GoMock package. 18 | package mock_cleanup 19 | 20 | import ( 21 | reflect "reflect" 22 | 23 | gomock "github.com/golang/mock/gomock" 24 | ) 25 | 26 | // MockResourceCleaner is a mock of ResourceCleaner interface. 27 | type MockResourceCleaner struct { 28 | ctrl *gomock.Controller 29 | recorder *MockResourceCleanerMockRecorder 30 | } 31 | 32 | // MockResourceCleanerMockRecorder is the mock recorder for MockResourceCleaner. 33 | type MockResourceCleanerMockRecorder struct { 34 | mock *MockResourceCleaner 35 | } 36 | 37 | // NewMockResourceCleaner creates a new mock instance. 38 | func NewMockResourceCleaner(ctrl *gomock.Controller) *MockResourceCleaner { 39 | mock := &MockResourceCleaner{ctrl: ctrl} 40 | mock.recorder = &MockResourceCleanerMockRecorder{mock} 41 | return mock 42 | } 43 | 44 | // EXPECT returns an object that allows the caller to indicate expected use. 45 | func (m *MockResourceCleaner) EXPECT() *MockResourceCleanerMockRecorder { 46 | return m.recorder 47 | } 48 | 49 | // DeleteLeakedResources mocks base method. 50 | func (m *MockResourceCleaner) DeleteLeakedResources() error { 51 | m.ctrl.T.Helper() 52 | ret := m.ctrl.Call(m, "DeleteLeakedResources") 53 | ret0, _ := ret[0].(error) 54 | return ret0 55 | } 56 | 57 | // DeleteLeakedResources indicates an expected call of DeleteLeakedResources. 58 | func (mr *MockResourceCleanerMockRecorder) DeleteLeakedResources() *gomock.Call { 59 | mr.mock.ctrl.T.Helper() 60 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLeakedResources", reflect.TypeOf((*MockResourceCleaner)(nil).DeleteLeakedResources)) 61 | } 62 | -------------------------------------------------------------------------------- /mocks/amazon-vcp-resource-controller-k8s/pkg/handler/mock_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Code generated by MockGen. DO NOT EDIT. 15 | // Source: github.com/aws/amazon-vpc-resource-controller-k8s/pkg/handler (interfaces: Handler) 16 | 17 | // Package mock_handler is a generated GoMock package. 18 | package mock_handler 19 | 20 | import ( 21 | reflect "reflect" 22 | 23 | gomock "github.com/golang/mock/gomock" 24 | v1 "k8s.io/api/core/v1" 25 | reconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" 26 | ) 27 | 28 | // MockHandler is a mock of Handler interface. 29 | type MockHandler struct { 30 | ctrl *gomock.Controller 31 | recorder *MockHandlerMockRecorder 32 | } 33 | 34 | // MockHandlerMockRecorder is the mock recorder for MockHandler. 35 | type MockHandlerMockRecorder struct { 36 | mock *MockHandler 37 | } 38 | 39 | // NewMockHandler creates a new mock instance. 40 | func NewMockHandler(ctrl *gomock.Controller) *MockHandler { 41 | mock := &MockHandler{ctrl: ctrl} 42 | mock.recorder = &MockHandlerMockRecorder{mock} 43 | return mock 44 | } 45 | 46 | // EXPECT returns an object that allows the caller to indicate expected use. 47 | func (m *MockHandler) EXPECT() *MockHandlerMockRecorder { 48 | return m.recorder 49 | } 50 | 51 | // HandleCreate mocks base method. 52 | func (m *MockHandler) HandleCreate(arg0 int, arg1 *v1.Pod) (reconcile.Result, error) { 53 | m.ctrl.T.Helper() 54 | ret := m.ctrl.Call(m, "HandleCreate", arg0, arg1) 55 | ret0, _ := ret[0].(reconcile.Result) 56 | ret1, _ := ret[1].(error) 57 | return ret0, ret1 58 | } 59 | 60 | // HandleCreate indicates an expected call of HandleCreate. 61 | func (mr *MockHandlerMockRecorder) HandleCreate(arg0, arg1 interface{}) *gomock.Call { 62 | mr.mock.ctrl.T.Helper() 63 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleCreate", reflect.TypeOf((*MockHandler)(nil).HandleCreate), arg0, arg1) 64 | } 65 | 66 | // HandleDelete mocks base method. 67 | func (m *MockHandler) HandleDelete(arg0 *v1.Pod) (reconcile.Result, error) { 68 | m.ctrl.T.Helper() 69 | ret := m.ctrl.Call(m, "HandleDelete", arg0) 70 | ret0, _ := ret[0].(reconcile.Result) 71 | ret1, _ := ret[1].(error) 72 | return ret0, ret1 73 | } 74 | 75 | // HandleDelete indicates an expected call of HandleDelete. 76 | func (mr *MockHandlerMockRecorder) HandleDelete(arg0 interface{}) *gomock.Call { 77 | mr.mock.ctrl.T.Helper() 78 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleDelete", reflect.TypeOf((*MockHandler)(nil).HandleDelete), arg0) 79 | } 80 | -------------------------------------------------------------------------------- /mocks/amazon-vcp-resource-controller-k8s/pkg/provider/branch/cooldown/mock_cooldown.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Code generated by MockGen. DO NOT EDIT. 15 | // Source: github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown (interfaces: CoolDown) 16 | 17 | // Package mock_cooldown is a generated GoMock package. 18 | package mock_cooldown 19 | 20 | import ( 21 | reflect "reflect" 22 | time "time" 23 | 24 | gomock "github.com/golang/mock/gomock" 25 | ) 26 | 27 | // MockCoolDown is a mock of CoolDown interface. 28 | type MockCoolDown struct { 29 | ctrl *gomock.Controller 30 | recorder *MockCoolDownMockRecorder 31 | } 32 | 33 | // MockCoolDownMockRecorder is the mock recorder for MockCoolDown. 34 | type MockCoolDownMockRecorder struct { 35 | mock *MockCoolDown 36 | } 37 | 38 | // NewMockCoolDown creates a new mock instance. 39 | func NewMockCoolDown(ctrl *gomock.Controller) *MockCoolDown { 40 | mock := &MockCoolDown{ctrl: ctrl} 41 | mock.recorder = &MockCoolDownMockRecorder{mock} 42 | return mock 43 | } 44 | 45 | // EXPECT returns an object that allows the caller to indicate expected use. 46 | func (m *MockCoolDown) EXPECT() *MockCoolDownMockRecorder { 47 | return m.recorder 48 | } 49 | 50 | // GetCoolDownPeriod mocks base method. 51 | func (m *MockCoolDown) GetCoolDownPeriod() time.Duration { 52 | m.ctrl.T.Helper() 53 | ret := m.ctrl.Call(m, "GetCoolDownPeriod") 54 | ret0, _ := ret[0].(time.Duration) 55 | return ret0 56 | } 57 | 58 | // GetCoolDownPeriod indicates an expected call of GetCoolDownPeriod. 59 | func (mr *MockCoolDownMockRecorder) GetCoolDownPeriod() *gomock.Call { 60 | mr.mock.ctrl.T.Helper() 61 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCoolDownPeriod", reflect.TypeOf((*MockCoolDown)(nil).GetCoolDownPeriod)) 62 | } 63 | 64 | // SetCoolDownPeriod mocks base method. 65 | func (m *MockCoolDown) SetCoolDownPeriod(arg0 time.Duration) { 66 | m.ctrl.T.Helper() 67 | m.ctrl.Call(m, "SetCoolDownPeriod", arg0) 68 | } 69 | 70 | // SetCoolDownPeriod indicates an expected call of SetCoolDownPeriod. 71 | func (mr *MockCoolDownMockRecorder) SetCoolDownPeriod(arg0 interface{}) *gomock.Call { 72 | mr.mock.ctrl.T.Helper() 73 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCoolDownPeriod", reflect.TypeOf((*MockCoolDown)(nil).SetCoolDownPeriod), arg0) 74 | } 75 | -------------------------------------------------------------------------------- /mocks/amazon-vcp-resource-controller-k8s/pkg/utils/mock_k8shelper.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Code generated by MockGen. DO NOT EDIT. 15 | // Source: github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils (interfaces: SecurityGroupForPodsAPI) 16 | 17 | // Package mock_utils is a generated GoMock package. 18 | package mock_utils 19 | 20 | import ( 21 | reflect "reflect" 22 | 23 | gomock "github.com/golang/mock/gomock" 24 | v1 "k8s.io/api/core/v1" 25 | ) 26 | 27 | // MockSecurityGroupForPodsAPI is a mock of SecurityGroupForPodsAPI interface. 28 | type MockSecurityGroupForPodsAPI struct { 29 | ctrl *gomock.Controller 30 | recorder *MockSecurityGroupForPodsAPIMockRecorder 31 | } 32 | 33 | // MockSecurityGroupForPodsAPIMockRecorder is the mock recorder for MockSecurityGroupForPodsAPI. 34 | type MockSecurityGroupForPodsAPIMockRecorder struct { 35 | mock *MockSecurityGroupForPodsAPI 36 | } 37 | 38 | // NewMockSecurityGroupForPodsAPI creates a new mock instance. 39 | func NewMockSecurityGroupForPodsAPI(ctrl *gomock.Controller) *MockSecurityGroupForPodsAPI { 40 | mock := &MockSecurityGroupForPodsAPI{ctrl: ctrl} 41 | mock.recorder = &MockSecurityGroupForPodsAPIMockRecorder{mock} 42 | return mock 43 | } 44 | 45 | // EXPECT returns an object that allows the caller to indicate expected use. 46 | func (m *MockSecurityGroupForPodsAPI) EXPECT() *MockSecurityGroupForPodsAPIMockRecorder { 47 | return m.recorder 48 | } 49 | 50 | // GetMatchingSecurityGroupForPods mocks base method. 51 | func (m *MockSecurityGroupForPodsAPI) GetMatchingSecurityGroupForPods(arg0 *v1.Pod) ([]string, error) { 52 | m.ctrl.T.Helper() 53 | ret := m.ctrl.Call(m, "GetMatchingSecurityGroupForPods", arg0) 54 | ret0, _ := ret[0].([]string) 55 | ret1, _ := ret[1].(error) 56 | return ret0, ret1 57 | } 58 | 59 | // GetMatchingSecurityGroupForPods indicates an expected call of GetMatchingSecurityGroupForPods. 60 | func (mr *MockSecurityGroupForPodsAPIMockRecorder) GetMatchingSecurityGroupForPods(arg0 interface{}) *gomock.Call { 61 | mr.mock.ctrl.T.Helper() 62 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMatchingSecurityGroupForPods", reflect.TypeOf((*MockSecurityGroupForPodsAPI)(nil).GetMatchingSecurityGroupForPods), arg0) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/api/wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package api 15 | 16 | import ( 17 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api" 18 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s/pod" 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" 21 | ) 22 | 23 | // Wrapper providers wrapper around all clients used by the controller 24 | type Wrapper struct { 25 | EC2API api.EC2APIHelper 26 | K8sAPI k8s.K8sWrapper 27 | PodAPI pod.PodClientAPIWrapper 28 | SGPAPI utils.SecurityGroupForPodsAPI 29 | } 30 | -------------------------------------------------------------------------------- /pkg/aws/ec2/api/cleanup/node_cleanup.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package cleanup 15 | 16 | import ( 17 | ec2API "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api" 18 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" 19 | "github.com/aws/aws-sdk-go-v2/aws" 20 | ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" 21 | ctrl "sigs.k8s.io/controller-runtime" 22 | ) 23 | 24 | // NodeTerminationCleanerto handle resource cleanup at node termination 25 | type NodeTerminationCleaner struct { 26 | NodeID string 27 | *ENICleaner 28 | } 29 | 30 | func (n *NodeTerminationCleaner) GetENITagFilters() []ec2types.Filter { 31 | return []ec2types.Filter{ 32 | { 33 | Name: aws.String("tag:" + config.NetworkInterfaceNodeIDKey), 34 | Values: []string{n.NodeID}, 35 | }, 36 | } 37 | } 38 | 39 | // Return true. As the node is terminating all available ENIs need to be deleted 40 | func (n *NodeTerminationCleaner) ShouldDeleteENI(eniID *string) bool { 41 | return true 42 | } 43 | 44 | func (n *NodeTerminationCleaner) UpdateAvailableENIsIfNeeded(eniMap *map[string]struct{}) { 45 | // Nothing to do for the node termination cleaner 46 | return 47 | } 48 | 49 | // Updating node termination metrics does not make much sense as it will be updated on each node deletion and does not give us much info 50 | func (n *NodeTerminationCleaner) UpdateCleanupMetrics(vpcrcAvailableCount *int, vpccniAvailableCount *int, leakedENICount *int) { 51 | return 52 | } 53 | 54 | func NewNodeResourceCleaner(nodeID string, eC2Wrapper ec2API.EC2Wrapper, vpcID string) ResourceCleaner { 55 | cleaner := &NodeTerminationCleaner{ 56 | NodeID: nodeID, 57 | } 58 | cleaner.ENICleaner = &ENICleaner{ 59 | EC2Wrapper: eC2Wrapper, 60 | Manager: cleaner, 61 | VpcId: vpcID, 62 | Log: ctrl.Log.WithName("eniCleaner").WithName("node"), 63 | } 64 | return cleaner.ENICleaner 65 | } 66 | -------------------------------------------------------------------------------- /pkg/aws/ec2/api/cleanup/resource_cleaner.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package cleanup 15 | 16 | // ResourceCleaner interface should be implemented by components that need to delete leaked AWS resources 17 | type ResourceCleaner interface { 18 | DeleteLeakedResources() error 19 | } 20 | -------------------------------------------------------------------------------- /pkg/aws/ec2/api/wrapper_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func getMockEC2Wrapper() ec2Wrapper { 8 | return ec2Wrapper{} 9 | } 10 | func Test_getRegionalStsEndpoint(t *testing.T) { 11 | 12 | ec2Wapper := getMockEC2Wrapper() 13 | 14 | type args struct { 15 | partitionID string 16 | region string 17 | } 18 | 19 | tests := []struct { 20 | name string 21 | args args 22 | want string 23 | wantErr bool 24 | }{ 25 | { 26 | name: "service doesn't exist in partition", 27 | args: args{ 28 | partitionID: "aws-iso-f", 29 | region: "testregions", 30 | }, 31 | want: "https://sts.testregions.csp.hci.ic.gov", 32 | wantErr: false, 33 | }, 34 | { 35 | name: "region doesn't exist in partition", 36 | args: args{ 37 | partitionID: "aws", 38 | region: "us-test-2", 39 | }, 40 | want: "https://sts.us-test-2.amazonaws.com", 41 | wantErr: false, 42 | }, 43 | { 44 | name: "region and service exist in partition", 45 | args: args{ 46 | partitionID: "aws", 47 | region: "us-west-2", 48 | }, 49 | want: "https://sts.us-west-2.amazonaws.com", 50 | wantErr: false, 51 | }, 52 | } 53 | for _, tt := range tests { 54 | t.Run(tt.name, func(t *testing.T) { 55 | got, err := ec2Wapper.getRegionalStsEndpoint(tt.args.partitionID, tt.args.region) 56 | if (err != nil) != tt.wantErr { 57 | t.Errorf("getRegionalStsEndpoint() error = %v, wantErr %v", err, tt.wantErr) 58 | return 59 | } 60 | if got.URL != tt.want { 61 | t.Errorf("getRegionalStsEndpoint() = %v, want %v", got, tt.want) 62 | } 63 | }) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/aws/errors/ec2_errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | const ( 4 | NotFoundAssociationID = "InvalidAssociationID.NotFound" 5 | NotFoundInterfaceID = "InvalidNetworkInterfaceID.NotFound" 6 | ) 7 | -------------------------------------------------------------------------------- /pkg/aws/vpc/README.md: -------------------------------------------------------------------------------- 1 | # supported EC2 instance types by amazon-vpc-resource-controller-k8s 2 | 3 | The limit.go file in master branch provides the latest supported EC2 instance types by the controller for [Security Group for Pods feature](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). In this file, you will find if your EC2 instance is supported (`IsTrunkingCompatible`) and how many pods using the feature (`BranchInterface`) can be created per instance when the next version of the controller is released. 4 | 5 | Note: If you want to check EC2 instance types currently supported by the controller, you should check the limit.go file in the release branch instead. Thank you! -------------------------------------------------------------------------------- /pkg/aws/vpc/limits_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vpc 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/stretchr/testify/assert" 20 | ) 21 | 22 | var exludedTypes = map[string]bool{"g5g.4xlarge": true, "i4i.16xlarge": true, "g5g.xlarge": true} 23 | 24 | func TestInstanceTypesHavingNetworkMetadata(t *testing.T) { 25 | for instanceType, instance := range Limits { 26 | assert.Truef(t, len(instance.NetworkCards) > 0, "Every instance should have network card in %s", instanceType) 27 | } 28 | } 29 | 30 | func TestInstanceInterfacesMatchNetworkCards(t *testing.T) { 31 | for instanceType, instance := range Limits { 32 | // currently we are seeing some instance types has unmatched max ENIs between instance and sum of network cards 33 | // have reached out to EC2 for the inconsistency 34 | if _, ok := exludedTypes[instanceType]; ok { 35 | continue 36 | } 37 | totalInterfaces := instance.Interface 38 | for _, nc := range instance.NetworkCards { 39 | totalInterfaces -= int(nc.MaximumNetworkInterfaces) 40 | } 41 | assert.Truef(t, totalInterfaces == 0, "Instance's max interfaces should match the sum of network cards' interfaces in %s", instanceType) 42 | } 43 | } 44 | 45 | func TestBranchInterfaceMatchSupportFlag(t *testing.T) { 46 | for instanceType, instance := range Limits { 47 | if instance.IsTrunkingCompatible { 48 | assert.Truef(t, instance.BranchInterface > 0, "if instance support trunk it should have more than one branch interface in %s", instanceType) 49 | } 50 | } 51 | } 52 | 53 | func TestDefaultNetworkCardIndex(t *testing.T) { 54 | for instanceType, instance := range Limits { 55 | interfaces := 0 56 | for _, nc := range instance.NetworkCards { 57 | if nc.NetworkCardIndex == int64(instance.DefaultNetworkCardIndex) { 58 | interfaces = int(nc.MaximumNetworkInterfaces) 59 | } 60 | } 61 | assert.Truef(t, interfaces > 0, "instance default network card should be valid in %s", instanceType) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pkg/handler/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package handler 15 | 16 | import ( 17 | v1 "k8s.io/api/core/v1" 18 | ctrl "sigs.k8s.io/controller-runtime" 19 | ) 20 | 21 | // Handler interface allows different types of resource implementation to be clubbed under a single handler. 22 | // For instance, warm resource handler would handle all the types of resources that support warm pools. An example 23 | // of warm pool resource is IPv4. Another example of handler is on demand handler, resources that can be only 24 | // processed on demand would fit into this category. For instance, Branch ENIs are tied to the Security 25 | // Group required by the pod which we would know only after receiving the pod request. 26 | type Handler interface { 27 | HandleCreate(requestCount int, pod *v1.Pod) (ctrl.Result, error) 28 | HandleDelete(pod *v1.Pod) (ctrl.Result, error) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/handler/on_demand.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package handler 15 | 16 | import ( 17 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider" 18 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker" 19 | 20 | "github.com/go-logr/logr" 21 | v1 "k8s.io/api/core/v1" 22 | ctrl "sigs.k8s.io/controller-runtime" 23 | ) 24 | 25 | type onDemandResourceHandler struct { 26 | resourceProvider provider.ResourceProvider 27 | resourceName string 28 | Log logr.Logger 29 | } 30 | 31 | // NewOnDemandHandler returns a new on demand handler with all the workers that can handle particular resource types 32 | func NewOnDemandHandler(log logr.Logger, resourceName string, 33 | ondDemandProvider provider.ResourceProvider) Handler { 34 | return &onDemandResourceHandler{ 35 | resourceProvider: ondDemandProvider, 36 | resourceName: resourceName, 37 | Log: log, 38 | } 39 | } 40 | 41 | // HandleCreate provides the resource to the on demand resource by passing the Create Job to the respective Worker 42 | func (h *onDemandResourceHandler) HandleCreate(requestCount int, pod *v1.Pod) (ctrl.Result, error) { 43 | job := worker.NewOnDemandCreateJob(pod.Namespace, pod.Name, requestCount) 44 | h.resourceProvider.SubmitAsyncJob(job) 45 | 46 | return ctrl.Result{}, nil 47 | } 48 | 49 | // HandleDelete reclaims the on demand resource by passing the Delete Job to the respective Worker 50 | func (h *onDemandResourceHandler) HandleDelete(pod *v1.Pod) (ctrl.Result, error) { 51 | deleteJob := worker.NewOnDemandDeletedJob(pod.Spec.NodeName, pod.UID) 52 | h.resourceProvider.SubmitAsyncJob(deleteJob) 53 | 54 | return ctrl.Result{}, nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/healthz/healthz.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package healthz 15 | 16 | import ( 17 | "errors" 18 | "fmt" 19 | "net/http" 20 | "time" 21 | 22 | "github.com/go-logr/logr" 23 | "sigs.k8s.io/controller-runtime/pkg/healthz" 24 | "sigs.k8s.io/controller-runtime/pkg/manager" 25 | ) 26 | 27 | type HealthzHandler struct { 28 | CheckersMap map[string]healthz.Checker 29 | } 30 | 31 | var ( 32 | HealthzTimeout time.Duration = 2 33 | ) 34 | 35 | func NewHealthzHandler(timeout int) *HealthzHandler { 36 | HealthzTimeout = time.Duration(timeout) 37 | return &HealthzHandler{ 38 | CheckersMap: make(map[string]healthz.Checker), 39 | } 40 | } 41 | 42 | func (hh *HealthzHandler) AddControllerHealthChecker(name string, controllerCheck healthz.Checker) { 43 | // only add health check if the map doesn't already contain 44 | if _, ok := hh.CheckersMap[name]; !ok { 45 | hh.CheckersMap[name] = controllerCheck 46 | } 47 | } 48 | 49 | func (hh *HealthzHandler) AddControllersHealthCheckers(controllerCheckers map[string]healthz.Checker) { 50 | for key, value := range controllerCheckers { 51 | hh.AddControllerHealthChecker(key, value) 52 | } 53 | } 54 | 55 | func (hh *HealthzHandler) AddControllersHealthStatusChecksToManager(mgr manager.Manager) error { 56 | if len(hh.CheckersMap) > 0 { 57 | return hh.checkControllersHealthStatus(mgr, hh.CheckersMap) 58 | } else { 59 | return errors.New("couldn't find any controller's check to add for healthz endpoint") 60 | } 61 | } 62 | 63 | func (hh *HealthzHandler) checkControllersHealthStatus(mgr manager.Manager, checkers map[string]healthz.Checker) error { 64 | var err error 65 | for name, check := range checkers { 66 | err = mgr.AddHealthzCheck(name, check) 67 | fmt.Printf("Added Health check for %s with error %v\n", name, err) 68 | if err != nil { 69 | break 70 | } 71 | } 72 | return err 73 | } 74 | 75 | func SimplePing(controllerName string, log logr.Logger) healthz.Checker { 76 | log.Info(fmt.Sprintf("%s's healthz subpath was added", controllerName)) 77 | return func(req *http.Request) error { 78 | log.V(1).Info(fmt.Sprintf("***** %s healthz endpoint was pinged *****", controllerName)) 79 | return nil 80 | } 81 | } 82 | 83 | func PingWithTimeout(healthCheck func(c chan<- error), logger logr.Logger) error { 84 | status := make(chan error, 1) 85 | var err error 86 | go healthCheck(status) 87 | 88 | select { 89 | case err = <-status: 90 | logger.V(1).Info("finished healthz check on controller before probing times out", "TimeoutInSecond", HealthzTimeout*time.Second) 91 | case <-time.After(HealthzTimeout * time.Second): 92 | err = errors.New("healthz check failed due to timeout") 93 | logger.Error(err, "healthz check has a preset timeout to fail no responding probing", "TimeoutInSecond", HealthzTimeout*time.Second) 94 | } 95 | return err 96 | } 97 | -------------------------------------------------------------------------------- /pkg/healthz/healthz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package healthz 15 | 16 | import ( 17 | "testing" 18 | "time" 19 | 20 | "github.com/stretchr/testify/assert" 21 | "sigs.k8s.io/controller-runtime/pkg/healthz" 22 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 23 | ) 24 | 25 | var testTimeout = 3 26 | 27 | // TestHealthzHandler tests creating a new healthz handler with timeout value passed to it 28 | func TestHealthzHandler(t *testing.T) { 29 | handler := NewHealthzHandler(testTimeout) 30 | assert.True(t, handler != nil) 31 | assert.True(t, HealthzTimeout == time.Duration(testTimeout)) 32 | } 33 | 34 | // TestAddControllerHealthChecker tests adding individual healthz checker 35 | func TestAddControllerHealthChecker(t *testing.T) { 36 | handler := NewHealthzHandler(testTimeout) 37 | checker := healthz.Ping 38 | name := "test-ping" 39 | handler.AddControllerHealthChecker(name, checker) 40 | assert.True(t, len(handler.CheckersMap) == 1, "Should be only one healthz checker") 41 | _, ok := handler.CheckersMap[name] 42 | assert.True(t, ok) 43 | } 44 | 45 | // TestAddControllersHealthCheckers tests adding the map of healthz checkers 46 | func TestAddControllersHealthCheckers(t *testing.T) { 47 | handler := NewHealthzHandler(testTimeout) 48 | checkers := map[string]healthz.Checker{ 49 | "test-checker-1": healthz.Ping, 50 | "test-checker-2": SimplePing("test", zap.New()), 51 | } 52 | handler.AddControllersHealthCheckers(checkers) 53 | assert.True(t, len(handler.CheckersMap) == 2, "Two checkers should be added") 54 | } 55 | 56 | // TestPingWithTimeout_Success tests ping responding before timeout 57 | func TestPingWithTimeout_Success(t *testing.T) { 58 | err := PingWithTimeout(func(c chan<- error) { 59 | time.Sleep(1 * time.Second) 60 | c <- nil 61 | }, zap.New()) 62 | time.Sleep(5 * time.Second) 63 | assert.NoError(t, err) 64 | } 65 | 66 | // TestPingWithTimeout_Failure tests ping responding after timeout 67 | func TestPingWithTimeout_Failure(t *testing.T) { 68 | err := PingWithTimeout(func(c chan<- error) { 69 | time.Sleep(4 * time.Second) 70 | c <- nil 71 | }, zap.New()) 72 | time.Sleep(5 * time.Second) 73 | assert.Error(t, err) 74 | assert.EqualErrorf(t, err, "healthz check failed due to timeout", "Healthz check should fail due to timeout") 75 | } 76 | -------------------------------------------------------------------------------- /pkg/k8s/finalizer.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package k8s 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/go-logr/logr" 20 | "sigs.k8s.io/controller-runtime/pkg/client" 21 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 22 | ) 23 | 24 | type FinalizerManager interface { 25 | AddFinalizers(ctx context.Context, object client.Object, finalizers ...string) error 26 | RemoveFinalizers(ctx context.Context, object client.Object, finalizers ...string) error 27 | } 28 | 29 | func NewDefaultFinalizerManager(k8sClient client.Client, log logr.Logger) FinalizerManager { 30 | return &defaultFinalizerManager{ 31 | k8sClient: k8sClient, 32 | log: log, 33 | } 34 | } 35 | 36 | type defaultFinalizerManager struct { 37 | k8sClient client.Client 38 | log logr.Logger 39 | } 40 | 41 | func (m *defaultFinalizerManager) AddFinalizers(ctx context.Context, obj client.Object, finalizers ...string) error { 42 | if err := m.k8sClient.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { 43 | return err 44 | } 45 | 46 | oldObj := obj.DeepCopyObject().(client.Object) 47 | needsUpdate := false 48 | for _, finalizer := range finalizers { 49 | if !controllerutil.ContainsFinalizer(obj, finalizer) { 50 | m.log.Info("adding finalizer", "object", obj.GetObjectKind().GroupVersionKind().Kind, "name", obj.GetName(), "finalizer", finalizer) 51 | controllerutil.AddFinalizer(obj, finalizer) 52 | needsUpdate = true 53 | } 54 | } 55 | if !needsUpdate { 56 | return nil 57 | } 58 | return m.k8sClient.Patch(ctx, obj, client.MergeFromWithOptions(oldObj, client.MergeFromWithOptimisticLock{})) 59 | } 60 | 61 | func (m *defaultFinalizerManager) RemoveFinalizers(ctx context.Context, obj client.Object, finalizers ...string) error { 62 | if err := m.k8sClient.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { 63 | return err 64 | } 65 | 66 | oldObj := obj.DeepCopyObject().(client.Object) 67 | needsUpdate := false 68 | for _, finalizer := range finalizers { 69 | if controllerutil.ContainsFinalizer(obj, finalizer) { 70 | m.log.Info("removing finalizer", "object", obj.GetObjectKind().GroupVersionKind().Kind, "name", obj.GetName(), "finalizer", finalizer) 71 | controllerutil.RemoveFinalizer(obj, finalizer) 72 | needsUpdate = true 73 | } 74 | } 75 | if !needsUpdate { 76 | return nil 77 | } 78 | return m.k8sClient.Patch(ctx, obj, client.MergeFromWithOptions(oldObj, client.MergeFromWithOptimisticLock{})) 79 | } 80 | -------------------------------------------------------------------------------- /pkg/pool/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package pool 15 | 16 | import ( 17 | "github.com/go-logr/logr" 18 | 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/api" 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" 21 | ) 22 | 23 | // GetWinWarmPoolConfig retrieves Windows warmpool configuration from ConfigMap, falls back to using default values on failure 24 | func GetWinWarmPoolConfig(log logr.Logger, w api.Wrapper, isPDEnabled bool) *config.WarmPoolConfig { 25 | var resourceConfig map[string]config.ResourceConfig 26 | vpcCniConfigMap, err := w.K8sAPI.GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace) 27 | if err == nil { 28 | resourceConfig = config.LoadResourceConfigFromConfigMap(log, vpcCniConfigMap) 29 | } else { 30 | log.Error(err, "failed to read from config map, will use default resource config") 31 | resourceConfig = config.LoadResourceConfig() 32 | } 33 | 34 | if isPDEnabled { 35 | return resourceConfig[config.ResourceNameIPAddressFromPrefix].WarmPoolConfig 36 | } else { 37 | return resourceConfig[config.ResourceNameIPAddress].WarmPoolConfig 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/provider/branch/cooldown/cooldown.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package cooldown 15 | 16 | import ( 17 | "fmt" 18 | "strconv" 19 | "sync" 20 | "time" 21 | 22 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" 23 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" 24 | "github.com/go-logr/logr" 25 | ) 26 | 27 | // Global variable for CoolDownPeriod allows packages to Get and Set the coolDown period 28 | var coolDown *cooldown 29 | 30 | type cooldown struct { 31 | mu sync.RWMutex 32 | // CoolDownPeriod is the period to wait before deleting the branch ENI for propagation of ip tables rule for deleted pod 33 | coolDownPeriod time.Duration 34 | } 35 | 36 | type CoolDown interface { 37 | GetCoolDownPeriod() time.Duration 38 | SetCoolDownPeriod(time.Duration) 39 | } 40 | 41 | const ( 42 | DefaultCoolDownPeriod = time.Second * 60 43 | MinimalCoolDownPeriod = time.Second * 30 44 | ) 45 | 46 | // Initialize coolDown period by setting the value in configmap or to default 47 | func InitCoolDownPeriod(k8sApi k8s.K8sWrapper, log logr.Logger) { 48 | coolDown = &cooldown{} 49 | coolDownPeriod, err := GetVpcCniConfigMapCoolDownPeriodOrDefault(k8sApi, log) 50 | if err != nil { 51 | log.Info("setting coolDown period to default", "cool down period", coolDownPeriod) 52 | } 53 | coolDown.SetCoolDownPeriod(coolDownPeriod) 54 | } 55 | 56 | func GetCoolDown() CoolDown { 57 | return coolDown 58 | } 59 | 60 | func GetVpcCniConfigMapCoolDownPeriodOrDefault(k8sApi k8s.K8sWrapper, log logr.Logger) (time.Duration, error) { 61 | vpcCniConfigMap, err := k8sApi.GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace) 62 | if err == nil && vpcCniConfigMap.Data != nil { 63 | if val, ok := vpcCniConfigMap.Data[config.BranchENICooldownPeriodKey]; ok { 64 | coolDownPeriodInt, err := strconv.Atoi(val) 65 | if err != nil { 66 | log.Error(err, "failed to parse branch ENI coolDown period", "cool down period", val) 67 | } else { 68 | return time.Second * time.Duration(coolDownPeriodInt), nil 69 | } 70 | } 71 | } 72 | // If configmap not found, or configmap data not found, or error in parsing coolDown period, return default coolDown period and error 73 | return DefaultCoolDownPeriod, fmt.Errorf("failed to get cool down period:%v", err) 74 | } 75 | 76 | func (c *cooldown) GetCoolDownPeriod() time.Duration { 77 | if c.coolDownPeriod < 30*time.Second { 78 | return MinimalCoolDownPeriod 79 | } 80 | return c.coolDownPeriod 81 | } 82 | 83 | func (c *cooldown) SetCoolDownPeriod(newCoolDownPeriod time.Duration) { 84 | c.mu.Lock() 85 | defer c.mu.Unlock() 86 | c.coolDownPeriod = newCoolDownPeriod 87 | } 88 | -------------------------------------------------------------------------------- /pkg/provider/provider.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package provider 15 | 16 | import ( 17 | ctrl "sigs.k8s.io/controller-runtime" 18 | "sigs.k8s.io/controller-runtime/pkg/healthz" 19 | 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2" 21 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/pool" 22 | ) 23 | 24 | // ResourceProvider is the provider interface that each resource managed by the controller has to implement 25 | type ResourceProvider interface { 26 | // InitResource initializes the resource provider 27 | InitResource(instance ec2.EC2Instance) error 28 | // DeInitResources de initializes the resource provider 29 | DeInitResource(instance ec2.EC2Instance) error 30 | // UpdateResourceCapacity updates the resource capacity 31 | UpdateResourceCapacity(instance ec2.EC2Instance) error 32 | // SubmitAsyncJob submits a job to the worker 33 | SubmitAsyncJob(job interface{}) 34 | // ProcessAsyncJob processes a job form the worker queue 35 | ProcessAsyncJob(job interface{}) (ctrl.Result, error) 36 | // GetPool returns the warm pool for resources that support warm pool 37 | GetPool(nodeName string) (pool.Pool, bool) 38 | // IsInstanceSupported returns true if an instance type is supported by the provider 39 | IsInstanceSupported(instance ec2.EC2Instance) bool 40 | // Introspect allows introspection of all nodes for the given resource 41 | Introspect() interface{} 42 | // IntrospectNode allows introspection of a node for the given resource 43 | IntrospectNode(node string) interface{} 44 | // GetHealthChecker provider a health check subpath for pinging provider 45 | GetHealthChecker() healthz.Checker 46 | // IntrospectSummary allows introspection of resources summary per node 47 | IntrospectSummary() interface{} 48 | ReconcileNode(nodeName string) bool 49 | } 50 | -------------------------------------------------------------------------------- /pkg/utils/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import ( 17 | "errors" 18 | "strings" 19 | ) 20 | 21 | var ( 22 | ErrNotFound = errors.New("resource was not found") 23 | ErrInsufficientCidrBlocks = errors.New("InsufficientCidrBlocks: The specified subnet does not have enough free cidr blocks to satisfy the request") 24 | ErrMsgProviderAndPoolNotFound = "cannot find the instance provider and pool from the cache" 25 | NotRetryErrors = []string{InsufficientCidrBlocksReason} 26 | PauseHealthCheckErrors = []string{"RequestLimitExceeded"} 27 | ) 28 | 29 | // ShouldRetryOnError returns true if the error is retryable, else returns false 30 | func ShouldRetryOnError(err error) bool { 31 | for _, e := range NotRetryErrors { 32 | if strings.HasPrefix(err.Error(), e) { 33 | return false 34 | } 35 | } 36 | return true 37 | } 38 | -------------------------------------------------------------------------------- /pkg/utils/events.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import ( 17 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" 18 | "github.com/go-logr/logr" 19 | v1 "k8s.io/api/core/v1" 20 | "k8s.io/apimachinery/pkg/types" 21 | ) 22 | 23 | const ( 24 | UnsupportedInstanceTypeReason = "Unsupported" 25 | InsufficientCidrBlocksReason = "InsufficientCidrBlocks" 26 | NodeTrunkInitiatedReason = "NodeTrunkInitiated" 27 | NodeTrunkFailedInitializationReason = "NodeTrunkFailedInit" 28 | EniConfigNameNotFoundReason = "EniConfigNameNotFound" 29 | VersionNotice = "ControllerVersionNotice" 30 | BranchENICoolDownUpdateReason = "BranchENICoolDownPeriodUpdated" 31 | CNINodeDeleteFailed = "CNINodeDeletionFailed" 32 | CNINodeCreateFailed = "CNINodeCreationFailed" 33 | ) 34 | 35 | func SendNodeEventWithNodeName(client k8s.K8sWrapper, nodeName, reason, msg, eventType string, logger logr.Logger) { 36 | if node, err := client.GetNode(nodeName); err == nil { 37 | // set UID to node name for kubelet filter the event to node description 38 | node.SetUID(types.UID(nodeName)) 39 | client.BroadcastEvent(node, reason, msg, eventType) 40 | } else { 41 | logger.Error(err, "had an error to get the node for sending unsupported event", "Node", nodeName) 42 | } 43 | } 44 | 45 | func SendNodeEventWithNodeObject(client k8s.K8sWrapper, node *v1.Node, reason, msg, eventType string, logger logr.Logger) { 46 | client.BroadcastEvent(node, reason, msg, eventType) 47 | } 48 | 49 | func SendBroadcastNodeEvent(client k8s.K8sWrapper, reason, msg, eventType string, logger logr.Logger) { 50 | if nodeList, err := client.ListNodes(); err == nil { 51 | for _, node := range nodeList.Items { 52 | node := node // Fix gosec G601, so we can use &node 53 | client.BroadcastEvent(&node, reason, msg, eventType) 54 | } 55 | } else { 56 | logger.Info("failed to list nodes when broadcasting node event", "Reason", reason, "Message", msg) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pkg/utils/httpClient.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import ( 17 | "fmt" 18 | "net/http" 19 | 20 | "golang.org/x/time/rate" 21 | ) 22 | 23 | // NewRateLimitedClient returns a new HTTP client with rate limiter. 24 | func NewRateLimitedClient(qps int, burst int) (*http.Client, error) { 25 | if qps == 0 { 26 | return http.DefaultClient, nil 27 | } 28 | if burst < 1 { 29 | return nil, fmt.Errorf("burst expected >0, got %d", burst) 30 | } 31 | return &http.Client{ 32 | Transport: &rateLimitedRoundTripper{ 33 | rt: http.DefaultTransport, 34 | rl: rate.NewLimiter(rate.Limit(qps), burst), 35 | }, 36 | }, nil 37 | } 38 | 39 | type rateLimitedRoundTripper struct { 40 | rt http.RoundTripper 41 | rl *rate.Limiter 42 | } 43 | 44 | func (rr *rateLimitedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 45 | if err := rr.rl.Wait(req.Context()); err != nil { 46 | return nil, err 47 | } 48 | return rr.rt.RoundTrip(req) 49 | } 50 | -------------------------------------------------------------------------------- /pkg/utils/map.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | func CopyMap(original map[string]string) map[string]string { 17 | copy := make(map[string]string) 18 | for key, val := range original { 19 | copy[key] = val 20 | } 21 | return copy 22 | } 23 | -------------------------------------------------------------------------------- /pkg/utils/math.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import ( 17 | "golang.org/x/exp/constraints" 18 | ) 19 | 20 | func Minimum[T constraints.Ordered](a, b T) T { 21 | if a < b { 22 | return a 23 | } 24 | return b 25 | } 26 | 27 | func Maximum[T constraints.Ordered](a, b T) T { 28 | if a > b { 29 | return a 30 | } 31 | return b 32 | } 33 | 34 | func MinOf[T constraints.Ordered](vars ...T) T { 35 | result := vars[0] 36 | for _, v := range vars { 37 | if result > v { 38 | result = v 39 | } 40 | } 41 | return result 42 | } 43 | 44 | func MaxOf[T constraints.Ordered](vars ...T) T { 45 | result := vars[0] 46 | for _, v := range vars { 47 | if result < v { 48 | result = v 49 | } 50 | } 51 | return result 52 | } 53 | 54 | // CeilDivision returns the ceiling result of numerator x divided by denominator y. y should be non-zero 55 | func CeilDivision(x, y int) int { 56 | return (x + y - 1) / y 57 | } 58 | 59 | // IntPower calculates n to the mth power. m should be a non-negative power to have an int return value 60 | func IntPower(n, m int) int { 61 | if m == 0 { 62 | return 1 63 | } 64 | result := n 65 | for i := 1; i < m; i++ { 66 | result *= n 67 | } 68 | return result 69 | } 70 | -------------------------------------------------------------------------------- /pkg/utils/math_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/stretchr/testify/assert" 20 | ) 21 | 22 | func TestTwoElements(t *testing.T) { 23 | a := 10 24 | b := 11 25 | assert.True(t, Minimum(a, b) == a) 26 | assert.True(t, Maximum(a, b) == b) 27 | 28 | x := "ab" 29 | y := "ba" 30 | assert.True(t, Minimum(x, y) == x) 31 | assert.True(t, Maximum(x, y) == y) 32 | } 33 | 34 | func TestMultipleElements(t *testing.T) { 35 | a := 10 36 | b := 11 37 | c := 5 38 | vars := []int{a, b, c} 39 | assert.True(t, MinOf(vars...) == c) 40 | assert.True(t, MaxOf(vars...) == b) 41 | 42 | x := "ab" 43 | y := "aba" 44 | z := "aa" 45 | varStr := []string{x, y, z} 46 | 47 | assert.True(t, MinOf(varStr...) == z) 48 | assert.True(t, MaxOf(varStr...) == y) 49 | } 50 | 51 | func TestCeilDivision(t *testing.T) { 52 | assert.True(t, CeilDivision(0, 16) == 0) 53 | assert.True(t, CeilDivision(32, 1) == 32) 54 | assert.True(t, CeilDivision(32, 16) == 2) 55 | assert.True(t, CeilDivision(32, 30) == 2) 56 | assert.True(t, CeilDivision(50, 16) == 4) 57 | } 58 | 59 | func TestIntPower(t *testing.T) { 60 | assert.True(t, IntPower(2, 0) == 1) 61 | assert.True(t, IntPower(2, 4) == 16) 62 | assert.True(t, IntPower(-2, 2) == 4) 63 | assert.True(t, IntPower(-2, 3) == -8) 64 | assert.True(t, IntPower(-2, 0) == 1) 65 | } 66 | -------------------------------------------------------------------------------- /pkg/utils/set.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import ( 17 | ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" 18 | ) 19 | 20 | // Difference returns a-b, elements present in a and not in b 21 | func Difference[T comparable](a, b []T) (diff []T) { 22 | m := make(map[T]struct{}) 23 | 24 | for _, item := range b { 25 | m[item] = struct{}{} 26 | } 27 | for _, item := range a { 28 | if _, ok := m[item]; !ok { 29 | diff = append(diff, item) 30 | } 31 | } 32 | return 33 | } 34 | 35 | func GetKeyValSlice(m map[string]string) (key []string, val []string) { 36 | for k, v := range m { 37 | key = append(key, k) 38 | val = append(val, v) 39 | } 40 | return 41 | } 42 | 43 | func GetTagKeyValueMap(tagSet []ec2types.Tag) map[string]string { 44 | m := make(map[string]string) 45 | for _, tag := range tagSet { 46 | m[*tag.Key] = *tag.Value 47 | } 48 | return m 49 | } 50 | -------------------------------------------------------------------------------- /pkg/utils/set_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/stretchr/testify/assert" 20 | ) 21 | 22 | type TestResource struct { 23 | GroupID string 24 | ResourceID string 25 | } 26 | 27 | func TestDifference(t *testing.T) { 28 | a := []string{"X", "Y", "Z"} 29 | b := []string{"X", "Z", "Q"} 30 | 31 | assert.ElementsMatch(t, Difference(a, b), []string{"Y"}) 32 | assert.ElementsMatch(t, Difference(b, a), []string{"Q"}) 33 | assert.ElementsMatch(t, Difference(a, a), []string{}) 34 | } 35 | 36 | func TestDifferenceResource(t *testing.T) { 37 | res1 := TestResource{GroupID: "res1", ResourceID: "res1"} 38 | res2 := TestResource{GroupID: "res2", ResourceID: "res2"} 39 | res3 := TestResource{GroupID: "res3", ResourceID: "res3"} 40 | a := []TestResource{res1, res2} 41 | b := []TestResource{res1, res3} 42 | c := []TestResource{res3} 43 | var d []TestResource 44 | 45 | assert.ElementsMatch(t, Difference(a, b), []TestResource{res2}) 46 | assert.ElementsMatch(t, Difference(b, a), []TestResource{res3}) 47 | assert.ElementsMatch(t, Difference(a, a), []TestResource{}) 48 | assert.ElementsMatch(t, Difference(a, c), []TestResource{res1, res2}) 49 | assert.ElementsMatch(t, Difference(c, a), []TestResource{res3}) 50 | assert.ElementsMatch(t, Difference(d, a), d) 51 | assert.ElementsMatch(t, Difference(a, d), a) 52 | assert.ElementsMatch(t, Difference(d, d), d) 53 | } 54 | 55 | func TestGetKeySet(t *testing.T) { 56 | keys := []string{"a", "b", "c"} 57 | m := map[string]string{} 58 | 59 | for _, key := range keys { 60 | m[key] = key 61 | } 62 | 63 | k, v := GetKeyValSlice(m) 64 | assert.ElementsMatch(t, k, keys) 65 | assert.ElementsMatch(t, v, keys) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package version 15 | 16 | var ( 17 | GitVersion string 18 | GitCommit string 19 | BuildDate string 20 | ) 21 | -------------------------------------------------------------------------------- /pkg/worker/jobs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package worker 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/stretchr/testify/assert" 20 | "k8s.io/apimachinery/pkg/types" 21 | ) 22 | 23 | var ( 24 | podName = "pod-name" 25 | podNamespace = "pod-namespace" 26 | podUid = "pod-uid" 27 | UID = types.UID(podUid) 28 | reqCount = 2 29 | nodeName = "node-name" 30 | ) 31 | 32 | // TestNewOnDemandCreateJob tests the fields of Create Job 33 | func TestNewOnDemandCreateJob(t *testing.T) { 34 | onDemandJob := NewOnDemandCreateJob(podNamespace, podName, reqCount) 35 | 36 | assert.Equal(t, OperationCreate, onDemandJob.Operation) 37 | assert.Equal(t, podName, onDemandJob.PodName) 38 | assert.Equal(t, podNamespace, onDemandJob.PodNamespace) 39 | assert.Equal(t, reqCount, onDemandJob.RequestCount) 40 | } 41 | 42 | // TestNewOnDemandDeleteJob tests the fields of Deleted Job 43 | func TestNewOnDemandDeletedJob(t *testing.T) { 44 | onDemandJob := NewOnDemandDeletedJob(nodeName, UID) 45 | 46 | assert.Equal(t, OperationDeleted, onDemandJob.Operation) 47 | assert.Equal(t, podUid, onDemandJob.UID) 48 | } 49 | 50 | func TestNewOnDemandReconcileJob(t *testing.T) { 51 | onDemandJob := NewOnDemandReconcileNodeJob(nodeName) 52 | 53 | assert.Equal(t, OperationReconcileNode, onDemandJob.Operation) 54 | assert.Equal(t, nodeName, onDemandJob.NodeName) 55 | } 56 | 57 | func TestNewOnDemandProcessDeleteQueueJob(t *testing.T) { 58 | onDemandJob := NewOnDemandProcessDeleteQueueJob(nodeName) 59 | 60 | assert.Equal(t, OperationProcessDeleteQueue, onDemandJob.Operation) 61 | assert.Equal(t, nodeName, onDemandJob.NodeName) 62 | } 63 | 64 | func TestNewWarmPoolCreateJob(t *testing.T) { 65 | warmPoolJob := NewWarmPoolCreateJob(nodeName, 2) 66 | 67 | assert.Equal(t, OperationCreate, warmPoolJob.Operations) 68 | assert.Equal(t, 2, warmPoolJob.ResourceCount) 69 | assert.Equal(t, nodeName, warmPoolJob.NodeName) 70 | } 71 | 72 | func TestNewWarmPoolDeleteJob(t *testing.T) { 73 | resources := []string{"res-1", "res-2"} 74 | WarmPoolJob := NewWarmPoolDeleteJob(nodeName, resources) 75 | 76 | assert.Equal(t, OperationDeleted, WarmPoolJob.Operations) 77 | assert.Equal(t, nodeName, WarmPoolJob.NodeName) 78 | assert.Equal(t, resources, WarmPoolJob.Resources) 79 | assert.Equal(t, len(resources), WarmPoolJob.ResourceCount) 80 | } 81 | 82 | func TestNewWarmPoolReSyncJob(t *testing.T) { 83 | WarmPoolJob := NewWarmPoolReSyncJob(nodeName) 84 | 85 | assert.Equal(t, OperationReSyncPool, WarmPoolJob.Operations) 86 | assert.Equal(t, nodeName, WarmPoolJob.NodeName) 87 | } 88 | -------------------------------------------------------------------------------- /scripts/templates/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. -------------------------------------------------------------------------------- /scripts/templates/copyright.txt: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | not use this file except in compliance with the License. A copy of the 5 | License is located at 6 | 7 | http://aws.amazon.com/apache2.0/ 8 | 9 | or in the "license" file accompanying this file. This file is distributed 10 | on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | express or implied. See the License for the specific language governing 12 | permissions and limitations under the License. -------------------------------------------------------------------------------- /scripts/test/README.md: -------------------------------------------------------------------------------- 1 | ## Integration Script 2 | The Integration test script creates an eksctl cluster and runs the Ginkgo Integration tests on the current build from the repository. 3 | 4 | ### Usage 5 | The Integration test script will **fail to run** on accounts that is not allowlisted for ENI Trunking feature. The test script is currently used in **CI Setup** for the repository `amazon-vpc-resource-controller-k8s`. 6 | 7 | Users can still consume this script indirectly by opening a Pull Request on this repository. Once the PR is labelled okay-to-test, the users can check the status of the test under GitHub Actions. 8 | 9 | ### Script Execution Order 10 | ``` 11 | CLUSTER_NAME= 12 | K8S_VERSION= 13 | ``` 14 | 15 | - Create the EKS Cluster. 16 | ``` 17 | ./scripts/test/create-cluster.sh -n $CLUSTER_NAME -v $K8S_VERSION 18 | ``` 19 | - Create the necessary IAM Policies and Roles 20 | ``` 21 | ./scripts/test/iam-resources.sh -o create -n $CLUSTER_NAME 22 | ``` 23 | - Start test Execution 24 | ``` 25 | ./scripts/test/test-with-eksctl.sh -n $CLUSTER_NAME 26 | ``` 27 | - Delete the IAM Role and Policies 28 | ``` 29 | ./scripts/test/iam-resources.sh -o delete -n $CLUSTER_NAME 30 | ``` 31 | - Delete the EKS Cluster 32 | ``` 33 | ./scripts/test/delete-cluster.sh 34 | ``` 35 | 36 | ### Integration Test Scripts 37 | `run-canary-test.sh` runs integration tests against an existing cluster with the "CANARY" focus. 38 | `run-integration-tests.sh` runs non-local integration tests against an existing cluster. 39 | 40 | To run the above scripts against a cluster without sufficient Windows nodes, the "SKIP_WINDOWS_TEST" environment variable can be passed. 41 | 42 | ### Design 43 | 44 | #### IAM Roles 45 | 46 | The controller uses two different IAM Roles. 47 | 48 | 1. The VPCResourceControllerRole on Account A for managing (create, delete, modify..) Trunk and Branch ENI. This account doesn't have to be allowlisted for ENI Trunking. 49 | 2. The Instance/Node Role on Account B where the controller runs. It has permissions to Associate Trunk to Branch. This account must be allowlisted for ENI Trunking. 50 | 51 | In order to manage the Trunk/Branch ENI on user's behalf the Role 2 must be allowed to assume Role 1. For simplicity, Role 1 and 2 are created in the same account in this test setup. 52 | 53 | ### Future Enhancement 54 | The test script can create a Kops Kubernetes Cluster with VPC CNI Plugin. This would eliminate the workarounds we have to temporarily disable the controller on EKS while the test executes. 55 | -------------------------------------------------------------------------------- /scripts/test/create-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # The Scripts creates an eksctl cluster using a predefined template. 4 | # This cluster should have the required nodegroups for running the 5 | # Security Group for Pods and Windows IPAM integration tests. 6 | 7 | set -eo pipefail 8 | 9 | SCRIPTS_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) 10 | TEMPLATE_FILE="$SCRIPTS_DIR/template/eksctl/eks-cluster.yaml" 11 | 12 | USAGE=$(cat << 'EOM' 13 | Usage: create-cluster.sh -n [cluster-name] -v [k8s-version] 14 | 15 | Creates an EKS Cluster with Nodegroups. The Cluster is 16 | created using eksctl with a pre-defined eksctl template. 17 | 18 | Required: 19 | -n Name of the EKS cluster 20 | -v K8s Version 21 | Optional: 22 | -r Region of the EKS Cluster. Defaults to us-west-2 23 | -c eksctl Cluster config path. Defaults to build/eks-cluster.yaml 24 | -s Suffix that will be added to each resource. This is useful when 25 | running in CI Setup to prevent parallel runs from modifying same 26 | resources 27 | EOM 28 | ) 29 | 30 | source "$SCRIPTS_DIR/lib/common.sh" 31 | 32 | while getopts "n:v:r:c:s:" o; do 33 | case "${o}" in 34 | n) # Name of the EKS Cluster 35 | CLUSTER_NAME=${OPTARG} 36 | ;; 37 | v) # K8s Version of the EKS Cluster 38 | K8S_VERSION=${OPTARG} 39 | ;; 40 | r) # Region where EKS Cluster will be created 41 | AWS_REGION=${OPTARG} 42 | ;; 43 | c) # eksctl cluster config file path 44 | CLUSTER_CONFIG_PATH=${OPTARG} 45 | ;; 46 | s) # Suffix that will be attached to each AWS Resource 47 | RESOURCE_SUFFIX=${OPTARG} 48 | ;; 49 | *) 50 | echoerr "${USAGE}" 51 | exit 1 52 | ;; 53 | esac 54 | done 55 | shift $((OPTIND-1)) 56 | 57 | if [[ -z "$CLUSTER_NAME" ]]; then 58 | echoerr "${USAGE}\n\nmissing: -n is a required flag\n" 59 | exit 1 60 | fi 61 | 62 | if [[ -z "$K8S_VERSION" ]]; then 63 | echoerr "${USAGE}\n\nmissing: -v is a required flag\n" 64 | exit 1 65 | fi 66 | 67 | if [[ -z "$AWS_REGION" ]]; then 68 | AWS_REGION="us-west-2" 69 | echo "no regions defined, will fallback to default region $AWS_REGION" 70 | fi 71 | 72 | if [[ -z "$CLUSTER_CONFIG_PATH" ]]; then 73 | CLUSTER_CONFIG_DIR="$SCRIPTS_DIR/build" 74 | CLUSTER_CONFIG_PATH="$CLUSTER_CONFIG_DIR/eks-cluster.yaml" 75 | 76 | echo "cluster config path not defined, will generate it at default path $CLUSTER_CONFIG_PATH" 77 | 78 | mkdir -p "$CLUSTER_CONFIG_DIR" 79 | fi 80 | 81 | source "$SCRIPTS_DIR/lib/config.sh" 82 | 83 | check_is_installed eksctl 84 | 85 | CLUSTER_NAME=$(add_suffix "$CLUSTER_NAME") 86 | 87 | function create_eksctl_cluster() { 88 | echo "creating a $K8S_VERSION cluster named $CLUSTER_NAME using eksctl" 89 | 90 | # Using a static Instance Role Name so IAM Policies 91 | # can be attached to it later without having to retrieve 92 | # the auto generated eksctl name. 93 | sed "s/CLUSTER_NAME/$CLUSTER_NAME/g; 94 | s/CLUSTER_REGION/$AWS_REGION/g; 95 | s/K8S_VERSION/$K8S_VERSION/g; 96 | s/INSTANCE_ROLE_NAME/$INSTANCE_ROLE_NAME/g" \ 97 | "$TEMPLATE_FILE" > "$CLUSTER_CONFIG_PATH" 98 | 99 | eksctl create cluster -f "$CLUSTER_CONFIG_PATH" 100 | } 101 | 102 | create_eksctl_cluster 103 | -------------------------------------------------------------------------------- /scripts/test/delete-cluster-karpenter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Delete EKS cluster & related resources created via script create-cluster-karpenter.sh 4 | set -eo pipefail 5 | 6 | SCRIPTS_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) 7 | source "$SCRIPTS_DIR/lib/common.sh" 8 | check_is_installed helm 9 | check_is_installed eksctl 10 | check_is_installed jq 11 | check_is_installed aws 12 | 13 | export KARPENTER_NAMESPACE="kube-system" 14 | export CLUSTER_NAME="${USER}-sgp-scaletest" # Update cluster name if it is different 15 | echo "Uninstalling Karpenter" 16 | helm uninstall karpenter --namespace "${KARPENTER_NAMESPACE}" 17 | echo "Deleting Karpenter CFN stack" 18 | aws cloudformation delete-stack --stack-name "Karpenter-${CLUSTER_NAME}" 19 | aws ec2 describe-launch-templates --filters "Name=tag:karpenter.k8s.aws/cluster,Values=${CLUSTER_NAME}" | 20 | jq -r ".LaunchTemplates[].LaunchTemplateName" | 21 | xargs -I{} aws ec2 delete-launch-template --launch-template-name {} 22 | echo "Deleting EKS cluster" 23 | eksctl delete cluster --name "${CLUSTER_NAME}" -------------------------------------------------------------------------------- /scripts/test/delete-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Deletes an EKS Cluster and it's nodegroup created using 4 | # eksctl. The script requires the cluster config path to delete 5 | # all resources associated with the cluster. 6 | 7 | set -eo pipefail 8 | 9 | USAGE=$(cat << 'EOM' 10 | Usage: delete-cluster.sh -c [cluster-config] 11 | 12 | Deletes an EKS Cluster along with the Nodegroups. Takes in 13 | the cluster config as an optional argument. If no arugment is 14 | provided then it will use the cluster config in the default 15 | path i.e /build/eks-cluster.yaml 16 | 17 | Optional: 18 | -c eksctl Cluster config path. Defaults to build/eks-cluster.yaml 19 | EOM 20 | ) 21 | 22 | SCRIPTS_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) 23 | CLUSTER_CONFIG_DIR="$SCRIPTS_DIR/build" 24 | CLUSTER_CONFIG_PATH="$CLUSTER_CONFIG_DIR/eks-cluster.yaml" 25 | 26 | source "$SCRIPTS_DIR/lib/common.sh" 27 | 28 | check_is_installed eksctl 29 | 30 | while getopts "c:" o; do 31 | case "${o}" in 32 | c) # eksctl cluster config file path 33 | CLUSTER_CONFIG_PATH=${OPTARG} 34 | ;; 35 | *) 36 | echoerr "${USAGE}" 37 | exit 1 38 | ;; 39 | esac 40 | done 41 | shift $((OPTIND-1)) 42 | 43 | eksctl delete cluster -f "$CLUSTER_CONFIG_PATH" --wait 44 | -------------------------------------------------------------------------------- /scripts/test/install-cert-manager.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Installs a stable version of cert-manager 4 | 5 | set -eo pipefail 6 | 7 | SCRIPTS_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) 8 | DEFAULT_CERT_MANAGER_VERSION="v1.4.0" 9 | 10 | source "$SCRIPTS_DIR/lib/k8s.sh" 11 | source "$SCRIPTS_DIR/lib/common.sh" 12 | 13 | check_is_installed kubectl "You can install kubectl with the helper scripts/install-kubectl.sh" 14 | 15 | __cert_manager_version="$1" 16 | if [ "z$__cert_manager_version" == "z" ]; then 17 | __cert_manager_version=${CERT_MANAGER_VERSION:-$DEFAULT_CERT_MANAGER_VERSION} 18 | fi 19 | 20 | install() { 21 | echo -n "installing cert-manager ... " 22 | __cert_manager_url="https://github.com/jetstack/cert-manager/releases/download/${__cert_manager_version}/cert-manager.yaml" 23 | echo -n "installing cert-manager from $__cert_manager_url ... " 24 | kubectl apply --validate=false -f $__cert_manager_url 25 | echo "ok." 26 | } 27 | 28 | check() { 29 | echo -n "checking cert-manager deployments have rolled out ... " 30 | local __ns="cert-manager" 31 | local __timeout="4m" 32 | check_deployment_rollout cert-manager-webhook $__ns $__timeout 33 | check_deployment_rollout cert-manager $__ns 34 | check_deployment_rollout cert-manager-cainjector $__ns 35 | echo "ok." 36 | } 37 | 38 | install 39 | check 40 | -------------------------------------------------------------------------------- /scripts/test/install-vpc-cni.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Installs a stable version of vpc-cni 4 | 5 | set -eo pipefail 6 | 7 | SCRIPTS_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) 8 | DEFAULT_VPC_CNI_VERSION="1.11" 9 | 10 | source "$SCRIPTS_DIR/lib/k8s.sh" 11 | source "$SCRIPTS_DIR/lib/common.sh" 12 | 13 | check_is_installed kubectl 14 | 15 | __vpc_cni_version="$1" 16 | if [ -z "$__vpc_cni_version" ]; then 17 | __vpc_cni_version=${VPC_CNI_VERSION:-$DEFAULT_VPC_CNI_VERSION} 18 | fi 19 | 20 | install() { 21 | 22 | echo "installing vpc cni $__vpc_cni_version" 23 | # install the latest patch version for the release 24 | __vpc_cni_url="https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-$__vpc_cni_version/config/master/aws-k8s-cni.yaml" 25 | kubectl apply -f $__vpc_cni_url 26 | echo "ok.." 27 | } 28 | 29 | check() { 30 | echo "checking vpc cni version has rolled out" 31 | check_ds_rollout "aws-node" "kube-system" 32 | echo "ok.." 33 | } 34 | 35 | install 36 | check -------------------------------------------------------------------------------- /scripts/test/lib/aws.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | AWS_CLI_VERSION=$(aws --version 2>&1 | cut -d/ -f2 | cut -d. -f1) 4 | 5 | ecr_login() { 6 | echo -n "ecr login ... " 7 | local __aws_region=$1 8 | local __ecr_url=$2 9 | local __err_msg="Failed ECR login. Please make sure you have IAM permissions to access ECR." 10 | if [ $AWS_CLI_VERSION -gt 1 ]; then 11 | ( aws ecr get-login-password --region $__aws_region | \ 12 | docker login --username AWS --password-stdin $__ecr_url ) || 13 | ( echo "\n$__err_msg" && exit 1 ) 14 | else 15 | $( aws ecr get-login --no-include-email ) || ( echo "\n$__err_msg" && exit 1 ) 16 | fi 17 | echo "ok." 18 | } 19 | 20 | create_repository() { 21 | local __repository_name=$1 22 | 23 | if ! (aws ecr describe-repositories --repository-name "$__repository_name"); then 24 | echo "creating new repository, $__repository_name" 25 | aws ecr create-repository --repository-name "$__repository_name" 26 | else 27 | echo "repository already exists $__repository_name" 28 | fi 29 | } 30 | 31 | delete_ecr_image() { 32 | local __ecr_repository=$1 33 | local __image_tag=$2 34 | 35 | aws ecr batch-delete-image \ 36 | --repository-name "$__ecr_repository" \ 37 | --image-ids imageTag="$__image_tag" 38 | } 39 | 40 | create_policy() { 41 | local __policy_name=$1 42 | local __policy_document=$2 43 | 44 | echo "creating IAM policy $__policy_arn" 45 | aws iam create-policy --policy-name "$__policy_name" --policy-document "file:///$__policy_document" 46 | } 47 | 48 | delete_policy() { 49 | local __policy_arn=$1 50 | 51 | echo "deleting IAM policy $__policy_arn" 52 | aws iam delete-policy --policy-arn "$__policy_arn" 53 | } 54 | 55 | create_role() { 56 | local __role_name=$1 57 | local __trust_policy_doc=$2 58 | 59 | if [[ -z $__trust_policy_doc ]]; then 60 | echo "creating IAM role $__role_name" 61 | aws iam create-role --role-name "$__role_name" 62 | else 63 | echo "creating IAM role $__role_name with trust policy $__trust_policy_doc" 64 | aws iam create-role --role-name "$__role_name" --assume-role-policy-document "file:///$__trust_policy_doc" 65 | fi 66 | } 67 | 68 | delete_role() { 69 | local __role_name=$1 70 | 71 | echo "deleting IAM role $__role_name" 72 | aws iam delete-role --role-name "$__role_name" 73 | } 74 | 75 | attach_policy() { 76 | local __policy_arn=$1 77 | local __role_name=$2 78 | 79 | echo "attaching policy $__policy_arn to role $__role_name" 80 | aws iam attach-role-policy --policy-arn "$__policy_arn" --role-name "$__role_name" 81 | } 82 | 83 | detach_policy() { 84 | local __policy_arn=$1 85 | local __role_name=$2 86 | 87 | echo "detaching policy $__policy_arn to role $__role_name" 88 | aws iam detach-role-policy --policy-arn "$__policy_arn" --role-name "$__role_name" 89 | } 90 | -------------------------------------------------------------------------------- /scripts/test/lib/cluster.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | LIB_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 4 | ENDPOINT_FLAG="" 5 | 6 | source "$LIB_DIR"/k8s.sh 7 | 8 | if [[ -n "${ENDPOINT}" ]]; then 9 | ENDPOINT_FLAG="--endpoint $ENDPOINT" 10 | fi 11 | 12 | function load_cluster_details() { 13 | CLUSTER_INFO=$(aws eks describe-cluster --name $CLUSTER_NAME --region $REGION $ENDPOINT_FLAG) 14 | VPC_ID=$(echo $CLUSTER_INFO | jq -r '.cluster.resourcesVpcConfig.vpcId') 15 | SERVICE_ROLE_ARN=$(echo $CLUSTER_INFO | jq -r '.cluster.roleArn') 16 | K8S_VERSION=$(echo $CLUSTER_INFO | jq -r '.cluster.version') 17 | ROLE_NAME=${SERVICE_ROLE_ARN##*/} 18 | 19 | echo "VPC ID: $VPC_ID, Service Role ARN: $SERVICE_ROLE_ARN, Role Name: $ROLE_NAME" 20 | } 21 | 22 | function load_deveks_cluster_details() { 23 | 24 | PROVIDER_ID=$(kubectl get nodes -ojson | jq -r '.items[0].spec.providerID') 25 | INSTANCE_ID=${PROVIDER_ID##*/} 26 | VPC_ID=$(aws ec2 describe-instances --instance-ids ${INSTANCE_ID} --no-cli-pager | jq -r '.Reservations[].Instances[].VpcId') 27 | REGION_NAME=$(echo $REGION | tr -d '-') 28 | ROLE_NAME="deveks-${CLUSTER_NAME}-cluster-ServiceRole-${REGION_NAME}" 29 | ACCOUNT_ID=$(aws sts get-caller-identity | jq -r '.Account') 30 | SERVICE_ROLE_ARN="arn:aws:iam::${ACCOUNT_ID}:role/${ROLE_NAME}" 31 | 32 | echo "VPC ID: $VPC_ID, Service Role ARN: $SERVICE_ROLE_ARN, Role Name: $ROLE_NAME" 33 | } 34 | 35 | # This operation fails with rate limit exceeded when test is running for multiple K8s 36 | # version at same time, hence we increase the exponential retries on the aws call 37 | function attach_controller_policy_cluster_role() { 38 | echo "Attaching IAM Policy to Cluster Service Role" 39 | AWS_MAX_ATTEMPTS=10 aws iam attach-role-policy \ 40 | --policy-arn arn:aws:iam::aws:policy/AmazonEKSVPCResourceController \ 41 | --role-name "$ROLE_NAME" > /dev/null 42 | } 43 | 44 | function detach_controller_policy_cluster_role() { 45 | echo "Detaching the IAM Policy from Cluster Service Role" 46 | aws iam detach-role-policy \ 47 | --policy-arn arn:aws:iam::aws:policy/AmazonEKSVPCResourceController \ 48 | --role-name $ROLE_NAME > /dev/null 49 | } 50 | 51 | function set_env_aws_node() { 52 | local KEY=$1 53 | local VAL=$2 54 | 55 | echo "Setting environment variable $KEY to $VAL on aws-node" 56 | kubectl set env daemonset aws-node -n kube-system $KEY=$VAL 57 | check_ds_rollout "aws-node" "kube-system" 58 | } 59 | -------------------------------------------------------------------------------- /scripts/test/lib/common.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echoerr() { echo -e "$@" 1>&2; } 4 | 5 | check_is_installed() { 6 | local __name="$1" 7 | local __extra_msg="$2" 8 | if ! is_installed "$__name"; then 9 | echo "FATAL: Missing requirement '$__name'" 10 | echo "Please install $__name before running this script." 11 | if [[ -n $__extra_msg ]]; then 12 | echo "" 13 | echo "$__extra_msg" 14 | echo "" 15 | fi 16 | exit 1 17 | fi 18 | } 19 | 20 | is_installed() { 21 | local __name="$1" 22 | if $(which $__name >/dev/null 2>&1); then 23 | return 0 24 | else 25 | return 1 26 | fi 27 | } 28 | -------------------------------------------------------------------------------- /scripts/test/lib/config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Add Suffix to each AWS Resource Name if supplied. 4 | # This is done to allow multiple concurrent runs on 5 | # same accounts from modifying same resources 6 | function add_suffix() { 7 | local __resource=$1 8 | if [[ $RESOURCE_SUFFIX ]]; then 9 | echo "$__resource-$RESOURCE_SUFFIX" 10 | else 11 | echo "$__resource" 12 | fi 13 | } 14 | 15 | # IAM Role Name for Linux Node Role where VPC Resource Controller Runs. It should 16 | # have the Trunk Association Policy 17 | TRUNK_ASSOC_POLICY_NAME=$(add_suffix "AssociateTrunkInterfacePolicy") 18 | INSTANCE_ROLE_NAME=$(add_suffix "LinuxNodeRole") 19 | 20 | # IAM Role and it's Policy Names which have the permission to manage Trunk/Branch 21 | # Interfaces 22 | VPC_RC_POLICY_NAME=$(add_suffix "VPCResourceControllerPolicy") 23 | VPC_RC_ROLE_NAME=$(add_suffix "VPCResourceControllerRole") 24 | -------------------------------------------------------------------------------- /scripts/test/lib/k8s.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # resource_exists returns 0 when the supplied resource can be found, 1 4 | # otherwise. An optional second parameter overrides the Kubernetes namespace 5 | # argument 6 | k8s_resource_exists() { 7 | local __res_name=${1:-} 8 | local __namespace=${2:-} 9 | local __args="" 10 | if [ -n "$__namespace" ]; then 11 | __args="$__args-n $__namespace" 12 | fi 13 | kubectl get $__args "$__res_name" >/dev/null 2>&1 14 | } 15 | 16 | 17 | # check_deployment_rollout watches the status of the latest rollout 18 | # until it's done or until the timeout. Namespace and timeout are optional 19 | # parameters 20 | check_deployment_rollout() { 21 | local __dep_name=${1:-} 22 | local __namespace=${2:-} 23 | local __timeout=${3:-"2m"} 24 | local __args="" 25 | if [ -n "$__namespace" ]; then 26 | __args="$__args-n $__namespace" 27 | fi 28 | kubectl rollout status deployment/"$__dep_name" $__args --timeout=$__timeout 29 | } 30 | 31 | # check_ds_rollout watches the status of the latest rollout until it's done or 32 | # until the timeout. Namespace and timeout are optional parameters 33 | check_ds_rollout() { 34 | local __ds_name=${1:-} 35 | local __namespace=${2:-} 36 | local __timeout=${3:-"2m"} 37 | local __args="" 38 | if [ -n "$__namespace" ]; then 39 | __args="$__args-n $__namespace" 40 | fi 41 | kubectl rollout status ds/"$__ds_name" $__args --timeout=$__timeout 42 | } -------------------------------------------------------------------------------- /scripts/test/run-integration-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Script to run vpc-resource-controller release tests: webhook, perpodsg, windows integration tests 4 | # This script does not install any addons nor update vpc-resource-controller. Please install all 5 | # required versions to be tests prior to running the script. 6 | 7 | # Parameters: 8 | # CLUSTER_NAME: name of the cluster 9 | # KUBE_CONFIG_PATH: path to the kubeconfig file, default ~/.kube/config 10 | # REGION: default us-west-2 11 | # RUN_DEVEKS_TEST: false 12 | 13 | set -euoE pipefail 14 | 15 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 16 | INTEGRATION_TEST_DIR="$SCRIPT_DIR/../../test/integration" 17 | SECONDS=0 18 | 19 | : "${RUN_DEVEKS_TEST:=false}" 20 | : "${ENDPOINT:=""}" 21 | : "${SKIP_WINDOWS_TEST:=""}" 22 | : "${EXTRA_GINKGO_FLAGS:=""}" 23 | 24 | source "$SCRIPT_DIR"/lib/cluster.sh 25 | 26 | cleanup(){ 27 | 28 | if [[ $? == 0 ]]; then 29 | echo "Successfully ran all tests in $(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds" 30 | else 31 | echo "[Error] Integration tests failed" 32 | fi 33 | 34 | echo "Cleaning up the setup" 35 | set_env_aws_node "ENABLE_POD_ENI" "false" 36 | detach_controller_policy_cluster_role 37 | } 38 | 39 | trap cleanup EXIT 40 | 41 | function run_integration_tests(){ 42 | TEST_RESULT=success 43 | (cd $INTEGRATION_TEST_DIR/perpodsg && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=35m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail 44 | if [[ -z "${SKIP_WINDOWS_TEST}" ]]; then 45 | (cd $INTEGRATION_TEST_DIR/windows && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=150m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail 46 | else 47 | echo "skipping Windows tests" 48 | fi 49 | (cd $INTEGRATION_TEST_DIR/webhook && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=5m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail 50 | (cd $INTEGRATION_TEST_DIR/cninode && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=10m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail 51 | 52 | if [[ "$TEST_RESULT" == fail ]]; then 53 | exit 1 54 | fi 55 | } 56 | 57 | echo "Running VPC Resource Controller integration test with the following variables 58 | KUBE CONFIG: $KUBE_CONFIG_PATH 59 | CLUSTER_NAME: $CLUSTER_NAME 60 | REGION: $REGION" 61 | 62 | if [[ "${RUN_DEVEKS_TEST}" == "true" ]];then 63 | load_deveks_cluster_details 64 | else 65 | load_cluster_details 66 | fi 67 | 68 | attach_controller_policy_cluster_role 69 | set_env_aws_node "ENABLE_POD_ENI" "true" 70 | run_integration_tests 71 | -------------------------------------------------------------------------------- /scripts/test/template/eksctl/eks-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: eksctl.io/v1alpha5 2 | kind: ClusterConfig 3 | metadata: 4 | name: CLUSTER_NAME 5 | region: CLUSTER_REGION 6 | version: "K8S_VERSION" 7 | nodeGroups: 8 | - name: linux-instance 9 | instanceType: c5.large # For SGP Tests, this must be a Nitro-based Instance 10 | desiredCapacity: 3 11 | iam: 12 | instanceRoleName: INSTANCE_ROLE_NAME # Static IAM Role Name for easily adding IAM Policies after creation 13 | attachPolicyARNs: 14 | - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly 15 | - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy 16 | - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy 17 | - name: windows-instance 18 | amiFamily: WindowsServer2019FullContainer 19 | instanceType: c5.4xlarge # Larger instance type since the Pod density on Windows node is less 20 | desiredCapacity: 3 21 | 22 | -------------------------------------------------------------------------------- /scripts/test/template/iam/associate-trunk-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "ec2:AssociateTrunkInterface" 8 | ], 9 | "Resource": [ 10 | "*" 11 | ] 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /scripts/test/template/iam/aws-trust-relationship.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Principal": { 7 | "AWS": "AWS_ROLE_ARN" 8 | }, 9 | "Action": "sts:AssumeRole" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /scripts/test/template/iam/vpc-resource-controller-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": "ec2:CreateNetworkInterfacePermission", 7 | "Resource": "*", 8 | "Condition": { 9 | "ForAnyValue:StringEquals": { 10 | "ec2:ResourceTag/eks:eni:owner": "eks-vpc-resource-controller" 11 | } 12 | } 13 | }, 14 | { 15 | "Effect": "Allow", 16 | "Action": [ 17 | "ec2:CreateNetworkInterface", 18 | "ec2:DetachNetworkInterface", 19 | "ec2:ModifyNetworkInterfaceAttribute", 20 | "ec2:DeleteNetworkInterface", 21 | "ec2:AttachNetworkInterface", 22 | "ec2:UnassignPrivateIpAddresses", 23 | "ec2:AssignPrivateIpAddresses" 24 | ], 25 | "Resource": "*" 26 | }, 27 | { 28 | "Effect": "Allow", 29 | "Action": [ 30 | "ec2:CreateTags", 31 | "ec2:DescribeNetworkInterfaces", 32 | "ec2:DescribeInstances", 33 | "ec2:DescribeSubnets" 34 | ], 35 | "Resource": "*" 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /scripts/test/template/rbac/cp-vpc-leader-election-role-patch.yaml: -------------------------------------------------------------------------------- 1 | # The patch doesn't have policies to patch/update the leader lease, thereby disabling the 2 | # EKS controller to take the leader lease while the test runs 3 | rules: 4 | - apiGroups: 5 | - "" 6 | resources: 7 | - configmaps 8 | verbs: 9 | - create 10 | - list 11 | - watch 12 | - apiGroups: 13 | - "" 14 | resourceNames: 15 | - cp-vpc-resource-controller 16 | resources: 17 | - configmaps 18 | verbs: 19 | - get 20 | - apiGroups: 21 | - "" 22 | resources: 23 | - events 24 | verbs: 25 | - create 26 | -------------------------------------------------------------------------------- /test/framework/manifest/configmap.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package manifest 15 | 16 | import ( 17 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" 18 | v1 "k8s.io/api/core/v1" 19 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 | ) 21 | 22 | type ConfigMapBuilder struct { 23 | namespace string 24 | name string 25 | data map[string]string 26 | } 27 | 28 | func NewConfigMapBuilder() *ConfigMapBuilder { 29 | return &ConfigMapBuilder{ 30 | namespace: "kube-system", 31 | name: "amazon-vpc-cni", 32 | data: map[string]string{config.EnableWindowsIPAMKey: "true"}, 33 | } 34 | } 35 | func (c *ConfigMapBuilder) Build() *v1.ConfigMap { 36 | return &v1.ConfigMap{ 37 | ObjectMeta: metav1.ObjectMeta{ 38 | Name: c.name, 39 | Namespace: c.namespace, 40 | }, 41 | Data: c.data, 42 | } 43 | } 44 | 45 | func (c *ConfigMapBuilder) Name(name string) *ConfigMapBuilder { 46 | c.name = name 47 | return c 48 | } 49 | func (c *ConfigMapBuilder) Namespace(namespace string) *ConfigMapBuilder { 50 | c.namespace = namespace 51 | return c 52 | } 53 | func (c *ConfigMapBuilder) Data(data map[string]string) *ConfigMapBuilder { 54 | c.data = data 55 | return c 56 | } 57 | -------------------------------------------------------------------------------- /test/framework/manifest/container.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package manifest 15 | 16 | import ( 17 | v1 "k8s.io/api/core/v1" 18 | ) 19 | 20 | type Container struct { 21 | name string 22 | image string 23 | imagePullPolicy v1.PullPolicy 24 | command []string 25 | args []string 26 | containerPorts []v1.ContainerPort 27 | requirements v1.ResourceRequirements 28 | } 29 | 30 | func NewBusyBoxContainerBuilder() *Container { 31 | return &Container{ 32 | name: "busybox", 33 | image: "busybox", 34 | imagePullPolicy: v1.PullIfNotPresent, 35 | command: []string{"sleep", "3600"}, 36 | args: []string{}, 37 | } 38 | } 39 | 40 | func NewWindowsContainerBuilder() *Container { 41 | return &Container{ 42 | name: "windows-container", 43 | image: "mcr.microsoft.com/windows/servercore:ltsc2019", 44 | imagePullPolicy: v1.PullIfNotPresent, 45 | command: []string{"powershell.exe"}, 46 | args: []string{"Start-Sleep -s 3600"}, 47 | } 48 | } 49 | 50 | func (w *Container) Name(name string) *Container { 51 | w.name = name 52 | return w 53 | } 54 | 55 | func (w *Container) Image(image string) *Container { 56 | w.image = image 57 | return w 58 | } 59 | 60 | func (w *Container) ImagePullPolicy(policy v1.PullPolicy) *Container { 61 | w.imagePullPolicy = policy 62 | return w 63 | } 64 | 65 | func (w *Container) Command(cmd []string) *Container { 66 | w.command = cmd 67 | return w 68 | } 69 | 70 | func (w *Container) Args(arg []string) *Container { 71 | w.args = arg 72 | return w 73 | } 74 | 75 | func (w *Container) AddContainerPort(containerPort v1.ContainerPort) *Container { 76 | w.containerPorts = append(w.containerPorts, containerPort) 77 | return w 78 | } 79 | 80 | func (w *Container) Resources(requirements v1.ResourceRequirements) *Container { 81 | w.requirements = requirements 82 | return w 83 | } 84 | 85 | func (w *Container) Build() v1.Container { 86 | return v1.Container{ 87 | Name: w.name, 88 | Image: w.image, 89 | Command: w.command, 90 | Args: w.args, 91 | ImagePullPolicy: w.imagePullPolicy, 92 | Ports: w.containerPorts, 93 | Resources: w.requirements, 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /test/framework/manifest/eniconfig.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package manifest 15 | 16 | import ( 17 | "fmt" 18 | 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 20 | 21 | "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" 22 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | ) 24 | 25 | type ENIConfigBuilder struct { 26 | name string 27 | subnetID string 28 | securityGroup []string 29 | } 30 | 31 | func NewENIConfigBuilder() *ENIConfigBuilder { 32 | return &ENIConfigBuilder{ 33 | name: utils.ResourceNamePrefix + "eniConfig", 34 | } 35 | } 36 | 37 | func (e *ENIConfigBuilder) Name(name string) *ENIConfigBuilder { 38 | e.name = name 39 | return e 40 | } 41 | 42 | func (e *ENIConfigBuilder) SubnetID(subnetID string) *ENIConfigBuilder { 43 | e.subnetID = subnetID 44 | return e 45 | } 46 | 47 | func (e *ENIConfigBuilder) SecurityGroup(securityGroup []string) *ENIConfigBuilder { 48 | e.securityGroup = securityGroup 49 | return e 50 | } 51 | 52 | func (e *ENIConfigBuilder) Build() (*v1alpha1.ENIConfig, error) { 53 | if e.subnetID == "" { 54 | return nil, fmt.Errorf("subnet id is a required field") 55 | } 56 | 57 | return &v1alpha1.ENIConfig{ 58 | ObjectMeta: v1.ObjectMeta{ 59 | Name: e.name, 60 | }, 61 | Spec: v1alpha1.ENIConfigSpec{ 62 | SecurityGroups: e.securityGroup, 63 | Subnet: e.subnetID, 64 | }, 65 | }, nil 66 | } 67 | -------------------------------------------------------------------------------- /test/framework/manifest/service.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package manifest 15 | 16 | import ( 17 | v1 "k8s.io/api/core/v1" 18 | metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" 19 | "k8s.io/apimachinery/pkg/util/intstr" 20 | ) 21 | 22 | type ServiceBuilder struct { 23 | name string 24 | namespace string 25 | port int32 26 | nodePort int32 27 | protocol v1.Protocol 28 | selector map[string]string 29 | serviceType v1.ServiceType 30 | } 31 | 32 | func NewHTTPService() *ServiceBuilder { 33 | return &ServiceBuilder{ 34 | port: 80, 35 | protocol: v1.ProtocolTCP, 36 | selector: map[string]string{}, 37 | } 38 | } 39 | 40 | func (s *ServiceBuilder) Name(name string) *ServiceBuilder { 41 | s.name = name 42 | return s 43 | } 44 | 45 | func (s *ServiceBuilder) Namespace(namespace string) *ServiceBuilder { 46 | s.namespace = namespace 47 | return s 48 | } 49 | 50 | func (s *ServiceBuilder) Port(port int32) *ServiceBuilder { 51 | s.port = port 52 | return s 53 | } 54 | 55 | func (s *ServiceBuilder) NodePort(nodePort int32) *ServiceBuilder { 56 | s.nodePort = nodePort 57 | return s 58 | } 59 | 60 | func (s *ServiceBuilder) Protocol(protocol v1.Protocol) *ServiceBuilder { 61 | s.protocol = protocol 62 | return s 63 | } 64 | 65 | func (s *ServiceBuilder) Selector(labelKey string, labelVal string) *ServiceBuilder { 66 | s.selector[labelKey] = labelVal 67 | return s 68 | } 69 | 70 | func (s *ServiceBuilder) ServiceType(serviceType v1.ServiceType) *ServiceBuilder { 71 | s.serviceType = serviceType 72 | return s 73 | } 74 | 75 | func (s *ServiceBuilder) Build() v1.Service { 76 | return v1.Service{ 77 | ObjectMeta: metaV1.ObjectMeta{ 78 | Name: s.name, 79 | Namespace: s.namespace, 80 | }, 81 | Spec: v1.ServiceSpec{ 82 | Ports: []v1.ServicePort{{ 83 | Name: "", 84 | Protocol: v1.ProtocolTCP, 85 | Port: s.port, 86 | TargetPort: intstr.IntOrString{IntVal: s.port}, 87 | NodePort: s.nodePort, 88 | }}, 89 | Selector: s.selector, 90 | Type: s.serviceType, 91 | }, 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /test/framework/manifest/serviceaccount.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package manifest 15 | 16 | import ( 17 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 18 | 19 | v1 "k8s.io/api/core/v1" 20 | metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | type ServiceAccountBuilder struct { 24 | namespace string 25 | name string 26 | labels map[string]string 27 | } 28 | 29 | func NewServiceAccountBuilder() *ServiceAccountBuilder { 30 | return &ServiceAccountBuilder{ 31 | namespace: "default", 32 | name: utils.ResourceNamePrefix + "sa", 33 | } 34 | } 35 | 36 | func (s *ServiceAccountBuilder) Name(name string) *ServiceAccountBuilder { 37 | s.name = name 38 | return s 39 | } 40 | 41 | func (s *ServiceAccountBuilder) Namespace(namespace string) *ServiceAccountBuilder { 42 | s.namespace = namespace 43 | return s 44 | } 45 | 46 | func (s *ServiceAccountBuilder) Label(labelKey string, labelValue string) *ServiceAccountBuilder { 47 | if s.labels == nil { 48 | s.labels = map[string]string{} 49 | } 50 | s.labels[labelKey] = labelValue 51 | return s 52 | } 53 | 54 | func (s *ServiceAccountBuilder) Build() *v1.ServiceAccount { 55 | return &v1.ServiceAccount{ 56 | ObjectMeta: metaV1.ObjectMeta{ 57 | Name: s.name, 58 | Namespace: s.namespace, 59 | Labels: s.labels, 60 | }, 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /test/framework/options.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package framework 15 | 16 | import ( 17 | "flag" 18 | "os" 19 | "strings" 20 | 21 | "github.com/pkg/errors" 22 | "k8s.io/client-go/tools/clientcmd" 23 | ) 24 | 25 | var GlobalOptions Options 26 | 27 | func init() { 28 | GlobalOptions.BindFlags() 29 | } 30 | 31 | type Options struct { 32 | KubeConfig string 33 | ClusterName string 34 | AWSRegion string 35 | AWSVPCID string 36 | ReleasedImageVersion string 37 | ClusterRoleArn string 38 | } 39 | 40 | func (options *Options) BindFlags() { 41 | flag.StringVar(&options.KubeConfig, "cluster-kubeconfig", "", "Path to kubeconfig containing embedded authinfo (required)") 42 | flag.StringVar(&options.ClusterName, "cluster-name", "", `Kubernetes cluster name (required)`) 43 | flag.StringVar(&options.AWSRegion, "aws-region", "", `AWS Region for the kubernetes cluster`) 44 | flag.StringVar(&options.AWSVPCID, "aws-vpc-id", "", `AWS VPC ID for the kubernetes cluster`) 45 | flag.StringVar(&options.ReleasedImageVersion, "latest-released-rc-image-tag", "v1.1.3", `VPC RC latest released image`) 46 | flag.StringVar(&options.ClusterRoleArn, "cluster-role-arn", "", "EKS Cluster role ARN") 47 | } 48 | 49 | func (options *Options) Validate() error { 50 | if len(options.KubeConfig) == 0 { 51 | return errors.Errorf("%s must be set!", clientcmd.RecommendedConfigPathFlag) 52 | } 53 | if len(options.ClusterName) == 0 { 54 | return errors.Errorf("%s must be set!", "cluster-name") 55 | } 56 | if len(options.AWSRegion) == 0 { 57 | return errors.Errorf("%s must be set!", "aws-region") 58 | } 59 | if len(options.AWSVPCID) == 0 { 60 | return errors.Errorf("%s must be set!", "aws-vpc-id") 61 | } 62 | if len(options.ReleasedImageVersion) == 0 { 63 | return errors.Errorf("%s must be set!", "latest-released-rc-image-tag") 64 | } 65 | dir, err := os.Executable() 66 | if err == nil && len(options.ClusterRoleArn) == 0 && strings.Contains(dir, "ec2api") { 67 | return errors.Errorf("%s must be set when running ec2api tests", "cluster-role-arn") 68 | } 69 | 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /test/framework/resource/aws/autoscaling/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package autoscaling 15 | 16 | import ( 17 | "context" 18 | "fmt" 19 | 20 | "github.com/aws/aws-sdk-go-v2/aws" 21 | "github.com/aws/aws-sdk-go-v2/service/autoscaling" 22 | autoscalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" 23 | ) 24 | 25 | type Manager interface { 26 | DescribeAutoScalingGroup(autoScalingGroupName string) ([]autoscalingtypes.AutoScalingGroup, error) 27 | UpdateAutoScalingGroup(asgName string, desiredSize, minSize, maxSize int32) error 28 | } 29 | 30 | type defaultManager struct { 31 | AutoScalingAPI *autoscaling.Client 32 | } 33 | 34 | func NewManager(cfg aws.Config) Manager { 35 | return &defaultManager{ 36 | AutoScalingAPI: autoscaling.NewFromConfig(cfg), 37 | } 38 | } 39 | 40 | func (d defaultManager) DescribeAutoScalingGroup(autoScalingGroupName string) ([]autoscalingtypes.AutoScalingGroup, error) { 41 | describeAutoScalingGroupIp := &autoscaling.DescribeAutoScalingGroupsInput{ 42 | AutoScalingGroupNames: []string{autoScalingGroupName}, 43 | } 44 | asg, err := d.AutoScalingAPI.DescribeAutoScalingGroups(context.TODO(), describeAutoScalingGroupIp) 45 | if err != nil { 46 | return nil, err 47 | } 48 | if len(asg.AutoScalingGroups) == 0 { 49 | return nil, fmt.Errorf("failed to find asg %s", autoScalingGroupName) 50 | } 51 | 52 | return asg.AutoScalingGroups, nil 53 | } 54 | 55 | func (d defaultManager) UpdateAutoScalingGroup(asgName string, desiredSize, minSize, maxSize int32) error { 56 | updateASGInput := &autoscaling.UpdateAutoScalingGroupInput{ 57 | AutoScalingGroupName: aws.String(asgName), 58 | DesiredCapacity: aws.Int32(desiredSize), 59 | MaxSize: aws.Int32(maxSize), 60 | MinSize: aws.Int32(minSize), 61 | } 62 | _, err := d.AutoScalingAPI.UpdateAutoScalingGroup(context.TODO(), updateASGInput) 63 | return err 64 | } 65 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/configmap/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package configmap 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 20 | v1 "k8s.io/api/core/v1" 21 | "sigs.k8s.io/controller-runtime/pkg/client" 22 | ) 23 | 24 | type Manager interface { 25 | CreateConfigMap(ctx context.Context, configMap *v1.ConfigMap) error 26 | GetConfigMap(ctx context.Context, configMap *v1.ConfigMap) (*v1.ConfigMap, error) 27 | UpdateConfigMap(ctx context.Context, configMap *v1.ConfigMap) error 28 | DeleteConfigMap(ctx context.Context, configMap *v1.ConfigMap) error 29 | } 30 | 31 | type defaultManager struct { 32 | k8sClient client.Client 33 | } 34 | 35 | func NewManager(k8sClient client.Client) Manager { 36 | return &defaultManager{k8sClient: k8sClient} 37 | } 38 | 39 | func (d *defaultManager) GetConfigMap(ctx context.Context, configMap *v1.ConfigMap) (*v1.ConfigMap, error) { 40 | // Create configmap if it doesn't exist and if flag not true 41 | observedConfigMap := &v1.ConfigMap{} 42 | err := d.k8sClient.Get(ctx, utils.NamespacedName(configMap), observedConfigMap) 43 | return observedConfigMap, err 44 | 45 | } 46 | 47 | func (d *defaultManager) CreateConfigMap(ctx context.Context, configMap *v1.ConfigMap) error { 48 | return d.k8sClient.Create(ctx, configMap) 49 | } 50 | 51 | func (d *defaultManager) UpdateConfigMap(ctx context.Context, configMap *v1.ConfigMap) error { 52 | return d.k8sClient.Update(ctx, configMap) 53 | } 54 | 55 | func (d *defaultManager) DeleteConfigMap(ctx context.Context, configMap *v1.ConfigMap) error { 56 | return d.k8sClient.Delete(ctx, configMap) 57 | } 58 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/configmap/wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package configmap 15 | 16 | import ( 17 | "context" 18 | "time" 19 | 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 21 | . "github.com/onsi/ginkgo/v2" 22 | . "github.com/onsi/gomega" 23 | v1 "k8s.io/api/core/v1" 24 | ) 25 | 26 | // sleep to allow cache to sync and nodes to be updated after all configmap operations 27 | // TODO: Ideally we need to wait till the nodes are updated, need to check for this 28 | 29 | func CreateConfigMap(manager Manager, ctx context.Context, configmap *v1.ConfigMap) { 30 | By("creating the configmap") 31 | err := manager.CreateConfigMap(ctx, configmap) 32 | Expect(err).NotTo(HaveOccurred()) 33 | 34 | time.Sleep(utils.PollIntervalMedium) 35 | } 36 | 37 | func DeleteConfigMap(manager Manager, ctx context.Context, configmap *v1.ConfigMap) { 38 | By("deleting the configmap") 39 | err := manager.DeleteConfigMap(ctx, configmap) 40 | Expect(err).NotTo(HaveOccurred()) 41 | 42 | time.Sleep(utils.PollIntervalMedium) 43 | } 44 | 45 | func UpdateConfigMap(manager Manager, ctx context.Context, configmap *v1.ConfigMap) { 46 | By("updating the configmap") 47 | err := manager.UpdateConfigMap(ctx, configmap) 48 | Expect(err).NotTo(HaveOccurred()) 49 | 50 | time.Sleep(utils.PollIntervalMedium) 51 | } 52 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/controller/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package controller 15 | 16 | import ( 17 | "context" 18 | "encoding/json" 19 | "fmt" 20 | "strings" 21 | 22 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 23 | 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/labels" 26 | "k8s.io/apimachinery/pkg/types" 27 | "k8s.io/apimachinery/pkg/util/wait" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | ) 30 | 31 | const ( 32 | DeploymentName = "vpc-resource-local-controller" 33 | Namespace = "kube-system" 34 | LeaderLeaseMapName = "cp-vpc-resource-controller" 35 | 36 | PodLabelKey = "app" 37 | PodLabelVal = "vpc-resource-controller" 38 | 39 | ClusterRoleName = "vpc-resource-controller-role" 40 | ) 41 | 42 | type LeaderLease struct { 43 | HolderIdentity string 44 | } 45 | 46 | type Manager interface { 47 | WaitTillControllerHasLeaderLease(ctx context.Context) (string, error) 48 | } 49 | 50 | func NewManager(k8sClient client.Client) Manager { 51 | return &defaultManager{k8sClient: k8sClient} 52 | } 53 | 54 | type defaultManager struct { 55 | k8sClient client.Client 56 | } 57 | 58 | func (d *defaultManager) WaitTillControllerHasLeaderLease(ctx context.Context) (string, error) { 59 | leaderLeaseMap := &v1.ConfigMap{} 60 | 61 | controllerPods := &v1.PodList{} 62 | err := d.k8sClient.List(ctx, controllerPods, &client.ListOptions{ 63 | LabelSelector: labels.SelectorFromSet(labels.Set{PodLabelKey: PodLabelVal}), 64 | }) 65 | if err != nil { 66 | return "", err 67 | } 68 | 69 | var leaderPodName string 70 | err = wait.PollUntil(utils.PollIntervalMedium, func() (done bool, err error) { 71 | err = d.k8sClient.Get(ctx, types.NamespacedName{ 72 | Namespace: Namespace, 73 | Name: LeaderLeaseMapName, 74 | }, leaderLeaseMap) 75 | if err != nil { 76 | return false, err 77 | } 78 | key, found := leaderLeaseMap.ObjectMeta. 79 | Annotations["control-plane.alpha.kubernetes.io/leader"] 80 | if !found { 81 | return false, fmt.Errorf("failed to find leader election configmap") 82 | } 83 | 84 | lease := &LeaderLease{} 85 | err = json.Unmarshal([]byte(key), lease) 86 | if err != nil { 87 | return false, err 88 | } 89 | 90 | for _, pod := range controllerPods.Items { 91 | // The Holder identity consists of the Leader Pod name + random UID 92 | if strings.Contains(lease.HolderIdentity, pod.Name) { 93 | leaderPodName = pod.Name 94 | return true, nil 95 | } 96 | } 97 | return false, nil 98 | }, ctx.Done()) 99 | 100 | return leaderPodName, err 101 | } 102 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/deployment/wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package deployment 15 | 16 | import ( 17 | "context" 18 | 19 | . "github.com/onsi/ginkgo/v2" 20 | . "github.com/onsi/gomega" 21 | appsv1 "k8s.io/api/apps/v1" 22 | ) 23 | 24 | func CreateAndWaitForDeploymentToStart(deploymentManager Manager, ctx context.Context, 25 | deployment *appsv1.Deployment) *appsv1.Deployment { 26 | 27 | By("creating and waiting for the deployment to start") 28 | deployment, err := deploymentManager.CreateAndWaitUntilDeploymentReady(ctx, deployment) 29 | Expect(err).ToNot(HaveOccurred()) 30 | 31 | return deployment 32 | } 33 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/namespace/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package namespace 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 20 | "k8s.io/apimachinery/pkg/api/errors" 21 | "k8s.io/apimachinery/pkg/util/wait" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "sigs.k8s.io/controller-runtime/pkg/client" 26 | ) 27 | 28 | type Manager interface { 29 | CreateNamespace(ctx context.Context, namespace string) error 30 | DeleteAndWaitTillNamespaceDeleted(ctx context.Context, namespace string) error 31 | } 32 | 33 | func NewManager(k8sClient client.Client) Manager { 34 | return &defaultManager{k8sClient: k8sClient} 35 | } 36 | 37 | type defaultManager struct { 38 | k8sClient client.Client 39 | } 40 | 41 | func (m *defaultManager) CreateNamespace(ctx context.Context, namespace string) error { 42 | if namespace == "default" { 43 | return nil 44 | } 45 | return m.k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metaV1.ObjectMeta{Name: namespace}}) 46 | } 47 | 48 | func (m *defaultManager) DeleteAndWaitTillNamespaceDeleted(ctx context.Context, namespace string) error { 49 | if namespace == "default" { 50 | return nil 51 | } 52 | namespaceObj := &v1.Namespace{ObjectMeta: metaV1.ObjectMeta{Name: namespace, Namespace: ""}} 53 | err := m.k8sClient.Delete(ctx, namespaceObj) 54 | if err != nil { 55 | return client.IgnoreNotFound(err) 56 | } 57 | 58 | observedNamespace := &v1.Namespace{} 59 | return wait.PollUntil(utils.PollIntervalShort, func() (done bool, err error) { 60 | err = m.k8sClient.Get(ctx, utils.NamespacedName(namespaceObj), observedNamespace) 61 | if errors.IsNotFound(err) { 62 | return true, nil 63 | } 64 | return false, err 65 | }, ctx.Done()) 66 | } 67 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/node/wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package node 15 | 16 | import ( 17 | "context" 18 | "fmt" 19 | 20 | cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1" 21 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 22 | . "github.com/onsi/ginkgo/v2" 23 | . "github.com/onsi/gomega" 24 | "github.com/samber/lo" 25 | v1 "k8s.io/api/core/v1" 26 | "k8s.io/apimachinery/pkg/util/wait" 27 | ) 28 | 29 | func GetNodeAndWaitTillCapacityPresent(manager Manager, os string, expectedResource string) *v1.NodeList { 30 | observedNodeList := &v1.NodeList{} 31 | var err error 32 | err = wait.PollUntilContextTimeout(context.Background(), utils.PollIntervalShort, utils.ResourceCreationTimeout, true, 33 | func(ctx context.Context) (bool, error) { 34 | By("checking nodes have capacity present") 35 | observedNodeList, err = manager.GetNodesWithOS(os) 36 | Expect(err).ToNot(HaveOccurred()) 37 | for _, node := range observedNodeList.Items { 38 | _, found := node.Status.Allocatable[v1.ResourceName(expectedResource)] 39 | if !found { 40 | return false, nil 41 | } 42 | } 43 | return true, nil 44 | }) 45 | Expect(err).ToNot(HaveOccurred()) 46 | return observedNodeList 47 | } 48 | 49 | // VerifyCNINode checks if the number of CNINodes is equal to number of nodes in the cluster, and verifies 1:1 mapping between CNINode and Node objects 50 | // Returns nil if count and 1:1 mapping exists, else returns error 51 | func VerifyCNINode(manager Manager) error { 52 | var cniNodeList *cninode.CNINodeList 53 | var nodeList *v1.NodeList 54 | var err error 55 | By("checking number of CNINodes match number of nodes in the cluster") 56 | err = wait.PollUntilContextTimeout(context.Background(), utils.PollIntervalShort, utils.PollTimeout, true, 57 | func(ctx context.Context) (bool, error) { 58 | if cniNodeList, err = manager.GetCNINodeList(); err != nil { 59 | return false, nil 60 | } 61 | if nodeList, err = manager.GetNodeList(); err != nil { 62 | return false, nil 63 | } 64 | if len(nodeList.Items) != len(cniNodeList.Items) { 65 | return false, nil 66 | } 67 | return true, nil 68 | }) 69 | if err != nil { 70 | return fmt.Errorf("number of CNINodes does not match number of nodes in the cluster") 71 | } 72 | By("checking CNINode list matches node list") 73 | nameMatched := true 74 | for _, node := range nodeList.Items { 75 | if !lo.ContainsBy(cniNodeList.Items, func(cniNode cninode.CNINode) bool { 76 | return cniNode.Name == node.Name 77 | }) { 78 | nameMatched = false 79 | } 80 | } 81 | if !nameMatched { 82 | return fmt.Errorf("CNINode list does not match node list") 83 | } 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/pod/wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package pod 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 20 | 21 | . "github.com/onsi/ginkgo/v2" 22 | . "github.com/onsi/gomega" 23 | v1 "k8s.io/api/core/v1" 24 | ) 25 | 26 | func CreateAndWaitForPodToStart(podManager Manager, ctx context.Context, pod *v1.Pod) *v1.Pod { 27 | By("create the pod") 28 | pod, err := podManager.CreateAndWaitTillPodIsRunning(ctx, pod, utils.ResourceCreationTimeout) 29 | Expect(err).NotTo(HaveOccurred()) 30 | 31 | return pod 32 | } 33 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/rbac/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package rbac 15 | 16 | import ( 17 | "context" 18 | v1 "k8s.io/api/rbac/v1" 19 | "k8s.io/apimachinery/pkg/types" 20 | "sigs.k8s.io/controller-runtime/pkg/client" 21 | ) 22 | 23 | type Manager interface { 24 | GetClusterRole(name string) (*v1.ClusterRole, error) 25 | PatchClusterRole(updatedRole *v1.ClusterRole) error 26 | } 27 | 28 | type defaultManager struct { 29 | k8sClient client.Client 30 | } 31 | 32 | func NewManager(k8sClient client.Client) Manager { 33 | return &defaultManager{k8sClient: k8sClient} 34 | } 35 | 36 | func (d *defaultManager) GetClusterRole(name string) (*v1.ClusterRole, error) { 37 | clusterRole := &v1.ClusterRole{} 38 | err := d.k8sClient.Get(context.TODO(), types.NamespacedName{ 39 | Name: name, 40 | }, clusterRole) 41 | return clusterRole, err 42 | } 43 | 44 | func (d *defaultManager) PatchClusterRole(updatedRole *v1.ClusterRole) error { 45 | clusterRole, err := d.GetClusterRole(updatedRole.Name) 46 | if err != nil { 47 | return err 48 | } 49 | err = d.k8sClient.Patch(context.TODO(), updatedRole, client.MergeFrom(clusterRole)) 50 | return err 51 | } 52 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/service/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package service 15 | 16 | import ( 17 | "context" 18 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 19 | 20 | v1 "k8s.io/api/core/v1" 21 | "k8s.io/apimachinery/pkg/types" 22 | "k8s.io/apimachinery/pkg/util/wait" 23 | "sigs.k8s.io/controller-runtime/pkg/client" 24 | ) 25 | 26 | type Manager interface { 27 | GetService(ctx context.Context, namespace string, name string) (*v1.Service, error) 28 | CreateService(ctx context.Context, service *v1.Service) (*v1.Service, error) 29 | DeleteService(ctx context.Context, service *v1.Service) error 30 | } 31 | 32 | type defaultManager struct { 33 | k8sClient client.Client 34 | } 35 | 36 | func NewManager(k8sClient client.Client) Manager { 37 | return &defaultManager{k8sClient: k8sClient} 38 | } 39 | 40 | func (s *defaultManager) GetService(ctx context.Context, namespace string, 41 | name string) (*v1.Service, error) { 42 | 43 | service := &v1.Service{} 44 | err := s.k8sClient.Get(ctx, types.NamespacedName{ 45 | Namespace: namespace, 46 | Name: name, 47 | }, service) 48 | 49 | return service, err 50 | } 51 | 52 | func (s *defaultManager) CreateService(ctx context.Context, service *v1.Service) (*v1.Service, error) { 53 | err := s.k8sClient.Create(ctx, service) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | observedService := &v1.Service{} 59 | return observedService, wait.PollUntil(utils.PollIntervalShort, func() (bool, error) { 60 | if err := s.k8sClient.Get(ctx, utils.NamespacedName(service), observedService); err != nil { 61 | return false, err 62 | } 63 | return true, nil 64 | }, ctx.Done()) 65 | } 66 | 67 | func (s *defaultManager) DeleteService(ctx context.Context, service *v1.Service) error { 68 | err := s.k8sClient.Delete(ctx, service) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | return nil 74 | } 75 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/sgp/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package sgp 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1" 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 21 | 22 | "k8s.io/apimachinery/pkg/api/errors" 23 | "k8s.io/apimachinery/pkg/util/wait" 24 | "sigs.k8s.io/controller-runtime/pkg/client" 25 | ) 26 | 27 | type Manager struct { 28 | k8sClient client.Client 29 | } 30 | 31 | func NewManager(k8sClient client.Client) *Manager { 32 | return &Manager{k8sClient: k8sClient} 33 | } 34 | 35 | func (d *Manager) DeleteAndWaitTillSecurityGroupIsDeleted(ctx context.Context, sgp *v1beta1.SecurityGroupPolicy) error { 36 | err := d.k8sClient.Delete(ctx, sgp) 37 | if err != nil { 38 | return client.IgnoreNotFound(err) 39 | } 40 | 41 | observedSgp := &v1beta1.SecurityGroupPolicy{} 42 | return wait.PollUntil(utils.PollIntervalShort, func() (done bool, err error) { 43 | err = d.k8sClient.Get(ctx, utils.NamespacedName(sgp), observedSgp) 44 | if errors.IsNotFound(err) { 45 | return true, nil 46 | } 47 | return false, err 48 | }, ctx.Done()) 49 | } 50 | -------------------------------------------------------------------------------- /test/framework/resource/k8s/sgp/wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package sgp 15 | 16 | import ( 17 | "context" 18 | "time" 19 | 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1" 21 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 22 | 23 | . "github.com/onsi/ginkgo/v2" 24 | . "github.com/onsi/gomega" 25 | "sigs.k8s.io/controller-runtime/pkg/client" 26 | ) 27 | 28 | func CreateSecurityGroupPolicy(k8sClient client.Client, ctx context.Context, 29 | securityGroupPolicy *v1beta1.SecurityGroupPolicy) { 30 | By("creating the security group policy") 31 | err := k8sClient.Create(ctx, securityGroupPolicy) 32 | Expect(err).NotTo(HaveOccurred()) 33 | } 34 | 35 | func UpdateSecurityGroupPolicy(k8sClient client.Client, ctx context.Context, 36 | securityGroupPolicy *v1beta1.SecurityGroupPolicy, securityGroups []string) { 37 | 38 | // Before updating the SGP wait for some time to allow the cache to update with 39 | // recently created SGP 40 | time.Sleep(utils.PollIntervalShort) 41 | 42 | By("updating the security group policy") 43 | updatedSgp := &v1beta1.SecurityGroupPolicy{} 44 | err := k8sClient.Get(ctx, client.ObjectKey{ 45 | Name: securityGroupPolicy.Name, 46 | Namespace: securityGroupPolicy.Namespace, 47 | }, updatedSgp) 48 | Expect(err).ToNot(HaveOccurred()) 49 | 50 | updatedSgp.Spec.SecurityGroups.Groups = securityGroups 51 | err = k8sClient.Update(ctx, updatedSgp) 52 | Expect(err).NotTo(HaveOccurred()) 53 | } 54 | -------------------------------------------------------------------------------- /test/framework/utils/metrics_helper.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | 6 | dto "github.com/prometheus/client_model/go" 7 | "github.com/prometheus/common/expfmt" 8 | ) 9 | 10 | func metricsConvert(rawData []byte) (map[string]*dto.MetricFamily, error) { 11 | parser := &expfmt.TextParser{} 12 | origFamilies, err := parser.TextToMetricFamilies(bytes.NewReader(rawData)) 13 | if err != nil { 14 | return nil, err 15 | } 16 | 17 | return origFamilies, nil 18 | } 19 | 20 | func RetrieveTestedMetricValue(rawData []byte, testedMetricName string, testedMetricType dto.MetricType) ([]*dto.Metric, error) { 21 | if metricFamilies, err := metricsConvert(rawData); err != nil { 22 | return nil, err 23 | } else { 24 | for name, metricFamily := range metricFamilies { 25 | if name == testedMetricName && metricFamily.GetType() == testedMetricType { 26 | return metricFamily.GetMetric(), nil 27 | } 28 | } 29 | return nil, nil 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/framework/utils/poll.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import "time" 17 | 18 | const ( 19 | PollIntervalShort = 2 * time.Second 20 | PollIntervalMedium = 10 * time.Second 21 | PollIntervalLong = 20 * time.Second 22 | PollTimeout = 30 * time.Second 23 | // ResourceCreationTimeout is the number of seconds till the controller waits 24 | // for the resource creation to complete 25 | ResourceCreationTimeout = 180 * time.Second 26 | // Windows Container Images are much larger in size and pulling them the first 27 | // time takes much longer, so have higher timeout for Windows Pod to be Ready 28 | WindowsPodsCreationTimeout = 240 * time.Second 29 | WindowsPodsDeletionTimeout = 60 * time.Second 30 | ) 31 | -------------------------------------------------------------------------------- /test/framework/utils/resource.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | const ( 17 | ResourceNamePrefix = "vpc-resource-controller-integration-" 18 | TestNameSpace = "test-ns" 19 | ) 20 | -------------------------------------------------------------------------------- /test/framework/utils/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package utils 15 | 16 | import ( 17 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 18 | "k8s.io/apimachinery/pkg/types" 19 | ) 20 | 21 | // NamespacedName returns the namespaced name for k8s objects 22 | func NamespacedName(obj metav1.Object) types.NamespacedName { 23 | return types.NamespacedName{ 24 | Namespace: obj.GetNamespace(), 25 | Name: obj.GetName(), 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /test/integration/cninode/cninode_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package cninode_test 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework" 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/node" 21 | . "github.com/onsi/ginkgo/v2" 22 | . "github.com/onsi/gomega" 23 | ) 24 | 25 | func TestCNINode(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecs(t, "CNINode Test Suite") 28 | } 29 | 30 | var frameWork *framework.Framework 31 | var _ = BeforeSuite(func() { 32 | By("creating a framework") 33 | frameWork = framework.New(framework.GlobalOptions) 34 | 35 | By("verify at least 2 nodes are available") 36 | nodeList, err := frameWork.NodeManager.GetNodeList() 37 | Expect(err).ToNot(HaveOccurred()) 38 | Expect(len(nodeList.Items)).To(BeNumerically(">", 1)) 39 | 40 | By("verify CNINode count") 41 | err = node.VerifyCNINode(frameWork.NodeManager) 42 | Expect(err).ToNot(HaveOccurred()) 43 | }) 44 | 45 | // Verify CNINode count before and after test remains same 46 | var _ = AfterSuite(func() { 47 | By("verify CNINode count") 48 | err := node.VerifyCNINode(frameWork.NodeManager) 49 | Expect(err).ToNot(HaveOccurred()) 50 | }) 51 | -------------------------------------------------------------------------------- /test/integration/ec2api/ec2api_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | package ec2api_test 14 | 15 | import ( 16 | "testing" 17 | 18 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework" 20 | . "github.com/onsi/ginkgo/v2" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestEc2api(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "EC2API Suite") 27 | } 28 | 29 | var frameWork *framework.Framework 30 | var nodeListLen int 31 | var _ = BeforeSuite(func() { 32 | By("creating a framework") 33 | frameWork = framework.New(framework.GlobalOptions) 34 | By("verify node count before test") 35 | nodeList, err := frameWork.NodeManager.GetNodesWithOS(config.OSLinux) 36 | Expect(err).ToNot(HaveOccurred()) 37 | nodeListLen = len(nodeList.Items) 38 | Expect(nodeListLen).To(BeNumerically(">", 1)) 39 | }) 40 | 41 | var _ = AfterSuite(func() { 42 | nodeList, err := frameWork.NodeManager.GetNodesWithOS(config.OSLinux) 43 | Expect(err).ToNot(HaveOccurred()) 44 | By("verifying node count after test is unchanged") 45 | Expect(len(nodeList.Items)).To(Equal(nodeListLen)) 46 | }) 47 | -------------------------------------------------------------------------------- /test/integration/metrics/metrics_test_config.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package metrics 15 | 16 | const ( 17 | describeENICallCount = "ec2_describe_network_interface_api_req_count" 18 | totalMemAllocatedBytes = "go_memstats_alloc_bytes_total" 19 | goRoutineCount = "go_goroutines" 20 | ) 21 | 22 | type TestMetric struct { 23 | MetricsName string 24 | MetricsThreshold float64 25 | } 26 | 27 | // NewTestingMetrics is used to add more metrics and monitoring thresholds for regression tests. 28 | // To add another metrics 29 | // 30 | // 1, add the metrics name into constant variable 31 | // 2, add expected upper threshold into this map 32 | func NewTestingMetrics() map[string]TestMetric { 33 | return map[string]TestMetric{ 34 | describeENICallCount: { 35 | MetricsName: "ec2_describe_network_interface_api_req_count", 36 | MetricsThreshold: 10.0, 37 | }, 38 | totalMemAllocatedBytes: { 39 | MetricsName: "go_memstats_alloc_bytes_total", 40 | MetricsThreshold: 1.5, 41 | }, 42 | goRoutineCount: { 43 | MetricsName: "go_goroutines", 44 | MetricsThreshold: 1.25, 45 | }, 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /test/integration/perpodsg/perpodsg_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package perpodsg_test 15 | 16 | import ( 17 | "context" 18 | "testing" 19 | 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" 21 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework" 22 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/node" 23 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 24 | verifier "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/verify" 25 | 26 | . "github.com/onsi/ginkgo/v2" 27 | . "github.com/onsi/gomega" 28 | v1 "k8s.io/api/core/v1" 29 | ) 30 | 31 | var frameWork *framework.Framework 32 | var verify *verifier.PodVerification 33 | var securityGroupID1 string 34 | var securityGroupID2 string 35 | var ctx context.Context 36 | var err error 37 | var nodeList *v1.NodeList 38 | 39 | func TestPerPodGG(t *testing.T) { 40 | RegisterFailHandler(Fail) 41 | RunSpecs(t, "Per Pod Security Group Suite") 42 | } 43 | 44 | var _ = BeforeSuite(func() { 45 | By("creating a framework") 46 | frameWork = framework.New(framework.GlobalOptions) 47 | ctx = context.Background() 48 | verify = verifier.NewPodVerification(frameWork, ctx) 49 | 50 | securityGroupID1, err = frameWork.EC2Manager.ReCreateSG(utils.ResourceNamePrefix+"sg-1", ctx) 51 | Expect(err).ToNot(HaveOccurred()) 52 | securityGroupID2, err = frameWork.EC2Manager.ReCreateSG(utils.ResourceNamePrefix+"sg-2", ctx) 53 | Expect(err).ToNot(HaveOccurred()) 54 | 55 | nodeList = node.GetNodeAndWaitTillCapacityPresent(frameWork.NodeManager, "linux", 56 | config.ResourceNamePodENI) 57 | err = node.VerifyCNINode(frameWork.NodeManager) 58 | Expect(err).ToNot(HaveOccurred()) 59 | }) 60 | 61 | var _ = AfterSuite(func() { 62 | Expect(frameWork.EC2Manager.DeleteSecurityGroup(ctx, securityGroupID1)).To(Succeed()) 63 | Expect(frameWork.EC2Manager.DeleteSecurityGroup(ctx, securityGroupID2)).To(Succeed()) 64 | }) 65 | -------------------------------------------------------------------------------- /test/integration/scale/pod_scale_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package scale_test 15 | 16 | import ( 17 | "time" 18 | 19 | "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1" 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/manifest" 21 | deploymentWrapper "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/deployment" 22 | sgpWrapper "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/sgp" 23 | . "github.com/onsi/ginkgo/v2" 24 | . "github.com/onsi/gomega" 25 | v1 "k8s.io/api/apps/v1" 26 | ) 27 | 28 | var _ = Describe("Security group per pod scale test", func() { 29 | var ( 30 | sgpLabelKey string 31 | sgpLabelValue string 32 | securityGroups []string 33 | securityGroupPolicy *v1beta1.SecurityGroupPolicy 34 | err error 35 | ) 36 | 37 | BeforeEach(func() { 38 | sgpLabelKey = "role" 39 | sgpLabelValue = "db" 40 | securityGroups = []string{securityGroupID} 41 | }) 42 | 43 | JustBeforeEach(func() { 44 | // create SGP 45 | securityGroupPolicy, err = manifest.NewSGPBuilder(). 46 | Namespace(namespace). 47 | PodMatchLabel(sgpLabelKey, sgpLabelValue). 48 | SecurityGroup(securityGroups).Build() 49 | Expect(err).NotTo(HaveOccurred()) 50 | }) 51 | 52 | JustAfterEach(func() { 53 | By("deleting security group policy") 54 | err = frameWork.SGPManager.DeleteAndWaitTillSecurityGroupIsDeleted(ctx, securityGroupPolicy) 55 | Expect(err).NotTo(HaveOccurred()) 56 | }) 57 | 58 | Describe("creating deployment", func() { 59 | var deployment *v1.Deployment 60 | 61 | JustBeforeEach(func() { 62 | deployment = manifest.NewDefaultDeploymentBuilder(). 63 | Namespace(namespace). 64 | Replicas(1000). 65 | PodLabel(sgpLabelKey, sgpLabelValue).Build() 66 | }) 67 | 68 | JustAfterEach(func() { 69 | By("deleting the deployment") 70 | err = frameWork.DeploymentManager.DeleteAndWaitUntilDeploymentDeleted(ctx, deployment) 71 | Expect(err).ToNot(HaveOccurred()) 72 | time.Sleep(time.Minute) // allow time for pods to terminate 73 | }) 74 | 75 | Context("when deployment is created", func() { 76 | It("should have all the pods running", MustPassRepeatedly(3), func() { 77 | start := time.Now() 78 | sgpWrapper.CreateSecurityGroupPolicy(frameWork.K8sClient, ctx, securityGroupPolicy) 79 | deploymentWrapper. 80 | CreateAndWaitForDeploymentToStart(frameWork.DeploymentManager, ctx, deployment) 81 | duration := time.Since(start) 82 | verify.VerifyNetworkingOfAllPodUsingENI(namespace, sgpLabelKey, sgpLabelValue, 83 | securityGroups) 84 | Expect(duration.Minutes()).To(BeNumerically("<", 5.5)) 85 | }) 86 | }) 87 | }) 88 | 89 | }) 90 | -------------------------------------------------------------------------------- /test/integration/scale/scale_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package scale_test 15 | 16 | import ( 17 | "context" 18 | "testing" 19 | 20 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework" 21 | "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" 22 | verifier "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/verify" 23 | . "github.com/onsi/ginkgo/v2" 24 | . "github.com/onsi/gomega" 25 | ) 26 | 27 | var frameWork *framework.Framework 28 | var verify *verifier.PodVerification 29 | var ctx context.Context 30 | var securityGroupID string 31 | var err error 32 | var namespace = "podsg-scale-" + utils.TestNameSpace 33 | 34 | func TestScale(t *testing.T) { 35 | RegisterFailHandler(Fail) 36 | RunSpecs(t, "Scale Test Suite") 37 | } 38 | 39 | var _ = BeforeSuite(func() { 40 | By("creating a framework") 41 | frameWork = framework.New(framework.GlobalOptions) 42 | ctx = context.Background() 43 | verify = verifier.NewPodVerification(frameWork, ctx) 44 | 45 | // create test namespace 46 | Expect(frameWork.NSManager.CreateNamespace(ctx, namespace)).To(Succeed()) 47 | // create test security group 48 | securityGroupID, err = frameWork.EC2Manager.ReCreateSG(utils.ResourceNamePrefix+"sg", ctx) 49 | Expect(err).ToNot(HaveOccurred()) 50 | }) 51 | 52 | var _ = AfterSuite(func() { 53 | // delete test namespace 54 | Expect(frameWork.NSManager.DeleteAndWaitTillNamespaceDeleted(ctx, namespace)).To(Succeed()) 55 | // delete test security group 56 | Expect(frameWork.EC2Manager.DeleteSecurityGroup(ctx, securityGroupID)).To(Succeed()) 57 | }) 58 | -------------------------------------------------------------------------------- /webhooks/idle/webhooks.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package idle 15 | 16 | import ( 17 | "context" 18 | 19 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 20 | ) 21 | 22 | type PodMutationWebHook struct { 23 | } 24 | 25 | func NewPodMutationWebHook() *PodMutationWebHook { 26 | return &PodMutationWebHook{} 27 | } 28 | 29 | func (i *PodMutationWebHook) Handle(_ context.Context, req admission.Request) admission.Response { 30 | return admission.Allowed("the controller is disabled") 31 | } 32 | 33 | type NodeUpdateWebhook struct { 34 | } 35 | 36 | func NewNodeUpdateWebhook() *NodeUpdateWebhook { 37 | return &NodeUpdateWebhook{} 38 | } 39 | 40 | func (a *NodeUpdateWebhook) Handle(_ context.Context, req admission.Request) admission.Response { 41 | return admission.Allowed("the controller is disabled") 42 | } 43 | 44 | type AnnotationValidator struct { 45 | } 46 | 47 | func NewAnnotationValidator() *AnnotationValidator { 48 | return &AnnotationValidator{} 49 | } 50 | 51 | func (a *AnnotationValidator) Handle(_ context.Context, req admission.Request) admission.Response { 52 | return admission.Allowed("the controller is disabled") 53 | } 54 | --------------------------------------------------------------------------------