├── .dockerignore ├── .github ├── dependabot.yml ├── linters │ └── .markdown-lint.yml ├── release-change-log.json └── workflows │ ├── build-policy.yml │ ├── build.yml │ ├── check.yml │ ├── codeql-analysis.yml │ └── release.yml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── Makefile ├── README-zh_CN.md ├── README.md ├── SECURITY.md ├── charts └── terway │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ ├── terway-controlplane │ │ ├── _helpers.tpl │ │ ├── clusterrole.yaml │ │ ├── clusterrolebinding.yaml │ │ ├── configmap.yaml │ │ ├── deployment.yaml │ │ ├── role.yaml │ │ ├── rolebinding.yaml │ │ ├── secret.yaml │ │ ├── service.yaml │ │ ├── serviceaccount.yaml │ │ └── webhook.yaml │ └── terwayd │ │ ├── _helpers.tpl │ │ ├── clusterrole.yaml │ │ ├── clusterrolebinding.yaml │ │ ├── configmap.yaml │ │ ├── daemonset.yaml │ │ ├── role.yml │ │ ├── rolebinding.yml │ │ └── serviceaccount.yaml │ └── values.yaml ├── cmd ├── terway-cli │ ├── cmd.go │ ├── cni.go │ ├── cni_linux.go │ ├── cni_linux_test.go │ ├── cni_test.go │ ├── cni_unsupport.go │ ├── common.go │ ├── common_test.go │ ├── main.go │ ├── net.go │ ├── node.go │ ├── node_test.go │ ├── policy.go │ ├── policy_test.go │ └── tree.go ├── terway-controlplane │ ├── terway-controlplane.go │ └── terway-controlplane_test.go └── terway │ └── main.go ├── daemon ├── builder.go ├── builder_test.go ├── config.go ├── config_test.go ├── daemon.go ├── daemon_linux.go ├── daemon_test.go ├── daemon_unsupported.go ├── daemon_unwindows.go ├── daemon_windows.go ├── resource_manager.go ├── rule_linux.go ├── rule_linux_test.go ├── rule_unsupported.go ├── server.go ├── server_linux.go ├── server_unsupported.go └── server_windows.go ├── deploy └── images │ ├── policy │ └── Dockerfile │ ├── terway-controlplane │ └── Dockerfile │ ├── terway │ └── Dockerfile │ └── windows │ └── Dockerfile ├── deviceplugin └── eni.go ├── docs ├── centralized-ipam.md ├── cni-chain.md ├── design.md ├── dynamic-config.md ├── ecs.md ├── eip.md ├── host-stack-cidrs.md ├── hubble-intergration.md ├── images │ ├── ding_group_qrcode.jpg │ ├── eni_connection.jpg │ ├── eniip_connection.png │ ├── eniip_connection_ipvlan.png │ ├── host_stack_cidrs.png │ ├── terway-dynamic-config.png │ ├── terway.png │ ├── terway_cli_mapping.png │ ├── terway_cli_metadata.png │ ├── terway_deploy_diagram.png │ ├── terway_resource_pool.png │ ├── terway_tracing.png │ ├── terway_with_cilium.png │ └── vpc_connection.jpg ├── ipv6.md ├── linjun.md ├── qos.md ├── terway-cli.md ├── terway-trunk.md └── terway-with-cilium.md ├── eni.conf ├── examples └── maxpods │ └── maxpods.go ├── go.mod ├── go.sum ├── hack ├── boilerplate.go.txt ├── entrypoint.ps1 ├── init.sh ├── iptables-wrapper-installer.sh ├── tools.go ├── update-codegen.sh ├── update.sh ├── verify-codegen.sh └── windows-multirelease-build.sh ├── monitoring ├── terway-metric-proxy.yml └── terway_grafana_dashboard.json ├── pkg ├── aliyun │ ├── client │ │ ├── api_facade.go │ │ ├── ecs.go │ │ ├── ecs_2.go │ │ ├── ecs_service.go │ │ ├── eflo_service.go │ │ ├── errors │ │ │ ├── errors.go │ │ │ └── errors_test.go │ │ ├── hdeni.go │ │ ├── interface.go │ │ ├── limit.go │ │ ├── limit_test.go │ │ ├── mocks │ │ │ ├── ECS.go │ │ │ ├── EFLO.go │ │ │ ├── LimitProvider.go │ │ │ ├── OpenAPI.go │ │ │ └── VPC.go │ │ ├── options.go │ │ ├── options_test.go │ │ ├── ratelimit.go │ │ ├── ratelimit_test.go │ │ ├── token.go │ │ ├── token_test.go │ │ ├── types.go │ │ └── vpc_service.go │ ├── credential │ │ ├── aliyun_client_mgr.go │ │ ├── clients.go │ │ ├── wire.go │ │ └── wire_gen.go │ ├── eni │ │ └── eni.go │ ├── instance │ │ ├── ecs.go │ │ ├── instance.go │ │ └── mocks │ │ │ └── Interface.go │ └── metadata │ │ ├── metadata.go │ │ ├── metadata_test.go │ │ └── metrics.go ├── apis │ ├── crds │ │ ├── network.alibabacloud.com_networkinterfaces.yaml │ │ ├── network.alibabacloud.com_noderuntimes.yaml │ │ ├── network.alibabacloud.com_nodes.yaml │ │ ├── network.alibabacloud.com_podenis.yaml │ │ ├── network.alibabacloud.com_podnetworkings.yaml │ │ └── register.go │ └── network.alibabacloud.com │ │ ├── register.go │ │ └── v1beta1 │ │ ├── doc.go │ │ ├── eni.go │ │ ├── helper.go │ │ ├── node_runtime.go │ │ ├── node_types.go │ │ ├── register.go │ │ ├── types.go │ │ └── zz_generated.deepcopy.go ├── backoff │ └── backoff.go ├── cert │ └── webhook.go ├── controller │ ├── all │ │ └── all.go │ ├── common │ │ ├── ctx_default.go │ │ ├── eni.go │ │ ├── eni_test.go │ │ ├── suite_test.go │ │ ├── types.go │ │ ├── types_default.go │ │ └── types_default_test.go │ ├── eni │ │ ├── bo.go │ │ ├── bo_test.go │ │ ├── eni.go │ │ ├── eni_test.go │ │ └── suite_test.go │ ├── multi-ip │ │ ├── node │ │ │ ├── eni.go │ │ │ ├── eni_test.go │ │ │ ├── metric.go │ │ │ ├── pod.go │ │ │ ├── pod_test.go │ │ │ ├── pool.go │ │ │ ├── pool_test.go │ │ │ └── suite_test.go │ │ └── pod │ │ │ ├── pod.go │ │ │ ├── pod_test.go │ │ │ ├── predict.go │ │ │ └── suite_test.go │ ├── node │ │ ├── node.go │ │ ├── node_controller_test.go │ │ ├── predict.go │ │ ├── predict_test.go │ │ └── suite_test.go │ ├── pod-eni │ │ ├── eni_controller.go │ │ ├── eni_controller_test.go │ │ ├── predict.go │ │ └── suite_test.go │ ├── pod-networking │ │ ├── networking.go │ │ ├── networking_test.go │ │ ├── predict.go │ │ ├── predict_test.go │ │ └── suite_test.go │ ├── pod │ │ ├── pod_controller.go │ │ ├── pod_controller_default.go │ │ ├── pod_controller_test.go │ │ ├── predict.go │ │ └── suite_test.go │ ├── preheating │ │ ├── dummy.go │ │ ├── dummy_test.go │ │ └── suite_test.go │ ├── register.go │ ├── status │ │ ├── status.go │ │ └── status_test.go │ └── webhook │ │ ├── mutating.go │ │ ├── mutating_test.go │ │ ├── validate.go │ │ ├── validate_test.go │ │ ├── webhook_integration_test.go │ │ └── webhook_suite_test.go ├── eni │ ├── conditiontype_string.go │ ├── crdv2.go │ ├── crdv2_test.go │ ├── enistatus_string.go │ ├── ipstatus_string.go │ ├── local.go │ ├── local_test.go │ ├── local_unwindows.go │ ├── local_windows.go │ ├── manager.go │ ├── manager_test.go │ ├── node_reconcile.go │ ├── node_reconcile_test.go │ ├── remote.go │ ├── remote_test.go │ ├── suite_test.go │ ├── trunk.go │ ├── types.go │ └── types_test.go ├── factory │ ├── aliyun │ │ └── aliyun.go │ ├── mocks │ │ └── Factory.go │ └── types.go ├── feature │ └── feature.go ├── generated │ ├── clientset │ │ └── versioned │ │ │ ├── clientset.go │ │ │ ├── doc.go │ │ │ ├── fake │ │ │ ├── clientset_generated.go │ │ │ ├── doc.go │ │ │ └── register.go │ │ │ ├── scheme │ │ │ ├── doc.go │ │ │ └── register.go │ │ │ └── typed │ │ │ └── network.alibabacloud.com │ │ │ └── v1beta1 │ │ │ ├── doc.go │ │ │ ├── fake │ │ │ ├── doc.go │ │ │ ├── fake_network.alibabacloud.com_client.go │ │ │ ├── fake_networkinterface.go │ │ │ ├── fake_node.go │ │ │ ├── fake_noderuntime.go │ │ │ ├── fake_podeni.go │ │ │ └── fake_podnetworking.go │ │ │ ├── generated_expansion.go │ │ │ ├── network.alibabacloud.com_client.go │ │ │ ├── networkinterface.go │ │ │ ├── node.go │ │ │ ├── noderuntime.go │ │ │ ├── podeni.go │ │ │ └── podnetworking.go │ ├── informers │ │ └── externalversions │ │ │ ├── factory.go │ │ │ ├── generic.go │ │ │ ├── internalinterfaces │ │ │ └── factory_interfaces.go │ │ │ └── network.alibabacloud.com │ │ │ ├── interface.go │ │ │ └── v1beta1 │ │ │ ├── interface.go │ │ │ ├── networkinterface.go │ │ │ ├── node.go │ │ │ ├── noderuntime.go │ │ │ ├── podeni.go │ │ │ └── podnetworking.go │ └── listers │ │ └── network.alibabacloud.com │ │ └── v1beta1 │ │ ├── expansion_generated.go │ │ ├── networkinterface.go │ │ ├── node.go │ │ ├── noderuntime.go │ │ ├── podeni.go │ │ └── podnetworking.go ├── ip │ ├── ip.go │ ├── ip_cilium.go │ ├── ip_cilium_test.go │ └── ip_test.go ├── k8s │ ├── k8s.go │ ├── k8s_test.go │ └── mocks │ │ └── Kubernetes.go ├── link │ ├── interface_linux.go │ ├── interface_linux_test.go │ ├── interface_unsupport.go │ ├── interface_windows.go │ ├── type.go │ ├── veth.go │ └── veth_test.go ├── metric │ ├── aliyun.go │ ├── factory.go │ ├── resource_pool.go │ ├── rpc.go │ └── util.go ├── smc │ └── pnet.go ├── storage │ └── store.go ├── sysctl │ └── sysctl.go ├── tc │ ├── tc.go │ ├── u32.go │ └── u32_test.go ├── tracing │ ├── rpc.go │ └── tracing.go ├── utils │ ├── k8s.go │ ├── k8s_test.go │ ├── k8sclient │ │ └── client.go │ ├── map.go │ ├── map_test.go │ ├── nodecap │ │ ├── mocks │ │ │ └── NodeCapabilitiesStore.go │ │ ├── node_capabilities.go │ │ └── node_capabilities_test.go │ ├── os.go │ └── path.go ├── version │ └── version.go ├── vswitch │ ├── vswitch.go │ └── vswitch_test.go └── windows │ ├── apis │ ├── endpoint.go │ ├── network.go │ ├── network_bridge.go │ ├── network_transparent.go │ ├── network_tunnel.go │ └── utils.go │ ├── converters │ ├── binary.go │ └── byte.go │ ├── endian │ └── endianess.go │ ├── iface │ └── iface.go │ ├── ip │ └── ipnet.go │ ├── ipforward │ └── route.go │ ├── powershell │ └── command.go │ └── syscalls │ └── iphlpapi.go ├── plugin ├── datapath │ ├── consts_linux.go │ ├── consts_linux_test.go │ ├── exclusive_eni_linux.go │ ├── exclusive_eni_linux_test.go │ ├── exclusive_eni_windows.go │ ├── ipvlan_linux.go │ ├── ipvlan_linux_test.go │ ├── policy_router_linux.go │ ├── policy_router_linux_test.go │ ├── policy_router_windows.go │ └── vlan_linux.go ├── driver │ ├── ipvlan │ │ └── ipvlan.go │ ├── nic │ │ └── nic.go │ ├── types │ │ ├── types.go │ │ ├── types_linux.go │ │ ├── types_unspport.go │ │ └── types_windows.go │ ├── utils │ │ ├── netlink_linux.go │ │ ├── utils.go │ │ ├── utils_linux.go │ │ └── utils_linux_test.go │ ├── veth │ │ └── veth.go │ ├── vf │ │ └── vf.go │ └── vlan │ │ ├── vlan.go │ │ └── vlan_test.go └── terway │ ├── cni.go │ ├── cni │ └── runtime_config.go │ ├── cni_linux.go │ ├── cni_unsupport.go │ └── cni_windows.go ├── policy ├── cilium │ ├── 0001-cni-add-terway-cni.patch │ ├── 0002-bypass-the-node-local-dns-ip.patch │ ├── 0003-cep-optimize-cep-watch.patch │ ├── 0004-lb-enable-in-cluster-load-balancer.patch │ ├── 0005-deprecated-disable-per-package-lb.patch │ ├── 0006-gops-allow-disable-gops.patch │ ├── 0007-ctmap-log-ct-gc-status.patch │ ├── 0008-Ignore-the-link-local-IPv6-addresses-as-it-is-genera.patch │ ├── 0009-bandwidth-support-ingress-QoS-using-eBPF-token-bucke.patch │ ├── 0010-Revert-fix-Adding-ipv6-check-on-the-node-init-functi.patch │ └── 0011-fix-viper-flag.patch ├── felix │ ├── 0001-terway.patch │ ├── 0002-performance-improve.patch │ ├── 0003-Use-Aliyun-CNI-annotation-to-get-pod-IPs-if-set.patch │ └── 0004-update-mod.patch ├── policy.ps1 ├── policyinit.sh └── uninstall_policy.sh ├── rpc ├── generate.go ├── rpc.pb.go ├── rpc.proto ├── rpc_grpc.pb.go ├── tracing.pb.go ├── tracing.proto └── tracing_grpc.pb.go ├── terraform ├── terraform.tf ├── terraform_alicloud.tf └── terraform_windbag.tf ├── terway-metric.yml ├── terway-windows-eni.yml ├── terway-windows-eniip.yml ├── terway-windows.yml ├── tests ├── connective_test.go ├── eflo │ └── elfo_test.go ├── erdma │ ├── erdma_test.go │ └── setup_test.go ├── kind │ ├── Makefile │ ├── cluster.yml │ ├── conf │ │ ├── eniip_datapathv2_cmdline │ │ ├── eniip_default_cmdline │ │ └── eniip_legacy_ciliumargs_cmdline │ └── run.sh ├── main_test.go ├── stress │ ├── setup_test.go │ └── stress_test.go ├── trunk_test.go ├── upgrade_test.go ├── utils │ └── k8s.go └── utils_test.go └── types ├── aliyun.go ├── controlplane ├── annotations.go ├── annotations_default.go ├── annotations_test.go ├── config.go ├── config_default.go └── config_test.go ├── daemon ├── cni.go ├── config.go ├── config_test.go ├── dynamicconfig.go ├── dynamicconfig_test.go ├── res.go ├── res_test.go └── types.go ├── errors.go ├── help_test.go ├── helper.go ├── helper_test.go ├── k8s.go ├── k8s_test.go ├── route └── route.go ├── scheme.go ├── secret ├── secret.go └── secret_test.go ├── types.go └── types_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | *.log 8 | 9 | # Directories 10 | ## Binaries of build service 11 | sbin 12 | .cache 13 | ## Cache of docker build 14 | .docker 15 | ## Editor and IDE paraphernalia 16 | .idea 17 | ## Workspace of Terraform 18 | .terraform 19 | 20 | 21 | # Files 22 | *.swp 23 | *.swo 24 | *~ 25 | .vim* 26 | .bash* 27 | ## Mac storage information 28 | .DS_Store 29 | ## Lock file 30 | *.lock 31 | ## Terraform artifacts 32 | *.lock.hcl 33 | *.tfstate 34 | *.tfplan 35 | *.auto.tfvars 36 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | open-pull-requests-limit: 2 8 | rebase-strategy: "disabled" 9 | ignore: 10 | - dependency-name: "google.golang.org/grpc" 11 | - dependency-name: "github.com/miekg/dns" 12 | - dependency-name: "k8s.io/*" 13 | - dependency-name: "sigs.k8s.io/*" 14 | labels: 15 | - kind/enhancement 16 | - release-note/misc -------------------------------------------------------------------------------- /.github/release-change-log.json: -------------------------------------------------------------------------------- 1 | { 2 | "categories": [ 3 | { 4 | "title": "## 🚀 Features", 5 | "labels": ["feature"] 6 | }, 7 | { 8 | "title": "## 📦 Uncategorized", 9 | "labels": [] 10 | } 11 | ], 12 | "template": "${{CHANGELOG}}\n\nUncategorized:\n${{UNCATEGORIZED}}\n\nIgnored:\n${{IGNORED}}\n", 13 | "empty_template": "${{OWNER}}\n${{REPO}}\n${{FROM_TAG}}\n${{TO_TAG}}" 14 | } -------------------------------------------------------------------------------- /.github/workflows/build-policy.yml: -------------------------------------------------------------------------------- 1 | name: build-policy 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*-policy' 7 | 8 | env: 9 | REGISTRY: ghcr.io 10 | OWNER: AliyunContainerService 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | build-policy: 18 | runs-on: ubuntu-22.04 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Set up QEMU 22 | uses: docker/setup-qemu-action@v3 23 | with: 24 | image: tonistiigi/binfmt:qemu-v7.0.0 25 | 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3 28 | 29 | - name: Cache Docker layers 30 | uses: actions/cache@v3 31 | with: 32 | path: /tmp/.buildx-cache 33 | key: ${{ runner.os }}-buildx-${{ github.sha }} 34 | restore-keys: | 35 | ${{ runner.os }}-buildx- 36 | 37 | - name: Docker meta 38 | id: meta 39 | uses: docker/metadata-action@v4 40 | with: 41 | images: ${{ env.REGISTRY }}/${{ env.OWNER }}/terway-policy 42 | tags: | 43 | type=raw,value={{date 'YYYYMMDD'}}-{{sha}} 44 | type=semver,pattern={{version}} 45 | type=semver,pattern={{raw}} 46 | 47 | - name: Login to DockerHub 48 | uses: docker/login-action@v2 49 | with: 50 | registry: ${{ env.REGISTRY }} 51 | username: ${{ github.actor }} 52 | password: ${{ secrets.GITHUB_TOKEN }} 53 | if: ${{ github.event_name != 'pull_request' && github.event.action != 'unassigned' }} 54 | 55 | - name: Build and push 56 | uses: docker/build-push-action@v6 57 | with: 58 | context: . 59 | file: ./Dockerfile.policy 60 | platforms: linux/amd64,linux/arm64 61 | push: ${{ github.event_name != 'pull_request' }} 62 | tags: ${{ steps.meta.outputs.tags }} 63 | labels: ${{ steps.meta.outputs.labels }} 64 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | tags: 7 | - 'v*' 8 | pull_request: 9 | branches: [main] 10 | 11 | env: 12 | REGISTRY: ghcr.io 13 | OWNER: AliyunContainerService 14 | 15 | concurrency: 16 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }} 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | build-terway: 21 | runs-on: ubuntu-22.04 22 | steps: 23 | - uses: actions/checkout@v4 24 | 25 | - name: Set up QEMU 26 | uses: docker/setup-qemu-action@v3 27 | with: 28 | image: tonistiigi/binfmt:qemu-v7.0.0 29 | 30 | - name: Set up Docker Buildx 31 | uses: docker/setup-buildx-action@v3 32 | 33 | - name: Cache Docker layers 34 | uses: actions/cache@v4 35 | with: 36 | path: /tmp/.buildx-cache 37 | key: ${{ runner.os }}-buildx-${{ github.sha }} 38 | restore-keys: | 39 | ${{ runner.os }}-buildx- 40 | 41 | - name: Docker meta 42 | id: meta 43 | uses: docker/metadata-action@v4 44 | with: 45 | images: ${{ env.REGISTRY }}/${{ env.OWNER }}/terway 46 | tags: | 47 | type=raw,value={{date 'YYYYMMDD'}}-{{sha}} 48 | type=semver,pattern={{version}} 49 | type=semver,pattern={{raw}} 50 | 51 | - name: Login to DockerHub 52 | uses: docker/login-action@v2 53 | with: 54 | registry: ${{ env.REGISTRY }} 55 | username: ${{ github.actor }} 56 | password: ${{ secrets.GITHUB_TOKEN }} 57 | if: ${{ github.event_name != 'pull_request' && github.event.action != 'unassigned' }} 58 | 59 | - name: Build 60 | run: | 61 | make build 62 | 63 | - name: Build and push 64 | run: | 65 | make REGISTRY=ghcr.io/aliyuncontainerservice build-push 66 | if: ${{ github.event_name != 'pull_request' && github.event.action != 'unassigned' }} -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: check 2 | 3 | on: 4 | push: { } 5 | pull_request: { } 6 | 7 | concurrency: 8 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }} 9 | cancel-in-progress: true 10 | 11 | jobs: 12 | go-test: 13 | runs-on: ubuntu-22.04 14 | steps: 15 | - uses: actions/checkout@v4 16 | - uses: actions/setup-go@v5 17 | with: 18 | go-version: 1.24.0 19 | - name: Test 20 | run: | 21 | go=$(which go) 22 | sudo make test 23 | - uses: codecov/codecov-action@v4 24 | with: 25 | token: ${{ secrets.CODECOV_TOKEN }} 26 | files: ./coverage.txt 27 | flags: unittests 28 | name: codecov-terway 29 | fail_ci_if_error: true 30 | verbose: true 31 | 32 | go-mod: 33 | runs-on: ubuntu-22.04 34 | steps: 35 | - uses: actions/checkout@v4 36 | - uses: actions/setup-go@v5 37 | with: 38 | go-version: 1.24.0 39 | - name: Check module vendoring 40 | run: | 41 | go mod tidy 42 | go mod vendor 43 | git diff --exit-code || (echo "please run 'go mod tidy && go mod vendor', and submit your changes"; exit 1) 44 | 45 | go-lint: 46 | name: lint 47 | runs-on: ubuntu-22.04 48 | steps: 49 | - uses: actions/checkout@v4 50 | - uses: actions/setup-go@v5 51 | with: 52 | go-version: 1.24.0 53 | cache: false 54 | - name: Run golangci-lint 55 | uses: golangci/golangci-lint-action@v6 56 | with: 57 | version: v1.64.5 58 | args: --config=.golangci.yml 59 | 60 | super-linter: 61 | runs-on: ubuntu-22.04 62 | steps: 63 | - uses: actions/checkout@v4 64 | - name: Lint Code Base 65 | uses: super-linter/super-linter@v5.1.1 66 | env: 67 | VALIDATE_ALL_CODEBASE: true 68 | VALIDATE_MARKDOWN: true 69 | VALIDATE_BASH: true 70 | DEFAULT_BRANCH: main 71 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 72 | FILTER_REGEX_EXCLUDE: .*(test|tests).* 73 | SHELLCHECK_OPTS: "-e SC2166" -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '18 15 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-22.04 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | 37 | steps: 38 | - name: Checkout repository 39 | uses: actions/checkout@v4 40 | 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v3 43 | with: 44 | languages: ${{ matrix.language }} 45 | 46 | 47 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 48 | # If this step fails, then you should remove it and run the build manually (see below) 49 | - name: Autobuild 50 | uses: github/codeql-action/autobuild@v3 51 | 52 | # ℹ️ Command-line programs to run using the OS shell. 53 | # 📚 https://git.io/JvXDl 54 | 55 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 56 | # and modify them (or add more) to build your code if your project 57 | # uses a compiled language 58 | 59 | #- run: | 60 | # make bootstrap 61 | # make release 62 | 63 | - name: Perform CodeQL Analysis 64 | uses: github/codeql-action/analyze@v3 65 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | concurrency: 9 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }} 10 | cancel-in-progress: true 11 | 12 | jobs: 13 | release: 14 | runs-on: ubuntu-22.04 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Build Changelog 18 | id: github_release 19 | uses: mikepenz/release-changelog-builder-action@v1 20 | with: 21 | configuration: ".github/release-change-log.json" 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | 25 | - name: Create Release 26 | uses: actions/create-release@v1 27 | with: 28 | tag_name: ${{ github.ref }} 29 | release_name: ${{ github.ref }} 30 | body: ${{steps.github_release.outputs.changelog}} 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | vendor 3 | **/*.exe 4 | .run 5 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | concurrency: 4 3 | timeout: 5m 4 | tests: false 5 | build-tags: 6 | - default_build 7 | - privileged 8 | 9 | issues: 10 | exclude-rules: 11 | - path: _test\.go 12 | linters: 13 | - dupl 14 | - goconst 15 | linters: 16 | enable: 17 | - goconst 18 | - goimports 19 | - govet 20 | - errcheck 21 | - ineffassign 22 | - staticcheck 23 | - goconst 24 | - stylecheck 25 | - misspell 26 | linters-settings: 27 | errcheck: 28 | check-blank: false 29 | -------------------------------------------------------------------------------- /README-zh_CN.md: -------------------------------------------------------------------------------- 1 | # Terway 网络插件 2 | 3 | CNI plugin for alibaba cloud VPC/ENI 4 | 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/AliyunContainerService/terway)](https://goreportcard.com/report/github.com/AliyunContainerService/terway) 6 | [![codecov](https://codecov.io/gh/AliyunContainerService/terway/branch/main/graph/badge.svg)](https://codecov.io/gh/AliyunContainerService/terway) 7 | [![Linter](https://github.com/AliyunContainerService/terway/workflows/check/badge.svg)](https://github.com/marketplace/actions/super-linter) 8 | 9 | [English](./README.md) | 简体中文 10 | 11 | ## 简介 12 | 13 | Terway网络插件是ACK自研的容器网络接口（CNI）插件，基于阿里云的弹性网卡（ENI）构建网络，可以充分利用云上资源。Terway支持eBPF对网络流量进行加速，降低延迟，支持基于Kubernetes标准的网络策略（Network Policy）来定义容器间的访问策略。 14 | 15 | 在Terway网络插件中，每个Pod都拥有自己的网络栈和IP地址。同一台ECS内的Pod之间通信，直接通过机器内部的转发，跨ECS的Pod通信、报文通过VPC的弹性网卡直接转发。由于不需要使用VxLAN等的隧道技术封装报文，因此Terway模式网络具有较高的通信性能。 16 | 17 | ## 特性 18 | 19 | - ENI网络模式：分配 Elastic Network Interfaces (ENIs) 给Pod，优化资源利用率和网络性能。 20 | - Trunking功能：允许Pod配置独立的ENI，支持灵活安全组、交换机配置。 21 | - 节点池维度网络模式配置：支持节点池配置为独占ENI。 22 | - 安全策略：支持NetworkPolicy和传统的安全组，提供多维度的网络安全控制。 23 | - 高性能：使用eBPF加速协议栈，确保低延迟和高吞吐量。 24 | - IPv6: 支持IPv4/IPv6双栈。 25 | - 灵骏: 支持智能计算灵骏。 26 | 27 | ### 以下功能已经废弃 28 | 29 | - VPC网络模式：利用VPC路由，实现容器与VPC内其他资源的直接通信。 30 | - 独占ENI模式：将ENI直通进Pod，最大化网络性能。（替换为通过节点池维度网络模式配置为独占ENI） 31 | 32 | ## 版本差异 33 | 34 | ACK 提供的版本和开源一致。仅Trunking功能无法在自建集群使用。 35 | 36 | ## 贡献 37 | 38 | 我们非常欢迎社区的贡献！无论是修复bug、新增功能、改进文档，或者仅仅是对现有代码的改进，你的帮助都将被我们珍视。 39 | 40 | [报告问题](https://github.com/AliyunContainerService/terway/issues/new) 41 | [提交Pull Request](https://github.com/AliyunContainerService/terway/compare) 42 | 43 | ## 安全 44 | 45 | 如果您发现了代码中的安全漏洞，请联系[kubernetes-security@service.aliyun.com](mailto:kubernetes-security@service.aliyun.com)。详见 [SECURITY.md](SECURITY.md) 46 | 47 | ## 社区 48 | 49 | ### 钉钉群 50 | 51 | 通过钉钉群号 "35924643" 加入`钉钉`群组。 52 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | | ------- | ------------------ | 7 | | >= v1.6.0 | :white_check_mark: | 8 | | < 1.6.0 | :x: | 9 | 10 | ## Reporting a Vulnerability 11 | 12 | DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to [kubernetes-security@service.aliyun.com](mailto:kubernetes-security@service.aliyun.com) 13 | 14 | ## Vulnerability Management Plans 15 | 16 | ### Critical Updates And Security Notices 17 | 18 | We learn about critical software updates and security threats from these sources 19 | 20 | 1. GitHub Security Alerts 21 | 2. [Dependabot](https://dependabot.com/) Dependency Updates 22 | -------------------------------------------------------------------------------- /charts/terway/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/terway/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: terway 3 | description: A Helm chart for Kubernetes 4 | type: application 5 | version: 1.13.0 6 | appVersion: "1.13.0" 7 | -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "terway-controlplane.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "terway-controlplane.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "terway-controlplane.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "terway-controlplane.labels" -}} 37 | helm.sh/chart: {{ include "terway-controlplane.chart" . }} 38 | {{ include "terway-controlplane.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "terway-controlplane.selectorLabels" -}} 49 | app.kubernetes.io/name: terway-controlplane 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: terway-controlplane 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: terway-controlplane 9 | subjects: 10 | - kind: ServiceAccount 11 | name: terway-controlplane 12 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: terway-controlplane 5 | labels: 6 | {{- include "terway-controlplane.labels" . | nindent 4 }} 7 | data: 8 | cilium-config.yaml: | 9 | k8s-namespace: kube-system 10 | identity-gc-interval: 10m 11 | identity-heartbeat-timeout: 20m 12 | ctrl-config.yaml: | 13 | leaseLockName: "terway-controller-lock" 14 | leaseLockNamespace: "kube-system" 15 | controllerNamespace: "kube-system" 16 | controllerName: "terway-controlplane" 17 | healthzBindAddress: "{{.Values.terwayControlplane.healthzBindAddress}}" 18 | clusterDomain: "{{.Values.terwayControlplane.clusterDomain}}" 19 | leaderElection: true 20 | webhookPort: {{.Values.terwayControlplane.webhookPort}} 21 | certDir: "/var/run/webhook-cert" 22 | regionID: "{{ .Values.terwayControlplane.regionID }}" 23 | clusterID: "{{ .Values.terwayControlplane.clusterID }}" 24 | vpcID: "{{ .Values.terwayControlplane.vpcID }}" 25 | ipStack: "{{ .Values.terwayControlplane.ipStack }}" 26 | enableTrunk: {{.Values.terwayControlplane.enableTrunk}} 27 | enableTrace: {{.Values.terwayControlplane.enableTrace}} 28 | centralizedIPAM: {{.Values.centralizedIPAM}} 29 | {{- with .Values.terwayControlplane.controllers }} 30 | controllers: 31 | {{- toYaml . | nindent 4 }} 32 | {{- end }} -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: terway-controlplane 5 | rules: 6 | - apiGroups: [ "" ] 7 | resources: 8 | - secrets 9 | verbs: 10 | - create 11 | - apiGroups: [ "" ] 12 | resources: 13 | - secrets 14 | verbs: 15 | - get 16 | - update 17 | - patch 18 | - watch 19 | resourceNames: 20 | - terway-controlplane-webhook-cert 21 | - apiGroups: 22 | - coordination.k8s.io 23 | resources: 24 | - leases 25 | verbs: 26 | - get 27 | - list 28 | - watch 29 | - create 30 | - update 31 | -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: terway-controlplane 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: terway-controlplane 9 | subjects: 10 | - kind: ServiceAccount 11 | name: terway-controlplane 12 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: terway-controlplane-credential 5 | labels: 6 | {{- include "terway-controlplane.labels" . | nindent 4 }} 7 | stringData: 8 | ctrl-secret.yaml: | 9 | accessKey: "{{ .Values.terwayControlplane.accessKey }}" 10 | accessSecret: "{{ .Values.terwayControlplane.accessSecret }}" 11 | otelEndpoint: "{{ .Values.terwayControlplane.otelEndpoint }}" 12 | otelToken: "{{ .Values.terwayControlplane.otelToken }}" 13 | 14 | -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: terway-controlplane 5 | labels: 6 | {{- include "terway-controlplane.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.terwayControlplane.service.type }} 9 | ports: 10 | - port: {{ .Values.terwayControlplane.service.port }} 11 | targetPort: 4443 12 | protocol: TCP 13 | name: https 14 | selector: 15 | {{- include "terway-controlplane.selectorLabels" . | nindent 4 }} 16 | -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: terway-controlplane 5 | labels: 6 | {{- include "terway-controlplane.labels" . | nindent 4 }} 7 | -------------------------------------------------------------------------------- /charts/terway/templates/terway-controlplane/webhook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: terway-controlplane 5 | labels: 6 | {{- include "terway-controlplane.labels" . | nindent 4 }} 7 | webhooks: 8 | - name: {{ .Chart.Name }}.mutating.k8s.io 9 | namespaceSelector: 10 | matchExpressions: 11 | - key: k8s.aliyun.com/pod-eni 12 | operator: NotIn 13 | values: ["false"] 14 | rules: 15 | - apiGroups: [""] 16 | apiVersions: ["v1"] 17 | operations: ["CREATE"] 18 | resources: ["pods"] 19 | scope: "Namespaced" 20 | - apiGroups: ["network.alibabacloud.com"] 21 | apiVersions: ["*"] 22 | operations: ["CREATE"] 23 | resources: ["podnetworkings"] 24 | scope: "Cluster" 25 | clientConfig: 26 | service: 27 | namespace: {{ .Release.Namespace }} 28 | name: terway-controlplane 29 | path: /mutating 30 | admissionReviewVersions: ["v1", "v1beta1"] 31 | sideEffects: None 32 | timeoutSeconds: {{ .Values.terwayControlplane.webhookTimeoutSeconds }} 33 | failurePolicy: {{ .Values.terwayControlplane.webhookFailurePolicy }} 34 | --- 35 | apiVersion: admissionregistration.k8s.io/v1 36 | kind: ValidatingWebhookConfiguration 37 | metadata: 38 | name: terway-controlplane 39 | labels: 40 | {{- include "terway-controlplane.labels" . | nindent 4 }} 41 | webhooks: 42 | - name: {{ .Chart.Name }}.validate.k8s.io 43 | rules: 44 | - apiGroups: ["network.alibabacloud.com"] 45 | apiVersions: ["*"] 46 | operations: ["CREATE"] 47 | resources: ["podnetworkings"] 48 | scope: "Cluster" 49 | clientConfig: 50 | service: 51 | namespace: {{ .Release.Namespace }} 52 | name: terway-controlplane 53 | path: /validate 54 | admissionReviewVersions: ["v1", "v1beta1"] 55 | sideEffects: None 56 | timeoutSeconds: {{ .Values.terwayControlplane.webhookTimeoutSeconds }} 57 | failurePolicy: Fail -------------------------------------------------------------------------------- /charts/terway/templates/terwayd/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "terway.name" -}} 5 | {{- default .Chart.Name .Values.terway.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "terway.fullname" -}} 14 | {{- if .Values.terway.fullnameOverride }} 15 | {{- .Values.terway.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.terway.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "terway.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "terway.labels" -}} 37 | helm.sh/chart: {{ include "terway.chart" . }} 38 | {{ include "terway.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "terway.selectorLabels" -}} 49 | app.kubernetes.io/name: terway 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} -------------------------------------------------------------------------------- /charts/terway/templates/terwayd/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: terway-binding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: terway-pod-reader 9 | subjects: 10 | - kind: ServiceAccount 11 | name: terway 12 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /charts/terway/templates/terwayd/configmap.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: eni-config 5 | data: 6 | eni_conf: | 7 | { 8 | "version": "1", 9 | "max_pool_size": 5, 10 | "min_pool_size": 0, 11 | "enable_eni_trunking": false, 12 | "access_key": "{{.Values.terway.accessKey}}", 13 | "access_secret": "{{.Values.terway.accessSecret}}", 14 | "vswitches": {{- toJson .Values.terway.vSwitchIDs }}, 15 | "service_cidr": "{{.Values.terway.serviceCIDR}}", 16 | "security_groups": {{- toJson .Values.terway.securityGroupIDs }}, 17 | "ip_stack": "{{.Values.terway.ipStack}}", 18 | {{- if .Values.centralizedIPAM }} 19 | "ipam_type": "crd", 20 | {{- end }} 21 | "vswitch_selection_policy": "ordered" 22 | } 23 | 10-terway.conf: | 24 | { 25 | "cniVersion": "0.4.0", 26 | "name": "terway", 27 | "capabilities": {"bandwidth": true}, 28 | "network_policy_provider": "{{.Values.terway.networkPolicyProvider}}", 29 | {{- if .Values.terway.ciliumArgs }} 30 | "cilium_args": "{{.Values.terway.ciliumArgs}}", 31 | {{- end}} 32 | {{- if .Values.terway.enableDatapathV2 }} 33 | "eniip_virtual_type": "datapathv2", 34 | "host_stack_cidrs": ["169.254.20.10/32"], 35 | {{- end}} 36 | "type": "terway" 37 | } 38 | disable_network_policy: {{ not .Values.terway.enableNetworkPolicy | quote }} 39 | in_cluster_loadbalance: "true" 40 | -------------------------------------------------------------------------------- /charts/terway/templates/terwayd/role.yml: -------------------------------------------------------------------------------- 1 | kind: Role 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: terway 5 | rules: 6 | - apiGroups: 7 | - '' 8 | resources: 9 | - configmaps 10 | verbs: 11 | - get 12 | - watch 13 | - list -------------------------------------------------------------------------------- /charts/terway/templates/terwayd/rolebinding.yml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: terway-binding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: terway 9 | subjects: 10 | - kind: ServiceAccount 11 | name: terway 12 | namespace: {{ .Release.Namespace }} -------------------------------------------------------------------------------- /charts/terway/templates/terwayd/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: terway 5 | automountServiceAccountToken: true 6 | -------------------------------------------------------------------------------- /cmd/terway-cli/cni_linux_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_parseRelease(t *testing.T) { 10 | type args struct { 11 | rel string 12 | } 13 | tests := []struct { 14 | name string 15 | args args 16 | wantMajor int 17 | wantMinor int 18 | wantPatch int 19 | wantOk bool 20 | }{ 21 | { 22 | name: "test1", 23 | args: args{ 24 | rel: "6.8.0-51-generic", 25 | }, 26 | wantMajor: 6, 27 | wantMinor: 8, 28 | wantPatch: 0, 29 | wantOk: true, 30 | }, 31 | } 32 | for _, tt := range tests { 33 | t.Run(tt.name, func(t *testing.T) { 34 | gotMajor, gotMinor, gotPatch, gotOk := parseRelease(tt.args.rel) 35 | assert.Equalf(t, tt.wantMajor, gotMajor, "parseRelease(%v)", tt.args.rel) 36 | assert.Equalf(t, tt.wantMinor, gotMinor, "parseRelease(%v)", tt.args.rel) 37 | assert.Equalf(t, tt.wantPatch, gotPatch, "parseRelease(%v)", tt.args.rel) 38 | assert.Equalf(t, tt.wantOk, gotOk, "parseRelease(%v)", tt.args.rel) 39 | }) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /cmd/terway-cli/cni_unsupport.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | 3 | package main 4 | 5 | func switchDataPathV2() bool { 6 | return true 7 | } 8 | 9 | func checkKernelVersion(k, major, minor int) bool { 10 | return false 11 | } 12 | 13 | func allowEBPFNetworkPolicy(enable bool) (bool, error) { 14 | return enable, nil 15 | } 16 | 17 | func isOldNode() (bool, error) { 18 | return false, nil 19 | } 20 | -------------------------------------------------------------------------------- /cmd/terway-cli/common.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | const ( 9 | pluginTypeTerway = "terway" 10 | pluginTypeCilium = "cilium-cni" 11 | ) 12 | const eniConfBasePath = "/etc/eni" 13 | 14 | const ( 15 | True = "true" 16 | False = "false" 17 | ) 18 | 19 | type TerwayConfig struct { 20 | enableNetworkPolicy bool 21 | enableInClusterLB bool 22 | 23 | eniConfig []byte 24 | cniConfig []byte 25 | cniConfigList []byte 26 | } 27 | 28 | // getAllConfig ready terway configmap mounted on path 29 | func getAllConfig(base string) (*TerwayConfig, error) { 30 | cfg := &TerwayConfig{ 31 | enableNetworkPolicy: true, 32 | } 33 | 34 | r, err := os.ReadFile(filepath.Join(base, "10-terway.conf")) 35 | if err != nil { 36 | // this file must exist 37 | return nil, err 38 | } 39 | 40 | cfg.cniConfig = r 41 | 42 | r, err = os.ReadFile(filepath.Join(base, "10-terway.conflist")) 43 | if err != nil { 44 | if !os.IsNotExist(err) { 45 | return nil, err 46 | } 47 | } else { 48 | cfg.cniConfigList = r 49 | } 50 | 51 | r, err = os.ReadFile(filepath.Join(base, "disable_network_policy")) 52 | if err != nil { 53 | if !os.IsNotExist(err) { 54 | return nil, err 55 | } 56 | // default enable policy 57 | } else { 58 | switch string(r) { 59 | case "false", "0", "": 60 | cfg.enableNetworkPolicy = true 61 | default: 62 | cfg.enableNetworkPolicy = false 63 | } 64 | } 65 | 66 | r, err = os.ReadFile(filepath.Join(base, "eni_conf")) 67 | if err != nil { 68 | // this file must exist 69 | return nil, err 70 | } 71 | cfg.eniConfig = r 72 | 73 | r, err = os.ReadFile(filepath.Join(base, "in_cluster_loadbalance")) 74 | if err != nil { 75 | if !os.IsNotExist(err) { 76 | return nil, err 77 | } 78 | } 79 | if string(r) == True { 80 | cfg.enableInClusterLB = true 81 | } 82 | 83 | return cfg, nil 84 | } 85 | -------------------------------------------------------------------------------- /cmd/terway-cli/net.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "net" 6 | ) 7 | 8 | var ( 9 | netInterfaces []net.Interface 10 | ) 11 | 12 | func getNetInterfaces() ([]net.Interface, error) { 13 | if netInterfaces != nil { 14 | return netInterfaces, nil 15 | } 16 | 17 | var err error 18 | netInterfaces, err = net.Interfaces() 19 | 20 | return netInterfaces, err 21 | } 22 | 23 | func getInterfaceByMAC(mac string) (net.Interface, error) { 24 | interfaces, err := getNetInterfaces() 25 | if err != nil { 26 | return net.Interface{}, err 27 | } 28 | 29 | for _, i := range interfaces { 30 | if i.HardwareAddr.String() == mac { 31 | return i, nil 32 | } 33 | } 34 | 35 | return net.Interface{}, errors.New("not found") 36 | } 37 | -------------------------------------------------------------------------------- /cmd/terway-cli/tree.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/AliyunContainerService/terway/rpc" 8 | 9 | "github.com/pterm/pterm" 10 | "github.com/pterm/pterm/putils" 11 | ) 12 | 13 | type tree struct { 14 | Key string 15 | Value string // value only exists in leaf 16 | IsLeaf bool 17 | Leaves []*tree 18 | } 19 | 20 | // add a leaf to the tree 21 | func (t *tree) addLeaf(path []string, value string) { 22 | if len(path) == 0 { // noop 23 | return 24 | } 25 | 26 | if len(path) == 1 { // is leaf 27 | t.Leaves = append(t.Leaves, &tree{ 28 | Key: path[0], 29 | Value: value, 30 | IsLeaf: true, 31 | }) 32 | return 33 | } 34 | 35 | for _, v := range t.Leaves { 36 | if v.Key == path[0] && !v.IsLeaf { 37 | v.addLeaf(path[1:], value) 38 | return 39 | } 40 | } 41 | 42 | tree := &tree{ 43 | Key: path[0], 44 | IsLeaf: false, 45 | Leaves: nil, 46 | } 47 | t.Leaves = append(t.Leaves, tree) 48 | 49 | tree.addLeaf(path[1:], value) 50 | } 51 | 52 | func (t *tree) leveledList(list pterm.LeveledList, level int) pterm.LeveledList { 53 | if t.IsLeaf { 54 | return append(list, pterm.LeveledListItem{ 55 | Level: level, 56 | Text: fmt.Sprintf("%s: %s", t.Key, pterm.ThemeDefault.WarningMessageStyle.Sprint(t.Value)), 57 | }) 58 | } 59 | 60 | list = append(list, pterm.LeveledListItem{ 61 | Level: level, 62 | Text: t.Key, 63 | }) 64 | 65 | for _, v := range t.Leaves { 66 | list = v.leveledList(list, level+1) 67 | } 68 | 69 | return list 70 | } 71 | 72 | func printPTermTree(m []*rpc.MapKeyValueEntry) error { 73 | // build a tree 74 | t := &tree{} 75 | for _, v := range m { 76 | t.addLeaf(strings.Split(v.Key, "/"), v.Value) 77 | } 78 | 79 | list := pterm.LeveledList{} 80 | list = t.leveledList(list, 0) 81 | 82 | root := putils.TreeFromLeveledList(list) 83 | return pterm.DefaultTree. 84 | WithTextStyle(&pterm.ThemeDefault.BarLabelStyle). 85 | WithRoot(root). 86 | Render() 87 | } 88 | -------------------------------------------------------------------------------- /cmd/terway/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "math/rand" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "k8s.io/klog/v2/textlogger" 11 | 12 | _ "go.uber.org/automaxprocs" 13 | 14 | "github.com/AliyunContainerService/terway/daemon" 15 | "github.com/AliyunContainerService/terway/pkg/utils" 16 | "github.com/AliyunContainerService/terway/pkg/version" 17 | 18 | "k8s.io/klog/v2" 19 | ctrl "sigs.k8s.io/controller-runtime" 20 | ) 21 | 22 | var ( 23 | log = ctrl.Log.WithName("setup") 24 | ) 25 | 26 | const defaultSocketPath = "/var/run/eni/eni.socket" 27 | const debugSocketPath = "unix:///var/run/eni/eni_debug.socket" 28 | 29 | var ( 30 | logLevel string 31 | daemonMode string 32 | readonlyListen string 33 | configFilePath string 34 | ) 35 | 36 | func main() { 37 | rand.New(rand.NewSource(time.Now().UnixNano())) 38 | 39 | fs := flag.NewFlagSet("terway", flag.ExitOnError) 40 | fs.StringVar(&daemonMode, "daemon-mode", "VPC", "terway network mode") 41 | fs.StringVar(&logLevel, "log-level", "info", "terway log level") 42 | fs.StringVar(&readonlyListen, "readonly-listen", utils.NormalizePath(debugSocketPath), "terway readonly listen") 43 | fs.StringVar(&configFilePath, "config", "/etc/eni/eni.json", "terway config file") 44 | ctrl.RegisterFlags(fs) 45 | 46 | err := fs.Parse(os.Args[1:]) 47 | if err != nil { 48 | panic(err) 49 | } 50 | var opts []textlogger.ConfigOption 51 | 52 | if strings.ToLower(logLevel) == "debug" { 53 | opts = append(opts, textlogger.Verbosity(4)) 54 | } 55 | ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig(opts...))) 56 | 57 | log.Info(version.Version) 58 | 59 | ctx := ctrl.SetupSignalHandler() 60 | ctx = ctrl.LoggerInto(ctx, ctrl.Log) 61 | err = daemon.Run(ctx, utils.NormalizePath(defaultSocketPath), readonlyListen, utils.NormalizePath(configFilePath), daemonMode) 62 | 63 | if err != nil { 64 | klog.Fatal(err) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /daemon/daemon_unsupported.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | 3 | package daemon 4 | 5 | import ( 6 | "context" 7 | 8 | "k8s.io/apimachinery/pkg/util/sets" 9 | 10 | "github.com/AliyunContainerService/terway/types" 11 | ) 12 | 13 | func gcLeakedRules(existIP sets.Set[string]) {} 14 | 15 | func gcPolicyRoutes(ctx context.Context, mac string, containerIPNet *types.IPNetSet, namespace, name string) error { 16 | return nil 17 | } 18 | -------------------------------------------------------------------------------- /daemon/daemon_unwindows.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package daemon 4 | 5 | import ( 6 | "github.com/AliyunContainerService/terway/pkg/k8s" 7 | ) 8 | 9 | func preStartResourceManager(daemonMode string, k8s k8s.Kubernetes) error { 10 | return nil 11 | } 12 | -------------------------------------------------------------------------------- /daemon/daemon_windows.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/pkg/errors" 8 | 9 | "github.com/AliyunContainerService/terway/pkg/aliyun/metadata" 10 | "github.com/AliyunContainerService/terway/pkg/k8s" 11 | "github.com/AliyunContainerService/terway/pkg/windows/apis" 12 | "github.com/AliyunContainerService/terway/pkg/windows/iface" 13 | "github.com/AliyunContainerService/terway/pkg/windows/ip" 14 | "github.com/AliyunContainerService/terway/types/daemon" 15 | ) 16 | 17 | func preStartResourceManager(daemonMode string, k8s k8s.Kubernetes) error { 18 | ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) 19 | defer cancel() 20 | 21 | primaryMac, err := metadata.GetPrimaryENIMAC() 22 | if err != nil { 23 | return errors.Wrap(err, "error getting primary mac") 24 | } 25 | 26 | switch daemonMode { 27 | case daemon.ModeENIOnly, daemon.ModeENIMultiIP: 28 | // NB(thxCode): create a fake network to allow service connection 29 | var assistantIface, err = iface.GetInterfaceByMAC(primaryMac, true) 30 | if err != nil { 31 | return errors.Wrap(err, "error getting assistant interface") 32 | } 33 | var assistantNwSubnet = ip.FromIPNet(k8s.GetServiceCIDR().IPv4).Next().ToIPNet() 34 | var assistantNw = &apis.Network{ 35 | Name: "cb0", 36 | AdapterName: assistantIface.Alias, 37 | AdapterMAC: assistantIface.MacAddress, 38 | Subnet: *assistantNwSubnet, 39 | Gateway: apis.GetDefaultNetworkGateway(assistantNwSubnet), 40 | } 41 | err = apis.AddBridgeHNSNetwork(ctx, assistantNw) 42 | if err != nil { 43 | return errors.Wrapf(err, "error adding assistant network: %s", assistantNw.Format(apis.HNS)) 44 | } 45 | } 46 | 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /daemon/resource_manager.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | const ( 4 | resDBPath = "/var/lib/cni/terway/ResRelation.db" 5 | resDBName = "relation" 6 | ) 7 | -------------------------------------------------------------------------------- /daemon/rule_unsupported.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | 3 | package daemon 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/AliyunContainerService/terway/types/daemon" 9 | ) 10 | 11 | func ruleSync(ctx context.Context, res daemon.PodResources) error { 12 | return nil 13 | } 14 | -------------------------------------------------------------------------------- /daemon/server_linux.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "os" 5 | "syscall" 6 | ) 7 | 8 | var stackTriggerSignals = []os.Signal{ 9 | syscall.SIGUSR1, 10 | } 11 | 12 | var syscallUmask = syscall.Umask 13 | -------------------------------------------------------------------------------- /daemon/server_unsupported.go: -------------------------------------------------------------------------------- 1 | //go:build !linux && !windows 2 | 3 | package daemon 4 | 5 | import ( 6 | "os" 7 | ) 8 | 9 | var stackTriggerSignals = []os.Signal{} 10 | 11 | var syscallUmask = func(int) int { 12 | // nothing to do 13 | return 0 14 | } 15 | -------------------------------------------------------------------------------- /daemon/server_windows.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "os" 5 | "syscall" 6 | ) 7 | 8 | var stackTriggerSignals = []os.Signal{ 9 | syscall.SIGINT, 10 | } 11 | 12 | var syscallUmask = func(int) int { 13 | // nothing to do 14 | return 0 15 | } 16 | -------------------------------------------------------------------------------- /deploy/images/terway-controlplane/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1-labs 2 | ARG TERWAY_POLICY_IMAGE=registry-cn-zhangjiakou.ack.aliyuncs.com/acs/terway:policy-fcb36c33@sha256:e5c73e167c4d7c68837b522df6c8223fb94f94e457970d2bc41b1e629ee2ab16 3 | 4 | FROM --platform=$TARGETPLATFORM ${TERWAY_POLICY_IMAGE} AS policy-dist 5 | 6 | FROM --platform=$BUILDPLATFORM golang:1.24.2 AS builder 7 | ARG GOPROXY 8 | ARG TARGETOS 9 | ARG TARGETARCH 10 | ENV GOPROXY=$GOPROXY 11 | WORKDIR /go/src/github.com/AliyunContainerService/terway/ 12 | COPY --parents go.mod go.sum .git cmd daemon deploy deviceplugin pkg plugin rpc tests types ./ 13 | RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags default_build \ 14 | -ldflags \ 15 | "-X \"github.com/AliyunContainerService/terway/pkg/version.gitCommit=`git rev-parse HEAD`\" \ 16 | -X \"github.com/AliyunContainerService/terway/pkg/version.gitVersion=`git describe --tags --match='v*' --abbrev=14`\" \ 17 | -X \"github.com/AliyunContainerService/terway/pkg/version.buildDate=`date -u +'%Y-%m-%dT%H:%M:%SZ'`\" \ 18 | -X \"github.com/AliyunContainerService/terway/pkg/aliyun/credential.kubernetesAlicloudIdentity=terway-controlplane/`git rev-parse --short HEAD 2>/dev/null`\"" \ 19 | -o terway-controlplane cmd/terway-controlplane/terway-controlplane.go 20 | 21 | FROM --platform=$TARGETPLATFORM debian:stable-slim AS cert 22 | RUN apt-get update && apt-get -uy upgrade 23 | RUN apt-get -y install ca-certificates && update-ca-certificates 24 | 25 | FROM --platform=$TARGETPLATFORM scratch 26 | WORKDIR / 27 | COPY --from=cert /etc/ssl/certs /etc/ssl/certs 28 | COPY --from=builder /go/src/github.com/AliyunContainerService/terway/terway-controlplane /usr/bin/terway-controlplane 29 | COPY --from=policy-dist /tmp/install/usr/bin/cilium-operator-generic /usr/bin/cilium-operator-generic 30 | USER 1000:1000 31 | 32 | ENTRYPOINT ["/usr/bin/terway-controlplane"] -------------------------------------------------------------------------------- /docs/centralized-ipam.md: -------------------------------------------------------------------------------- 1 | # Terway 中心化 IAPM 2 | 3 | ## 简介 4 | 5 | 之前 IAPM 设计是由 terway daemonSet 运行在每个节点上自主完成的。 6 | 有几个缺点： 7 | 8 | - 数据面权限高： 每个节点都能操作 openAPI ，风险比较高。 9 | - 流控管理：每个节点独立运作，不能做集群维度的流控管理。 10 | - IPAM 记录可见性：不好观察 IPAM 记录，分配记录都在节点的 DB 里面。 11 | 12 | ## 基本流程 13 | 14 | ```mermaid 15 | sequenceDiagram 16 | autonumber 17 | participant terwaycontrolplane 18 | participant api 19 | participant terwayd 20 | 21 | terwaycontrolplane ->> api: create CR node 22 | terwayd ->> api: watch CR node 23 | terwayd ->> api: update spec.eni spec.pool 24 | api -->> terwaycontrolplane: wait updated 25 | terwaycontrolplane ->> api: sync node and pods 26 | ``` 27 | -------------------------------------------------------------------------------- /docs/cni-chain.md: -------------------------------------------------------------------------------- 1 | # Custom CNI Chain 2 | 3 | ## Overview 4 | 5 | Terway will set CNI config on the startup. The config (`/etc/cni/net.d/10-terway.conf`) looks like: 6 | 7 | ```json 8 | { 9 | "cniVersion": "0.4.0", 10 | "name": "terway", 11 | "type": "terway", 12 | "eniip_virtual_type": "IPVlan" 13 | } 14 | ``` 15 | 16 | ## Configuration 17 | 18 | Terway will take the cni config in this order `10-terway.conflist` then `10-terway.conf`. 19 | If you need to use custom CNI plugin, you need to add the new `10-terway.conflist` field. 20 | 21 | > Please note that there is no guarantee that custom plugin functionality will work. 22 | 23 | ```yaml 24 | kind: ConfigMap 25 | apiVersion: v1 26 | metadata: 27 | name: eni-config 28 | namespace: kube-system 29 | data: 30 | 10-terway.conflist: | 31 | { 32 | "plugins": [ 33 | { 34 | "type": "terway" 35 | }, 36 | { 37 | "type": "cilium-cni" 38 | }, 39 | { 40 | "type": "portmap", 41 | "capabilities": {"portMappings": true}, 42 | "externalSetMarkChain":"KUBE-MARK-MASQ" 43 | } 44 | ] 45 | } 46 | 47 | 10-terway.conf: | 48 | { 49 | "cniVersion": "0.4.0", 50 | "name": "terway", 51 | "type": "terway" 52 | } 53 | 54 | 55 | ``` 56 | -------------------------------------------------------------------------------- /docs/ecs.md: -------------------------------------------------------------------------------- 1 | # Terway ECS节点 2 | 3 | ## RAM 4 | 5 | 确保Terway 使用的凭据，具备下面的 RAM 权限： 6 | 7 | Terway需要授权中包含以下 [`RAM 权限`](https://ram.console.aliyun.com/) 8 | 9 | ```json 10 | { 11 | "Version": "1", 12 | "Statement": [ 13 | { 14 | "Action": [ 15 | "ecs:CreateNetworkInterface", 16 | "ecs:DescribeNetworkInterfaces", 17 | "ecs:AttachNetworkInterface", 18 | "ecs:DetachNetworkInterface", 19 | "ecs:DeleteNetworkInterface", 20 | "ecs:DescribeInstanceTypes", 21 | "ecs:AssignPrivateIpAddresses", 22 | "ecs:UnassignPrivateIpAddresses", 23 | "ecs:AssignIpv6Addresses", 24 | "ecs:UnassignIpv6Addresses" 25 | ], 26 | "Resource": [ 27 | "*" 28 | ], 29 | "Effect": "Allow" 30 | }, 31 | { 32 | "Action": [ 33 | "vpc:DescribeVSwitches" 34 | ], 35 | "Resource": [ 36 | "*" 37 | ], 38 | "Effect": "Allow" 39 | } 40 | ] 41 | } 42 | ``` 43 | 44 | ## 节点配置 45 | 46 | | Label Key | Description | Example Value | 47 | |------------------------------------|-------------|-------------------| 48 | | `alibabacloud.com/lingjun-worker` | 灵骏节点标记。 | `true` | 49 | | `topology.kubernetes.io/region` | 地域。 | `cn-wulanchabu` | 50 | | `node.kubernetes.io/instance-type` | 实例规格。 | `ecs.i2ne.xlarge` | 51 | | `topology.kubernetes.io/zone` | 可用区。 | `cn-wulanchabu-c` | 52 | 53 | | Spec | Description | Example Value | 54 | |-------------|-------------|------------------------| 55 | | Provider ID | ECS实例 ID | `cn-wulanchabu.i-xxxx` | 56 | 57 | ## 部署 Terway 58 | 59 | ```bash 60 | echo " 61 | centralizedIPAM:true 62 | terway: 63 | securityGroupIDs: 64 | - sg-1 65 | - sg-2 66 | vSwitchIDs: 67 | cn-hangzhou-k: 68 | - vsw-1 69 | - vsw-2 70 | " | helm template --namespace kube-system terway-eniip . --values - 71 | ``` 72 | -------------------------------------------------------------------------------- /docs/host-stack-cidrs.md: -------------------------------------------------------------------------------- 1 | # Terway 主机网络栈路由 2 | 3 | ## 背景 4 | 5 | - 在 Terway IPVLAN 模式中，使用 Terway 网络的 Pod 网络流量会被转发至主机网络中 IPVLAN 接口，通过其 `tc egress filter` 策略转发流量。 6 | - 默认情况下，`tc egress` 仅会将 `ServiceCIDR` 的流量转发至主机网络栈，交由 IPVS 等策略进行处理。如果希望将部分网段额外路由至主机网络栈，您可以参考本文进行设置。 7 | - 常见场景：DNS 缓存方案 `node-local-dns` 中，Local DNS 缓存作为 DaemonSet 部署在每个集群节点上，通过 Link-Local Address 暴露缓存服务。如需在 IPVLAN 模式下使用本地 8 | DNS 缓存，可以将 Link-Local Address 设置到主机网络栈路由中。 9 | - 在 Terway ENIONLY 模式中，使用 Terway 网络的 Pod 中会默认添加 `ServiceCIDR` 路由至主机，通过设置主机网络栈，可以将指定网络同样路由至主机，配置方式同下。 10 | 11 | ![host_stack_cidrs](images/host_stack_cidrs.png) 12 | 13 | ## 操作流程 14 | 15 | 1. 执行以下命令，打开 Terway 配置文件修改 16 | 17 | ```bash 18 | kubectl -n kube-system edit cm eni-config -o yaml 19 | ``` 20 | 21 | 2. 可以看到默认的配置包含以下内容，如需增加主机网络栈路由，请添加 `host_stack_cidrs` 字段，并填入需要转发的网段，保存退出 22 | 23 | ```json 24 | 10-terway.conf: | 25 | { 26 | "cniVersion": "0.4.0", 27 | "name": "terway", 28 | "eniip_virtual_type": "IPVlan", 29 | "host_stack_cidrs": ["169.254.0.0/16"], // 此处为您希望添加的主机网络栈路由 30 | "type": "terway" 31 | } 32 | ``` 33 | 34 | 3. 执行以下命令，过滤出当前 Terway 的所有 DaemonSet 容器组 35 | 36 | ```bash 37 | kubectl -n kube-system get pod | grep terway-eniip 38 | terway-eniip-7d56l 2/2 Running 0 30m 39 | terway-eniip-s7m2t 2/2 Running 0 30m 40 | ``` 41 | 42 | 4. 执行以下命令，触发 Terway 容器组重建 43 | 44 | ```bash 45 | kubectl -n kube-system delete pod terway-eniip-7d56l terway-eniip-s7m2t 46 | ``` 47 | 48 | 5. 登录任意集群节点，执行以下命令，查看 Terway 配置文件，如果包含添加的网段，则说明变更成功 49 | 50 | ```bash 51 | cat /etc/cni/net.d/* 52 | { 53 | "cniVersion": "0.4.0", 54 | "name": "terway-chainer", 55 | "plugins": [ 56 | { 57 | "eniip_virtual_type": "IPVlan", 58 | "host_stack_cidrs": [ // 此处为新增的主机网络栈路由 59 | "169.254.0.0/16", 60 | ], 61 | "type": "terway" 62 | }, 63 | { 64 | "type": "cilium-cni" 65 | } 66 | ] 67 | } 68 | ``` 69 | 70 | 6. 重建任意一个使用 Terway 网络的 Pod 后，Pod 所在节点即可在容器中访问该新网段。 71 | -------------------------------------------------------------------------------- /docs/images/ding_group_qrcode.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/ding_group_qrcode.jpg -------------------------------------------------------------------------------- /docs/images/eni_connection.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/eni_connection.jpg -------------------------------------------------------------------------------- /docs/images/eniip_connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/eniip_connection.png -------------------------------------------------------------------------------- /docs/images/eniip_connection_ipvlan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/eniip_connection_ipvlan.png -------------------------------------------------------------------------------- /docs/images/host_stack_cidrs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/host_stack_cidrs.png -------------------------------------------------------------------------------- /docs/images/terway-dynamic-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/terway-dynamic-config.png -------------------------------------------------------------------------------- /docs/images/terway.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/terway.png -------------------------------------------------------------------------------- /docs/images/terway_cli_mapping.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/terway_cli_mapping.png -------------------------------------------------------------------------------- /docs/images/terway_cli_metadata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/terway_cli_metadata.png -------------------------------------------------------------------------------- /docs/images/terway_deploy_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/terway_deploy_diagram.png -------------------------------------------------------------------------------- /docs/images/terway_resource_pool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/terway_resource_pool.png -------------------------------------------------------------------------------- /docs/images/terway_tracing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/terway_tracing.png -------------------------------------------------------------------------------- /docs/images/terway_with_cilium.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/terway_with_cilium.png -------------------------------------------------------------------------------- /docs/images/vpc_connection.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliyunContainerService/terway/7a73bc02a2b08968eb7e351584976ce86d072144/docs/images/vpc_connection.jpg -------------------------------------------------------------------------------- /docs/ipv6.md: -------------------------------------------------------------------------------- 1 | # 开启 IPv6 双栈 2 | 3 | ## 配置 4 | 5 | 启用双栈模式，需要在 terway 配置中配置 `ip_stack` 为 `dual`，并重启 Terway Pods 6 | 7 | ```yaml 8 | ❯ kubectl get cm -n kube-system eni-config -oyaml 9 | apiVersion: v1 10 | data: 11 | 10-terway.conf: | 12 | { 13 | "cniVersion": "0.4.0", 14 | "name": "terway", 15 | "eniip_virtual_type": "IPVlan", 16 | "ip_stack": "dual", <----- 启用双栈支持 17 | "type": "terway" 18 | } 19 | disable_network_policy: "false" 20 | eni_conf: | 21 | { 22 | "min_pool_size": 0 23 | ... 24 | } 25 | kind: ConfigMap 26 | ``` 27 | 28 | 确保 Terway 使用的 RAM 包含下面 `RAM` 权限 29 | 30 | ```json 31 | { 32 | "Version": "1", 33 | "Statement": [{ 34 | "Action": [ 35 | "ecs:AssignIpv6Addresses", 36 | "ecs:UnassignIpv6Addresses" 37 | ], 38 | "Resource": [ 39 | "*" 40 | ], 41 | "Effect": "Allow" 42 | } 43 | } 44 | ``` 45 | -------------------------------------------------------------------------------- /docs/linjun.md: -------------------------------------------------------------------------------- 1 | # Terway 灵骏节点 2 | 3 | ## 概述 4 | 5 | PAI灵骏是一种高密度计算服务，专为大规模计算场景设计。Terway CNI 能够管理灵骏节点，并支持节点上容器的网络通信。 6 | 灵骏弹性网卡（Lingjun Elastic Network Interface，简称 LENI）是灵骏 GPU 实例接入专有网络 VPC 的虚拟网络接口。它连接灵骏节点与 7 | VPC，实现与 VPC 内其他云资源的高效互联互通。 8 | 与 ECS 弹性网卡类似，LENI 也支持辅助 IP 配置。 9 | 10 | ## RAM 11 | 12 | Terway需要授权中包含以下 [`RAM 权限`](https://ram.console.aliyun.com/) 13 | 14 | ```json 15 | { 16 | "Version": "1", 17 | "Statement": [ 18 | { 19 | "Action": [ 20 | "eflo:CreateElasticNetworkInterface", 21 | "eflo:DeleteElasticNetworkInterface", 22 | "eflo:AssignLeniPrivateIpAddress", 23 | "eflo:UnassignLeniPrivateIpAddress", 24 | "eflo:GetElasticNetworkInterface", 25 | "eflo:ListLeniPrivateIpAddresses", 26 | "eflo:ListElasticNetworkInterfaces", 27 | "eflo:GetNodeInfoForPod" 28 | ], 29 | "Resource": [ 30 | "*" 31 | ], 32 | "Effect": "Allow" 33 | }, 34 | { 35 | "Action": [ 36 | "vpc:DescribeVSwitches" 37 | ], 38 | "Resource": [ 39 | "*" 40 | ], 41 | "Effect": "Allow" 42 | } 43 | ] 44 | } 45 | ``` 46 | 47 | ## 节点配置 48 | 49 | | Label Key | Description | Example Value | 50 | |------------------------------------|-------------|-------------------| 51 | | `alibabacloud.com/lingjun-worker` | 灵骏节点标记。 | `true` | 52 | | `topology.kubernetes.io/region` | 地域。 | `cn-wulanchabu` | 53 | | `node.kubernetes.io/instance-type` | 实例规格。 | `ecs.i2ne.xlarge` | 54 | | `topology.kubernetes.io/zone` | 可用区。 | `cn-wulanchabu-c` | 55 | 56 | | Spec | Description | Example Value | 57 | |-------------|-------------|-------------------| 58 | | Provider ID | 灵骏实例 ID | `e01-ow037ccv4ux` | 59 | 60 | - 确保节点上arp 设置正确 61 | 62 | ```text 63 | sysctl -w net.ipv4.conf.all.arp_announce = 0 64 | sysctl -w net.ipv4.conf.all.arp_ignore = 0 65 | ``` 66 | 67 | ## 部署 Terway 68 | 69 | ```bash 70 | echo " 71 | centralizedIPAM:true 72 | featureGates=\"EFLO=true\" 73 | terway: 74 | securityGroupIDs: 75 | - sg-1 76 | - sg-2 77 | vSwitchIDs: 78 | cn-hangzhou-k: 79 | - vsw-1 80 | - vsw-2 81 | " | helm template --namespace kube-system terway-eniip . --values - 82 | ``` 83 | -------------------------------------------------------------------------------- /docs/terway-with-cilium.md: -------------------------------------------------------------------------------- 1 | # Terway + Cilium整合文档 2 | 3 | ## 考虑 4 | 5 | - 我们基本只使用 `Cilium` 的路由和网络策略功能，将 `Cilium` 的全部组件一起运行比较多余 6 | - `IPVLan` 和 `eBPF` 都对内核版本有要求，应该做一些判断 7 | - Terway的不同网络模式( `Veth/IPVlan/IPVlan+Cilium` )是否能够在同一集群中共存，并且保证连通性 8 | - Plugin Chaining 的最小 `CNI` 版本为 `3.0` ，如果版本不满足条件，是否有兼容措施 9 | - 不同网络模式之间的切换是否简单 10 | - 对 `Terway` 部署的改动尽量小，并且使用同一个镜像便于分发 11 | 12 | ## 设计 13 | 14 | 在`IPVlan`模式，基于`Calico`的网络策略失效，并且`Cilium`具有网络策略的相关功能。 15 | 16 | 可以将 `Cilium` 放入 `policy` 容器，通过启动脚本来判断应该启动 `Calico` 或是 `Cilium` ；同时，`Cilium` 的运行还需要安装`BPF` ，生成对应的 `conflist` 文件等步骤，可以在初始化容器中完成这些工作。 17 | 18 | ![image-20200717113416705](images/terway_with_cilium.png) 19 | 20 | 除此之外，为了保证低内核版本下的兼容性(创建网络时不会出现错误)，在 `Terway CNI` 中还需要加入额外的判断，在不满足IPVlan要求的情况下回退到策略路由模式，并以 `Pod Event` 的形式上报。 21 | 22 | ## 使用流程 23 | 24 | - 安装支持 `Cilium` 的 `Terway` 版本(修改并应用仓库目录下的 `charts/terway` ) 25 | 26 | ```shell 27 | helm package charts/terway -d charts 28 | 29 | helm install terway charts/terway --namespace kube-system --set enableIPvlan=true 30 | ``` 31 | 32 | - 重启节点以应用配置 33 | 34 | ## 验证计划 35 | 36 | - `Veth` 模式连通性正常，`network_policy` 开关正常 37 | - 切换到 `IPVlan` 之后连通性正常，`network_policy` 开关正常 38 | - 切换回 `Veth` 模式连通性正常 39 | - 在低内核版本下(`CentOS 7`)触发配置回退 40 | 41 | ## 问题 42 | 43 | - 需要重启节点 44 | - 由于 `Cilium` 需要的库较多，更换 `ubuntu` 镜像打包后体积由180MB到了500+MB 45 | - 丢弃了 `Cilium` 的较多组件(`Operator`、`Envoy Proxy`、基于 `ConfigMap` 的配置)，稳定性需要进一步测试 46 | 47 | ## 之后的工作 48 | 49 | - 针对ENI独占模式的 `Cilium` 整合 50 | - `Cilium` 的大部分组件在当前场景下不需要，能否单独将 `routing` 和 `network policy`部分抽出 51 | - `Cilium` 的参数能否透出给客户进行修改 52 | - 在大规模集群上需要做进一步的性能测试 53 | - Override kernel版本检查 54 | -------------------------------------------------------------------------------- /eni.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.4.0", 3 | "name": "eni", 4 | "type": "terway-plugin", 5 | "prefix": "eth" 6 | } 7 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /hack/init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | # install CNIs 7 | cp -f /usr/bin/terway /opt/cni/bin/ 8 | chmod +x /opt/cni/bin/terway 9 | 10 | cp -f /usr/bin/cilium-cni /opt/cni/bin/ 11 | chmod +x /opt/cni/bin/cilium-cni 12 | 13 | EXTRA_FLAGS="${1:-}" 14 | 15 | # init cni config 16 | terway-cli cni --output /etc/cni/net.d/10-terway.conflist "$EXTRA_FLAGS" 17 | terway-cli nodeconfig "$EXTRA_FLAGS" 18 | cat /etc/cni/net.d/10-terway.conflist 19 | 20 | node_capabilities=/var/run/eni/node_capabilities 21 | if [ ! -f "$node_capabilities" ]; then 22 | echo "Init node capabilities" 23 | mkdir -p /var/run/eni 24 | touch "$node_capabilities" 25 | fi 26 | 27 | require_erdma=$(jq '.enable_erdma' -r >"$node_capabilities" 33 | if ! grep -q "erdma *= *true" "$node_capabilities"; then 34 | sed -i '/erdma *=/d' "$node_capabilities" 35 | echo "erdma = true" >> "$node_capabilities" 36 | fi 37 | else 38 | sed -i '/erdma *= *true/d' "$node_capabilities" 39 | echo "node not support erdma, pls install the latest erdma driver" 40 | fi 41 | fi 42 | 43 | # copy node capabilities to tmpfs so policy container can read it 44 | cp $node_capabilities /var-run-eni/node_capabilities 45 | cat $node_capabilities 46 | sysctl -w net.ipv4.conf.eth0.rp_filter=0 47 | modprobe sch_htb || true 48 | 49 | set +o errexit 50 | 51 | chroot /host systemctl disable eni.service 52 | chroot /host rm -f /etc/udev/rules.d/75-persistent-net-generator.rules /lib/udev/rules.d/60-net.rules /lib/udev/rules.d/61-eni.rules /lib/udev/write_net_rules 53 | -------------------------------------------------------------------------------- /hack/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | // +build tools 3 | 4 | /* 5 | Copyright 2019 The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // This package imports things required by build scripts, to force `go mod` to see them as dependencies 21 | package tools 22 | 23 | import _ "k8s.io/code-generator" 24 | -------------------------------------------------------------------------------- /hack/update-codegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2017 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | # corresponding to go mod init 22 | MODULE=github.com/AliyunContainerService/terway 23 | # api package 24 | APIS_PKG=pkg/apis 25 | # generated output package 26 | OUTPUT_PKG=pkg/generated 27 | # group-version such as foo:v1alpha1 28 | GROUP_VERSION=network.alibabacloud.com:v1beta1 29 | 30 | 31 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 32 | CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)} 33 | 34 | # generate the code with: 35 | # --output-base because this script should also be able to run inside the vendor dir of 36 | # k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir 37 | # instead of the $GOPATH directly. For normal projects this can be dropped. 38 | bash "${CODEGEN_PKG}"/generate-groups.sh "deepcopy,client,informer,lister" \ 39 | ${MODULE}/${OUTPUT_PKG} ${MODULE}/${APIS_PKG} \ 40 | ${GROUP_VERSION} \ 41 | --go-header-file "${SCRIPT_ROOT}"/hack/boilerplate.go.txt 42 | -------------------------------------------------------------------------------- /hack/update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | go mod tidy 4 | go mod vendor 5 | bash hack/update-codegen.sh 6 | go install -tags tools ./ 7 | controller-gen crd paths=./pkg/apis/network.alibabacloud.com/v1beta1/ output:dir=./pkg/apis/crds 8 | rm -rf vendor 9 | -------------------------------------------------------------------------------- /hack/verify-codegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2017 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 22 | 23 | DIFFROOT="${SCRIPT_ROOT}/pkg" 24 | TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp/pkg" 25 | _tmp="${SCRIPT_ROOT}/_tmp" 26 | 27 | cleanup() { 28 | rm -rf "${_tmp}" 29 | } 30 | trap "cleanup" EXIT SIGINT 31 | 32 | cleanup 33 | 34 | mkdir -p "${TMP_DIFFROOT}" 35 | cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" 36 | 37 | "${SCRIPT_ROOT}/hack/update-codegen.sh" 38 | echo "diffing ${DIFFROOT} against freshly generated codegen" 39 | ret=0 40 | diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? 41 | cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" 42 | if [[ $ret -eq 0 ]] 43 | then 44 | echo "${DIFFROOT} up to date." 45 | else 46 | echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" 47 | exit 1 48 | fi 49 | -------------------------------------------------------------------------------- /monitoring/terway-metric-proxy.yml: -------------------------------------------------------------------------------- 1 | # use this config to expose terway metrics api on node 2 | # and register it to prometheus 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | name: terway-metric-svcmonitor 7 | # use your namespace where prometheus in 8 | # arms-prom for Aliyun ARMS Prometheus 9 | namespace: arms-prom 10 | spec: 11 | jobLabel: terway-metric 12 | selector: 13 | matchLabels: 14 | app: terway-metric-proxy 15 | namespaceSelector: 16 | matchNames: 17 | - kube-system 18 | endpoints: 19 | - port: metric 20 | path: /metrics 21 | # pull interval 22 | interval: 15s 23 | 24 | --- 25 | 26 | apiVersion: apps/v1 27 | kind: DaemonSet 28 | metadata: 29 | name: terway-metric-proxy 30 | namespace: kube-system 31 | labels: 32 | app: terway-metric-proxy 33 | spec: 34 | selector: 35 | matchLabels: 36 | app: terway-metric-proxy 37 | template: 38 | metadata: 39 | labels: 40 | app: terway-metric-proxy 41 | annotations: 42 | scheduler.alpha.kubernetes.io/critical-pod: '' 43 | prometheus.io/scrape: "true" 44 | prometheus.io/path: "/metrics" 45 | prometheus.io/port: "15432" 46 | spec: 47 | nodeSelector: 48 | beta.kubernetes.io/arch: amd64 49 | tolerations: 50 | - operator: "Exists" 51 | hostNetwork: true 52 | containers: 53 | - name: proxy 54 | image: alpine/socat 55 | command: [ "socat", "-d", "-d", "TCP4-LISTEN:15432,fork", "UNIX-CONNECT:/var/run/eni/eni_debug.socket" ] 56 | volumeMounts: 57 | - name: terway-metric 58 | mountPath: /var/run/eni/ 59 | volumes: 60 | - name: terway-metric 61 | hostPath: 62 | path: /var/run/eni/ 63 | --- 64 | 65 | apiVersion: v1 66 | kind: Service 67 | metadata: 68 | name: terway-metric 69 | namespace: kube-system 70 | spec: 71 | selector: 72 | app: terway-metric-proxy 73 | clusterIP: None 74 | ports: 75 | - name: metric 76 | protocol: TCP 77 | port: 15432 78 | targetPort: 15432 79 | -------------------------------------------------------------------------------- /pkg/aliyun/client/ecs_service.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "github.com/AliyunContainerService/terway/pkg/aliyun/credential" 5 | "go.opentelemetry.io/otel/trace" 6 | ) 7 | 8 | var _ ECS = &ECSService{} 9 | 10 | type ECSService struct { 11 | ClientSet credential.Client 12 | IdempotentKeyGen IdempotentKeyGen 13 | RateLimiter *RateLimiter 14 | Tracer trace.Tracer 15 | } 16 | 17 | func NewECSService(clientSet credential.Client, rateLimiter *RateLimiter, tracer trace.Tracer) *ECSService { 18 | return &ECSService{ 19 | ClientSet: clientSet, 20 | IdempotentKeyGen: NewIdempotentKeyGenerator(), 21 | RateLimiter: rateLimiter, 22 | Tracer: tracer, 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /pkg/aliyun/client/hdeni.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | ) 7 | 8 | func (a *EFLOService) CreateHDENI(ctx context.Context, opts ...CreateNetworkInterfaceOption) (*NetworkInterface, error) { 9 | return nil, fmt.Errorf("unsupported") 10 | } 11 | 12 | func (a *EFLOService) DeleteHDENI(ctx context.Context, eniID string) error { 13 | return fmt.Errorf("unsupported") 14 | } 15 | 16 | func (a *EFLOService) DescribeHDENI(ctx context.Context, opts ...DescribeNetworkInterfaceOption) ([]*NetworkInterface, error) { 17 | return nil, fmt.Errorf("unsupported") 18 | } 19 | 20 | func (a *EFLOService) AttachHDENI(ctx context.Context, opts ...AttachNetworkInterfaceOption) error { 21 | return fmt.Errorf("unsupported") 22 | } 23 | 24 | func (a *EFLOService) DetachHDENI(ctx context.Context, opts ...DetachNetworkInterfaceOption) error { 25 | return fmt.Errorf("unsupported") 26 | } 27 | -------------------------------------------------------------------------------- /pkg/aliyun/client/mocks/VPC.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.53.4. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | vpc "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // VPC is an autogenerated mock type for the VPC type 13 | type VPC struct { 14 | mock.Mock 15 | } 16 | 17 | // DescribeVSwitchByID provides a mock function with given fields: ctx, vSwitchID 18 | func (_m *VPC) DescribeVSwitchByID(ctx context.Context, vSwitchID string) (*vpc.VSwitch, error) { 19 | ret := _m.Called(ctx, vSwitchID) 20 | 21 | if len(ret) == 0 { 22 | panic("no return value specified for DescribeVSwitchByID") 23 | } 24 | 25 | var r0 *vpc.VSwitch 26 | var r1 error 27 | if rf, ok := ret.Get(0).(func(context.Context, string) (*vpc.VSwitch, error)); ok { 28 | return rf(ctx, vSwitchID) 29 | } 30 | if rf, ok := ret.Get(0).(func(context.Context, string) *vpc.VSwitch); ok { 31 | r0 = rf(ctx, vSwitchID) 32 | } else { 33 | if ret.Get(0) != nil { 34 | r0 = ret.Get(0).(*vpc.VSwitch) 35 | } 36 | } 37 | 38 | if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { 39 | r1 = rf(ctx, vSwitchID) 40 | } else { 41 | r1 = ret.Error(1) 42 | } 43 | 44 | return r0, r1 45 | } 46 | 47 | // NewVPC creates a new instance of VPC. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 48 | // The first argument is typically a *testing.T value. 49 | func NewVPC(t interface { 50 | mock.TestingT 51 | Cleanup(func()) 52 | }) *VPC { 53 | mock := &VPC{} 54 | mock.Mock.Test(t) 55 | 56 | t.Cleanup(func() { mock.AssertExpectations(t) }) 57 | 58 | return mock 59 | } 60 | -------------------------------------------------------------------------------- /pkg/aliyun/client/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestNewRateLimiter(t *testing.T) { 13 | type args struct { 14 | cfg LimitConfig 15 | } 16 | tests := []struct { 17 | name string 18 | args args 19 | checkFunc func(t *testing.T, r *RateLimiter) 20 | }{ 21 | { 22 | name: "test default", 23 | args: args{ 24 | cfg: nil, 25 | }, 26 | checkFunc: func(t *testing.T, r *RateLimiter) { 27 | assert.Equal(t, 400, r.store["DescribeInstanceTypes"].Burst()) 28 | }, 29 | }, 30 | { 31 | name: "test override", 32 | args: args{ 33 | cfg: map[string]Limit{ 34 | "DescribeInstanceTypes": { 35 | QPS: float64(600 / 60), 36 | Burst: 600, 37 | }, 38 | }, 39 | }, 40 | checkFunc: func(t *testing.T, r *RateLimiter) { 41 | assert.Equal(t, 600, r.store["DescribeInstanceTypes"].Burst()) 42 | }, 43 | }, 44 | } 45 | for _, tt := range tests { 46 | t.Run(tt.name, func(t *testing.T) { 47 | tt.checkFunc(t, NewRateLimiter(tt.args.cfg)) 48 | }) 49 | } 50 | } 51 | 52 | func TestRateLimiter_Wait(t *testing.T) { 53 | r := NewRateLimiter(map[string]Limit{ 54 | "foo": { 55 | QPS: 1, 56 | Burst: 1, 57 | }, 58 | }) 59 | 60 | start := time.Now() 61 | wg := sync.WaitGroup{} 62 | 63 | for i := 0; i < 2; i++ { 64 | wg.Add(1) 65 | 66 | go func() { 67 | defer wg.Done() 68 | err := r.Wait(context.Background(), "foo") 69 | assert.NoError(t, err) 70 | }() 71 | } 72 | wg.Wait() 73 | assert.True(t, 1*time.Second < time.Since(start)) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/aliyun/client/token.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/hex" 6 | "encoding/json" 7 | "os" 8 | "strconv" 9 | "sync" 10 | 11 | "github.com/google/uuid" 12 | "k8s.io/utils/lru" 13 | ) 14 | 15 | type IdempotentKeyGen interface { 16 | GenerateKey(paramHash string) string 17 | PutBack(paramHash string, uuid string) 18 | } 19 | 20 | // SimpleIdempotentKeyGenerator implements the generation and management of idempotency keys. 21 | type SimpleIdempotentKeyGenerator struct { 22 | mu sync.Mutex 23 | cache *lru.Cache 24 | } 25 | 26 | func NewIdempotentKeyGenerator() *SimpleIdempotentKeyGenerator { 27 | size, err := strconv.Atoi(os.Getenv("IDEMPOTENT_KEY_CACHE_SIZE")) 28 | if err != nil || size <= 0 { 29 | size = 500 30 | } 31 | 32 | return &SimpleIdempotentKeyGenerator{ 33 | cache: lru.New(size), 34 | } 35 | } 36 | 37 | // GenerateKey generates an idempotency key based on the given parameter hash. 38 | // multiple key is supported 39 | func (g *SimpleIdempotentKeyGenerator) GenerateKey(paramHash string) string { 40 | g.mu.Lock() 41 | defer g.mu.Unlock() 42 | 43 | v, ok := g.cache.Get(paramHash) 44 | if ok { 45 | uuids := v.([]string) 46 | if len(uuids) > 0 { 47 | id := "" 48 | id, uuids = uuids[len(uuids)-1], uuids[:len(uuids)-1] 49 | 50 | if len(uuids) == 0 { 51 | g.cache.Remove(paramHash) 52 | } else { 53 | g.cache.Add(paramHash, uuids) 54 | } 55 | return id 56 | } 57 | } 58 | 59 | return uuid.NewString() 60 | } 61 | 62 | // PutBack adds the specified idempotency key back into the cache for reuse, associating it with the given parameter hash. 63 | func (g *SimpleIdempotentKeyGenerator) PutBack(paramHash string, uuid string) { 64 | g.mu.Lock() 65 | defer g.mu.Unlock() 66 | 67 | v, ok := g.cache.Get(paramHash) 68 | if ok { 69 | uuids := v.([]string) 70 | uuids = append(uuids, uuid) 71 | 72 | g.cache.Add(paramHash, uuids) 73 | } else { 74 | g.cache.Add(paramHash, []string{uuid}) 75 | } 76 | } 77 | 78 | func md5Hash(obj any) string { 79 | out, _ := json.Marshal(obj) 80 | hash := md5.Sum(out) 81 | return hex.EncodeToString(hash[:]) 82 | } 83 | -------------------------------------------------------------------------------- /pkg/aliyun/client/token_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGenerateKey(t *testing.T) { 10 | generator := NewIdempotentKeyGenerator() 11 | 12 | // Test case 1: paramHash does not exist in cache 13 | paramHash1 := "paramHash1" 14 | key1 := generator.GenerateKey(paramHash1) 15 | assert.NotEmpty(t, key1) 16 | 17 | // Test case 2: paramHash exists in cache with multiple UUIDs 18 | paramHash2 := "paramHash2" 19 | 20 | uuids := []string{"uuid1", "uuid2", "uuid3"} 21 | generator.cache.Add(paramHash2, uuids) 22 | 23 | assert.Equal(t, "uuid3", generator.GenerateKey(paramHash2)) 24 | uuids2, ok := generator.cache.Get(paramHash2) 25 | assert.True(t, ok) 26 | assert.Len(t, uuids2.([]string), 2) 27 | 28 | assert.Equal(t, "uuid2", generator.GenerateKey(paramHash2)) 29 | uuids2, ok = generator.cache.Get(paramHash2) 30 | assert.True(t, ok) 31 | assert.Len(t, uuids2.([]string), 1) 32 | } 33 | 34 | func TestPutBack(t *testing.T) { 35 | // Create a new instance of SimpleIdempotentKeyGenerator 36 | generator := NewIdempotentKeyGenerator() 37 | 38 | // Generate a paramHash and uuid for testing 39 | paramHash := "testParamHash" 40 | uuid1 := "testUUID1" 41 | uuid2 := "testUUID2" 42 | 43 | // Call PutBack method 44 | generator.PutBack(paramHash, uuid1) 45 | generator.PutBack(paramHash, uuid2) 46 | 47 | // Retrieve the value associated with the paramHash from the cache 48 | value, ok := generator.cache.Get(paramHash) 49 | assert.True(t, ok) 50 | 51 | // Check if the retrieved value is a slice of strings 52 | uuids, ok := value.([]string) 53 | assert.True(t, ok) 54 | assert.Len(t, uuids, 2) 55 | 56 | assert.Equal(t, uuids, []string{uuid1, uuid2}) 57 | } 58 | -------------------------------------------------------------------------------- /pkg/aliyun/client/vpc_service.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors" 9 | "github.com/AliyunContainerService/terway/pkg/aliyun/credential" 10 | "github.com/AliyunContainerService/terway/pkg/metric" 11 | "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" 12 | "go.opentelemetry.io/otel/trace" 13 | logf "sigs.k8s.io/controller-runtime/pkg/log" 14 | ) 15 | 16 | const ( 17 | APIDescribeVSwitches = "DescribeVSwitches" 18 | ) 19 | 20 | var _ VPC = &VPCService{} 21 | 22 | type VPCService struct { 23 | ClientSet credential.Client 24 | IdempotentKeyGen IdempotentKeyGen 25 | RateLimiter *RateLimiter 26 | Tracer trace.Tracer 27 | } 28 | 29 | func NewVPCService(clientSet credential.Client, rateLimiter *RateLimiter, tracer trace.Tracer) *VPCService { 30 | return &VPCService{ 31 | ClientSet: clientSet, 32 | IdempotentKeyGen: NewIdempotentKeyGenerator(), 33 | RateLimiter: rateLimiter, 34 | Tracer: tracer, 35 | } 36 | } 37 | 38 | // DescribeVSwitchByID get vsw by id 39 | func (a *VPCService) DescribeVSwitchByID(ctx context.Context, vSwitchID string) (*vpc.VSwitch, error) { 40 | ctx, span := a.Tracer.Start(ctx, APIDescribeVSwitches) 41 | defer span.End() 42 | 43 | err := a.RateLimiter.Wait(ctx, APIDescribeVSwitches) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | req := vpc.CreateDescribeVSwitchesRequest() 49 | req.VSwitchId = vSwitchID 50 | 51 | l := LogFields(logf.FromContext(ctx), req) 52 | 53 | start := time.Now() 54 | resp, err := a.ClientSet.VPC().DescribeVSwitches(req) 55 | metric.OpenAPILatency.WithLabelValues(APIDescribeVSwitches, fmt.Sprint(err != nil)).Observe(metric.MsSince(start)) 56 | if err != nil { 57 | err = apiErr.WarpError(err) 58 | l.WithValues(LogFieldRequestID, apiErr.ErrRequestID(err)).Error(err, "DescribeVSwitches failed") 59 | return nil, err 60 | } 61 | if len(resp.VSwitches.VSwitch) == 0 { 62 | return nil, apiErr.ErrNotFound 63 | } 64 | if len(resp.VSwitches.VSwitch) > 0 { 65 | return &resp.VSwitches.VSwitch[0], nil 66 | } 67 | return nil, err 68 | } 69 | -------------------------------------------------------------------------------- /pkg/aliyun/credential/wire.go: -------------------------------------------------------------------------------- 1 | //go:build wireinject 2 | 3 | package credential 4 | 5 | import ( 6 | "os" 7 | 8 | "github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/provider" 9 | "github.com/google/wire" 10 | ) 11 | 12 | // InitializeClientMgr init ClientMgr 13 | func InitializeClientMgr(regionID string, credProvider provider.CredentialsProvider) (*ClientMgr, error) { 14 | wire.Build( 15 | NewScheme, 16 | NewNetworkType, 17 | NewClientConfig, 18 | NewECSClient, 19 | NewVPCClient, 20 | NewEFLOClient, 21 | NewEFLOV2Client, 22 | NewClientMgr, 23 | ProviderV2, 24 | ProviderV1, 25 | ) 26 | return &ClientMgr{}, nil 27 | } 28 | 29 | func NewClientMgr(credProvider provider.CredentialsProvider, 30 | ecsClient ECSClient, vpcClient VPCClient, efloClient EFLOClient, 31 | efloV2Client EFLOV2Client) *ClientMgr { 32 | 33 | return &ClientMgr{ 34 | 35 | provider: credProvider, 36 | ecsClient: ecsClient, 37 | vpcClient: vpcClient, 38 | efloClient: efloClient, 39 | efloV2Client: efloV2Client, 40 | } 41 | } 42 | 43 | func NewNetworkType() NetworkType { 44 | networkType := "vpc" 45 | if os.Getenv("ALICLOUD_ENDPOINT_TYPE") == "public" { 46 | networkType = "public" 47 | } 48 | return NetworkType(networkType) 49 | } 50 | 51 | func NewScheme() ClientScheme { 52 | scheme := "HTTPS" 53 | if os.Getenv("ALICLOUD_CLIENT_SCHEME") == "HTTP" { 54 | scheme = "HTTP" 55 | } 56 | return ClientScheme(scheme) 57 | } 58 | 59 | func NewClientConfig(regionID string, scheme ClientScheme, networkType NetworkType) ClientConfig { 60 | return ClientConfig{ 61 | RegionID: regionID, 62 | Scheme: string(scheme), 63 | EndpointType: "regional", 64 | NetworkType: string(networkType), 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/aliyun/instance/ecs.go: -------------------------------------------------------------------------------- 1 | package instance 2 | 3 | import ( 4 | "sync/atomic" 5 | 6 | "github.com/AliyunContainerService/terway/pkg/aliyun/metadata" 7 | ) 8 | 9 | type ECS struct { 10 | regionID atomic.Value 11 | zoneID atomic.Value 12 | vSwitchID atomic.Value 13 | primaryMAC atomic.Value 14 | instanceID atomic.Value 15 | instanceType atomic.Value 16 | } 17 | 18 | func (e *ECS) GetRegionID() (string, error) { 19 | if v := e.regionID.Load(); v != nil { 20 | return v.(string), nil 21 | } 22 | regionID, err := metadata.GetLocalRegion() 23 | if err != nil { 24 | return "", err 25 | } 26 | e.regionID.Store(regionID) 27 | return regionID, nil 28 | } 29 | 30 | func (e *ECS) GetZoneID() (string, error) { 31 | if v := e.zoneID.Load(); v != nil { 32 | return v.(string), nil 33 | } 34 | zoneID, err := metadata.GetLocalZone() 35 | if err != nil { 36 | return "", err 37 | } 38 | e.zoneID.Store(zoneID) 39 | return zoneID, nil 40 | } 41 | 42 | func (e *ECS) GetVSwitchID() (string, error) { 43 | if v := e.vSwitchID.Load(); v != nil { 44 | return v.(string), nil 45 | } 46 | vSwitchID, err := metadata.GetLocalVswitch() 47 | if err != nil { 48 | return "", err 49 | } 50 | e.vSwitchID.Store(vSwitchID) 51 | return vSwitchID, nil 52 | } 53 | 54 | func (e *ECS) GetPrimaryMAC() (string, error) { 55 | if v := e.primaryMAC.Load(); v != nil { 56 | return v.(string), nil 57 | } 58 | primaryMAC, err := metadata.GetPrimaryENIMAC() 59 | if err != nil { 60 | return "", err 61 | } 62 | e.primaryMAC.Store(primaryMAC) 63 | return primaryMAC, nil 64 | } 65 | 66 | func (e *ECS) GetInstanceID() (string, error) { 67 | if v := e.instanceID.Load(); v != nil { 68 | return v.(string), nil 69 | } 70 | instanceID, err := metadata.GetLocalInstanceID() 71 | if err != nil { 72 | return "", err 73 | } 74 | e.instanceID.Store(instanceID) 75 | return instanceID, nil 76 | } 77 | 78 | func (e *ECS) GetInstanceType() (string, error) { 79 | if v := e.instanceType.Load(); v != nil { 80 | return v.(string), nil 81 | } 82 | instanceType, err := metadata.GetInstanceType() 83 | if err != nil { 84 | return "", err 85 | } 86 | e.instanceType.Store(instanceType) 87 | return instanceType, nil 88 | } 89 | -------------------------------------------------------------------------------- /pkg/aliyun/instance/instance.go: -------------------------------------------------------------------------------- 1 | package instance 2 | 3 | //go:generate mockery --name Interface 4 | 5 | type Interface interface { 6 | GetRegionID() (string, error) 7 | GetZoneID() (string, error) 8 | GetVSwitchID() (string, error) 9 | GetPrimaryMAC() (string, error) 10 | GetInstanceID() (string, error) 11 | GetInstanceType() (string, error) 12 | } 13 | 14 | var defaultIns Interface = &ECS{} 15 | 16 | func Init(in Interface) { 17 | defaultIns = in 18 | } 19 | 20 | func GetInstanceMeta() Interface { 21 | return defaultIns 22 | } 23 | -------------------------------------------------------------------------------- /pkg/aliyun/metadata/metrics.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | ) 8 | 9 | var ( 10 | // MetadataLatency aliyun metadata latency 11 | MetadataLatency = prometheus.NewHistogramVec( 12 | prometheus.HistogramOpts{ 13 | Name: "aliyun_metadata_latency", 14 | Help: "aliyun metadata latency in ms", 15 | Buckets: prometheus.ExponentialBuckets(1, 2, 10), 16 | }, 17 | []string{"url", "error"}, 18 | ) 19 | ) 20 | 21 | func MsSince(start time.Time) float64 { 22 | return float64(time.Since(start) / time.Millisecond) 23 | } 24 | -------------------------------------------------------------------------------- /pkg/apis/network.alibabacloud.com/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package network 18 | 19 | // GroupName is the group name used in this package 20 | const ( 21 | GroupName = "network.alibabacloud.com" 22 | ) 23 | -------------------------------------------------------------------------------- /pkg/apis/network.alibabacloud.com/v1beta1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // +k8s:deepcopy-gen=package 18 | // +groupName=network.alibabacloud.com 19 | 20 | // Package v1beta1 Package is the v1beta1 version of the API. 21 | package v1beta1 22 | -------------------------------------------------------------------------------- /pkg/apis/network.alibabacloud.com/v1beta1/helper.go: -------------------------------------------------------------------------------- 1 | package v1beta1 2 | 3 | func (p *PodENISpec) HaveFixedIP() bool { 4 | for _, a := range p.Allocations { 5 | if a.AllocationType.Type == IPAllocTypeFixed { 6 | return true 7 | } 8 | } 9 | return false 10 | } 11 | -------------------------------------------------------------------------------- /pkg/apis/network.alibabacloud.com/v1beta1/node_runtime.go: -------------------------------------------------------------------------------- 1 | package v1beta1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | type RuntimePodSpec struct { 8 | UID string `json:"uid"` 9 | Name string `json:"name"` 10 | Namespace string `json:"namespace"` 11 | IPv4 string `json:"ipv4"` 12 | IPv6 string `json:"ipv6"` 13 | } 14 | 15 | // +kubebuilder:validation:Enum=initial;deleted 16 | type CNIStatus string 17 | 18 | const ( 19 | CNIStatusInitial CNIStatus = "initial" 20 | CNIStatusDeleted CNIStatus = "deleted" 21 | ) 22 | 23 | type CNIStatusInfo struct { 24 | LastUpdateTime metav1.Time `json:"lastUpdateTime"` 25 | } 26 | 27 | type RuntimePodStatus struct { 28 | PodID string `json:"podID"` 29 | // when pod is added 30 | Status map[CNIStatus]*CNIStatusInfo `json:"status"` 31 | } 32 | 33 | type NodeRuntimeSpec struct { 34 | } 35 | 36 | type NodeRuntimeStatus struct { 37 | // runtime status, indexed by pod uid 38 | Pods map[string]*RuntimePodStatus `json:"pods,omitempty"` 39 | } 40 | 41 | // +genclient 42 | // +genclient:nonNamespaced 43 | //+kubebuilder:object:root=true 44 | //+kubebuilder:subresource:status 45 | //+kubebuilder:resource:scope=Cluster 46 | 47 | // NodeRuntime is the Schema for the per node runtime API 48 | type NodeRuntime struct { 49 | metav1.TypeMeta `json:",inline"` 50 | metav1.ObjectMeta `json:"metadata,omitempty"` 51 | 52 | Spec NodeRuntimeSpec `json:"spec,omitempty"` 53 | Status NodeRuntimeStatus `json:"status,omitempty"` 54 | } 55 | 56 | //+kubebuilder:object:root=true 57 | 58 | // NodeRuntimeList contains a list of Node 59 | type NodeRuntimeList struct { 60 | metav1.TypeMeta `json:",inline"` 61 | metav1.ListMeta `json:"metadata,omitempty"` 62 | Items []NodeRuntime `json:"items"` 63 | } 64 | -------------------------------------------------------------------------------- /pkg/apis/network.alibabacloud.com/v1beta1/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | "k8s.io/apimachinery/pkg/runtime/schema" 23 | 24 | "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com" 25 | ) 26 | 27 | // SchemeGroupVersion is group version used to register these objects 28 | var SchemeGroupVersion = schema.GroupVersion{Group: network.GroupName, Version: "v1beta1"} 29 | 30 | // Kind takes an unqualified kind and returns back a Group qualified GroupKind 31 | func Kind(kind string) schema.GroupKind { 32 | return SchemeGroupVersion.WithKind(kind).GroupKind() 33 | } 34 | 35 | // Resource takes an unqualified resource and returns a Group qualified GroupResource 36 | func Resource(resource string) schema.GroupResource { 37 | return SchemeGroupVersion.WithResource(resource).GroupResource() 38 | } 39 | 40 | var ( 41 | // SchemeBuilder initializes a scheme builder 42 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 43 | // AddToScheme is a global function that registers this API group & version to a scheme 44 | AddToScheme = SchemeBuilder.AddToScheme 45 | ) 46 | 47 | // Adds the list of known types to Scheme. 48 | func addKnownTypes(scheme *runtime.Scheme) error { 49 | scheme.AddKnownTypes(SchemeGroupVersion, 50 | &PodENI{}, 51 | &PodENIList{}, 52 | &PodNetworking{}, 53 | &PodNetworkingList{}, 54 | &Node{}, 55 | &NodeList{}, 56 | &NodeRuntime{}, 57 | &NodeRuntimeList{}, 58 | &NetworkInterface{}, 59 | &NetworkInterfaceList{}, 60 | ) 61 | metav1.AddToGroupVersion(scheme, SchemeGroupVersion) 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /pkg/controller/all/all.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package all register all controllers 18 | package all 19 | 20 | import ( 21 | // register all controllers 22 | _ "github.com/AliyunContainerService/terway/pkg/controller/eni" 23 | _ "github.com/AliyunContainerService/terway/pkg/controller/multi-ip/node" 24 | _ "github.com/AliyunContainerService/terway/pkg/controller/multi-ip/pod" 25 | _ "github.com/AliyunContainerService/terway/pkg/controller/node" 26 | _ "github.com/AliyunContainerService/terway/pkg/controller/pod" 27 | _ "github.com/AliyunContainerService/terway/pkg/controller/pod-eni" 28 | _ "github.com/AliyunContainerService/terway/pkg/controller/pod-networking" 29 | ) 30 | -------------------------------------------------------------------------------- /pkg/controller/common/ctx_default.go: -------------------------------------------------------------------------------- 1 | //go:build default_build 2 | 3 | package common 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" 9 | ) 10 | 11 | // WithCtx extract fields from v1beta1.Allocation and set to context.Context 12 | func WithCtx(ctx context.Context, alloc *v1beta1.Allocation) context.Context { 13 | return ctx 14 | } 15 | -------------------------------------------------------------------------------- /pkg/controller/common/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | // NodeInfo is the type describe the eni config for this pod 20 | type NodeInfo struct { 21 | NodeName string 22 | InstanceID string 23 | InstanceType string 24 | ZoneID string 25 | TrunkENIID string 26 | RegionID string 27 | } 28 | -------------------------------------------------------------------------------- /pkg/controller/common/types_default.go: -------------------------------------------------------------------------------- 1 | //go:build default_build 2 | 3 | /* 4 | Copyright 2021 Terway Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package common 20 | 21 | import ( 22 | "fmt" 23 | "strings" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | 27 | "github.com/AliyunContainerService/terway/pkg/utils" 28 | "github.com/AliyunContainerService/terway/types" 29 | ) 30 | 31 | func NewNodeInfo(node *corev1.Node) (*NodeInfo, error) { 32 | res := &NodeInfo{NodeName: node.Name} 33 | 34 | if utils.ISLinJunNode(node.Labels) { 35 | res.InstanceID = node.Spec.ProviderID 36 | } else { 37 | ids := strings.Split(node.Spec.ProviderID, ".") 38 | if len(ids) < 2 { 39 | return nil, fmt.Errorf("error parse providerID %s", node.Spec.ProviderID) 40 | } 41 | res.InstanceID = ids[1] 42 | } 43 | if res.InstanceID == "" { 44 | return nil, fmt.Errorf("can not found instanceID from node %s", node.Name) 45 | } 46 | 47 | res.TrunkENIID = node.GetAnnotations()[types.TrunkOn] 48 | 49 | res.RegionID = node.Labels[corev1.LabelTopologyRegion] 50 | if res.RegionID == "" { 51 | return nil, fmt.Errorf("can not found regionID from node %s", node.Name) 52 | } 53 | 54 | res.InstanceType = node.Labels[corev1.LabelInstanceTypeStable] 55 | if res.InstanceType == "" { 56 | return nil, fmt.Errorf("can not found instance type from node %s", node.Name) 57 | } 58 | 59 | zone, ok := node.GetLabels()[corev1.LabelTopologyZone] 60 | if ok { 61 | res.ZoneID = zone 62 | return res, nil 63 | } 64 | zone, ok = node.GetLabels()[corev1.LabelZoneFailureDomain] 65 | if ok { 66 | res.ZoneID = zone 67 | return res, nil 68 | } 69 | return nil, fmt.Errorf("can not found zone label from node %s", node.Name) 70 | } 71 | -------------------------------------------------------------------------------- /pkg/controller/eni/bo.go: -------------------------------------------------------------------------------- 1 | package eni 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "time" 7 | 8 | "golang.org/x/sync/singleflight" 9 | "k8s.io/apimachinery/pkg/util/wait" 10 | ) 11 | 12 | var errTimeOut = errors.New("timeout") 13 | 14 | type BackoffManager struct { 15 | store sync.Map 16 | 17 | single singleflight.Group 18 | } 19 | 20 | func NewBackoffManager() *BackoffManager { 21 | return &BackoffManager{} 22 | } 23 | 24 | func (b *BackoffManager) Get(key string, bo wait.Backoff) (time.Duration, error) { 25 | v, err, _ := b.single.Do(key, func() (interface{}, error) { 26 | 27 | vv, _ := b.store.LoadOrStore(key, &ResourceBackoff{ 28 | Bo: bo, 29 | }) 30 | actual := vv.(*ResourceBackoff) 31 | 32 | du := time.Until(actual.NextTS) 33 | if du > 0 { 34 | // don't do backoff, as the executing is too soon 35 | return du, nil 36 | } 37 | 38 | if actual.Bo.Steps > 0 { 39 | next := actual.Bo.Step() 40 | actual.NextTS = time.Now().Add(next) 41 | return next, nil 42 | } 43 | return time.Duration(0), errTimeOut 44 | }) 45 | 46 | return v.(time.Duration), err 47 | } 48 | 49 | // GetNextTS test only 50 | func (b *BackoffManager) GetNextTS(key string) (time.Time, bool) { 51 | v, ok := b.store.Load(key) 52 | if !ok { 53 | return time.Time{}, false 54 | } 55 | actual := v.(*ResourceBackoff) 56 | return actual.NextTS, true 57 | } 58 | 59 | func (b *BackoffManager) Del(key string) { 60 | b.store.Delete(key) 61 | } 62 | 63 | type ResourceBackoff struct { 64 | NextTS time.Time 65 | Bo wait.Backoff 66 | } 67 | -------------------------------------------------------------------------------- /pkg/controller/multi-ip/node/metric.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | var ( 8 | // ResourcePoolTotal terway total source amount in the pool 9 | ResourcePoolTotal = prometheus.NewGaugeVec( 10 | prometheus.GaugeOpts{ 11 | Name: "gc_count_total", 12 | Help: "controlplane gc total", 13 | }, 14 | []string{"node"}, 15 | ) 16 | 17 | SyncOpenAPITotal = prometheus.NewGaugeVec( 18 | prometheus.GaugeOpts{ 19 | Name: "sync_openapi_count_total", 20 | Help: "controlplane sync data with openapi total", 21 | }, 22 | []string{"node"}, 23 | ) 24 | ) 25 | -------------------------------------------------------------------------------- /pkg/controller/multi-ip/node/pod.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "net/netip" 5 | 6 | networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" 7 | ) 8 | 9 | type PodRequest struct { 10 | // requirements 11 | PodUID string 12 | 13 | RequireIPv4 bool 14 | RequireIPv6 bool 15 | 16 | RequireERDMA bool 17 | 18 | // status form pod status, only used in takeover 19 | IPv4 string 20 | IPv6 string 21 | 22 | // ref for eni & ip 23 | ipv4Ref, ipv6Ref *EniIP 24 | } 25 | 26 | type EniIP struct { 27 | NetworkInterface *networkv1beta1.Nic 28 | IP *networkv1beta1.IP 29 | } 30 | 31 | func podIPs(ips []string) (string, string, error) { 32 | var ipv4, ipv6 string 33 | for _, v := range ips { 34 | if v == "" { 35 | continue 36 | } 37 | addr, err := netip.ParseAddr(v) 38 | if err != nil { 39 | return "", "", err 40 | } 41 | if addr.Is4() { 42 | ipv4 = v 43 | } else { 44 | ipv6 = v 45 | } 46 | } 47 | return ipv4, ipv6, nil 48 | } 49 | -------------------------------------------------------------------------------- /pkg/controller/multi-ip/node/pod_test.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestPodIPs(t *testing.T) { 8 | tests := []struct { 9 | name string 10 | ips []string 11 | wantIPv4 string 12 | wantIPv6 string 13 | wantErr bool 14 | }{ 15 | { 16 | name: "Empty slice", 17 | ips: []string{}, 18 | wantIPv4: "", 19 | wantIPv6: "", 20 | wantErr: false, 21 | }, 22 | { 23 | name: "Only IPv4", 24 | ips: []string{"192.168.1.1"}, 25 | wantIPv4: "192.168.1.1", 26 | wantIPv6: "", 27 | wantErr: false, 28 | }, 29 | { 30 | name: "Only IPv6", 31 | ips: []string{"2001:db8::1"}, 32 | wantIPv4: "", 33 | wantIPv6: "2001:db8::1", 34 | wantErr: false, 35 | }, 36 | { 37 | name: "Mixed IPv4 and IPv6", 38 | ips: []string{"192.168.1.1", "2001:db8::1"}, 39 | wantIPv4: "192.168.1.1", 40 | wantIPv6: "2001:db8::1", 41 | wantErr: false, 42 | }, 43 | { 44 | name: "Invalid IP address", 45 | ips: []string{"192.168.1", "2001:db8::1"}, 46 | wantIPv4: "", 47 | wantIPv6: "", 48 | wantErr: true, 49 | }, 50 | } 51 | 52 | for _, tt := range tests { 53 | t.Run(tt.name, func(t *testing.T) { 54 | gotIPv4, gotIPv6, err := podIPs(tt.ips) 55 | if (err != nil) != tt.wantErr { 56 | t.Errorf("podIPs() error = %v, wantErr %v", err, tt.wantErr) 57 | return 58 | } 59 | if gotIPv4 != tt.wantIPv4 { 60 | t.Errorf("podIPs() gotIPv4 = %v, want %v", gotIPv4, tt.wantIPv4) 61 | } 62 | if gotIPv6 != tt.wantIPv6 { 63 | t.Errorf("podIPs() gotIPv6 = %v, want %v", gotIPv6, tt.wantIPv6) 64 | } 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /pkg/controller/multi-ip/pod/predict.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod 18 | 19 | import ( 20 | "github.com/AliyunContainerService/terway/pkg/utils" 21 | "github.com/AliyunContainerService/terway/types" 22 | 23 | corev1 "k8s.io/api/core/v1" 24 | "sigs.k8s.io/controller-runtime/pkg/event" 25 | "sigs.k8s.io/controller-runtime/pkg/predicate" 26 | ) 27 | 28 | type predicateForPodEvent struct { 29 | predicate.Funcs 30 | } 31 | 32 | func (p *predicateForPodEvent) Create(e event.CreateEvent) bool { 33 | return needProcess(e.Object) 34 | } 35 | 36 | func (p *predicateForPodEvent) Update(e event.UpdateEvent) bool { 37 | return needProcess(e.ObjectNew) 38 | } 39 | 40 | func (p *predicateForPodEvent) Delete(e event.DeleteEvent) bool { 41 | return needProcess(e.Object) 42 | } 43 | 44 | func (p *predicateForPodEvent) Generic(e event.GenericEvent) bool { 45 | return needProcess(e.Object) 46 | } 47 | 48 | // needProcess filter pod which is ready to process 49 | func needProcess(obj interface{}) bool { 50 | pod, ok := obj.(*corev1.Pod) 51 | if !ok { 52 | return false 53 | } 54 | 55 | if pod.Spec.NodeName == "" { 56 | return false 57 | } 58 | 59 | if pod.Spec.HostNetwork { 60 | return false 61 | } 62 | 63 | if types.IgnoredByTerway(pod.Labels) { 64 | return false 65 | } 66 | 67 | if types.PodUseENI(pod) { 68 | return false 69 | } 70 | 71 | if !utils.PodSandboxExited(pod) && pod.Status.PodIP != "" { 72 | return false 73 | } 74 | return true 75 | } 76 | -------------------------------------------------------------------------------- /pkg/controller/pod-eni/predict.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package podeni 18 | 19 | import ( 20 | "reflect" 21 | 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | "sigs.k8s.io/controller-runtime/pkg/event" 24 | 25 | "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" 26 | ) 27 | 28 | func updateFunc(e event.TypedUpdateEvent[*v1beta1.PodENI]) bool { 29 | oldPodENICopy := e.ObjectOld.DeepCopy() 30 | newPodENICopy := e.ObjectNew.DeepCopy() 31 | 32 | oldPodENICopy.ResourceVersion = "" 33 | newPodENICopy.ResourceVersion = "" 34 | oldPodENICopy.Status.PodLastSeen = metav1.Unix(0, 0) 35 | newPodENICopy.Status.PodLastSeen = metav1.Unix(0, 0) 36 | 37 | return !reflect.DeepEqual(&oldPodENICopy, &newPodENICopy) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/controller/pod-networking/predict.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package podnetworking 18 | 19 | import ( 20 | "github.com/samber/lo" 21 | "k8s.io/apimachinery/pkg/util/sets" 22 | "sigs.k8s.io/controller-runtime/pkg/event" 23 | "sigs.k8s.io/controller-runtime/pkg/predicate" 24 | 25 | "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" 26 | ) 27 | 28 | type predicateForPodnetwokringEvent struct { 29 | predicate.Funcs 30 | } 31 | 32 | func (p *predicateForPodnetwokringEvent) Update(e event.UpdateEvent) bool { 33 | newPodNetworking, ok := e.ObjectNew.(*v1beta1.PodNetworking) 34 | if !ok { 35 | return false 36 | } 37 | 38 | switch newPodNetworking.Status.Status { 39 | case "", v1beta1.NetworkingStatusFail: 40 | return true 41 | } 42 | 43 | return changed(newPodNetworking) 44 | } 45 | 46 | func changed(pn *v1beta1.PodNetworking) bool { 47 | expect := sets.New[string](pn.Spec.VSwitchOptions...) 48 | got := sets.New[string](lo.Map(pn.Status.VSwitches, func(item v1beta1.VSwitch, index int) string { 49 | return item.ID 50 | })...) 51 | return !expect.Equal(got) 52 | } 53 | -------------------------------------------------------------------------------- /pkg/controller/pod-networking/predict_test.go: -------------------------------------------------------------------------------- 1 | package podnetworking 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" 7 | ) 8 | 9 | func Test_changed(t *testing.T) { 10 | type args struct { 11 | pn *v1beta1.PodNetworking 12 | } 13 | tests := []struct { 14 | name string 15 | args args 16 | want bool 17 | }{ 18 | { 19 | name: "un sorted", 20 | args: args{ 21 | pn: &v1beta1.PodNetworking{ 22 | Spec: v1beta1.PodNetworkingSpec{ 23 | VSwitchOptions: []string{"foo", "bar"}, 24 | }, 25 | Status: v1beta1.PodNetworkingStatus{ 26 | VSwitches: []v1beta1.VSwitch{ 27 | { 28 | ID: "bar", 29 | }, 30 | { 31 | ID: "foo", 32 | }, 33 | }, 34 | }, 35 | }, 36 | }, 37 | want: false, 38 | }, 39 | { 40 | name: "modify", 41 | args: args{ 42 | pn: &v1beta1.PodNetworking{ 43 | Spec: v1beta1.PodNetworkingSpec{ 44 | VSwitchOptions: []string{"foo", "bar"}, 45 | }, 46 | Status: v1beta1.PodNetworkingStatus{ 47 | VSwitches: []v1beta1.VSwitch{ 48 | { 49 | ID: "var", 50 | }, 51 | }, 52 | }, 53 | }, 54 | }, 55 | want: true, 56 | }, 57 | } 58 | for _, tt := range tests { 59 | t.Run(tt.name, func(t *testing.T) { 60 | if got := changed(tt.args.pn); got != tt.want { 61 | t.Errorf("changed() = %v, want %v", got, tt.want) 62 | } 63 | }) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/controller/pod/predict.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | "sigs.k8s.io/controller-runtime/pkg/client" 22 | "sigs.k8s.io/controller-runtime/pkg/event" 23 | "sigs.k8s.io/controller-runtime/pkg/predicate" 24 | 25 | "github.com/AliyunContainerService/terway/pkg/utils" 26 | "github.com/AliyunContainerService/terway/types" 27 | ) 28 | 29 | type predicateForPodEvent struct { 30 | predicate.Funcs 31 | } 32 | 33 | func (p *predicateForPodEvent) Create(e event.CreateEvent) bool { 34 | return processPod(e.Object) 35 | } 36 | 37 | func (p *predicateForPodEvent) Update(e event.UpdateEvent) bool { 38 | return processPod(e.ObjectNew) 39 | } 40 | 41 | func (p *predicateForPodEvent) Delete(e event.DeleteEvent) bool { 42 | return processPod(e.Object) 43 | } 44 | 45 | func (p *predicateForPodEvent) Generic(e event.GenericEvent) bool { 46 | return processPod(e.Object) 47 | } 48 | 49 | func processPod(o client.Object) bool { 50 | pod, ok := o.(*corev1.Pod) 51 | if !ok { 52 | return false 53 | } 54 | 55 | if pod.Spec.NodeName == "" || 56 | pod.Spec.HostNetwork { 57 | return false 58 | } 59 | 60 | if types.IgnoredByTerway(pod.Labels) { 61 | return false 62 | } 63 | return true 64 | } 65 | 66 | func processNode(node *corev1.Node) bool { 67 | if types.IgnoredByTerway(node.Labels) || 68 | utils.ISVKNode(node) { 69 | return false 70 | } 71 | 72 | return true 73 | } 74 | -------------------------------------------------------------------------------- /pkg/controller/preheating/dummy.go: -------------------------------------------------------------------------------- 1 | package preheating 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | 7 | ctrl "sigs.k8s.io/controller-runtime" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | "sigs.k8s.io/controller-runtime/pkg/controller" 10 | "sigs.k8s.io/controller-runtime/pkg/handler" 11 | "sigs.k8s.io/controller-runtime/pkg/manager" 12 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 13 | ) 14 | 15 | const ControllerName = "no-op" 16 | 17 | // ReconcilePod implements reconcile.Reconciler 18 | var _ reconcile.Reconciler = &DummyReconcile{} 19 | 20 | // DummyReconcile reconciles a AutoRepair object 21 | type DummyReconcile struct { 22 | RegisterResource []client.Object 23 | } 24 | 25 | func (r *DummyReconcile) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { 26 | return reconcile.Result{}, nil 27 | } 28 | 29 | func (r *DummyReconcile) SetupWithManager(mgr manager.Manager) error { 30 | NeedLeaderElection := false 31 | 32 | builder := ctrl.NewControllerManagedBy(mgr). 33 | WithOptions(controller.Options{ 34 | NeedLeaderElection: &NeedLeaderElection, 35 | }). 36 | Named(ControllerName) 37 | used := map[string]struct{}{} 38 | 39 | for _, v := range r.RegisterResource { 40 | t := reflect.TypeOf(v) 41 | if t.Kind() == reflect.Ptr { 42 | t = t.Elem() 43 | } 44 | _, ok := used[t.Name()] 45 | if ok { 46 | continue 47 | } 48 | used[t.Name()] = struct{}{} 49 | builder = builder.Watches(v, &handler.TypedFuncs[client.Object, reconcile.Request]{}) 50 | } 51 | return builder.Complete(&DummyReconcile{}) 52 | } 53 | -------------------------------------------------------------------------------- /pkg/controller/preheating/dummy_test.go: -------------------------------------------------------------------------------- 1 | package preheating 2 | 3 | import ( 4 | "context" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 9 | ) 10 | 11 | var _ = Describe("DummyReconcile", func() { 12 | Context("DummyReconcile", func() { 13 | controllerReconciler := &DummyReconcile{} 14 | 15 | _, err := controllerReconciler.Reconcile(context.Background(), reconcile.Request{}) 16 | Expect(err).NotTo(HaveOccurred()) 17 | }) 18 | }) 19 | -------------------------------------------------------------------------------- /pkg/controller/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | //go:generate mockery --name Interface --tags default_build 18 | 19 | package register 20 | 21 | import ( 22 | "context" 23 | 24 | "go.opentelemetry.io/otel/trace" 25 | "k8s.io/apimachinery/pkg/util/wait" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | 28 | aliyunClient "github.com/AliyunContainerService/terway/pkg/aliyun/client" 29 | "github.com/AliyunContainerService/terway/pkg/controller/status" 30 | "github.com/AliyunContainerService/terway/pkg/vswitch" 31 | "github.com/AliyunContainerService/terway/types/controlplane" 32 | 33 | "sigs.k8s.io/controller-runtime/pkg/manager" 34 | ) 35 | 36 | type ControllerCtx struct { 37 | context.Context 38 | 39 | Config *controlplane.Config 40 | VSwitchPool *vswitch.SwitchPool 41 | AliyunClient aliyunClient.OpenAPI 42 | 43 | Wg *wait.Group 44 | 45 | TracerProvider trace.TracerProvider 46 | 47 | RegisterResource []client.Object 48 | 49 | NodeStatusCache *status.Cache[status.NodeStatus] 50 | } 51 | 52 | type Creator func(mgr manager.Manager, ctrlCtx *ControllerCtx) error 53 | 54 | // Controllers collect for all controller 55 | var Controllers = map[string]struct { 56 | Creator Creator 57 | Enable bool 58 | }{} 59 | 60 | // Add add controller by name 61 | func Add(name string, creator Creator, enable bool) { 62 | Controllers[name] = struct { 63 | Creator Creator 64 | Enable bool 65 | }{ 66 | Creator: creator, 67 | Enable: enable, 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /pkg/controller/status/status_test.go: -------------------------------------------------------------------------------- 1 | package status 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "k8s.io/apimachinery/pkg/util/wait" 9 | ) 10 | 11 | func TestRequestNetworkIndex(t *testing.T) { 12 | nodeStatus := NewNodeStatus(2) 13 | wg := wait.Group{} 14 | for i := 0; i < 100; i++ { 15 | wg.Start(func() { 16 | nodeStatus.RequestNetworkIndex(fmt.Sprintf("%d", i), nil, nil) 17 | }) 18 | } 19 | wg.Wait() 20 | 21 | assert.Equal(t, 2, len(nodeStatus.NetworkCards)) 22 | assert.Equal(t, 50, len(nodeStatus.NetworkCards[0].NetworkInterfaces)) 23 | assert.Equal(t, 50, len(nodeStatus.NetworkCards[1].NetworkInterfaces)) 24 | 25 | for i := 0; i < 100; i++ { 26 | if i%2 == 0 { 27 | continue 28 | } 29 | wg.Start(func() { 30 | nodeStatus.DetachNetworkIndex(fmt.Sprintf("%d", i)) 31 | }) 32 | } 33 | 34 | wg.Wait() 35 | assert.Equal(t, 50, nodeStatus.NetworkCards[1].NetworkInterfaces.Len()+nodeStatus.NetworkCards[0].NetworkInterfaces.Len()) 36 | } 37 | 38 | func TestRequestNetworkIndex_Numa(t *testing.T) { 39 | nodeStatus := NewNodeStatus(2) 40 | numa := 1 41 | wg := wait.Group{} 42 | for i := 0; i < 100; i++ { 43 | wg.Start(func() { 44 | nodeStatus.RequestNetworkIndex(fmt.Sprintf("%d", i), nil, &numa) 45 | }) 46 | } 47 | wg.Wait() 48 | 49 | assert.Equal(t, 2, len(nodeStatus.NetworkCards)) 50 | assert.Equal(t, 0, len(nodeStatus.NetworkCards[0].NetworkInterfaces)) 51 | assert.Equal(t, 100, len(nodeStatus.NetworkCards[1].NetworkInterfaces)) 52 | 53 | index := 0 54 | for i := 0; i < 100; i++ { 55 | wg.Start(func() { 56 | nodeStatus.RequestNetworkIndex(fmt.Sprintf("%d", i), &index, &numa) 57 | }) 58 | } 59 | 60 | wg.Wait() 61 | assert.Equal(t, 100, len(nodeStatus.NetworkCards[0].NetworkInterfaces)) 62 | assert.Equal(t, 0, len(nodeStatus.NetworkCards[1].NetworkInterfaces)) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/eni/conditiontype_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=ConditionType -trimprefix=Condition"; DO NOT EDIT. 2 | 3 | package eni 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[Full-0] 12 | _ = x[ResourceTypeMismatch-1] 13 | _ = x[NetworkInterfaceMismatch-2] 14 | _ = x[InsufficientVSwitchIP-3] 15 | } 16 | 17 | const _ConditionType_name = "FullResourceTypeMismatchNetworkInterfaceMismatchInsufficientVSwitchIP" 18 | 19 | var _ConditionType_index = [...]uint8{0, 4, 24, 48, 69} 20 | 21 | func (i ConditionType) String() string { 22 | if i < 0 || i >= ConditionType(len(_ConditionType_index)-1) { 23 | return "ConditionType(" + strconv.FormatInt(int64(i), 10) + ")" 24 | } 25 | return _ConditionType_name[_ConditionType_index[i]:_ConditionType_index[i+1]] 26 | } 27 | -------------------------------------------------------------------------------- /pkg/eni/enistatus_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=eniStatus -trimprefix=status"; DO NOT EDIT. 2 | 3 | package eni 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[statusInit-0] 12 | _ = x[statusCreating-1] 13 | _ = x[statusInUse-2] 14 | _ = x[statusDeleting-3] 15 | } 16 | 17 | const _eniStatus_name = "InitCreatingInUseDeleting" 18 | 19 | var _eniStatus_index = [...]uint8{0, 4, 12, 17, 25} 20 | 21 | func (i eniStatus) String() string { 22 | if i < 0 || i >= eniStatus(len(_eniStatus_index)-1) { 23 | return "eniStatus(" + strconv.FormatInt(int64(i), 10) + ")" 24 | } 25 | return _eniStatus_name[_eniStatus_index[i]:_eniStatus_index[i+1]] 26 | } 27 | -------------------------------------------------------------------------------- /pkg/eni/ipstatus_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=ipStatus -trimprefix=ipStatus"; DO NOT EDIT. 2 | 3 | package eni 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[ipStatusInit-0] 12 | _ = x[ipStatusValid-1] 13 | _ = x[ipStatusInvalid-2] 14 | _ = x[ipStatusDeleting-3] 15 | } 16 | 17 | const _ipStatus_name = "InitValidInvalidDeleting" 18 | 19 | var _ipStatus_index = [...]uint8{0, 4, 9, 16, 24} 20 | 21 | func (i ipStatus) String() string { 22 | if i < 0 || i >= ipStatus(len(_ipStatus_index)-1) { 23 | return "ipStatus(" + strconv.FormatInt(int64(i), 10) + ")" 24 | } 25 | return _ipStatus_name[_ipStatus_index[i]:_ipStatus_index[i+1]] 26 | } 27 | -------------------------------------------------------------------------------- /pkg/eni/local_unwindows.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | package eni 4 | 5 | import ( 6 | "github.com/AliyunContainerService/terway/types/daemon" 7 | ) 8 | 9 | func setupENICompartment(eni *daemon.ENI) error { 10 | return nil 11 | } 12 | 13 | func destroyENICompartment(eni *daemon.ENI) error { 14 | return nil 15 | } 16 | -------------------------------------------------------------------------------- /pkg/eni/trunk.go: -------------------------------------------------------------------------------- 1 | package eni 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "sigs.k8s.io/controller-runtime/pkg/client" 8 | 9 | "github.com/AliyunContainerService/terway/types/daemon" 10 | ) 11 | 12 | var _ NetworkInterface = &Trunk{} 13 | var _ Usage = &Trunk{} 14 | var _ ReportStatus = &Trunk{} 15 | 16 | type Trunk struct { 17 | trunkENI *daemon.ENI 18 | 19 | remote *Remote 20 | local *Local 21 | } 22 | 23 | func NewTrunk(client client.Client, local *Local) *Trunk { 24 | return &Trunk{ 25 | trunkENI: local.eni, 26 | local: local, 27 | remote: NewRemote(client, local.eni), 28 | } 29 | } 30 | 31 | func (r *Trunk) Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error { 32 | return r.local.Run(ctx, podResources, wg) 33 | } 34 | 35 | func (r *Trunk) Priority() int { 36 | return 100 37 | } 38 | 39 | func (r *Trunk) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace) { 40 | switch request.ResourceType() { 41 | case ResourceTypeLocalIP: 42 | return r.local.Allocate(ctx, cni, request) 43 | case ResourceTypeRemoteIP: 44 | return r.remote.Allocate(ctx, cni, request) 45 | default: 46 | return nil, []Trace{{Condition: ResourceTypeMismatch}} 47 | } 48 | } 49 | 50 | func (r *Trunk) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) (bool, error) { 51 | switch request.ResourceType() { 52 | case ResourceTypeLocalIP: 53 | return r.local.Release(ctx, cni, request) 54 | case ResourceTypeRemoteIP: 55 | return r.remote.Release(ctx, cni, request) 56 | default: 57 | return false, nil 58 | } 59 | } 60 | 61 | func (r *Trunk) Dispose(n int) int { 62 | return r.local.Dispose(n) 63 | } 64 | 65 | func (r *Trunk) Status() Status { 66 | return r.local.Status() 67 | } 68 | 69 | func (r *Trunk) Usage() (int, int, error) { 70 | return r.local.Usage() 71 | } 72 | -------------------------------------------------------------------------------- /pkg/factory/types.go: -------------------------------------------------------------------------------- 1 | //go:generate mockery --name Factory 2 | 3 | package factory 4 | 5 | import ( 6 | "net/netip" 7 | 8 | "github.com/AliyunContainerService/terway/types/daemon" 9 | ) 10 | 11 | type Factory interface { 12 | CreateNetworkInterface(ipv4, ipv6 int, eniType string) (*daemon.ENI, []netip.Addr, []netip.Addr, error) 13 | AssignNIPv4(eniID string, count int, mac string) ([]netip.Addr, error) 14 | AssignNIPv6(eniID string, count int, mac string) ([]netip.Addr, error) 15 | 16 | // UnAssignNIPv4 unassign ip from eni, the primary ip is not allowed to unassign 17 | UnAssignNIPv4(eniID string, ips []netip.Addr, mac string) error 18 | UnAssignNIPv6(eniID string, ips []netip.Addr, mac string) error 19 | 20 | DeleteNetworkInterface(eniID string) error 21 | 22 | LoadNetworkInterface(mac string) ([]netip.Addr, []netip.Addr, error) 23 | 24 | GetAttachedNetworkInterface(preferTrunkID string) ([]*daemon.ENI, error) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/feature/feature.go: -------------------------------------------------------------------------------- 1 | package feature 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/util/runtime" 5 | "k8s.io/component-base/featuregate" 6 | 7 | utilfeature "k8s.io/apiserver/pkg/util/feature" 8 | ) 9 | 10 | func init() { 11 | runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)) 12 | } 13 | 14 | const ( 15 | // AutoDataPathV2 enable the new datapath feature. 16 | AutoDataPathV2 featuregate.Feature = "AutoDataPathV2" 17 | 18 | EFLO featuregate.Feature = "EFLO" 19 | 20 | KubeProxyReplacement featuregate.Feature = "KubeProxyReplacement" 21 | ) 22 | 23 | var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ 24 | AutoDataPathV2: {Default: true, PreRelease: featuregate.Alpha}, 25 | EFLO: {Default: true, PreRelease: featuregate.Alpha}, 26 | KubeProxyReplacement: {Default: false, PreRelease: featuregate.Alpha}, 27 | } 28 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // This package has the automatically generated clientset. 19 | package versioned 20 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // This package has the automatically generated fake clientset. 19 | package fake 20 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/fake/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package fake 19 | 20 | import ( 21 | networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" 22 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | runtime "k8s.io/apimachinery/pkg/runtime" 24 | schema "k8s.io/apimachinery/pkg/runtime/schema" 25 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 26 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 27 | ) 28 | 29 | var scheme = runtime.NewScheme() 30 | var codecs = serializer.NewCodecFactory(scheme) 31 | 32 | var localSchemeBuilder = runtime.SchemeBuilder{ 33 | networkv1beta1.AddToScheme, 34 | } 35 | 36 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 37 | // of clientsets, like in: 38 | // 39 | // import ( 40 | // "k8s.io/client-go/kubernetes" 41 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 42 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 43 | // ) 44 | // 45 | // kclientset, _ := kubernetes.NewForConfig(c) 46 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 47 | // 48 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 49 | // correctly. 50 | var AddToScheme = localSchemeBuilder.AddToScheme 51 | 52 | func init() { 53 | v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) 54 | utilruntime.Must(AddToScheme(scheme)) 55 | } 56 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/scheme/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // This package contains the scheme of the automatically generated clientset. 19 | package scheme 20 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/scheme/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package scheme 19 | 20 | import ( 21 | networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" 22 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | runtime "k8s.io/apimachinery/pkg/runtime" 24 | schema "k8s.io/apimachinery/pkg/runtime/schema" 25 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 26 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 27 | ) 28 | 29 | var Scheme = runtime.NewScheme() 30 | var Codecs = serializer.NewCodecFactory(Scheme) 31 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 32 | var localSchemeBuilder = runtime.SchemeBuilder{ 33 | networkv1beta1.AddToScheme, 34 | } 35 | 36 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 37 | // of clientsets, like in: 38 | // 39 | // import ( 40 | // "k8s.io/client-go/kubernetes" 41 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 42 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 43 | // ) 44 | // 45 | // kclientset, _ := kubernetes.NewForConfig(c) 46 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 47 | // 48 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 49 | // correctly. 50 | var AddToScheme = localSchemeBuilder.AddToScheme 51 | 52 | func init() { 53 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 54 | utilruntime.Must(AddToScheme(Scheme)) 55 | } 56 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/network.alibabacloud.com/v1beta1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // This package has the automatically generated typed clients. 19 | package v1beta1 20 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/network.alibabacloud.com/v1beta1/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | // Package fake has the automatically generated clients. 19 | package fake 20 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/network.alibabacloud.com/v1beta1/fake/fake_network.alibabacloud.com_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package fake 19 | 20 | import ( 21 | v1beta1 "github.com/AliyunContainerService/terway/pkg/generated/clientset/versioned/typed/network.alibabacloud.com/v1beta1" 22 | rest "k8s.io/client-go/rest" 23 | testing "k8s.io/client-go/testing" 24 | ) 25 | 26 | type FakeNetworkV1beta1 struct { 27 | *testing.Fake 28 | } 29 | 30 | func (c *FakeNetworkV1beta1) NetworkInterfaces() v1beta1.NetworkInterfaceInterface { 31 | return &FakeNetworkInterfaces{c} 32 | } 33 | 34 | func (c *FakeNetworkV1beta1) Nodes() v1beta1.NodeInterface { 35 | return &FakeNodes{c} 36 | } 37 | 38 | func (c *FakeNetworkV1beta1) NodeRuntimes() v1beta1.NodeRuntimeInterface { 39 | return &FakeNodeRuntimes{c} 40 | } 41 | 42 | func (c *FakeNetworkV1beta1) PodENIs(namespace string) v1beta1.PodENIInterface { 43 | return &FakePodENIs{c, namespace} 44 | } 45 | 46 | func (c *FakeNetworkV1beta1) PodNetworkings() v1beta1.PodNetworkingInterface { 47 | return &FakePodNetworkings{c} 48 | } 49 | 50 | // RESTClient returns a RESTClient that is used to communicate 51 | // with API server by this client implementation. 52 | func (c *FakeNetworkV1beta1) RESTClient() rest.Interface { 53 | var ret *rest.RESTClient 54 | return ret 55 | } 56 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/network.alibabacloud.com/v1beta1/generated_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by client-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | type NetworkInterfaceExpansion interface{} 21 | 22 | type NodeExpansion interface{} 23 | 24 | type NodeRuntimeExpansion interface{} 25 | 26 | type PodENIExpansion interface{} 27 | 28 | type PodNetworkingExpansion interface{} 29 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by informer-gen. DO NOT EDIT. 17 | 18 | package internalinterfaces 19 | 20 | import ( 21 | time "time" 22 | 23 | versioned "github.com/AliyunContainerService/terway/pkg/generated/clientset/versioned" 24 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | cache "k8s.io/client-go/tools/cache" 27 | ) 28 | 29 | // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. 30 | type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer 31 | 32 | // SharedInformerFactory a small interface to allow for adding an informer without an import cycle 33 | type SharedInformerFactory interface { 34 | Start(stopCh <-chan struct{}) 35 | InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer 36 | } 37 | 38 | // TweakListOptionsFunc is a function that transforms a v1.ListOptions. 39 | type TweakListOptionsFunc func(*v1.ListOptions) 40 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/network.alibabacloud.com/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by informer-gen. DO NOT EDIT. 17 | 18 | package network 19 | 20 | import ( 21 | internalinterfaces "github.com/AliyunContainerService/terway/pkg/generated/informers/externalversions/internalinterfaces" 22 | v1beta1 "github.com/AliyunContainerService/terway/pkg/generated/informers/externalversions/network.alibabacloud.com/v1beta1" 23 | ) 24 | 25 | // Interface provides access to each of this group's versions. 26 | type Interface interface { 27 | // V1beta1 provides access to shared informers for resources in V1beta1. 28 | V1beta1() v1beta1.Interface 29 | } 30 | 31 | type group struct { 32 | factory internalinterfaces.SharedInformerFactory 33 | namespace string 34 | tweakListOptions internalinterfaces.TweakListOptionsFunc 35 | } 36 | 37 | // New returns a new Interface. 38 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 39 | return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 40 | } 41 | 42 | // V1beta1 returns a new v1beta1.Interface. 43 | func (g *group) V1beta1() v1beta1.Interface { 44 | return v1beta1.New(g.factory, g.namespace, g.tweakListOptions) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/generated/listers/network.alibabacloud.com/v1beta1/expansion_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by lister-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | // NetworkInterfaceListerExpansion allows custom methods to be added to 21 | // NetworkInterfaceLister. 22 | type NetworkInterfaceListerExpansion interface{} 23 | 24 | // NodeListerExpansion allows custom methods to be added to 25 | // NodeLister. 26 | type NodeListerExpansion interface{} 27 | 28 | // NodeRuntimeListerExpansion allows custom methods to be added to 29 | // NodeRuntimeLister. 30 | type NodeRuntimeListerExpansion interface{} 31 | 32 | // PodENIListerExpansion allows custom methods to be added to 33 | // PodENILister. 34 | type PodENIListerExpansion interface{} 35 | 36 | // PodENINamespaceListerExpansion allows custom methods to be added to 37 | // PodENINamespaceLister. 38 | type PodENINamespaceListerExpansion interface{} 39 | 40 | // PodNetworkingListerExpansion allows custom methods to be added to 41 | // PodNetworkingLister. 42 | type PodNetworkingListerExpansion interface{} 43 | -------------------------------------------------------------------------------- /pkg/ip/ip.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/netip" 7 | 8 | "k8s.io/apimachinery/pkg/util/sets" 9 | ) 10 | 11 | // ToIP parse str to net.IP and return error is parse failed 12 | func ToIP(addr string) (net.IP, error) { 13 | ip := net.ParseIP(addr) 14 | if ip == nil { 15 | return nil, fmt.Errorf("failed to parse ip %s", addr) 16 | } 17 | return ip, nil 18 | } 19 | 20 | func ToIPAddrs(addrs []string) ([]netip.Addr, error) { 21 | var result []netip.Addr 22 | for _, addr := range addrs { 23 | i, err := netip.ParseAddr(addr) 24 | if err != nil { 25 | return nil, err 26 | } 27 | result = append(result, i) 28 | } 29 | return result, nil 30 | } 31 | 32 | func IPv6(ip net.IP) bool { 33 | return ip.To4() == nil 34 | } 35 | 36 | func IPs2str(ips []net.IP) []string { 37 | var result []string 38 | for _, ip := range ips { 39 | result = append(result, ip.String()) 40 | } 41 | return result 42 | } 43 | 44 | func IPAddrs2str(ips []netip.Addr) []string { 45 | var result []string 46 | for _, ip := range ips { 47 | result = append(result, ip.String()) 48 | } 49 | return result 50 | } 51 | 52 | // IPsIntersect return is 2 set is intersect 53 | func IPsIntersect(a []net.IP, b []net.IP) bool { 54 | return sets.NewString(IPs2str(a)...).HasAny(IPs2str(b)...) 55 | } 56 | 57 | // DeriveGatewayIP gateway ip from cidr 58 | func DeriveGatewayIP(cidr string) string { 59 | if cidr == "" { 60 | return "" 61 | } 62 | _, ipNet, err := net.ParseCIDR(cidr) 63 | if err != nil { 64 | return "" 65 | } 66 | gw := GetIPAtIndex(*ipNet, int64(-3)) 67 | if gw == nil { 68 | return "" 69 | } 70 | return gw.String() 71 | } 72 | -------------------------------------------------------------------------------- /pkg/ip/ip_cilium_test.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | ) 7 | 8 | func TestGetIPAtIndex(t *testing.T) { 9 | type args struct { 10 | cidr string 11 | index int64 12 | want net.IP 13 | } 14 | 15 | tests := []args{ 16 | { 17 | cidr: "10.0.0.0/29", 18 | index: -1, 19 | want: net.ParseIP("10.0.0.7"), 20 | }, { 21 | cidr: "10.0.0.0/29", 22 | index: 0, 23 | want: net.ParseIP("10.0.0.0"), 24 | }, { 25 | cidr: "10.0.0.0/29", 26 | index: 1, 27 | want: net.ParseIP("10.0.0.1"), 28 | }, { 29 | cidr: "10.0.0.16/28", 30 | index: -3, 31 | want: net.ParseIP("10.0.0.29"), 32 | }, { 33 | cidr: "10.0.0.0/29", 34 | index: -3, 35 | want: net.ParseIP("10.0.0.5"), 36 | }, { 37 | cidr: "10.0.0.0/25", 38 | index: -3, 39 | want: net.ParseIP("10.0.0.125"), 40 | }, { 41 | cidr: "10.0.0.128/25", 42 | index: -3, 43 | want: net.ParseIP("10.0.0.253"), 44 | }, { 45 | cidr: "10.0.8.0/21", 46 | index: -3, 47 | want: net.ParseIP("10.0.15.253"), 48 | }, { 49 | cidr: "fd00::/64", 50 | index: -3, 51 | want: net.ParseIP("fd00::ffff:ffff:ffff:fffd"), 52 | }, 53 | } 54 | for _, tt := range tests { 55 | _, ipNet, _ := net.ParseCIDR(tt.cidr) 56 | if got := GetIPAtIndex(*ipNet, tt.index); !got.Equal(tt.want) { 57 | t.Errorf("GetIPAtIndex() = %v, want %v", got, tt.want) 58 | } 59 | 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /pkg/ip/ip_test.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | "net/netip" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_ipIntersect(t *testing.T) { 12 | type args struct { 13 | a []net.IP 14 | b []net.IP 15 | } 16 | tests := []struct { 17 | name string 18 | args args 19 | want bool 20 | }{ 21 | { 22 | name: "intersect", 23 | args: args{ 24 | a: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.3")}, 25 | b: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.2")}, 26 | }, 27 | want: true, 28 | }, { 29 | name: "not intersect", 30 | args: args{ 31 | a: []net.IP{net.ParseIP("127.0.0.4"), net.ParseIP("127.0.0.3")}, 32 | b: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.2")}, 33 | }, 34 | want: false, 35 | }, { 36 | name: "nil val 1", 37 | args: args{ 38 | a: []net.IP{net.ParseIP("127.0.0.3")}, 39 | b: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.2")}, 40 | }, 41 | want: false, 42 | }, { 43 | name: "nil val 2", 44 | args: args{ 45 | a: []net.IP{net.ParseIP("127.0.0.1")}, 46 | b: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("127.0.0.2")}, 47 | }, 48 | want: true, 49 | }, 50 | } 51 | for _, tt := range tests { 52 | t.Run(tt.name, func(t *testing.T) { 53 | if got := IPsIntersect(tt.args.a, tt.args.b); got != tt.want { 54 | t.Errorf("IPsIntersect() = %v, want %v", got, tt.want) 55 | } 56 | }) 57 | } 58 | } 59 | 60 | func TestIPAddrs2str_MultipleValidIPs_ReturnsCorrectStrings(t *testing.T) { 61 | ip1, _ := netip.ParseAddr("192.0.2.1") 62 | ip2, _ := netip.ParseAddr("192.0.2.2") 63 | input := []netip.Addr{ip1, ip2} 64 | expected := []string{"192.0.2.1", "192.0.2.2"} 65 | result := IPAddrs2str(input) 66 | if len(result) != len(expected) { 67 | t.Errorf("Expected %v, got %v", expected, result) 68 | } 69 | for i := range expected { 70 | if result[i] != expected[i] { 71 | t.Errorf("Expected %v, got %v", expected, result) 72 | } 73 | } 74 | } 75 | 76 | func TestDeriveGatewayIP(t *testing.T) { 77 | assert.Equal(t, "192.168.0.253", DeriveGatewayIP("192.168.0.0/24")) 78 | } 79 | -------------------------------------------------------------------------------- /pkg/link/interface_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package link 4 | 5 | import ( 6 | "fmt" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/vishvananda/netlink" 10 | ) 11 | 12 | // GetDeviceNumber get interface device number by mac address 13 | func GetDeviceNumber(mac string) (int32, error) { 14 | linkList, err := netlink.LinkList() 15 | if err != nil { 16 | return 0, fmt.Errorf("error get link list from netlink, %w", err) 17 | } 18 | 19 | for _, link := range linkList { 20 | // ignore virtual nic type. eg. ipvlan veth bridge 21 | if _, ok := link.(*netlink.Device); !ok { 22 | continue 23 | } 24 | if link.Attrs().HardwareAddr.String() == mac { 25 | return int32(link.Attrs().Index), nil 26 | } 27 | } 28 | return 0, errors.Wrapf(ErrNotFound, "can't found dev by mac %s", mac) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/link/interface_linux_test.go: -------------------------------------------------------------------------------- 1 | package link 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestGetDeviceNumber(t *testing.T) { 11 | id, err := GetDeviceNumber("00") 12 | assert.True(t, errors.Is(err, ErrNotFound)) 13 | assert.Equal(t, int32(0), id) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/link/interface_unsupport.go: -------------------------------------------------------------------------------- 1 | //go:build !linux && !windows 2 | // +build !linux,!windows 3 | 4 | package link 5 | 6 | // GetDeviceNumber get interface device number by mac address 7 | func GetDeviceNumber(mac string) (int32, error) { 8 | return 0, ErrUnsupported 9 | } 10 | -------------------------------------------------------------------------------- /pkg/link/interface_windows.go: -------------------------------------------------------------------------------- 1 | package link 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | 6 | "github.com/AliyunContainerService/terway/pkg/windows/iface" 7 | ) 8 | 9 | // GetDeviceNumber get interface device number by mac address 10 | func GetDeviceNumber(mac string) (int32, error) { 11 | macIface, err := iface.GetInterfaceByMAC(mac, true) 12 | if err != nil { 13 | return 0, errors.Wrapf(err, "failed to get dev by MAC %s", mac) 14 | } 15 | return int32(macIface.Index), nil 16 | } 17 | -------------------------------------------------------------------------------- /pkg/link/type.go: -------------------------------------------------------------------------------- 1 | package link 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // define err 8 | var ( 9 | ErrUnsupported = errors.New("not supported arch") 10 | ErrNotFound = errors.New("not found") 11 | ) 12 | -------------------------------------------------------------------------------- /pkg/link/veth.go: -------------------------------------------------------------------------------- 1 | package link 2 | 3 | import ( 4 | "crypto/sha1" 5 | "encoding/hex" 6 | "fmt" 7 | ) 8 | 9 | // VethNameForPod return host-side veth name for pod 10 | // max veth length is 15 11 | func VethNameForPod(name, namespace, ifName, prefix string) (string, error) { 12 | // A SHA1 is always 20 bytes long, and so is sufficient for generating the 13 | // veth name and mac addr. 14 | h := sha1.New() 15 | if ifName == "eth0" { 16 | ifName = "" 17 | } 18 | _, err := h.Write([]byte(namespace + "." + name + ifName)) 19 | if err != nil { 20 | return "", err 21 | } 22 | return fmt.Sprintf("%s%s", prefix, hex.EncodeToString(h.Sum(nil))[:11]), nil 23 | } 24 | -------------------------------------------------------------------------------- /pkg/link/veth_test.go: -------------------------------------------------------------------------------- 1 | package link 2 | 3 | import "testing" 4 | 5 | func TestVethNameForPod(t *testing.T) { 6 | if veth, _ := VethNameForPod("client-b6989bf87-2bgtc", "default", "", "cali"); veth != "calic95a4947e07" { 7 | t.Fatalf("veth name failed: expect: %s, actual: %s", "calic95a4947e07", veth) 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /pkg/metric/aliyun.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | var ( 8 | // OpenAPILatency aliyun open api latency 9 | OpenAPILatency = prometheus.NewHistogramVec( 10 | prometheus.HistogramOpts{ 11 | Name: "aliyun_openapi_latency", 12 | Help: "aliyun openapi latency in ms", 13 | Buckets: []float64{50, 100, 200, 400, 800, 1600, 3200, 6400, 12800, 13800, 14800, 16800, 20800, 28800, 44800}, 14 | }, 15 | []string{"api", "error"}, 16 | ) 17 | 18 | RateLimiterLatency = prometheus.NewHistogramVec( 19 | prometheus.HistogramOpts{ 20 | Name: "rate_limiter_latency", 21 | Help: "rate_limiter_latency in ms", 22 | Buckets: []float64{200, 400, 800, 1600, 3200, 6400, 12800, 13800, 14800, 16800, 20800, 28800, 44800}, 23 | }, 24 | []string{"api"}, 25 | ) 26 | ) 27 | -------------------------------------------------------------------------------- /pkg/metric/factory.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import "github.com/prometheus/client_golang/prometheus" 4 | 5 | var ( 6 | // ENIIPFactoryENICount amount of allocated terway multi-ip factory eni 7 | ENIIPFactoryENICount = prometheus.NewGaugeVec( 8 | prometheus.GaugeOpts{ 9 | Name: "terway_eniip_factory_eni_count", 10 | Help: "amount of allocated terway multi-ip factory eni", 11 | }, 12 | []string{"name", "max_eni"}, 13 | ) 14 | 15 | // ENIIPFactoryIPCount amount of allocated terway multi-ip secondary ip 16 | ENIIPFactoryIPCount = prometheus.NewGaugeVec( 17 | prometheus.GaugeOpts{ 18 | Name: "terway_eniip_factory_ip_count", 19 | Help: "amount of allocated terway multi-ip secondary ip", 20 | }, 21 | // eni is represented by its mac address 22 | []string{"name", "eni", "max_ip"}, 23 | ) 24 | 25 | // ENIIPFactoryIPAllocCount counter of eniip factory ip allocation 26 | ENIIPFactoryIPAllocCount = prometheus.NewCounterVec( 27 | prometheus.CounterOpts{ 28 | Name: "terway_eniip_factory_ip_alloc_count", 29 | Help: "counter of eniip factory ip allocation", 30 | }, 31 | // status in "succeed" or "fail" 32 | []string{"eni", "status"}, 33 | ) 34 | ) 35 | 36 | const ( 37 | // ENIIPAllocActionSucceed represents a succeeded ip alloc request 38 | ENIIPAllocActionSucceed = "succeed" 39 | // ENIIPAllocActionFail represents a failed ip alloc request 40 | ENIIPAllocActionFail = "fail" 41 | ) 42 | -------------------------------------------------------------------------------- /pkg/metric/resource_pool.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import "github.com/prometheus/client_golang/prometheus" 4 | 5 | const ( 6 | ResourcePoolTypeLocal string = "LocalIP" 7 | ResourcePoolTypeRemote string = "RemoteIP" 8 | ) 9 | 10 | var ( 11 | // ResourcePoolTotal terway total source amount in the pool 12 | ResourcePoolTotal = prometheus.NewGaugeVec( 13 | prometheus.GaugeOpts{ 14 | Name: "terway_resource_pool_total_count", 15 | Help: "terway total resources amount in the pool", 16 | }, 17 | // not accessory to put capacity, max_idle or min_idle into labels ? 18 | []string{"type", "ipStack"}, 19 | ) 20 | 21 | // ResourcePoolIdle terway amount of idle resource in the pool 22 | ResourcePoolIdle = prometheus.NewGaugeVec( 23 | prometheus.GaugeOpts{ 24 | Name: "terway_resource_pool_idle_count", 25 | Help: "terway amount of idle resources in the pool", 26 | }, 27 | []string{"type", "ipStack"}, 28 | ) 29 | 30 | // ResourcePoolDisposed terway resource count of begin disposed 31 | ResourcePoolDisposed = prometheus.NewCounterVec( 32 | prometheus.CounterOpts{ 33 | Name: "terway_resource_pool_disposed_count", 34 | Help: "terway resource count of being disposed", 35 | }, 36 | []string{"type", "ipStack"}, 37 | ) 38 | 39 | // ResourcePoolAllocatedCount terway resource allocation count 40 | ResourcePoolAllocated = prometheus.NewCounterVec( 41 | prometheus.CounterOpts{ 42 | Name: "terway_resource_pool_allocated_count", 43 | Help: "terway resource allocation count", 44 | }, 45 | []string{"type"}, 46 | ) 47 | ) 48 | -------------------------------------------------------------------------------- /pkg/metric/rpc.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import "github.com/prometheus/client_golang/prometheus" 4 | 5 | var ( 6 | // RPCLatency terway grpc latency for grpc by cni binary 7 | RPCLatency = prometheus.NewHistogramVec( 8 | prometheus.HistogramOpts{ 9 | Name: "terway_rpc_latency", 10 | Help: "terway rpc latency in ms", 11 | Buckets: []float64{50, 100, 200, 400, 800, 1600, 3200, 6400, 12800, 25600, 26600, 27600, 29600, 33600, 41600, 57600, 89600, 110000, 120000}, 12 | }, 13 | []string{"rpc_api", "error"}, 14 | ) 15 | ) 16 | -------------------------------------------------------------------------------- /pkg/metric/util.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // MsSince returns milliseconds since start. 8 | func MsSince(start time.Time) float64 { 9 | return float64(time.Since(start) / time.Millisecond) 10 | } 11 | -------------------------------------------------------------------------------- /pkg/sysctl/sysctl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Authors of Cilium 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sysctl 16 | 17 | import ( 18 | "bytes" 19 | "os" 20 | ) 21 | 22 | func EnsureConf(fPath string, cfg string) error { 23 | if content, err := os.ReadFile(fPath); err == nil { 24 | if bytes.Equal(bytes.TrimSpace(content), []byte(cfg)) { 25 | return nil 26 | } 27 | } 28 | return os.WriteFile(fPath, []byte(cfg), 0644) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/tc/tc.go: -------------------------------------------------------------------------------- 1 | package tc 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/vishvananda/netlink" 9 | ) 10 | 11 | // TrafficShapingRule the interface traffic shaping rule 12 | type TrafficShapingRule struct { 13 | // rate in bytes 14 | Rate uint64 15 | } 16 | 17 | func burst(rate uint64, mtu int) uint32 { 18 | return uint32(math.Ceil(math.Max(float64(rate)/milliSeconds, float64(mtu)))) 19 | } 20 | 21 | func time2Tick(time uint32) uint32 { 22 | return uint32(float64(time) * float64(netlink.TickInUsec())) 23 | } 24 | 25 | func buffer(rate uint64, burst uint32) uint32 { 26 | return time2Tick(uint32(float64(burst) * float64(netlink.TIME_UNITS_PER_SEC) / float64(rate))) 27 | } 28 | 29 | func limit(rate uint64, latency float64, buffer uint32) uint32 { 30 | return uint32(float64(rate)*latency/float64(netlink.TIME_UNITS_PER_SEC)) + buffer 31 | } 32 | 33 | func latencyInUsec(latencyInMillis float64) float64 { 34 | return float64(netlink.TIME_UNITS_PER_SEC) * (latencyInMillis / 1000.0) 35 | } 36 | 37 | const latencyInMillis = 25 38 | const hardwareHeaderLen = 1500 39 | const milliSeconds = 1000 40 | 41 | // SetRule set the traffic rule on interface 42 | func SetRule(dev netlink.Link, rule *TrafficShapingRule) error { 43 | if rule.Rate <= 0 { 44 | return fmt.Errorf("invalid rate %d", rule.Rate) 45 | } 46 | 47 | burst := burst(rule.Rate, dev.Attrs().MTU+hardwareHeaderLen) 48 | buffer := buffer(rule.Rate, burst) 49 | latency := latencyInUsec(latencyInMillis) 50 | limit := limit(rule.Rate, latency, burst) 51 | //log.Infof("set tc qdics add dev %v/%s root tbf rate %d burst %d", dev.Attrs().Namespace, dev.Attrs().Name, rule.Rate, burst) 52 | 53 | tbf := &netlink.Tbf{ 54 | QdiscAttrs: netlink.QdiscAttrs{ 55 | LinkIndex: dev.Attrs().Index, 56 | Handle: netlink.MakeHandle(1, 0), 57 | Parent: netlink.HANDLE_ROOT, 58 | }, 59 | Rate: rule.Rate, 60 | Limit: uint32(limit), 61 | Buffer: uint32(buffer), 62 | Minburst: uint32(dev.Attrs().MTU), 63 | } 64 | 65 | if err := netlink.QdiscReplace(tbf); err != nil { 66 | return errors.Wrapf(err, "can not replace qdics %+v on device %v/%s", tbf, dev.Attrs().Namespace, dev.Attrs().Name) 67 | } 68 | 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /pkg/utils/k8sclient/client.go: -------------------------------------------------------------------------------- 1 | package k8sclient 2 | 3 | import ( 4 | "k8s.io/client-go/kubernetes" 5 | "k8s.io/client-go/rest" 6 | 7 | networkingclientset "github.com/AliyunContainerService/terway/pkg/generated/clientset/versioned" 8 | ) 9 | 10 | // K8sClient k8s client set 11 | var K8sClient kubernetes.Interface 12 | 13 | // NetworkClient network client set 14 | var NetworkClient networkingclientset.Interface 15 | 16 | // RegisterClients create all k8s clients 17 | func RegisterClients(restConfig *rest.Config) { 18 | K8sClient = kubernetes.NewForConfigOrDie(restConfig) 19 | NetworkClient = networkingclientset.NewForConfigOrDie(restConfig) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/utils/map.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func ContainsAll[K comparable, V comparable](superset, subset map[K]V) bool { 4 | for key, value := range subset { 5 | if supersetValue, exists := superset[key]; !exists || supersetValue != value { 6 | return false 7 | } 8 | } 9 | return true 10 | } 11 | -------------------------------------------------------------------------------- /pkg/utils/map_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestContainsAll(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | subset map[string]int 13 | superset map[string]int 14 | expected bool 15 | }{ 16 | { 17 | name: "Empty subset", 18 | subset: map[string]int{}, 19 | superset: map[string]int{"a": 1, "b": 2}, 20 | expected: true, 21 | }, 22 | { 23 | name: "Subset is equal to superset", 24 | subset: map[string]int{"a": 1, "b": 2}, 25 | superset: map[string]int{"a": 1, "b": 2}, 26 | expected: true, 27 | }, 28 | { 29 | name: "Subset is contained in superset", 30 | subset: map[string]int{"a": 1}, 31 | superset: map[string]int{"a": 1, "b": 2}, 32 | expected: true, 33 | }, 34 | { 35 | name: "Subset is not contained in superset", 36 | subset: map[string]int{"a": 1, "c": 3}, 37 | superset: map[string]int{"a": 1, "b": 2}, 38 | expected: false, 39 | }, 40 | { 41 | name: "Subset has different values", 42 | subset: map[string]int{"a": 2}, 43 | superset: map[string]int{"a": 1, "b": 2}, 44 | expected: false, 45 | }, 46 | } 47 | 48 | for _, tt := range tests { 49 | t.Run(tt.name, func(t *testing.T) { 50 | result := ContainsAll(tt.superset, tt.subset) 51 | assert.Equal(t, tt.expected, result) 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pkg/utils/nodecap/mocks/NodeCapabilitiesStore.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.53.4. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import mock "github.com/stretchr/testify/mock" 6 | 7 | // NodeCapabilitiesStore is an autogenerated mock type for the NodeCapabilitiesStore type 8 | type NodeCapabilitiesStore struct { 9 | mock.Mock 10 | } 11 | 12 | // Get provides a mock function with given fields: capName 13 | func (_m *NodeCapabilitiesStore) Get(capName string) string { 14 | ret := _m.Called(capName) 15 | 16 | if len(ret) == 0 { 17 | panic("no return value specified for Get") 18 | } 19 | 20 | var r0 string 21 | if rf, ok := ret.Get(0).(func(string) string); ok { 22 | r0 = rf(capName) 23 | } else { 24 | r0 = ret.Get(0).(string) 25 | } 26 | 27 | return r0 28 | } 29 | 30 | // Load provides a mock function with no fields 31 | func (_m *NodeCapabilitiesStore) Load() error { 32 | ret := _m.Called() 33 | 34 | if len(ret) == 0 { 35 | panic("no return value specified for Load") 36 | } 37 | 38 | var r0 error 39 | if rf, ok := ret.Get(0).(func() error); ok { 40 | r0 = rf() 41 | } else { 42 | r0 = ret.Error(0) 43 | } 44 | 45 | return r0 46 | } 47 | 48 | // Save provides a mock function with no fields 49 | func (_m *NodeCapabilitiesStore) Save() error { 50 | ret := _m.Called() 51 | 52 | if len(ret) == 0 { 53 | panic("no return value specified for Save") 54 | } 55 | 56 | var r0 error 57 | if rf, ok := ret.Get(0).(func() error); ok { 58 | r0 = rf() 59 | } else { 60 | r0 = ret.Error(0) 61 | } 62 | 63 | return r0 64 | } 65 | 66 | // Set provides a mock function with given fields: capName, value 67 | func (_m *NodeCapabilitiesStore) Set(capName string, value string) { 68 | _m.Called(capName, value) 69 | } 70 | 71 | // NewNodeCapabilitiesStore creates a new instance of NodeCapabilitiesStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 72 | // The first argument is typically a *testing.T value. 73 | func NewNodeCapabilitiesStore(t interface { 74 | mock.TestingT 75 | Cleanup(func()) 76 | }) *NodeCapabilitiesStore { 77 | mock := &NodeCapabilitiesStore{} 78 | mock.Mock.Test(t) 79 | 80 | t.Cleanup(func() { mock.AssertExpectations(t) }) 81 | 82 | return mock 83 | } 84 | -------------------------------------------------------------------------------- /pkg/utils/os.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "runtime" 4 | 5 | func IsWindowsOS() bool { 6 | return runtime.GOOS == "windows" 7 | } 8 | -------------------------------------------------------------------------------- /pkg/utils/path.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | ) 8 | 9 | var windowsSystemDrive = mustGetWindowsSystemDrive() 10 | 11 | func mustGetWindowsSystemDrive() string { 12 | if !IsWindowsOS() { 13 | return "" 14 | } 15 | var systemDrive = os.Getenv("SYSTEMDRIVE") 16 | if systemDrive == "" { 17 | systemDrive = filepath.VolumeName(os.Getenv("SYSTEMROOT")) 18 | } 19 | if systemDrive == "" { 20 | panic("unable to get windows system driver") 21 | } 22 | return systemDrive 23 | } 24 | 25 | // NormalizePath returns the normal path in heterogeneous platform. 26 | func NormalizePath(path string) string { 27 | if IsWindowsOS() { 28 | // parses the root path with windows system driver. 29 | if strings.HasPrefix(path, "/") { 30 | path = filepath.FromSlash(path) 31 | return windowsSystemDrive + path 32 | } 33 | } 34 | return path 35 | } 36 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | "strings" 9 | ) 10 | 11 | const unknown = "unknown" 12 | 13 | var ( 14 | Version string 15 | UA string 16 | 17 | gitVersion = "v0.0.0-master+$Format:%H$" 18 | gitCommit = "$Format:%H$" // sha1 from git, output of $(git rev-parse HEAD) 19 | 20 | buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') 21 | ) 22 | 23 | func init() { 24 | Version = fmt.Sprintf("%s/%s (%s/%s) %s %s", adjustCommand(os.Args[0]), adjustVersion(gitVersion), runtime.GOOS, runtime.GOARCH, gitCommit, buildDate) 25 | UA = fmt.Sprintf("%s/%s (%s/%s) terway/%s", adjustCommand(os.Args[0]), adjustVersion(gitVersion), runtime.GOOS, runtime.GOARCH, adjustCommit(gitCommit)) 26 | } 27 | 28 | // adjustVersion strips "alpha", "beta", etc. from version in form 29 | // major.minor.patch-[alpha|beta|etc]. 30 | func adjustVersion(v string) string { 31 | if len(v) == 0 { 32 | return unknown 33 | } 34 | seg := strings.SplitN(v, "-", 2) 35 | return seg[0] 36 | } 37 | 38 | // adjustCommand returns the last component of the 39 | // OS-specific command path for use in User-Agent. 40 | func adjustCommand(p string) string { 41 | // Unlikely, but better than returning "". 42 | if len(p) == 0 { 43 | return unknown 44 | } 45 | return filepath.Base(p) 46 | } 47 | 48 | // adjustCommit returns sufficient significant figures of the commit's git hash. 49 | func adjustCommit(c string) string { 50 | if len(c) == 0 { 51 | return unknown 52 | } 53 | if len(c) > 7 { 54 | return c[:7] 55 | } 56 | return c 57 | } 58 | -------------------------------------------------------------------------------- /pkg/windows/converters/binary.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package converters 5 | 6 | import ( 7 | "net" 8 | 9 | "github.com/AliyunContainerService/terway/pkg/windows/endian" 10 | ) 11 | 12 | func Inet_ntoa(ipnr uint32) string { 13 | ip := net.IPv4(0, 0, 0, 0) 14 | var bo = endian.GetNativelyByteOrder() 15 | bo.PutUint32(ip.To4(), ipnr) 16 | return ip.String() 17 | } 18 | 19 | func Inet_aton(ip string) uint32 { 20 | var bo = endian.GetNativelyByteOrder() 21 | return bo.Uint32( 22 | net.ParseIP(ip).To4(), 23 | ) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/windows/converters/byte.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package converters 5 | 6 | import ( 7 | "bytes" 8 | "unsafe" 9 | ) 10 | 11 | func UnsafeBytesToString(bs []byte) string { 12 | if len(bs) == 0 { 13 | return "" 14 | } 15 | return *(*string)(unsafe.Pointer(&bs)) 16 | } 17 | 18 | func UnsafeUTF16BytesToString(bs []byte) string { 19 | if len(bs) == 0 { 20 | return "" 21 | } 22 | return UnsafeBytesToString(bytes.Trim(bs, "\x00")) 23 | } 24 | -------------------------------------------------------------------------------- /pkg/windows/endian/endianess.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | // Copyright 2015 flannel authors 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | 18 | package endian 19 | 20 | // Taken from a patch by David Anderson who submitted it 21 | // but got rejected by the golang team 22 | 23 | import ( 24 | "encoding/binary" 25 | "unsafe" 26 | ) 27 | 28 | // nativeEndian is the ByteOrder of the current system. 29 | var nativeEndian binary.ByteOrder 30 | 31 | func init() { 32 | // Examine the memory layout of an int16 to determine system 33 | // endianness. 34 | var one int16 = 1 35 | b := (*byte)(unsafe.Pointer(&one)) 36 | if *b == 0 { 37 | nativeEndian = binary.BigEndian 38 | } else { 39 | nativeEndian = binary.LittleEndian 40 | } 41 | } 42 | 43 | func NativelyLittle() bool { 44 | return nativeEndian == binary.LittleEndian 45 | } 46 | 47 | func GetNativelyByteOrder() binary.ByteOrder { 48 | return nativeEndian 49 | } 50 | -------------------------------------------------------------------------------- /pkg/windows/powershell/command.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | // Copyright 2015 flannel authors 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | 18 | package powershell 19 | 20 | import ( 21 | "encoding/json" 22 | "errors" 23 | "fmt" 24 | "os/exec" 25 | "strings" 26 | ) 27 | 28 | // commandWrapper ensures that exceptions are written to stdout and the powershell process exit code is -1 29 | const commandWrapper = `$ErrorActionPreference="Stop";try { %s } catch { Write-Host $_; os.Exit(-1) }` 30 | 31 | // RunCommand executes a given powershell command. 32 | func RunCommand(command string) ([]byte, error) { 33 | var s = fmt.Sprintf(commandWrapper, command) 34 | var cmd = exec.Command("powershell.exe", "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", s) 35 | var stdout, err = cmd.Output() 36 | if err != nil { 37 | if cmd.ProcessState.ExitCode() != 0 { 38 | var message = strings.TrimSpace(string(stdout)) 39 | return nil, errors.New(message) 40 | } 41 | return nil, err 42 | } 43 | return stdout, nil 44 | } 45 | 46 | // RunCommandWithJsonResult executes a given powershell command. 47 | func RunCommandWithJsonResult(command string, v interface{}) error { 48 | var s = fmt.Sprintf(commandWrapper, "ConvertTo-Json (%s)") 49 | s = fmt.Sprintf(s, command) 50 | var stdout, err = RunCommand(s) 51 | if err != nil { 52 | return err 53 | } 54 | err = json.Unmarshal(stdout, v) 55 | if err != nil { 56 | return err 57 | } 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /plugin/datapath/consts_linux.go: -------------------------------------------------------------------------------- 1 | package datapath 2 | 3 | import ( 4 | "net" 5 | 6 | "github.com/vishvananda/netlink" 7 | 8 | "github.com/AliyunContainerService/terway/types" 9 | ) 10 | 11 | const ( 12 | toContainerPriority = 512 13 | fromContainerPriority = 2048 14 | ) 15 | 16 | // default addrs 17 | var ( 18 | _, defaultRoute, _ = net.ParseCIDR("0.0.0.0/0") 19 | _, defaultRouteIPv6, _ = net.ParseCIDR("::/0") 20 | LinkIP = net.IPv4(169, 254, 1, 1) 21 | LinkIPv6 = net.ParseIP("fe80::1") 22 | LinkIPNet = &net.IPNet{ 23 | IP: LinkIP, 24 | Mask: net.CIDRMask(32, 32), 25 | } 26 | LinkIPNetv6 = &net.IPNet{ 27 | IP: LinkIPv6, 28 | Mask: net.CIDRMask(128, 128), 29 | } 30 | ) 31 | 32 | var PrioMap = map[string]uint32{ 33 | string(types.NetworkPrioGuaranteed): netlink.MakeHandle(1, 1), // band 0 34 | string(types.NetworkPrioBurstable): netlink.MakeHandle(1, 2), // band 1 35 | string(types.NetworkPrioBestEffort): netlink.MakeHandle(1, 3), // band 2 36 | "": netlink.MakeHandle(1, 2), 37 | } 38 | -------------------------------------------------------------------------------- /plugin/driver/ipvlan/ipvlan.go: -------------------------------------------------------------------------------- 1 | package ipvlan 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/AliyunContainerService/terway/plugin/driver/utils" 7 | 8 | "github.com/containernetworking/plugins/pkg/ns" 9 | "github.com/vishvananda/netlink" 10 | ) 11 | 12 | type IPVlan struct { 13 | Parent string 14 | PreName string 15 | IfName string 16 | MTU int 17 | } 18 | 19 | func Setup(ctx context.Context, cfg *IPVlan, netNS ns.NetNS) error { 20 | parentLink, err := netlink.LinkByName(cfg.Parent) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | pre, err := netlink.LinkByName(cfg.PreName) 26 | if err == nil { 27 | // del pre link 28 | err = utils.LinkDel(ctx, pre) 29 | if err != nil { 30 | return err 31 | } 32 | } 33 | 34 | if _, ok := err.(netlink.LinkNotFoundError); !ok { 35 | return err 36 | } 37 | 38 | v := &netlink.IPVlan{ 39 | LinkAttrs: netlink.LinkAttrs{ 40 | MTU: cfg.MTU, 41 | Name: cfg.PreName, 42 | Namespace: netlink.NsFd(int(netNS.Fd())), 43 | ParentIndex: parentLink.Attrs().Index, 44 | }, 45 | Mode: netlink.IPVLAN_MODE_L2, 46 | } 47 | err = utils.LinkAdd(ctx, v) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | return netNS.Do(func(netNS ns.NetNS) error { 53 | contLink, innerErr := netlink.LinkByName(cfg.PreName) 54 | if innerErr != nil { 55 | return innerErr 56 | } 57 | _, innerErr = utils.EnsureLinkName(ctx, contLink, cfg.IfName) 58 | return innerErr 59 | }) 60 | } 61 | -------------------------------------------------------------------------------- /plugin/driver/types/types_linux.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "github.com/containernetworking/plugins/pkg/ns" 5 | 6 | terwayTypes "github.com/AliyunContainerService/terway/types" 7 | ) 8 | 9 | type CheckConfig struct { 10 | DP DataPath 11 | 12 | RecordPodEvent 13 | 14 | NetNS ns.NetNS 15 | 16 | HostVETHName string 17 | ContainerIfName string 18 | 19 | ContainerIPNet *terwayTypes.IPNetSet 20 | HostIPSet *terwayTypes.IPNetSet 21 | GatewayIP *terwayTypes.IPSet 22 | 23 | ENIIndex int32 // phy device 24 | TrunkENI bool 25 | MTU int 26 | 27 | DefaultRoute bool 28 | MultiNetwork bool 29 | } 30 | -------------------------------------------------------------------------------- /plugin/driver/types/types_unspport.go: -------------------------------------------------------------------------------- 1 | //go:build !linux && !windows 2 | // +build !linux,!windows 3 | 4 | package types 5 | 6 | import ( 7 | terwayTypes "github.com/AliyunContainerService/terway/types" 8 | ) 9 | 10 | type CheckConfig struct { 11 | DP DataPath 12 | 13 | RecordPodEvent 14 | 15 | HostVETHName string 16 | ContainerIfName string 17 | 18 | ContainerIPNet *terwayTypes.IPNetSet 19 | HostIPSet *terwayTypes.IPNetSet 20 | GatewayIP *terwayTypes.IPSet 21 | 22 | ENIIndex int32 // phy device 23 | TrunkENI bool 24 | MTU int 25 | 26 | DefaultRoute bool 27 | MultiNetwork bool 28 | } 29 | -------------------------------------------------------------------------------- /plugin/driver/types/types_windows.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | terwayTypes "github.com/AliyunContainerService/terway/types" 5 | ) 6 | 7 | type CheckConfig struct { 8 | DP DataPath 9 | 10 | RecordPodEvent 11 | 12 | HostVETHName string 13 | ContainerIfName string 14 | 15 | ContainerIPNet *terwayTypes.IPNetSet 16 | HostIPSet *terwayTypes.IPNetSet 17 | GatewayIP *terwayTypes.IPSet 18 | 19 | ENIIndex int32 // phy device 20 | TrunkENI bool 21 | MTU int 22 | 23 | DefaultRoute bool 24 | MultiNetwork bool 25 | } 26 | -------------------------------------------------------------------------------- /plugin/driver/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "os" 9 | "sync" 10 | "time" 11 | 12 | "github.com/alexflint/go-filemutex" 13 | "github.com/go-logr/logr" 14 | "k8s.io/apimachinery/pkg/util/wait" 15 | "k8s.io/klog/v2/textlogger" 16 | 17 | "github.com/AliyunContainerService/terway/pkg/utils" 18 | ) 19 | 20 | const ( 21 | fileLockTimeOut = 11 * time.Second 22 | ) 23 | 24 | var Log = logr.Discard() 25 | var once sync.Once 26 | 27 | func InitLog(debug bool) logr.Logger { 28 | var opts []textlogger.ConfigOption 29 | once.Do(func() { 30 | if debug { 31 | var file, err = os.OpenFile(utils.NormalizePath("/var/log/terway.cni.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 32 | if err != nil { 33 | panic(err) 34 | } 35 | opts = append(opts, textlogger.Verbosity(4), textlogger.Output(io.MultiWriter(file, os.Stderr))) 36 | 37 | } 38 | Log = textlogger.NewLogger(textlogger.NewConfig(opts...)) 39 | }) 40 | 41 | return Log 42 | } 43 | 44 | // JSONStr json to str 45 | func JSONStr(v interface{}) string { 46 | b, err := json.Marshal(v) 47 | if err != nil { 48 | return "" 49 | } 50 | return string(b) 51 | } 52 | 53 | type Locker struct { 54 | m *filemutex.FileMutex 55 | } 56 | 57 | // Close close 58 | func (l *Locker) Close() error { 59 | if l.m != nil { 60 | return l.m.Unlock() 61 | } 62 | return nil 63 | } 64 | 65 | // GrabFileLock get file lock with timeout 11seconds 66 | func GrabFileLock(lockfilePath string) (*Locker, error) { 67 | var m, err = filemutex.New(utils.NormalizePath(lockfilePath)) 68 | if err != nil { 69 | return nil, fmt.Errorf("failed to open lock %s: %v", lockfilePath, err) 70 | } 71 | 72 | err = wait.PollUntilContextTimeout(context.Background(), 200*time.Millisecond, fileLockTimeOut, true, func(ctx context.Context) (bool, error) { 73 | if err := m.Lock(); err != nil { 74 | return false, nil 75 | } 76 | return true, nil 77 | }) 78 | if err != nil { 79 | return nil, fmt.Errorf("failed to acquire lock: %v", err) 80 | } 81 | 82 | return &Locker{ 83 | m: m, 84 | }, nil 85 | } 86 | -------------------------------------------------------------------------------- /plugin/driver/veth/veth.go: -------------------------------------------------------------------------------- 1 | package veth 2 | 3 | import ( 4 | "context" 5 | "net" 6 | 7 | "github.com/AliyunContainerService/terway/plugin/driver/utils" 8 | 9 | "github.com/containernetworking/plugins/pkg/ip" 10 | "github.com/containernetworking/plugins/pkg/ns" 11 | "github.com/vishvananda/netlink" 12 | ) 13 | 14 | type Veth struct { 15 | IfName string // cont in netns 16 | PeerName string 17 | HwAddr net.HardwareAddr 18 | MTU int 19 | } 20 | 21 | func Setup(ctx context.Context, cfg *Veth, netNS ns.NetNS) error { 22 | peer, err := netlink.LinkByName(cfg.PeerName) 23 | if err == nil { 24 | // del pre link 25 | err = utils.LinkDel(ctx, peer) 26 | if err != nil { 27 | return err 28 | } 29 | } 30 | 31 | if _, ok := err.(netlink.LinkNotFoundError); !ok { 32 | return err 33 | } 34 | contLinkName, err := ip.RandomVethName() 35 | if err != nil { 36 | return err 37 | } 38 | v := &netlink.Veth{ 39 | LinkAttrs: netlink.LinkAttrs{ 40 | MTU: cfg.MTU, 41 | Name: contLinkName, 42 | Namespace: netlink.NsFd(int(netNS.Fd())), 43 | }, 44 | PeerName: cfg.PeerName, 45 | } 46 | if cfg.HwAddr != nil { 47 | v.HardwareAddr = cfg.HwAddr 48 | } 49 | err = utils.LinkAdd(ctx, v) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | return netNS.Do(func(netNS ns.NetNS) error { 55 | contLink, innerErr := netlink.LinkByName(contLinkName) 56 | if innerErr != nil { 57 | return innerErr 58 | } 59 | _, innerErr = utils.EnsureLinkName(ctx, contLink, cfg.IfName) 60 | return innerErr 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /plugin/driver/vlan/vlan.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package vlan 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "github.com/AliyunContainerService/terway/plugin/driver/utils" 24 | 25 | "github.com/containernetworking/plugins/pkg/ns" 26 | "github.com/vishvananda/netlink" 27 | ) 28 | 29 | type Vlan struct { 30 | Master string 31 | IfName string 32 | Vid int 33 | MTU int 34 | } 35 | 36 | func Setup(ctx context.Context, cfg *Vlan, netNS ns.NetNS) error { 37 | master, err := netlink.LinkByName(cfg.Master) 38 | if err != nil { 39 | return fmt.Errorf("cannot found master link by name %s", master) 40 | } 41 | peerName := fmt.Sprintf("%s.%d", master.Attrs().Name, cfg.Vid) 42 | if len(peerName) > 15 { 43 | peerName = peerName[len(peerName)-15:] 44 | } 45 | peer, err := netlink.LinkByName(peerName) 46 | if err == nil { 47 | // del pre link 48 | err = utils.LinkDel(ctx, peer) 49 | if err != nil { 50 | return err 51 | } 52 | } 53 | 54 | if _, ok := err.(netlink.LinkNotFoundError); !ok { 55 | return err 56 | } 57 | 58 | v := &netlink.Vlan{ 59 | LinkAttrs: netlink.LinkAttrs{ 60 | MTU: cfg.MTU, 61 | Name: peerName, 62 | ParentIndex: master.Attrs().Index, 63 | Namespace: netlink.NsFd(int(netNS.Fd())), 64 | }, 65 | VlanId: cfg.Vid, 66 | } 67 | err = utils.LinkAdd(ctx, v) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | return netNS.Do(func(netNS ns.NetNS) error { 73 | contLink, innerErr := netlink.LinkByName(peerName) 74 | if innerErr != nil { 75 | return innerErr 76 | } 77 | _, innerErr = utils.EnsureLinkName(ctx, contLink, cfg.IfName) 78 | return innerErr 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /plugin/driver/vlan/vlan_test.go: -------------------------------------------------------------------------------- 1 | //go:build privileged 2 | 3 | package vlan 4 | -------------------------------------------------------------------------------- /plugin/terway/cni/runtime_config.go: -------------------------------------------------------------------------------- 1 | package cni 2 | 3 | import "github.com/containernetworking/cni/pkg/types" 4 | 5 | type RuntimeDNS struct { 6 | Nameservers []string `json:"servers,omitempty"` 7 | Search []string `json:"searches,omitempty"` 8 | Options []string `json:"options,omitempty"` 9 | } 10 | 11 | func (i *RuntimeDNS) AsCNIDns() types.DNS { 12 | return types.DNS{ 13 | Nameservers: i.Nameservers, 14 | Search: i.Search, 15 | Options: i.Options, 16 | } 17 | } 18 | 19 | type RuntimePortMapEntry struct { 20 | HostPort int `json:"hostPort"` 21 | ContainerPort int `json:"containerPort"` 22 | Protocol string `json:"protocol"` 23 | HostIP string `json:"hostIP,omitempty"` 24 | } 25 | 26 | type RuntimeBandwidthEntry struct { 27 | // IngressRate is the bandwidth rate in bits per second for traffic through container. 0 for no limit. If IngressRate is set, IngressBurst must also be set 28 | IngressRate int `json:"ingressRate,omitempty"` 29 | // IngressBurst is the bandwidth burst in bits for traffic through container. 0 for no limit. If IngressBurst is set, IngressRate must also be set 30 | // NOTE: it's not used for now and defaults to 0. If IngressRate is set IngressBurst will be math.MaxInt32 ~ 2Gbit 31 | IngressBurst int `json:"ingressBurst,omitempty"` 32 | // EgressRate is the bandwidth is the bandwidth rate in bits per second for traffic through container. 0 for no limit. If EgressRate is set, EgressBurst must also be set 33 | EgressRate int `json:"egressRate,omitempty"` 34 | // EgressBurst is the bandwidth burst in bits for traffic through container. 0 for no limit. If EgressBurst is set, EgressRate must also be set 35 | // NOTE: it's not used for now and defaults to 0. If EgressRate is set EgressBurst will be math.MaxInt32 ~ 2Gbit 36 | EgressBurst int `json:"egressBurst,omitempty"` 37 | } 38 | 39 | type RuntimeConfig struct { 40 | DNS RuntimeDNS `json:"dns"` 41 | PortMaps []RuntimePortMapEntry `json:"portMappings,omitempty"` 42 | Bandwidth RuntimeBandwidthEntry `json:"bandwidth,omitempty"` 43 | } 44 | -------------------------------------------------------------------------------- /plugin/terway/cni_unsupport.go: -------------------------------------------------------------------------------- 1 | //go:build !linux && !windows 2 | // +build !linux,!windows 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | 10 | "github.com/containernetworking/cni/pkg/skel" 11 | 12 | "github.com/AliyunContainerService/terway/plugin/driver/types" 13 | "github.com/AliyunContainerService/terway/rpc" 14 | terwayTypes "github.com/AliyunContainerService/terway/types" 15 | ) 16 | 17 | func getCmdArgs(args *skel.CmdArgs) (*cniCmdArgs, error) { 18 | panic("not implement") 19 | } 20 | 21 | type cniCmdArgs struct { 22 | } 23 | 24 | func (args *cniCmdArgs) GetCNIConf() *types.CNIConf { 25 | panic("not implement") 26 | } 27 | 28 | func (args *cniCmdArgs) GetK8SConfig() *types.K8SArgs { 29 | panic("not implement") 30 | } 31 | 32 | func (args *cniCmdArgs) GetInputArgs() *skel.CmdArgs { 33 | panic("not implement") 34 | } 35 | 36 | func (args *cniCmdArgs) GetNetNSPath() string { 37 | panic("not implement") 38 | } 39 | 40 | func (args *cniCmdArgs) Close() error { 41 | panic("not implement") 42 | } 43 | 44 | func isNSPathNotExist(err error) bool { 45 | panic("not implement") 46 | } 47 | 48 | func doCmdAdd(ctx context.Context, client rpc.TerwayBackendClient, cmdArgs *cniCmdArgs) (*terwayTypes.IPNetSet, *terwayTypes.IPSet, error) { 49 | panic("not implement") 50 | } 51 | 52 | func doCmdDel(ctx context.Context, client rpc.TerwayBackendClient, cmdArgs *cniCmdArgs) error { 53 | panic("not implement") 54 | } 55 | 56 | func doCmdCheck(ctx context.Context, client rpc.TerwayBackendClient, cmdArgs *cniCmdArgs) error { 57 | panic("not implement") 58 | } 59 | func prepareVF(ctx context.Context, id int, mac string) (int32, error) { 60 | return 0, fmt.Errorf("not support") 61 | } 62 | -------------------------------------------------------------------------------- /policy/cilium/0002-bypass-the-node-local-dns-ip.patch: -------------------------------------------------------------------------------- 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 2 | From: l1b0k 3 | Date: Mon, 24 Mar 2025 16:43:22 +0800 4 | Subject: bypass the node local dns ip 5 | 6 | Signed-off-by: l1b0k 7 | --- 8 | bpf/bpf_lxc.c | 5 +++++ 9 | 1 file changed, 5 insertions(+) 10 | 11 | diff --git a/bpf/bpf_lxc.c b/bpf/bpf_lxc.c 12 | index fd46558a86..0701bddf5e 100644 13 | --- a/bpf/bpf_lxc.c 14 | +++ b/bpf/bpf_lxc.c 15 | @@ -14,6 +14,7 @@ 16 | #define IS_BPF_LXC 1 17 | 18 | #define EVENT_SOURCE LXC_ID 19 | +#define LOCAL_DNS_IP 0x0A14FEA9 20 | 21 | #include "lib/auth.h" 22 | #include "lib/tailcall.h" 23 | @@ -1265,6 +1266,10 @@ maybe_pass_to_stack: __maybe_unused; 24 | #else 25 | int oif = 0; 26 | #endif 27 | + 28 | + if (ip4->daddr == LOCAL_DNS_IP ) 29 | + goto pass_to_stack; 30 | + 31 | if (oif > 0) { 32 | ret = ipv4_l3(ctx, ETH_HLEN, NULL, NULL, ip4); 33 | if (ret == CTX_ACT_OK) 34 | -- 35 | 2.39.5 (Apple Git-154) 36 | 37 | -------------------------------------------------------------------------------- /policy/cilium/0006-gops-allow-disable-gops.patch: -------------------------------------------------------------------------------- 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 2 | From: l1b0k 3 | Date: Wed, 13 Nov 2024 15:56:13 +0800 4 | Subject: gops: allow disable gops 5 | 6 | Signed-off-by: l1b0k 7 | --- 8 | pkg/gops/cell.go | 3 +++ 9 | 1 file changed, 3 insertions(+) 10 | 11 | diff --git a/pkg/gops/cell.go b/pkg/gops/cell.go 12 | index 707835d816..0e2da2efa6 100644 13 | --- a/pkg/gops/cell.go 14 | +++ b/pkg/gops/cell.go 15 | @@ -36,6 +36,9 @@ func (def GopsConfig) Flags(flags *pflag.FlagSet) { 16 | } 17 | 18 | func registerGopsHooks(lc cell.Lifecycle, log logrus.FieldLogger, cfg GopsConfig) { 19 | + if cfg.GopsPort == 0 { 20 | + return 21 | + } 22 | addr := fmt.Sprintf("127.0.0.1:%d", cfg.GopsPort) 23 | addrField := logrus.Fields{"address": addr, logfields.LogSubsys: "gops"} 24 | log = log.WithFields(addrField) 25 | -- 26 | 2.39.5 (Apple Git-154) 27 | 28 | -------------------------------------------------------------------------------- /policy/cilium/0007-ctmap-log-ct-gc-status.patch: -------------------------------------------------------------------------------- 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 2 | From: l1b0k 3 | Date: Tue, 26 Nov 2024 10:51:43 +0800 4 | Subject: ctmap: log ct gc status 5 | 6 | Signed-off-by: l1b0k 7 | --- 8 | pkg/maps/ctmap/ctmap.go | 12 ++++++++++++ 9 | 1 file changed, 12 insertions(+) 10 | 11 | diff --git a/pkg/maps/ctmap/ctmap.go b/pkg/maps/ctmap/ctmap.go 12 | index c4ebecc404..81555cfc8a 100644 13 | --- a/pkg/maps/ctmap/ctmap.go 14 | +++ b/pkg/maps/ctmap/ctmap.go 15 | @@ -443,6 +443,12 @@ func doGC6(m *Map, filter *GCFilter) gcStats { 16 | globalDeleteLock[m.mapType].Lock() 17 | stats.dumpError = m.DumpReliablyWithCallback(filterCallback, stats.DumpStats) 18 | globalDeleteLock[m.mapType].Unlock() 19 | + 20 | + log.Infof("gc map %s max %d deleted %d alived %d", m.Name(), m.MaxEntries(), stats.deleted, stats.aliveEntries) 21 | + cur := stats.aliveEntries + stats.deleted 22 | + if float64(cur)/float64(m.MaxEntries()) >= 0.9 { 23 | + log.Infof("ConntrackFull table %s current %d maxEntries %d", m.Name(), cur, m.MaxEntries()) 24 | + } 25 | return stats 26 | } 27 | 28 | @@ -556,6 +562,12 @@ func doGC4(m *Map, filter *GCFilter) gcStats { 29 | globalDeleteLock[m.mapType].Lock() 30 | stats.dumpError = m.DumpReliablyWithCallback(filterCallback, stats.DumpStats) 31 | globalDeleteLock[m.mapType].Unlock() 32 | + 33 | + log.Infof("gc map %s max %d deleted %d alived %d", m.Name(), m.MaxEntries(), stats.deleted, stats.aliveEntries) 34 | + cur := stats.aliveEntries + stats.deleted 35 | + if float64(cur)/float64(m.MaxEntries()) >= 0.9 { 36 | + log.Infof("ConntrackFull table %s current %d maxEntries %d", m.Name(), cur, m.MaxEntries()) 37 | + } 38 | return stats 39 | } 40 | 41 | -- 42 | 2.39.5 (Apple Git-154) 43 | 44 | -------------------------------------------------------------------------------- /policy/cilium/0008-Ignore-the-link-local-IPv6-addresses-as-it-is-genera.patch: -------------------------------------------------------------------------------- 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 2 | From: "bingshen.wbs" 3 | Date: Tue, 23 Jul 2024 11:03:09 +0800 4 | Subject: =?UTF-8?q?Ignore=20the=20link-local=20IPv6=20addresses=20as=20it?= 5 | =?UTF-8?q?=20is=20generated=20based=20on=20the=20MAC=20address=20and=20is?= 6 | =?UTF-8?q?=20not=20actually=20reachable=20within=20the=20network.=0AAddit?= 7 | =?UTF-8?q?ionally,=20in=20some=20network=20plugins,=20the=20MAC=20address?= 8 | =?UTF-8?q?es=20in=20pods=20may=20be=20the=20same,=20leading=20to=20confli?= 9 | =?UTF-8?q?cts=20in=20the=20generated=20IPv6=20addresses=20and=20preventin?= 10 | =?UTF-8?q?g=20the=20creation=20of=20Cilium=20endpoints.?= 11 | 12 | Signed-off-by: bingshen.wbs 13 | Signed-off-by: l1b0k 14 | --- 15 | plugins/cilium-cni/chaining/generic-veth/generic-veth.go | 7 ++++++- 16 | 1 file changed, 6 insertions(+), 1 deletion(-) 17 | 18 | diff --git a/plugins/cilium-cni/chaining/generic-veth/generic-veth.go b/plugins/cilium-cni/chaining/generic-veth/generic-veth.go 19 | index 0b1187c6dc..f003329fc9 100644 20 | --- a/plugins/cilium-cni/chaining/generic-veth/generic-veth.go 21 | +++ b/plugins/cilium-cni/chaining/generic-veth/generic-veth.go 22 | @@ -98,7 +98,12 @@ func (f *GenericVethChainer) Add(ctx context.Context, pluginCtx chainingapi.Plug 23 | 24 | addrsv6, err := safenetlink.AddrList(link, netlink.FAMILY_V6) 25 | if err == nil && len(addrsv6) > 0 { 26 | - vethIPv6 = addrsv6[0].IPNet.IP.String() 27 | + for _, addrv6 := range addrsv6 { 28 | + if addrv6.IP.IsGlobalUnicast() { 29 | + vethIPv6 = addrv6.IPNet.IP.String() 30 | + break 31 | + } 32 | + } 33 | } else if err != nil { 34 | pluginCtx.Logger.WithError(err).WithField(logfields.Interface, link.Attrs().Name).Warn("No valid IPv6 address found") 35 | } 36 | -- 37 | 2.39.5 (Apple Git-154) 38 | 39 | -------------------------------------------------------------------------------- /policy/cilium/0010-Revert-fix-Adding-ipv6-check-on-the-node-init-functi.patch: -------------------------------------------------------------------------------- 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 2 | From: l1b0k 3 | Date: Thu, 10 Apr 2025 20:53:26 +0800 4 | Subject: Revert "fix: Adding ipv6 check on the node init function" 5 | 6 | This reverts commit 2fc52eb444635c83843c45735766db0e338b581c. 7 | --- 8 | daemon/k8s/init.go | 10 ---------- 9 | 1 file changed, 10 deletions(-) 10 | 11 | diff --git a/daemon/k8s/init.go b/daemon/k8s/init.go 12 | index 70aae859f1..90d3d52b0d 100644 13 | --- a/daemon/k8s/init.go 14 | +++ b/daemon/k8s/init.go 15 | @@ -126,16 +126,6 @@ func WaitForNodeInformation(ctx context.Context, log logrus.FieldLogger, localNo 16 | logfields.K8sNodeIP: k8sNodeIP, 17 | }).Info("Received own node information from API server") 18 | 19 | - // If the host does not have an IPv6 address, return an error 20 | - if option.Config.EnableIPv6 && nodeIP6 == nil { 21 | - log.WithFields(logrus.Fields{ 22 | - logfields.NodeName: n.Name, 23 | - logfields.IPAddr + ".ipv4": nodeIP4, 24 | - logfields.IPAddr + ".ipv6": nodeIP6, 25 | - }).Error("No IPv6 support on node as ipv6 address is nil") 26 | - return fmt.Errorf("node %s does not have an IPv6 address", n.Name) 27 | - } 28 | - 29 | useNodeCIDR(n) 30 | } else { 31 | // if node resource could not be received, fail if 32 | -- 33 | 2.39.5 (Apple Git-154) 34 | 35 | -------------------------------------------------------------------------------- /policy/cilium/0011-fix-viper-flag.patch: -------------------------------------------------------------------------------- 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 2 | From: l1b0k 3 | Date: Wed, 28 May 2025 22:44:50 +0800 4 | Subject: fix: viper flag 5 | 6 | Signed-off-by: l1b0k 7 | --- 8 | pkg/option/config.go | 2 +- 9 | 1 file changed, 1 insertion(+), 1 deletion(-) 10 | 11 | diff --git a/pkg/option/config.go b/pkg/option/config.go 12 | index 87f4fe20dd..8107f94de4 100644 13 | --- a/pkg/option/config.go 14 | +++ b/pkg/option/config.go 15 | @@ -3063,7 +3063,7 @@ func (c *DaemonConfig) Populate(vp *viper.Viper) { 16 | c.EnableIPIPTermination = vp.GetBool(EnableIPIPTermination) 17 | c.EnableUnreachableRoutes = vp.GetBool(EnableUnreachableRoutes) 18 | c.EnableNodePort = vp.GetBool(EnableNodePort) 19 | - c.EnableInClusterLoadBalance = viper.GetBool(EnableInClusterLoadBalance) 20 | + c.EnableInClusterLoadBalance = vp.GetBool(EnableInClusterLoadBalance) 21 | c.EnableSVCSourceRangeCheck = vp.GetBool(EnableSVCSourceRangeCheck) 22 | c.EnableHostPort = vp.GetBool(EnableHostPort) 23 | c.EnableHostLegacyRouting = vp.GetBool(EnableHostLegacyRouting) 24 | -- 25 | 2.39.5 (Apple Git-154) 26 | 27 | -------------------------------------------------------------------------------- /policy/policyinit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o errexit 3 | 4 | mount -o remount rw /proc/sys 5 | 6 | terway-cli policy -------------------------------------------------------------------------------- /rpc/generate.go: -------------------------------------------------------------------------------- 1 | //go:generate protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative rpc.proto tracing.proto 2 | package rpc 3 | -------------------------------------------------------------------------------- /rpc/tracing.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package rpc; 3 | option go_package = ".;rpc"; 4 | 5 | service TerwayTracing { 6 | rpc GetResourceTypes(Placeholder) returns (ResourcesTypesReply); 7 | rpc GetResources(ResourceTypeRequest) returns (ResourcesNamesReply); 8 | rpc GetResourceConfig(ResourceTypeNameRequest) returns (ResourceConfigReply); 9 | rpc GetResourceTrace(ResourceTypeNameRequest) returns (ResourceTraceReply); 10 | rpc ResourceExecute(ResourceExecuteRequest) returns (stream ResourceExecuteReply); 11 | rpc GetResourceMapping(Placeholder) returns (ResourceMappingReply); 12 | } 13 | 14 | message Placeholder {} 15 | 16 | message ResourcesTypesReply { 17 | repeated string TypeNames = 1; 18 | } 19 | message ResourcesNamesReply { 20 | repeated string ResourceNames = 1; 21 | } 22 | 23 | message ResourceTypeRequest { 24 | string Name = 1; 25 | } 26 | 27 | message ResourceTypeNameRequest { 28 | string Type = 1; 29 | string Name = 2; 30 | } 31 | 32 | message ResourceExecuteRequest { 33 | string Type = 1; 34 | string Name = 2; 35 | string Command = 3; 36 | repeated string Args = 4; 37 | } 38 | 39 | message ResourceExecuteReply { 40 | string Message = 1; 41 | } 42 | 43 | message MapKeyValueEntry { 44 | string Key = 1; 45 | string Value = 2; 46 | } 47 | 48 | message ResourceConfigReply { 49 | repeated MapKeyValueEntry Config = 1; 50 | } 51 | 52 | message ResourceTraceReply { 53 | repeated MapKeyValueEntry Trace = 1; 54 | } 55 | 56 | message ResourceMapping { 57 | string NetworkInterfaceID = 1; 58 | string MAC = 2; 59 | string Type = 3; 60 | string AllocInhibitExpireAt = 4 ; 61 | string Status = 5; 62 | repeated string Info = 6; 63 | } 64 | 65 | message ResourceMappingReply { 66 | repeated ResourceMapping info = 1; 67 | } 68 | -------------------------------------------------------------------------------- /terraform/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | alicloud = { 4 | source = "aliyun/alicloud" 5 | version = "1.139.0" 6 | } 7 | windbag = { 8 | source = "thxcode/windbag" 9 | version = "0.3.1" 10 | } 11 | } 12 | } 13 | 14 | # tf plan \ 15 | # --var 'region=cn-hongkong' \ 16 | # --var 'access_key=...' \ 17 | # --var 'secret_key=...' \ 18 | # --var 'host_image_list=["win2019_1809_x64_dtc_en-us_40G_container_alibase_20210716.vhd","wincore_1909_x64_dtc_en-us_40G_container_alibase_20200723.vhd","wincore_2004_x64_dtc_en-us_40G_container_alibase_20210716.vhd"]' \ 19 | # --var 'host_password=Just4Test' \ 20 | # --var 'image_repository_list=["registry.cn-hangzhou.aliyuncs.com", "registry.cn-hongkong.aliyuncs.com"]' \ 21 | # --var 'image_name=terway' \ 22 | # --var 'image_tag=v0.0.0' \ 23 | # --var 'image_registry_username=...' \ 24 | # --var 'image_registry_password=...' 25 | -------------------------------------------------------------------------------- /tests/kind/Makefile: -------------------------------------------------------------------------------- 1 | # tests/kind/Makefile 2 | 3 | ##@ General 4 | 5 | .PHONY: help 6 | help: ## Display this help. 7 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 8 | 9 | ##@ Tests 10 | 11 | .PHONY: datapath-test 12 | datapath-test: ## Run datapath tests using run.sh script. 13 | chmod +x run.sh && ./run.sh 14 | -------------------------------------------------------------------------------- /tests/kind/cluster.yml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | networking: 4 | disableDefaultCNI: true 5 | nodes: 6 | - role: control-plane 7 | image: kindest/node:v1.30.8@sha256:17cd608b3971338d9180b00776cb766c50d0a0b6b904ab4ff52fd3fc5c6369bf 8 | -------------------------------------------------------------------------------- /tests/kind/conf/eniip_datapathv2_cmdline: -------------------------------------------------------------------------------- 1 | cilium-agent--routing-mode=native--cni-chaining-mode=terway-chainer--enable-ipv4-masquerade=false--enable-ipv6-masquerade=false--disable-envoy-version-check=true--local-router-ipv4=169.254.10.1--local-router-ipv6=fe80:2400:3200:baba::1--enable-local-node-route=false--enable-endpoint-health-checking=false--enable-health-checking=false--enable-service-topology=true--k8s-heartbeat-timeout=0--enable-session-affinity=true--install-iptables-rules=false--enable-l7-proxy=false--ipam=delegated-plugin--enable-bandwidth-manager=true--agent-health-port=9099--enable-policy=never--labels=k8s:io\.kubernetes\.pod\.namespace--datapath-mode=veth--enable-endpoint-routes=true--enable-l2-neigh-discovery=false--enable-in-cluster-loadbalance=true -------------------------------------------------------------------------------- /tests/kind/conf/eniip_default_cmdline: -------------------------------------------------------------------------------- 1 | cilium-agent--routing-mode=native--cni-chaining-mode=terway-chainer--enable-ipv4-masquerade=false--enable-ipv6-masquerade=false--disable-envoy-version-check=true--local-router-ipv4=169.254.10.1--local-router-ipv6=fe80:2400:3200:baba::1--enable-local-node-route=false--enable-endpoint-health-checking=false--enable-health-checking=false--enable-service-topology=true--k8s-heartbeat-timeout=0--enable-session-affinity=true--install-iptables-rules=false--enable-l7-proxy=false--ipam=delegated-plugin--enable-bandwidth-manager=true--agent-health-port=9099--enable-policy=default--datapath-mode=veth--enable-endpoint-routes=true--enable-l2-neigh-discovery=false--enable-in-cluster-loadbalance=true -------------------------------------------------------------------------------- /tests/kind/conf/eniip_legacy_ciliumargs_cmdline: -------------------------------------------------------------------------------- 1 | cilium-agent--routing-mode=native--cni-chaining-mode=terway-chainer--enable-ipv4-masquerade=false--enable-ipv6-masquerade=false--disable-envoy-version-check=true--local-router-ipv4=169.254.10.1--local-router-ipv6=fe80:2400:3200:baba::1--enable-local-node-route=false--enable-endpoint-health-checking=false--enable-health-checking=false--enable-service-topology=true--k8s-heartbeat-timeout=0--enable-session-affinity=true--install-iptables-rules=false--enable-l7-proxy=false--ipam=delegated-plugin--enable-bandwidth-manager=true--agent-health-port=9099--enable-policy=default--datapath-mode=veth--enable-endpoint-routes=true--enable-l2-neigh-discovery=false--enable-in-cluster-loadbalance=true -------------------------------------------------------------------------------- /tests/stress/setup_test.go: -------------------------------------------------------------------------------- 1 | //go:build e2e 2 | 3 | package stress 4 | 5 | import ( 6 | "context" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | 11 | "k8s.io/client-go/kubernetes/scheme" 12 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 13 | "sigs.k8s.io/e2e-framework/pkg/env" 14 | "sigs.k8s.io/e2e-framework/pkg/envconf" 15 | "sigs.k8s.io/e2e-framework/pkg/envfuncs" 16 | 17 | "github.com/AliyunContainerService/terway/tests/utils" 18 | ) 19 | 20 | var ( 21 | testenv env.Environment 22 | 23 | containerNetworkPods int 24 | ) 25 | 26 | func TestMain(m *testing.M) { 27 | home, err := os.UserHomeDir() 28 | if err != nil { 29 | panic("error get home path") 30 | } 31 | 32 | envCfg := envconf.NewWithKubeConfig(filepath.Join(home, ".kube", "config")). 33 | WithRandomNamespace() 34 | 35 | testenv = env.NewWithConfig(envCfg) 36 | 37 | _ = clientgoscheme.AddToScheme(scheme.Scheme) 38 | 39 | testenv.Setup( 40 | envfuncs.CreateNamespace(envCfg.Namespace()), 41 | func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { 42 | containerNetworkPods, err = utils.GetAvailableContainerNetworkPods(ctx, cfg.Client()) 43 | return ctx, err 44 | }, 45 | ) 46 | 47 | testenv.Finish( 48 | envfuncs.DeleteNamespace(envCfg.Namespace()), 49 | ) 50 | 51 | os.Exit(testenv.Run(m)) 52 | } 53 | -------------------------------------------------------------------------------- /types/aliyun.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // this keys is used in alibabacloud resource 4 | const ( 5 | TagKeyClusterID = "ack.aliyun.com" 6 | 7 | // NetworkInterfaceTagCreatorKey denotes the creator tag's key of network interface 8 | NetworkInterfaceTagCreatorKey = "creator" 9 | // NetworkInterfaceTagCreatorValue denotes the creator tag's value of network interface 10 | NetworkInterfaceTagCreatorValue = "terway" 11 | 12 | // TagTerwayController terway controller 13 | TagTerwayController = "terway-controller" 14 | 15 | TagENIAllocPolicy = "eni-alloc-policy" 16 | 17 | TagK8SNodeName = "node-name" 18 | 19 | TagKubernetesPodName = "k8s_pod_name" 20 | TagKubernetesPodNamespace = "k8s_pod_namespace" 21 | ) 22 | -------------------------------------------------------------------------------- /types/controlplane/annotations.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Terway Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controlplane 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | 23 | terwayTypes "github.com/AliyunContainerService/terway/types" 24 | "github.com/AliyunContainerService/terway/types/route" 25 | 26 | corev1 "k8s.io/api/core/v1" 27 | ) 28 | 29 | type PodNetworksAnnotation struct { 30 | PodNetworks []PodNetworks `json:"podNetworks"` 31 | } 32 | 33 | type PodNetworkRef struct { 34 | InterfaceName string `json:"interfaceName"` 35 | Network string `json:"network"` 36 | DefaultRoute bool `json:"defaultRoute,omitempty"` 37 | Routes []route.Route `json:"routes,omitempty"` 38 | } 39 | 40 | // ParsePodNetworksFromAnnotation parse annotation and convert to PodNetworksAnnotation 41 | func ParsePodNetworksFromAnnotation(pod *corev1.Pod) (*PodNetworksAnnotation, error) { 42 | v, ok := pod.GetAnnotations()[terwayTypes.PodNetworks] 43 | if !ok { 44 | return &PodNetworksAnnotation{}, nil 45 | } 46 | 47 | var annoConf PodNetworksAnnotation 48 | err := json.Unmarshal([]byte(v), &annoConf) 49 | if err != nil { 50 | return nil, fmt.Errorf("parse %s from pod annotataion, %w", terwayTypes.PodNetworks, err) 51 | } 52 | return &annoConf, nil 53 | } 54 | 55 | func ParsePodNetworksFromRequest(anno map[string]string) ([]PodNetworkRef, error) { 56 | v, ok := anno[terwayTypes.PodNetworksRequest] 57 | if !ok { 58 | return nil, nil 59 | } 60 | 61 | var annoConf []PodNetworkRef 62 | err := json.Unmarshal([]byte(v), &annoConf) 63 | if err != nil { 64 | return nil, fmt.Errorf("parse %s from pod annotataion, %w", terwayTypes.PodNetworksRequest, err) 65 | } 66 | return annoConf, nil 67 | } 68 | -------------------------------------------------------------------------------- /types/controlplane/annotations_default.go: -------------------------------------------------------------------------------- 1 | //go:build default_build 2 | 3 | /* 4 | Copyright 2021 Terway Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package controlplane 20 | 21 | import ( 22 | "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1" 23 | "github.com/AliyunContainerService/terway/types/route" 24 | ) 25 | 26 | type PodNetworks struct { 27 | VSwitchOptions []string `json:"vSwitchOptions"` 28 | SecurityGroupIDs []string `json:"securityGroupIDs"` 29 | Interface string `json:"interface"` 30 | ExtraRoutes []route.Route `json:"extraRoutes,omitempty"` 31 | ENIOptions v1beta1.ENIOptions `json:"eniOptions,omitempty"` 32 | VSwitchSelectOptions v1beta1.VSwitchSelectOptions `json:"vSwitchSelectOptions,omitempty"` 33 | ResourceGroupID string `json:"resourceGroupID"` 34 | NetworkInterfaceTrafficMode string `json:"networkInterfaceTrafficMode"` // "Standard"|"HighPerformance" 35 | DefaultRoute bool `json:"defaultRoute"` 36 | 37 | AllocationType *v1beta1.AllocationType `json:"allocationType"` 38 | } 39 | -------------------------------------------------------------------------------- /types/controlplane/annotations_test.go: -------------------------------------------------------------------------------- 1 | package controlplane 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | corev1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | ) 10 | 11 | func TestParsePodNetworksFromAnnotation(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | podAnnotations map[string]string 15 | expectedResult *PodNetworksAnnotation 16 | expectedError string 17 | }{ 18 | { 19 | name: "AnnotationDoesNotExist", 20 | podAnnotations: map[string]string{}, 21 | expectedResult: &PodNetworksAnnotation{}, 22 | expectedError: "", 23 | }, 24 | { 25 | name: "AnnotationExistsAndValid", 26 | podAnnotations: map[string]string{ 27 | "k8s.aliyun.com/pod-networks": `{ "podNetworks": [{"vSwitchOptions": [ 28 | "vsw-a","vsw-b","vsw-c" 29 | ], "interface": "eth0", 30 | "securityGroupIDs": [ 31 | "sg-1" 32 | ]}]}`, 33 | }, 34 | expectedResult: &PodNetworksAnnotation{ 35 | PodNetworks: []PodNetworks{ 36 | { 37 | Interface: "eth0", 38 | VSwitchOptions: []string{ 39 | "vsw-a", "vsw-b", "vsw-c", 40 | }, 41 | SecurityGroupIDs: []string{ 42 | "sg-1", 43 | }, 44 | }, 45 | }, 46 | }, 47 | expectedError: "", 48 | }, 49 | } 50 | 51 | for _, test := range tests { 52 | t.Run(test.name, func(t *testing.T) { 53 | pod := &corev1.Pod{ 54 | ObjectMeta: metav1.ObjectMeta{ 55 | Annotations: test.podAnnotations, 56 | }, 57 | } 58 | 59 | result, err := ParsePodNetworksFromAnnotation(pod) 60 | if test.expectedError != "" { 61 | assert.Error(t, err) 62 | assert.Contains(t, err.Error(), test.expectedError) 63 | } else { 64 | assert.NoError(t, err) 65 | assert.Equal(t, test.expectedResult, result) 66 | } 67 | }) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /types/daemon/cni.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | type CNI struct { 4 | PodName string 5 | PodNamespace string 6 | PodID string 7 | PodUID string 8 | NetNSPath string 9 | } 10 | -------------------------------------------------------------------------------- /types/daemon/dynamicconfig_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | corev1 "k8s.io/api/core/v1" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "sigs.k8s.io/controller-runtime/pkg/client/fake" 11 | ) 12 | 13 | func TestConfigFromConfigMapReturnsErrorWhenBaseConfigMapNotFound(t *testing.T) { 14 | client := fake.NewFakeClient() 15 | _, err := ConfigFromConfigMap(context.Background(), client, "") 16 | assert.Error(t, err) 17 | } 18 | 19 | func TestConfigFromConfigMapReturnsErrorWhenBaseConfigIsEmpty(t *testing.T) { 20 | client := fake.NewFakeClient(&corev1.ConfigMap{ 21 | ObjectMeta: metav1.ObjectMeta{ 22 | Name: "eni-config", 23 | Namespace: "kube-system", 24 | }, 25 | Data: map[string]string{"eni_conf": ""}, 26 | }) 27 | _, err := ConfigFromConfigMap(context.Background(), client, "") 28 | assert.Error(t, err) 29 | } 30 | 31 | func TestConfigFromConfigMapReturnsConfigWhenNodeNameIsNotEmpty(t *testing.T) { 32 | client := fake.NewFakeClient( 33 | &corev1.ConfigMap{ 34 | ObjectMeta: metav1.ObjectMeta{ 35 | Name: "eni-config", 36 | Namespace: "kube-system", 37 | }, 38 | Data: map[string]string{"eni_conf": `{"version": "1"}`}, 39 | }, 40 | ) 41 | cfg, err := ConfigFromConfigMap(context.Background(), client, "node-1") 42 | assert.NoError(t, err) 43 | assert.Equal(t, "1", cfg.Version) 44 | } 45 | 46 | func TestConfigFromConfigMapReturnsErrorWhenSecurityGroupsExceedLimit(t *testing.T) { 47 | client := fake.NewFakeClient(&corev1.ConfigMap{ 48 | ObjectMeta: metav1.ObjectMeta{ 49 | Name: "eni-config", 50 | Namespace: "kube-system", 51 | }, 52 | Data: map[string]string{"eni_conf": `{"security_groups": ["sg-1", "sg-2", "sg-3", "sg-4", "sg-5", "sg-6","sg-7","sg-8","sg-9","sg-10","sg-11"]}`}, 53 | }) 54 | _, err := ConfigFromConfigMap(context.Background(), client, "") 55 | assert.Error(t, err) 56 | } 57 | -------------------------------------------------------------------------------- /types/daemon/res_test.go: -------------------------------------------------------------------------------- 1 | package daemon 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | // TestGetResourceItemByType tests the GetResourceItemByType method of PodResources. 9 | func TestGetResourceItemByType(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | resType string 13 | res []ResourceItem 14 | expected []ResourceItem 15 | }{ 16 | { 17 | name: "MatchingType", 18 | resType: "network", 19 | res: []ResourceItem{ 20 | {Type: "network", ID: "1", ENIID: "eni-1", ENIMAC: "02:12:34:56:78:90", IPv4: "10.0.0.1", IPv6: "2001:0db8:85a3:0000:0000:8a2e:0700:7344"}, 21 | {Type: "storage", ID: "2", ENIID: "eni-2", ENIMAC: "02:12:34:56:78:91", IPv4: "10.0.0.2", IPv6: "2001:0db8:85a3:0000:0000:8a2e:0700:7345"}, 22 | }, 23 | expected: []ResourceItem{ 24 | {Type: "network", ID: "1", ENIID: "eni-1", ENIMAC: "02:12:34:56:78:90", IPv4: "10.0.0.1", IPv6: "2001:0db8:85a3:0000:0000:8a2e:0700:7344"}, 25 | }, 26 | }, 27 | { 28 | name: "MultipleMatchingTypes", 29 | resType: "network", 30 | res: []ResourceItem{ 31 | {Type: "network", ID: "1", ENIID: "eni-1", ENIMAC: "02:12:34:56:78:90", IPv4: "10.0.0.1", IPv6: "2001:0db8:85a3:0000:0000:8a2e:0700:7344"}, 32 | {Type: "network", ID: "2", ENIID: "eni-2", ENIMAC: "02:12:34:56:78:91", IPv4: "10.0.0.2", IPv6: "2001:0db8:85a3:0000:0000:8a2e:0700:7345"}, 33 | }, 34 | expected: []ResourceItem{ 35 | {Type: "network", ID: "1", ENIID: "eni-1", ENIMAC: "02:12:34:56:78:90", IPv4: "10.0.0.1", IPv6: "2001:0db8:85a3:0000:0000:8a2e:0700:7344"}, 36 | {Type: "network", ID: "2", ENIID: "eni-2", ENIMAC: "02:12:34:56:78:91", IPv4: "10.0.0.2", IPv6: "2001:0db8:85a3:0000:0000:8a2e:0700:7345"}, 37 | }, 38 | }, 39 | } 40 | 41 | for _, test := range tests { 42 | t.Run(test.name, func(t *testing.T) { 43 | podResources := PodResources{Resources: test.res} 44 | result := podResources.GetResourceItemByType(test.resType) 45 | if !reflect.DeepEqual(result, test.expected) { 46 | t.Errorf("GetResourceItemByType(%s) = %v, want %v", test.resType, result, test.expected) 47 | } 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /types/errors.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | const ( 8 | ErrInternalError ErrCode = "InternalError" 9 | ErrInvalidArgsErrCode ErrCode = "InvalidArgs" 10 | ErrInvalidDataType ErrCode = "InvalidDataType" 11 | 12 | ErrPodIsProcessing ErrCode = "PodIsProcessing" 13 | 14 | ErrResourceInvalid ErrCode = "ResourceInvalid" 15 | 16 | ErrOpenAPIErr ErrCode = "OpenAPIErr" 17 | ErrPodENINotReady ErrCode = "PodENINotReady" 18 | ErrIPNotAllocated ErrCode = "IPNotAllocated" 19 | 20 | ErrIPOutOfSyncErr ErrCode = "OutIPOfSync" 21 | ) 22 | 23 | type ErrCode string 24 | 25 | type Error struct { 26 | Code ErrCode 27 | Msg string 28 | 29 | R error 30 | } 31 | 32 | func (e *Error) Error() string { 33 | return fmt.Sprintf("code: %s, msg: %s", e.Code, e.Msg) 34 | } 35 | 36 | func (e *Error) Unwrap() error { 37 | return e.R 38 | } 39 | -------------------------------------------------------------------------------- /types/help_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/AliyunContainerService/terway/rpc" 9 | ) 10 | 11 | func TestBuildIPNetReturnsEmptyWhenIPAndSubnetAreNil(t *testing.T) { 12 | ipnet, err := BuildIPNet(nil, nil) 13 | assert.NoError(t, err) 14 | assert.NotNil(t, ipnet) 15 | assert.Nil(t, ipnet.IPv4) 16 | assert.Nil(t, ipnet.IPv6) 17 | } 18 | 19 | func TestBuildIPNetReturnsErrorWhenIPv4IsInvalid(t *testing.T) { 20 | ip := &rpc.IPSet{IPv4: "invalid-ip"} 21 | subnet := &rpc.IPSet{IPv4: "192.168.0.0/24"} 22 | _, err := BuildIPNet(ip, subnet) 23 | assert.Error(t, err) 24 | } 25 | 26 | func TestBuildIPNetReturnsErrorWhenIPv6IsInvalid(t *testing.T) { 27 | ip := &rpc.IPSet{IPv6: "invalid-ip"} 28 | subnet := &rpc.IPSet{IPv6: "fd00::/64"} 29 | _, err := BuildIPNet(ip, subnet) 30 | assert.Error(t, err) 31 | } 32 | 33 | func TestBuildIPNetReturnsErrorWhenSubnetIPv4IsInvalid(t *testing.T) { 34 | ip := &rpc.IPSet{IPv4: "192.168.0.1"} 35 | subnet := &rpc.IPSet{IPv4: "invalid-subnet"} 36 | _, err := BuildIPNet(ip, subnet) 37 | assert.Error(t, err) 38 | } 39 | 40 | func TestBuildIPNetReturnsErrorWhenSubnetIPv6IsInvalid(t *testing.T) { 41 | ip := &rpc.IPSet{IPv6: "fd00::1"} 42 | subnet := &rpc.IPSet{IPv6: "invalid-subnet"} 43 | _, err := BuildIPNet(ip, subnet) 44 | assert.Error(t, err) 45 | } 46 | 47 | func TestBuildIPNetReturnsCorrectIPNetForValidIPv4(t *testing.T) { 48 | ip := &rpc.IPSet{IPv4: "192.168.0.1"} 49 | subnet := &rpc.IPSet{IPv4: "192.168.0.0/24"} 50 | ipnet, err := BuildIPNet(ip, subnet) 51 | assert.NoError(t, err) 52 | assert.NotNil(t, ipnet.IPv4) 53 | assert.Equal(t, "192.168.0.1/24", ipnet.IPv4.String()) 54 | assert.Nil(t, ipnet.IPv6) 55 | } 56 | 57 | func TestBuildIPNetReturnsCorrectIPNetForValidIPv6(t *testing.T) { 58 | ip := &rpc.IPSet{IPv6: "fd00::1"} 59 | subnet := &rpc.IPSet{IPv6: "fd00::/64"} 60 | ipnet, err := BuildIPNet(ip, subnet) 61 | assert.NoError(t, err) 62 | assert.NotNil(t, ipnet.IPv6) 63 | assert.Equal(t, "fd00::1/64", ipnet.IPv6.String()) 64 | assert.Nil(t, ipnet.IPv4) 65 | } 66 | -------------------------------------------------------------------------------- /types/helper.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | 7 | terwayIP "github.com/AliyunContainerService/terway/pkg/ip" 8 | "github.com/AliyunContainerService/terway/rpc" 9 | ) 10 | 11 | func BuildIPNet(ip, subnet *rpc.IPSet) (*IPNetSet, error) { 12 | ipnet := &IPNetSet{} 13 | if ip == nil || subnet == nil { 14 | return ipnet, nil 15 | } 16 | exec := func(ip, subnet string) (*net.IPNet, error) { 17 | i, err := terwayIP.ToIP(ip) 18 | if err != nil { 19 | return nil, err 20 | } 21 | _, sub, err := net.ParseCIDR(subnet) 22 | if err != nil { 23 | return nil, err 24 | } 25 | sub.IP = i 26 | return sub, nil 27 | } 28 | var err error 29 | if ip.IPv4 != "" && subnet.IPv4 != "" { 30 | ipnet.IPv4, err = exec(ip.IPv4, subnet.IPv4) 31 | if err != nil { 32 | return nil, err 33 | } 34 | } 35 | if ip.IPv6 != "" && subnet.IPv6 != "" { 36 | ipnet.IPv6, err = exec(ip.IPv6, subnet.IPv6) 37 | if err != nil { 38 | return nil, err 39 | } 40 | } 41 | return ipnet, nil 42 | } 43 | 44 | func ToIPSet(ip *rpc.IPSet) (*IPSet, error) { 45 | if ip == nil { 46 | return nil, fmt.Errorf("ip is nil") 47 | } 48 | ipSet := &IPSet{} 49 | var err error 50 | if ip.IPv4 != "" { 51 | ipSet.IPv4, err = terwayIP.ToIP(ip.IPv4) 52 | if err != nil { 53 | return nil, err 54 | } 55 | } 56 | if ip.IPv6 != "" { 57 | ipSet.IPv6, err = terwayIP.ToIP(ip.IPv6) 58 | if err != nil { 59 | return nil, err 60 | } 61 | } 62 | return ipSet, nil 63 | } 64 | 65 | func ToIPNetSet(ip *rpc.IPSet) (*IPNetSet, error) { 66 | if ip == nil { 67 | return nil, fmt.Errorf("ip is nil") 68 | } 69 | ipNetSet := &IPNetSet{} 70 | var err error 71 | if ip.IPv4 != "" { 72 | _, ipNetSet.IPv4, err = net.ParseCIDR(ip.IPv4) 73 | if err != nil { 74 | return nil, err 75 | } 76 | } 77 | if ip.IPv6 != "" { 78 | _, ipNetSet.IPv6, err = net.ParseCIDR(ip.IPv6) 79 | if err != nil { 80 | return nil, err 81 | } 82 | } 83 | return ipNetSet, nil 84 | } 85 | -------------------------------------------------------------------------------- /types/k8s_test.go: -------------------------------------------------------------------------------- 1 | package types_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/AliyunContainerService/terway/types" 9 | ) 10 | 11 | func TestNodeExclusiveENIMode(t *testing.T) { 12 | t.Run("Returns default when label is missing", func(t *testing.T) { 13 | labels := map[string]string{} 14 | result := types.NodeExclusiveENIMode(labels) 15 | assert.Equal(t, types.ExclusiveDefault, result) 16 | }) 17 | 18 | t.Run("Returns default when label is empty", func(t *testing.T) { 19 | labels := map[string]string{ 20 | types.ExclusiveENIModeLabel: "", 21 | } 22 | result := types.NodeExclusiveENIMode(labels) 23 | assert.Equal(t, types.ExclusiveDefault, result) 24 | }) 25 | 26 | t.Run("Returns ENI only when label is set", func(t *testing.T) { 27 | labels := map[string]string{ 28 | types.ExclusiveENIModeLabel: "eniOnly", 29 | } 30 | result := types.NodeExclusiveENIMode(labels) 31 | assert.Equal(t, types.ExclusiveENIOnly, result) 32 | }) 33 | 34 | t.Run("Is case insensitive", func(t *testing.T) { 35 | labels := map[string]string{ 36 | types.ExclusiveENIModeLabel: "ENIONLY", 37 | } 38 | result := types.NodeExclusiveENIMode(labels) 39 | assert.Equal(t, types.ExclusiveENIOnly, result) 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /types/route/route.go: -------------------------------------------------------------------------------- 1 | package route 2 | 3 | type Route struct { 4 | Dst string `json:"dst"` 5 | } 6 | -------------------------------------------------------------------------------- /types/secret/secret.go: -------------------------------------------------------------------------------- 1 | package secret 2 | 3 | type Secret string 4 | 5 | func (s Secret) String() string { 6 | return "******" 7 | } 8 | 9 | func (s Secret) GoString() string { 10 | return "******" 11 | } 12 | 13 | func (s Secret) MarshalJSON() ([]byte, error) { 14 | return []byte(`"******"`), nil 15 | } 16 | -------------------------------------------------------------------------------- /types/secret/secret_test.go: -------------------------------------------------------------------------------- 1 | package secret 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestStringReturnsMaskedValue(t *testing.T) { 10 | s := Secret("mysecret") 11 | assert.Equal(t, "******", s.String()) 12 | assert.Equal(t, "mysecret", string((s))) 13 | } 14 | 15 | func TestGoStringReturnsMaskedValue(t *testing.T) { 16 | s := Secret("mysecret") 17 | assert.Equal(t, "******", s.GoString()) 18 | } 19 | 20 | func TestMarshalJSONReturnsMaskedValue(t *testing.T) { 21 | s := Secret("mysecret") 22 | json, err := s.MarshalJSON() 23 | assert.NoError(t, err) 24 | assert.Equal(t, `"******"`, string(json)) 25 | } 26 | 27 | func TestMarshalJSONHandlesEmptySecret(t *testing.T) { 28 | s := Secret("") 29 | json, err := s.MarshalJSON() 30 | assert.NoError(t, err) 31 | assert.Equal(t, `"******"`, string(json)) 32 | } 33 | --------------------------------------------------------------------------------