├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── codeql-analysis.yml │ ├── main.yml │ ├── stale.yml │ ├── sync2gitee.yml │ └── unittest.yaml ├── .gitignore ├── CHANGELOG ├── CHANGELOG-0.2.md ├── CHANGELOG-0.3.md ├── CHANGELOG-0.4.0.md ├── CHANGELOG-0.5.0.md ├── CHANGELOG-0.6.0.md ├── CHANGELOG-0.7.0.md ├── CHANGELOG-0.8.0.md ├── CHANGELOG-0.8.1.md └── CHANGELOG-1.0.0.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MAINTAINERS.md ├── Makefile ├── README.md ├── README_zh.md ├── build ├── agent │ ├── Dockerfile │ └── entrypoint.sh ├── base-image │ ├── Dockerfile │ └── README.md ├── cert │ └── Dockerfile ├── cloud-agent │ ├── Dockerfile │ └── entrypoint.sh ├── connector │ ├── Dockerfile │ ├── check-connector-leader.sh │ └── entrypoint.sh ├── node │ ├── Dockerfile │ ├── check-connector-leader.sh │ └── entrypoint.sh ├── operator │ └── Dockerfile ├── strongswan │ └── Dockerfile └── utils │ └── no_proxy.sh ├── cmd ├── agent │ └── main.go ├── cloud-agent │ └── main.go ├── connector │ └── main.go ├── node │ └── main.go └── operator │ └── main.go ├── coredns └── plugin │ └── kubernetes │ ├── README.md │ ├── autopath.go │ ├── controller.go │ ├── controller_test.go │ ├── external.go │ ├── external_test.go │ ├── handler.go │ ├── handler_case_test.go │ ├── handler_ignore_emptyservice_test.go │ ├── handler_pod_disabled_test.go │ ├── handler_pod_insecure_test.go │ ├── handler_pod_verified_test.go │ ├── handler_test.go │ ├── informer_test.go │ ├── kubernetes.go │ ├── kubernetes_apex_test.go │ ├── kubernetes_test.go │ ├── local.go │ ├── log_test.go │ ├── metadata.go │ ├── metadata_test.go │ ├── metrics.go │ ├── metrics_test.go │ ├── namespace.go │ ├── namespace_test.go │ ├── ns.go │ ├── ns_test.go │ ├── object │ ├── endpoint.go │ ├── informer.go │ ├── object.go │ ├── pod.go │ └── service.go │ ├── parse.go │ ├── parse_test.go │ ├── ready.go │ ├── reverse.go │ ├── reverse_test.go │ ├── setup.go │ ├── setup_reverse_test.go │ ├── setup_test.go │ ├── setup_ttl_test.go │ ├── watch.go │ ├── xfr.go │ └── xfr_test.go ├── deploy ├── calico │ └── ip-pool.yaml ├── cloud-agent.yaml ├── connector.yaml ├── crds │ ├── fabedge.io_clusters.yaml │ └── fabedge.io_communities.yaml ├── operator-svc.yaml ├── operator.yaml └── rbac.yaml ├── docs ├── FAQ.md ├── FAQ_zh.md ├── adopters │ ├── bocloud.png │ ├── ictnj.svg │ └── linklogis.png ├── deploy-ha.md ├── deploy-ha_zh.md ├── design │ ├── diagrams │ │ ├── fabedge-arch.gliffy │ │ ├── multi-cluster-commuication.gliffy │ │ └── network-topology.gliffy │ ├── fab-dns-show3.png │ ├── fabedge-design.md │ ├── images │ │ ├── fabedge-arch.png │ │ ├── multi-cluster-commuication.png │ │ └── network-topology.png │ ├── multi-cluster-communication.md │ └── 跨集群服务访问介绍.md ├── get-started.md ├── get-started_zh.md ├── images │ ├── FabEdge-Arch.gliffy │ ├── FabEdge-Arch.png │ └── wechat-group-qr-code.jpg ├── integration │ ├── integrate-with-k3s.md │ └── loadbalancer │ │ ├── fabedge-with-lb.gliffy │ │ ├── fabedge-with-lb.png │ │ └── use-nginx-to-proxy-connector.md ├── manually-install.md ├── manually-install_zh.md ├── obsoleted-doc │ ├── get-started-v0.5.0.md │ ├── get-started-v0.5.0_zh.md │ ├── get-started-v0.6.0.md │ ├── get-started-v0.6.0_zh.md │ ├── get-started-v0.7.0.md │ ├── get-started-v0.7.0_zh.md │ └── install-k8s-and-kubeedge.md ├── roadmap.md ├── troubleshooting-guide.md ├── troubleshooting-guide_zh.md ├── uninstall.md ├── uninstall_zh.md ├── user-guide.md └── user-guide_zh.md ├── examples ├── communities.yaml ├── mysql.yaml └── nginx.yaml ├── go.mod ├── go.sum ├── pkg ├── agent │ ├── broadcast.go │ ├── cmd.go │ ├── config.go │ ├── edgedns.go │ ├── endpoint.go │ ├── iptables.go │ ├── manager.go │ ├── proxy.go │ ├── route.go │ ├── types.go │ └── utils.go ├── apis │ └── v1alpha1 │ │ ├── cluster_types.go │ │ ├── community_types.go │ │ ├── doc.go │ │ ├── register.go │ │ └── zz_generated.deepcopy.go ├── cloud-agent │ ├── cloud_agent.go │ └── iptables.go ├── common │ ├── about │ │ └── about.go │ ├── constants │ │ └── default.go │ └── netconf │ │ ├── ipvs.go │ │ └── tunnels.go ├── connector │ ├── cmd.go │ ├── iptables.go │ ├── manager.go │ ├── routing │ │ └── routing.go │ ├── tunnel.go │ └── watcher.go ├── operator │ ├── allocator │ │ ├── allocator.go │ │ ├── allocator_suite_test.go │ │ └── allocator_test.go │ ├── apiserver │ │ ├── apiserver.go │ │ ├── apiserver_suite_test.go │ │ └── apiserver_test.go │ ├── client │ │ ├── client.go │ │ ├── client_test.go │ │ └── errors.go │ ├── cmd.go │ ├── controllers │ │ ├── agent │ │ │ ├── agentcontroller_suite_test.go │ │ │ ├── agentpodhandler.go │ │ │ ├── agentpodhandler_test.go │ │ │ ├── certhandler.go │ │ │ ├── certhandler_test.go │ │ │ ├── confighandler.go │ │ │ ├── confighandler_test.go │ │ │ ├── controller.go │ │ │ ├── controller_test.go │ │ │ ├── podcidrshandler.go │ │ │ └── podcidrshandler_test.go │ │ ├── cluster │ │ │ ├── cluster_suite_test.go │ │ │ ├── controller.go │ │ │ └── controller_test.go │ │ ├── community │ │ │ ├── community_suite_test.go │ │ │ ├── controller.go │ │ │ └── controller_test.go │ │ ├── connector │ │ │ ├── connector_suite_test.go │ │ │ ├── controller.go │ │ │ └── controller_test.go │ │ └── ipamblockmonitor │ │ │ ├── ipam_block_monitor.go │ │ │ ├── ipam_block_monitor_test.go │ │ │ └── ipamblockmonitor_suite_test.go │ ├── options.go │ ├── routines │ │ ├── endpoints.go │ │ ├── endpoints_test.go │ │ ├── ippool_keeper.go │ │ ├── ippool_keeper_test.go │ │ ├── local_cluster_reporter.go │ │ ├── local_cluster_reporter_test.go │ │ ├── periodic_runnable.go │ │ ├── periodic_runnable_test.go │ │ └── routines_suite_test.go │ ├── store │ │ ├── store.go │ │ ├── store_suite_test.go │ │ └── store_test.go │ └── types │ │ ├── agent_argument_map.go │ │ ├── agent_argument_map_test.go │ │ ├── cluster_cidrs_map.go │ │ ├── cluster_cidrs_map_test.go │ │ ├── community.go │ │ ├── funcs.go │ │ ├── funcs_test.go │ │ ├── podcidrstore.go │ │ ├── podcidrstore_test.go │ │ ├── safe_string_set.go │ │ ├── safe_string_set_test.go │ │ └── types_suite_test.go ├── tunnel │ ├── manager.go │ └── strongswan │ │ ├── options.go │ │ └── strongswan.go └── util │ ├── cert │ ├── cert_suite_test.go │ ├── certutil.go │ ├── certutil_test.go │ ├── manager.go │ ├── manager_test.go │ ├── remote_manager.go │ └── remote_manager_test.go │ ├── ginkgoext │ └── ginkgo_extension.go │ ├── ipset │ └── ipset.go │ ├── iptables │ └── iptables.go │ ├── log │ └── flags.go │ ├── memberlist │ └── memberlist.go │ ├── net │ └── net.go │ ├── node │ ├── nodeutil.go │ └── nodeutil_test.go │ ├── route │ └── route.go │ ├── secret │ ├── builder.go │ └── secretutil.go │ ├── test │ ├── purge.go │ └── test.go │ └── time │ └── time.go ├── test └── e2e │ ├── cluster.go │ ├── e2e.go │ ├── e2e_test.go │ ├── framework │ ├── README.md │ ├── cleanup.go │ ├── expect.go │ ├── ginkgowrapper │ │ └── wrapper.go │ ├── log.go │ ├── report.go │ ├── test_context.go │ └── util.go │ ├── pod.go │ └── service.go └── third_party ├── calicoapi ├── constants.go ├── crd │ ├── crd.projectcalico.org_ipamblocks.yaml │ └── crd.projectcalico.org_ippools.yaml ├── ipam_block.go ├── ippool.go ├── register.go └── zz_generated.deepcopy.go ├── ipset ├── LICENSE ├── README.md ├── ipset.go ├── ipset_test.go ├── testing │ ├── fake.go │ └── fake_test.go └── types.go ├── ipvs ├── LICENSE ├── README.md ├── ipvs.go ├── ipvs_linux.go ├── netlink.go ├── netlink_linux.go └── proxier.go └── sysctl └── sysctl.go /.dockerignore: -------------------------------------------------------------------------------- 1 | _output -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '**.go' 7 | - 'go.mod' 8 | - 'go.sum' 9 | 10 | jobs: 11 | analyze: 12 | name: Analyze 13 | runs-on: ubuntu-latest 14 | permissions: 15 | actions: read 16 | contents: read 17 | security-events: write 18 | 19 | steps: 20 | - name: Checkout repository 21 | uses: actions/checkout@v4 22 | 23 | - name: Initialize CodeQL 24 | uses: github/codeql-action/init@v2 25 | with: 26 | languages: go 27 | 28 | - name: Autobuild 29 | uses: github/codeql-action/autobuild@v2 30 | 31 | - name: Perform CodeQL Analysis 32 | uses: github/codeql-action/analyze@v2 -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | branches: 8 | - 'v*' 9 | paths-ignore: 10 | - 'docs/**' 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | include: 18 | - platform: linux/amd64,linux/arm64,linux/arm/v7 19 | target: agent-image 20 | 21 | - platform: linux/amd64,linux/arm64,linux/arm/v7 22 | target: connector-image 23 | 24 | - platform: linux/amd64,linux/arm64,linux/arm/v7 25 | target: operator-image 26 | 27 | - platform: linux/amd64,linux/arm64,linux/arm/v7 28 | target: cloud-agent-image 29 | 30 | steps: 31 | - uses: actions/checkout@v4 32 | with: 33 | fetch-depth: 0 34 | 35 | - name: Remove proxies 36 | run: | 37 | bash build/utils/no_proxy.sh 38 | 39 | - name: Set up QEMU 40 | uses: docker/setup-qemu-action@v3 41 | 42 | - name: Set up Docker Buildx 43 | uses: docker/setup-buildx-action@v3 44 | 45 | - name: Login to DockerHub 46 | uses: docker/login-action@v3 47 | with: 48 | username: ${{ secrets.DOCKER_USERNAME }} 49 | password: ${{ secrets.DOCKER_PASSWORD }} 50 | 51 | - name: Build and push 52 | run: | 53 | make ${target} PLATFORM=${platform} PUSH=true 54 | env: 55 | platform: ${{ matrix.platform }} 56 | target: ${{ matrix.target }} 57 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | # This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. 2 | # 3 | # You can adjust the behavior by modifying this file. 4 | # For more information, see: 5 | # https://github.com/actions/stale 6 | name: stale 7 | 8 | on: 9 | schedule: 10 | - cron: '0 1 * * *' 11 | 12 | jobs: 13 | stale: 14 | 15 | runs-on: ubuntu-latest 16 | permissions: 17 | issues: write 18 | pull-requests: write 19 | 20 | steps: 21 | - uses: actions/stale@v8.0.0 22 | with: 23 | repo-token: ${{ secrets.GITHUB_TOKEN }} 24 | stale-issue-message: 'Stale issue message' 25 | stale-pr-message: 'Stale pull request message' 26 | stale-issue-label: 'no-issue-activity' 27 | stale-pr-label: 'no-pr-activity' 28 | -------------------------------------------------------------------------------- /.github/workflows/sync2gitee.yml: -------------------------------------------------------------------------------- 1 | name: sync2gitee 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | branches: 8 | - 'main' 9 | 10 | jobs: 11 | repo-sync: 12 | env: 13 | dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} 14 | dst_token: ${{ secrets.GITEE_TOKEN }} 15 | gitee_user: ${{ secrets.GITEE_USER }} 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | with: 20 | persist-credentials: false 21 | 22 | - name: sync github -> gitee 23 | uses: Yikun/hub-mirror-action@master 24 | if: env.dst_key && env.dst_token && env.gitee_user 25 | with: 26 | src: 'github/${{ github.repository_owner }}' 27 | dst: 'gitee/${{ secrets.GITEE_USER }}' 28 | dst_key: ${{ secrets.GITEE_PRIVATE_KEY }} 29 | dst_token: ${{ secrets.GITEE_TOKEN }} 30 | account_type: org 31 | static_list: ${{ github.event.repository.name }} 32 | -------------------------------------------------------------------------------- /.github/workflows/unittest.yaml: -------------------------------------------------------------------------------- 1 | name: "Unit Test" 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '**.go' 7 | - 'go.mod' 8 | - 'go.sum' 9 | 10 | jobs: 11 | unittest: 12 | name: Unit Test 13 | runs-on: ubuntu-latest 14 | permissions: 15 | actions: read 16 | contents: read 17 | 18 | steps: 19 | - name: Checkout Repository 20 | uses: actions/checkout@v4 21 | 22 | - name: Setup Go 23 | uses: actions/setup-go@v4.1.0 24 | with: 25 | go-version: 1.17.13 26 | 27 | - name: Go vet 28 | run: | 29 | export GOPATH=~/go 30 | export PATH=$PATH:$GOPATH/bin 31 | go install github.com/onsi/ginkgo/ginkgo@v1.16.4 32 | make install-test-tools fmt vet test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .DS_Store 3 | _output/ 4 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-0.2.md: -------------------------------------------------------------------------------- 1 | # FabEdge V0.2 2 | 3 | 4 | 5 | ## 新特性 6 | 7 | 1. 一键部署K8S+KubeEdge 8 | 9 | FabEdge是一个边缘容器的网络方案，使用它的前提是有K8S+KubeEdge集群。但是K8S+KubeEdge的部署比较复杂，导致使用FabEdge的门槛过高。我们推出一键部署K8S+KubeEdge的功能，方便用户快速上手。 10 | 11 | 1. 自动管理证书 12 | 13 | Strongswan是一个开源的IPSec VPN管理软件，FabEdge底层使用它管理隧道。为了安全性，它使用证书验证边缘节点。但为每一个边缘节点分配证书是一个麻烦且容易出错的过程。我们在Operator里实现了证书的自动管理，在节点上线的时候自动分配证书，大大降低了运维工作量。 14 | 15 | 1. 使用Helm安装部署 16 | 17 | FabEdge有多个组件，组件的配置比较复杂。我们使用Helm （package manager for kubernetes）管理FabEdge，简化了安装部署过程，方便用户使用。 18 | 19 | 20 | 21 | ## 其它更新 22 | 23 | 1. 支持IPSec NAT-T 24 | 25 | 可以为云端connector设置外网地址，public_addresses, 支持公有云使用浮动IP或私有云使用防火墙地址映射的场景。 26 | 27 | 1. 完善了connector的iptables规则 28 | 29 | connector自动配置iptables规则，允许IPSec流量（ESP， UDP50/4500）。 30 | 31 | 1. 增加enable-proxy的开关 32 | 33 | 对于在边缘节点上使用原生kube-proxy的场景，可以选择关闭FabEdge自有proxy的实现。 34 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-0.3.md: -------------------------------------------------------------------------------- 1 | # FabEdge V0.3 2 | 3 | ## 新特性 4 | 5 | 1. 支持云端集群使用Flannel网络插件 6 | 7 | [Flannel](https://github.com/flannel-io/flannel)简单，易用，有大量的用户，本版本加入了对它的支持。到目前为止，FabEdge支持的插件有：Calico， Flannel。 8 | 9 | 1. 支持SuperEdge 10 | 11 | [SuperEdge](https://github.com/superedge/superedge/blob/main/README_CN.md)是Kubernetes原生的边缘容器方案，它将Kubernetes强大的容器管理能力扩展到边缘计算场景中，针对边缘计算场景中常见的技术挑战提供了解决方案。FabEdge本版本加入对SuperEdge的支持。 12 | 13 | 1. 支持OpenYurt 14 | 15 | [OpenYurt](https://openyurt.io/)是托管在 Cloud Native Computing Foundation (CNCF) 下的 [沙箱项目](https://www.cncf.io/sandbox-projects/). 它是基于原生 Kubernetes 构建的，目标是扩展 Kubernetes 以无缝支持边缘计算场景。FabEdge本版本加入OpenYurt的支持。 16 | 17 | ## 其它更新 18 | 19 | 1. 自动识别云端POD网段 20 | 21 | Operator自动识别云端集群POD网段，不再需要用户手动输入。 22 | 23 | 1. 支持用户自定义边缘节点标签 24 | 25 | 用户可以自定义用于标识FabEdge管理的边缘节点的标签组。 26 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-0.4.0.md: -------------------------------------------------------------------------------- 1 | # FabEdge V0.4 2 | 3 | [toc] 4 | 5 | ## 新特性 6 | 7 | 1. 支持多集群通讯 8 | 9 | 支持跨集群的，Pod/Service的直接访问。 10 | 11 | 1. 支持ARM架构 12 | 13 | 支持ARM32/64架构，而且支持异构环境，即一个环境里包含多种硬件架构（X86/ARM32/ARM64） 14 | 15 | 16 | ## 其它更新 17 | 18 | 1. 支持用户手动指定边缘节点公网地址 19 | 20 | 用户可以为边缘节点添加公网地址的注解，用于建立边缘节点到边缘节点的隧道 21 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-0.5.0.md: -------------------------------------------------------------------------------- 1 | # FabEdge V0.5.0 2 | 3 | [toc] 4 | 5 | ## 新特性 6 | 7 | 1. 支持多集群服务发现 8 | 9 | 支持对应用无侵入的，拓扑感知的跨集群服务访问。 10 | 11 | 12 | ## 其它更新 13 | 14 | 1. 修复了一些bug 15 | 2. 优化了配置比对的逻辑 16 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-0.6.0.md: -------------------------------------------------------------------------------- 1 | # FabEdge V0.6.0 2 | 3 | [toc] 4 | 5 | ## 新特性 6 | 7 | 1. 支持同一局域网内的边缘节点自动组网，无需Community配置 8 | 2. 支持双栈网络（仅限于flannel) 9 | 3. 更灵活的agent参数配置 10 | 11 | 12 | 13 | ## 其它更新 14 | 15 | 1. 修复了一些bug 16 | 2. 改善了对strongswan不活跃链接的清理和重建能力 17 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-0.7.0.md: -------------------------------------------------------------------------------- 1 | # FabEdge V0.7.0 2 | 3 | [toc] 4 | 5 | ## New features 6 | 7 | 1. Change the naming strategy of fabedge-agent pods; 8 | 2. Add commonName validation for fabedge-agent certificates; 9 | 3. Implement node-specific configuration of fabedge-agent arguments; 10 | 4. Let fabedge-agent configure sysctl parameters needed; 11 | 5. Let fabedge-operator manage calico ippools for CIDRs; 12 | 13 | 14 | 15 | ## Bug fixes 16 | 17 | 1. Fix wrong service port mapping of fab-proxy; 18 | 19 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-0.8.0.md: -------------------------------------------------------------------------------- 1 | # FabEdge V0.8.0 2 | 3 | [toc] 4 | 5 | ## New features 6 | 7 | 1. Integerate coredns and kube-proxy into fabedge-agent and remove fab-proxy component; 8 | 9 | 2. Allow user to set strongswan port on connector; 10 | 11 | 3. Implement hole-punching feature which help edge nodes behind NAT network to communicate each other ; 12 | 13 | 14 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-0.8.1.md: -------------------------------------------------------------------------------- 1 | # FabEdge V0.8.1 2 | 3 | 1. Fix infinitely generated route table 220 mentioned in [issue #386](https://github.com/FabEdge/fabedge/issues/386); 4 | 5 | 2. Use iptables-wrapper in images of fabedge-agent, fabedge-connector and fabedge-cloud-agent; 6 | 7 | 3. Improve startup process of fabedge-cloud-agent. 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /CHANGELOG/CHANGELOG-1.0.0.md: -------------------------------------------------------------------------------- 1 | # FabEdge V1.0.0 2 | 3 | Added: 4 | 5 | 1. Connector HA is implemented; 6 | 2. More calico modes are supported; 7 | 3. Flannel host-gw mode is supported; 8 | 9 | Fixed: 10 | 11 | 1. Fix the bug that nodePort service doesn't work on cloud side; 12 | 2. Fix the bug that cloud-agent lost connections to connector after connector reboot; 13 | 3. Fix the bug that fabedge-agent can't initialize tunnels if strongswan container reboot; -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # FabEdge Community Code of Conduct 2 | 3 | We follow the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 4 | 5 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [fabedge@beyondcent.com](mailto:fabedge@beyondcent.com). 6 | 7 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # FabEdge Maintainers 2 | 3 | ## Current 4 | 5 | | Maintainer | GitHub ID | Affiliation | Email | 6 | | -------------------- | ------------------------------------------------------- | ----------- |-----------------| 7 | | Jianbo Yan | [yanjianbo1983](https://github.com/yanjianbo1983) | BoCloud | yanjianbo@beyondcent.com | 8 | | Zhen Tang | [lostcharlie](https://github.com/lostcharlie) | [ISCAS](http://www.is.cas.cn/) | tangzhen12@otcaix.iscas.ac.cn | 9 | 10 | ## Emeritus Maintainers 11 | 12 | * [haotaogeng](https://github.com/haotaogeng) -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- 1 | # FabEdge 2 | 3 | [![main](https://github.com/FabEdge/fabedge/actions/workflows/main.yml/badge.svg)](https://github.com/FabEdge/fabedge/actions/workflows/main.yml) 4 | [![Releases](https://img.shields.io/github/release/fabedge/fabedge/all.svg?style=flat-square)](https://github.com/fabedge/fabedge/releases) 5 | [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/FabEdge/fabedge/blob/main/LICENSE) 6 | 7 | 8 | 9 | FabEdge是一个基于kubernetes构建的，专注于边缘计算的容器网络方案，支持KubeEdge/SuperEdge/OpenYurt等主流边缘计算框架。 FabEdge旨在解决边缘计算场景下网络管理复杂，跨集群通信困难，缺少能自动感知网络拓扑的服务发现等问题，使能云边、边边业务协同。FabEdge支持4/5G，WiFi等弱网环境，适用于物联网，车联网、智慧城市等场景。 10 | 11 | FabEdge不仅支持边缘节点（通过KubeEdge等边缘计算框架加入集群的远程节点），还支持边缘集群（独立的K8S集群）。 12 | 13 | FabEdge是托管在CNCF下的沙箱项目。 14 | 15 | ## 特性 16 | 17 | * **自动地址管理**：自动管理边缘节点网段，自动管理边缘容器IP地址。 18 | * **云边、边边协同**: 建立云边，边边安全隧道，使能云边，边边之间的业务协同。 19 | * **灵活的隧道管理**: 使用自定义资源“社区”，可以根据业务需要灵活控制边边隧道。 20 | * **拓扑感知路由**: 使用最近的可用服务端点，减少服务访问延时。 21 | 22 | ## 优势 23 | 24 | * **标准**: 遵从K8S CNI规范，适用于任何协议，任何应用 。 25 | * **安全**: 使用成熟稳定的IPSec技术，使用安全的证书认证体系。 26 | * **易用**: 使用Operator机制，自动管理地址，节点，证书等，最大程度减少人为干预。 27 | 28 | ## 工作原理 29 | 30 | fabedge-arch 31 | 32 | * KubeEdge等边缘计算框架建立了控制面，把边缘节点加入云端K8S集群，使得可以在边缘节点上下发Pod等资源；FabEdge在此基础上建立了一个三层的数据转发面，使得Pod和Pod之间可以直接通讯。 33 | * 云端可以是任何K8S集群，目前支持的CNI包括Calico， Flannel。 34 | * FabEdge使用安全隧道技术，目前支持IPSec。 35 | * FabEdge包括的组件：Operators, Connector，Agent和Cloud-Agent。 36 | * Operator运行在云端任意的节点，通过监听节点，服务等K8S资源，为每个Agent维护一个ConfigMap，包括了本Agent需要的路由信息，比如子网，端点，负载均衡规则等，同时为每个Agent维护一个Secret，包括CA证书，节点证书等。Operator也负责Agent自身的管理，包括创建，更新，删除等。 37 | * Connector运行在云端选定的节点，负责管理从边缘节点发起的隧道，在边缘节点和云端集群之间转发流量。从Connector节点到云端其它非Connector节点的流量转发仍然依靠云端CNI。 38 | * Cloud-Agent运行在集群中所有非边缘，非Connector的节点，它负责管理本节点到远端的路由。 39 | * Agent运行在每个边缘节点上， 它使用自己的ConfigMap和Secret的信息，发起到云端Connector和其它边缘节点的隧道，负责本节点的路由，负载均衡，iptables规则的管理。 40 | * Fab-DNS运行在所有FabEdge的集群里，它通过截获DNS请求，提供拓扑感知的跨集群服务发现能力。 41 | 42 | ## FabEdge和传统CNI的区别 43 | 44 | FabEdge和现有的CNI，比如Calico，Flannel，互为补充，解决不同的问题。就像前面架构图所示，Calico等传统的插件运行在云端K8S集群里，负责云内节点之间的流量转发，FabEdge作为它的一个补充，把网络的能力延伸到了边缘节点和边缘集群，使能了云边，边边通讯。 45 | 46 | ## 用户手册 47 | 48 | * [快速安装](docs/get-started_zh.md) 49 | * [使用指南](docs/user-guide.md) 50 | * [常见问题](docs/FAQ_zh.md) 51 | * [卸载FabEdge](docs/uninstall.md) 52 | * [问题排查指南](docs/troubleshooting-guide.md) 53 | 54 | ## 社区例会 55 | 56 | 双周例会（每个月的第一和第四周的周四下午） 57 | 58 | 会议资料: 59 | [Meeting notes and agenda](https://shimo.im/docs/Wwt9TdGqgVvpDHJt) 60 | [Meeting recordings：bilibili channel](https://space.bilibili.com/524926244?spm_id_from=333.1007.0.0) 61 | 62 | ## 联系方式 63 | 64 | · 邮箱: fabedge@beyondcent.com 65 | · 扫描加入微信群 66 | 67 | wechat-group 68 | 69 | ## 贡献 70 | 71 | 如果您有兴趣成为一个贡献者，也有兴趣加入FabEdge的开发，请查看[CONTRIBUTING](./CONTRIBUTING.md)获取更多关于如何提交 Patch 和贡献的流程 72 | 73 | 请务必阅读并遵守我们的[行为准则](./CODE_OF_CONDUCT.md) 74 | 75 | ## 软件许可 76 | 77 | FabEdge遵循Apache 2.0 许可。 78 | -------------------------------------------------------------------------------- /build/agent/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.13 as builder 2 | COPY . /fabedge 3 | RUN cd /fabedge && make agent QUICK=1 CGO_ENABLED=0 GOPROXY=https://goproxy.cn,direct 4 | 5 | FROM fabedge/cni-plugins:v1.4.0 as cni-plugins 6 | 7 | FROM fabedge/base-image:0.1.0 8 | 9 | COPY --from=builder /fabedge/build/agent/entrypoint.sh / 10 | COPY --from=builder /fabedge/_output/fabedge-agent /usr/local/bin 11 | COPY --from=cni-plugins /plugins/ /plugins 12 | 13 | ENTRYPOINT ["/entrypoint.sh"] -------------------------------------------------------------------------------- /build/agent/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | echo 'select between the two modes of iptables ("legacy" and "nft")' 5 | /iptables-wrapper-installer.sh 6 | 7 | cmd="/usr/local/bin/fabedge-agent $@" 8 | echo "entrypoint: run command: $cmd" 9 | exec $cmd -------------------------------------------------------------------------------- /build/base-image/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.13 as builder 2 | RUN mkdir /iptables-wrapper && \ 3 | cd /iptables-wrapper && \ 4 | git clone https://github.com/kubernetes-sigs/iptables-wrappers.git . && \ 5 | make build 6 | 7 | FROM alpine:3.15 8 | COPY --from=builder /iptables-wrapper/iptables-wrapper-installer.sh \ 9 | /iptables-wrapper/bin/iptables-wrapper / 10 | 11 | RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && \ 12 | apk --update add iptables && \ 13 | apk --update add ip6tables && \ 14 | apk --update add ipset && \ 15 | apk --update add ipvsadm && \ 16 | rm -rf /var/cache/apk/* 17 | -------------------------------------------------------------------------------- /build/base-image/README.md: -------------------------------------------------------------------------------- 1 | # Base Image 2 | 3 | This Dockerfile is used to make base-image which is mainly used by agent, connector, cloud-agent. Base image contains iptables/ip6tables, ipvsadm, ipset and iptables-wrapper, these are all needed by componets metioned before. 4 | 5 | [iptables-wrapper](https://github.com/kubernetes-sigs/iptables-wrappers) is the main reason for base-image, it can detect iptables version on host machine during runtime and change links to corresponding implementation in container, this is important, without it, those components might not create iptables rules correctly. -------------------------------------------------------------------------------- /build/cert/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.16.4 as builder 2 | COPY . /fabedge 3 | RUN cd /fabedge && make cert QUICK=1 CGO_ENABLED=0 GOPROXY=https://goproxy.cn,direct 4 | 5 | FROM alpine:3.15 6 | COPY --from=builder /fabedge/_output/fabedge-cert /usr/local/bin/ 7 | 8 | ENTRYPOINT ["/usr/local/bin/fabedge-cert"] 9 | -------------------------------------------------------------------------------- /build/cloud-agent/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.13 as builder 2 | COPY . /fabedge 3 | RUN cd /fabedge && make cloud-agent QUICK=1 CGO_ENABLED=0 GOPROXY=https://goproxy.cn,direct 4 | 5 | FROM fabedge/base-image:0.1.0 6 | 7 | COPY --from=builder /fabedge/_output/fabedge-cloud-agent /usr/local/bin/ 8 | COPY --from=builder /fabedge/build/cloud-agent/entrypoint.sh / 9 | 10 | RUN chmod +x /entrypoint.sh 11 | 12 | ENTRYPOINT ["/entrypoint.sh"] -------------------------------------------------------------------------------- /build/cloud-agent/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | echo 'select between the two modes of iptables ("legacy" and "nft")' 5 | /iptables-wrapper-installer.sh 6 | 7 | cmd="/usr/local/bin/fabedge-cloud-agent $@" 8 | echo "entrypoint: run command: $cmd" 9 | exec $cmd 10 | -------------------------------------------------------------------------------- /build/connector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.13 as builder 2 | COPY . /fabedge 3 | RUN cd /fabedge && make connector QUICK=1 CGO_ENABLED=0 GOPROXY=https://goproxy.cn,direct 4 | 5 | FROM fabedge/base-image:0.1.0 6 | 7 | COPY --from=builder /fabedge/_output/fabedge-connector /usr/local/bin/connector 8 | COPY --from=builder /fabedge/build/connector/*.sh / 9 | 10 | RUN apk --update add keepalived curl && \ 11 | rm -rf /var/cache/apk/* && \ 12 | chmod +x /entrypoint.sh /check-connector-leader.sh 13 | 14 | ENTRYPOINT ["/entrypoint.sh"] -------------------------------------------------------------------------------- /build/connector/check-connector-leader.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | is_leader=`curl -s http://127.0.0.1:30306/is-leader` 3 | exit_code=$? 4 | 5 | if [ $exit_code != 0 ]; then 6 | exit $exit_code 7 | fi 8 | 9 | if [ $is_leader == "true" ]; then 10 | exit 0 11 | else 12 | exit 1 13 | fi 14 | -------------------------------------------------------------------------------- /build/connector/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | echo 'select between the two modes of iptables ("legacy" and "nft")' 5 | /iptables-wrapper-installer.sh 6 | 7 | cmd="/usr/local/bin/connector $@" 8 | echo "entrypoint: run command: $cmd" 9 | 10 | exec $cmd 11 | -------------------------------------------------------------------------------- /build/node/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.13 as builder 2 | COPY . /fabedge 3 | RUN cd /fabedge && make node QUICK=1 CGO_ENABLED=0 GOPROXY=https://goproxy.cn,direct 4 | 5 | FROM fabedge/cni-plugins:v1.4.0 as cni-plugins 6 | FROM fabedge/base-image:0.1.0 7 | 8 | COPY --from=builder /fabedge/_output/fabedge-node /usr/local/bin/node 9 | COPY --from=builder /fabedge/build/node/*.sh / 10 | COPY --from=cni-plugins /plugins/ /plugins 11 | 12 | RUN apk --update add keepalived curl && \ 13 | rm -rf /var/cache/apk/* && \ 14 | chmod +x /entrypoint.sh /check-connector-leader.sh 15 | 16 | ENTRYPOINT ["/entrypoint.sh"] -------------------------------------------------------------------------------- /build/node/check-connector-leader.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | is_leader=`curl -s http://127.0.0.1:30306/is-leader` 3 | exit_code=$? 4 | 5 | if [ $exit_code != 0 ]; then 6 | exit $exit_code 7 | fi 8 | 9 | if [ $is_leader == "true" ]; then 10 | exit 0 11 | else 12 | exit 1 13 | fi 14 | -------------------------------------------------------------------------------- /build/node/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | echo 'select between the two modes of iptables ("legacy" and "nft")' 5 | /iptables-wrapper-installer.sh 6 | 7 | cmd="/usr/local/bin/node $@" 8 | echo "entrypoint: run command: $cmd" 9 | 10 | exec $cmd 11 | -------------------------------------------------------------------------------- /build/operator/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.13 as builder 2 | COPY . /fabedge 3 | RUN cd /fabedge && make operator QUICK=1 CGO_ENABLED=0 GOPROXY=https://goproxy.cn,direct 4 | 5 | FROM alpine:3.15 6 | COPY --from=builder /fabedge/_output/fabedge-operator /usr/local/bin/ 7 | 8 | ENTRYPOINT ["/usr/local/bin/fabedge-operator"] -------------------------------------------------------------------------------- /build/strongswan/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.15 2 | 3 | RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && \ 4 | apk add strongswan=5.9.1-r1 && \ 5 | rm -rf /var/cache/apk/* 6 | 7 | RUN sed -i 's/# install_routes = yes/install_routes = no/' /etc/strongswan.d/charon.conf 8 | 9 | EXPOSE 500/udp 4500/udp 10 | 11 | CMD ["/usr/sbin/ipsec", "start", "--nofork"] 12 | -------------------------------------------------------------------------------- /build/utils/no_proxy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # remove China stuff to speedup build on github 4 | 5 | set -ex 6 | 7 | for app in agent connector operator strongswan 8 | do 9 | sed -i 's/gitee/github/g' build/$app/Dockerfile 10 | sed -i 's/GOPROXY/NOGOPROXY/g' build/$app/Dockerfile 11 | sed -i 's/dl-cdn.alpinelinux.org/nothing-to-do/g' build/$app/Dockerfile 12 | done 13 | -------------------------------------------------------------------------------- /cmd/agent/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/fabedge/fabedge/pkg/agent" 19 | logutil "github.com/fabedge/fabedge/pkg/util/log" 20 | flag "github.com/spf13/pflag" 21 | "os" 22 | "time" 23 | ) 24 | 25 | func main() { 26 | fs := flag.CommandLine 27 | cfg := &agent.Config{} 28 | 29 | logutil.AddFlags(fs) 30 | cfg.AddFlags(fs) 31 | fs.DurationVar(&cfg.SyncPeriod, "sync-period", 30*time.Second, "The period to synchronize network configuration") 32 | fs.UintVar(&cfg.TunnelInitTimeout, "tunnel-init-timeout", 10, "The timeout of tunnel initiation. Uint: second") 33 | 34 | flag.Parse() 35 | 36 | if err := agent.Execute(cfg); err != nil { 37 | os.Exit(1) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /cmd/cloud-agent/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/fabedge/fabedge/pkg/cloud-agent" 5 | "github.com/fabedge/fabedge/pkg/common/about" 6 | logutil "github.com/fabedge/fabedge/pkg/util/log" 7 | flag "github.com/spf13/pflag" 8 | ) 9 | 10 | func main() { 11 | var initMembers []string 12 | 13 | flag.StringSliceVar(&initMembers, "connector-node-addresses", []string{}, "internal ip address of all connector nodes") 14 | logutil.AddFlags(flag.CommandLine) 15 | about.AddFlags(flag.CommandLine) 16 | 17 | flag.Parse() 18 | 19 | cloud_agent.Execute(initMembers) 20 | } 21 | -------------------------------------------------------------------------------- /cmd/connector/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/fabedge/fabedge/pkg/common/about" 19 | "github.com/fabedge/fabedge/pkg/connector" 20 | logutil "github.com/fabedge/fabedge/pkg/util/log" 21 | flag "github.com/spf13/pflag" 22 | "time" 23 | ) 24 | 25 | func main() { 26 | fs := flag.CommandLine 27 | cfg := &connector.Config{} 28 | 29 | logutil.AddFlags(fs) 30 | about.AddFlags(fs) 31 | cfg.AddFlags(fs) 32 | 33 | fs.StringVar(&cfg.CNIType, "cni-type", "flannel", "CNI type used in cloud") 34 | fs.DurationVar(&cfg.LeaderElection.LeaseDuration, "leader-lease-duration", 15*time.Second, "The duration that non-leader candidates will wait to force acquire leadership") 35 | fs.DurationVar(&cfg.LeaderElection.RenewDeadline, "leader-renew-deadline", 10*time.Second, "The duration that the acting controlplane will retry refreshing leadership before giving up") 36 | fs.DurationVar(&cfg.LeaderElection.RetryPeriod, "leader-retry-period", 2*time.Second, "The duration that the LeaderElector clients should wait between tries of actions") 37 | fs.StringSliceVar(&cfg.InitMembers, "connector-node-addresses", []string{}, "internal address of all connector nodes") 38 | fs.DurationVar(&cfg.SyncPeriod, "sync-period", 5*time.Minute, "period to sync routes/rules") 39 | fs.UintVar(&cfg.TunnelInitTimeout, "tunnel-init-timeout", 10, "The timeout of tunnel initiation. Unit: second") 40 | 41 | flag.Parse() 42 | 43 | connector.Execute(cfg) 44 | } 45 | -------------------------------------------------------------------------------- /cmd/node/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/fabedge/fabedge/pkg/agent" 6 | "github.com/fabedge/fabedge/pkg/cloud-agent" 7 | "github.com/fabedge/fabedge/pkg/common/about" 8 | "github.com/fabedge/fabedge/pkg/connector" 9 | "github.com/fabedge/fabedge/pkg/operator" 10 | logutil "github.com/fabedge/fabedge/pkg/util/log" 11 | flag "github.com/spf13/pflag" 12 | "os" 13 | "time" 14 | ) 15 | 16 | func main() { 17 | var component string 18 | 19 | fs := flag.CommandLine 20 | fs.StringVar(&component, "component", "", "the component to initiate") 21 | 22 | // common 23 | var cniType string 24 | var leaseDuration time.Duration 25 | var renewDeadline time.Duration 26 | var retryPeriod time.Duration 27 | var syncPeriod time.Duration 28 | var tunnelInitTimeout uint 29 | 30 | // cloud-agent 31 | var initMembers []string 32 | fs.StringSliceVar(&initMembers, "connector-node-addresses", []string{}, "internal ip address of all connector nodes") 33 | 34 | // common 35 | fs.StringVar(&cniType, "cni-type", "", "The CNI name in your kubernetes cluster") 36 | fs.DurationVar(&leaseDuration, "leader-lease-duration", 15*time.Second, "The duration that non-leader candidates will wait to force acquire leadership") 37 | fs.DurationVar(&renewDeadline, "leader-renew-deadline", 10*time.Second, "The duration that the acting controlplane will retry refreshing leadership before giving up") 38 | fs.DurationVar(&retryPeriod, "leader-retry-period", 2*time.Second, "The duration that the LeaderElector clients should wait between tries of actions") 39 | fs.UintVar(&tunnelInitTimeout, "tunnel-init-timeout", 10, "The timeout of tunnel initiation. Uint: second") 40 | // agent: 30s, connector: 5m 41 | fs.DurationVar(&syncPeriod, "sync-period", 30*time.Second, "The period to synchronize network configuration") 42 | 43 | // operator 44 | operatorConfig := &operator.Options{} 45 | operatorConfig.AddFlags(fs) 46 | 47 | // connector 48 | connectorConfig := &connector.Config{} 49 | connectorConfig.AddFlags(fs) 50 | 51 | // agent 52 | agentConfig := &agent.Config{} 53 | agentConfig.AddFlags(fs) 54 | 55 | logutil.AddFlags(fs) 56 | about.AddFlags(fs) 57 | 58 | flag.Parse() 59 | 60 | fmt.Printf("component: %s\n", component) 61 | 62 | switch component { 63 | case "cloud-agent": 64 | cloud_agent.Execute(initMembers) 65 | case "operator": 66 | operatorConfig.CNIType = cniType 67 | operatorConfig.ManagerOpts.LeaseDuration = &leaseDuration 68 | operatorConfig.ManagerOpts.RenewDeadline = &renewDeadline 69 | operatorConfig.ManagerOpts.RetryPeriod = &retryPeriod 70 | if err := operator.Execute(operatorConfig); err != nil { 71 | os.Exit(1) 72 | } 73 | case "connector": 74 | connectorConfig.CNIType = cniType 75 | connectorConfig.SyncPeriod = syncPeriod 76 | connectorConfig.InitMembers = initMembers 77 | connectorConfig.LeaderElection.LeaseDuration = leaseDuration 78 | connectorConfig.LeaderElection.RenewDeadline = renewDeadline 79 | connectorConfig.LeaderElection.RetryPeriod = retryPeriod 80 | connectorConfig.TunnelInitTimeout = tunnelInitTimeout 81 | connector.Execute(connectorConfig) 82 | case "agent": 83 | agentConfig.SyncPeriod = syncPeriod 84 | agentConfig.TunnelInitTimeout = tunnelInitTimeout 85 | if err := agent.Execute(agentConfig); err != nil { 86 | os.Exit(1) 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /cmd/operator/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/fabedge/fabedge/pkg/common/about" 19 | logutil "github.com/fabedge/fabedge/pkg/util/log" 20 | flag "github.com/spf13/pflag" 21 | "os" 22 | "time" 23 | 24 | "github.com/fabedge/fabedge/pkg/operator" 25 | ) 26 | 27 | func main() { 28 | opts := &operator.Options{} 29 | 30 | fs := flag.CommandLine 31 | logutil.AddFlags(fs) 32 | about.AddFlags(fs) 33 | opts.AddFlags(fs) 34 | 35 | flag.StringVar(&opts.CNIType, "cni-type", "", "The CNI name in your kubernetes cluster") 36 | opts.ManagerOpts.LeaseDuration = flag.Duration("leader-lease-duration", 15*time.Second, "The duration that non-leader candidates will wait to force acquire leadership") 37 | opts.ManagerOpts.RenewDeadline = flag.Duration("leader-renew-deadline", 10*time.Second, "The duration that the acting controlplane will retry refreshing leadership before giving up") 38 | opts.ManagerOpts.RetryPeriod = flag.Duration("leader-retry-period", 2*time.Second, "The duration that the LeaderElector clients should wait between tries of actions") 39 | 40 | flag.Parse() 41 | 42 | if err := operator.Execute(opts); err != nil { 43 | os.Exit(1) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/autopath.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "github.com/coredns/coredns/plugin" 5 | "github.com/coredns/coredns/plugin/kubernetes/object" 6 | "github.com/coredns/coredns/request" 7 | ) 8 | 9 | // AutoPath implements the AutoPathFunc call from the autopath plugin. 10 | // It returns a per-query search path or nil indicating no searchpathing should happen. 11 | func (k *Kubernetes) AutoPath(state request.Request) []string { 12 | // Check if the query falls in a zone we are actually authoritative for and thus if we want autopath. 13 | zone := plugin.Zones(k.Zones).Matches(state.Name()) 14 | if zone == "" { 15 | return nil 16 | } 17 | 18 | // cluster.local { 19 | // autopath @kubernetes 20 | // kubernetes { 21 | // pods verified # 22 | // } 23 | // } 24 | // if pods != verified will cause panic and return SERVFAIL, expect worked as normal without autopath function 25 | if !k.opts.initPodCache { 26 | return nil 27 | } 28 | 29 | ip := state.IP() 30 | 31 | pod := k.podWithIP(ip) 32 | if pod == nil { 33 | return nil 34 | } 35 | 36 | search := make([]string, 3) 37 | if zone == "." { 38 | search[0] = pod.Namespace + ".svc." 39 | search[1] = "svc." 40 | search[2] = "." 41 | } else { 42 | search[0] = pod.Namespace + ".svc." + zone 43 | search[1] = "svc." + zone 44 | search[2] = zone 45 | } 46 | 47 | search = append(search, k.autoPathSearch...) 48 | search = append(search, "") // sentinel 49 | return search 50 | } 51 | 52 | // podWithIP returns the api.Pod for source IP. It returns nil if nothing can be found. 53 | func (k *Kubernetes) podWithIP(ip string) *object.Pod { 54 | if k.podMode != podModeVerified { 55 | return nil 56 | } 57 | ps := k.APIConn.PodIndex(ip) 58 | if len(ps) == 0 { 59 | return nil 60 | } 61 | return ps[0] 62 | } 63 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/external.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/coredns/coredns/plugin/etcd/msg" 7 | "github.com/coredns/coredns/plugin/kubernetes/object" 8 | "github.com/coredns/coredns/plugin/pkg/dnsutil" 9 | "github.com/coredns/coredns/request" 10 | 11 | "github.com/miekg/dns" 12 | ) 13 | 14 | // External implements the ExternalFunc call from the external plugin. 15 | // It returns any services matching in the services' ExternalIPs. 16 | func (k *Kubernetes) External(state request.Request) ([]msg.Service, int) { 17 | base, _ := dnsutil.TrimZone(state.Name(), state.Zone) 18 | 19 | segs := dns.SplitDomainName(base) 20 | last := len(segs) - 1 21 | if last < 0 { 22 | return nil, dns.RcodeServerFailure 23 | } 24 | // We are dealing with a fairly normal domain name here, but we still need to have the service 25 | // and the namespace: 26 | // service.namespace. 27 | // 28 | // for service (and SRV) you can also say _tcp, and port (i.e. _http), we need those be picked 29 | // up, unless they are not specified, then we use an internal wildcard. 30 | port := "*" 31 | protocol := "*" 32 | namespace := segs[last] 33 | if !k.namespaceExposed(namespace) { 34 | return nil, dns.RcodeNameError 35 | } 36 | 37 | last-- 38 | if last < 0 { 39 | return nil, dns.RcodeSuccess 40 | } 41 | 42 | service := segs[last] 43 | last-- 44 | if last == 1 { 45 | protocol = stripUnderscore(segs[last]) 46 | port = stripUnderscore(segs[last-1]) 47 | last -= 2 48 | } 49 | 50 | if last != -1 { 51 | // too long 52 | return nil, dns.RcodeNameError 53 | } 54 | 55 | idx := object.ServiceKey(service, namespace) 56 | serviceList := k.APIConn.SvcIndex(idx) 57 | 58 | services := []msg.Service{} 59 | zonePath := msg.Path(state.Zone, coredns) 60 | rcode := dns.RcodeNameError 61 | 62 | for _, svc := range serviceList { 63 | if namespace != svc.Namespace { 64 | continue 65 | } 66 | if service != svc.Name { 67 | continue 68 | } 69 | 70 | for _, ip := range svc.ExternalIPs { 71 | for _, p := range svc.Ports { 72 | if !(match(port, p.Name) && match(protocol, string(p.Protocol))) { 73 | continue 74 | } 75 | rcode = dns.RcodeSuccess 76 | s := msg.Service{Host: ip, Port: int(p.Port), TTL: k.ttl} 77 | s.Key = strings.Join([]string{zonePath, svc.Namespace, svc.Name}, "/") 78 | 79 | services = append(services, s) 80 | } 81 | } 82 | } 83 | return services, rcode 84 | } 85 | 86 | // ExternalAddress returns the external service address(es) for the CoreDNS service. 87 | func (k *Kubernetes) ExternalAddress(state request.Request) []dns.RR { 88 | // If CoreDNS is running inside the Kubernetes cluster: k.nsAddrs() will return the external IPs of the services 89 | // targeting the CoreDNS Pod. 90 | // If CoreDNS is running outside of the Kubernetes cluster: k.nsAddrs() will return the first non-loopback IP 91 | // address seen on the local system it is running on. This could be the wrong answer if coredns is using the *bind* 92 | // plugin to bind to a different IP address. 93 | return k.nsAddrs(true, state.Zone) 94 | } 95 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/handler.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/coredns/coredns/plugin" 7 | "github.com/coredns/coredns/request" 8 | 9 | "github.com/miekg/dns" 10 | ) 11 | 12 | // ServeDNS implements the plugin.Handler interface. 13 | func (k Kubernetes) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) { 14 | state := request.Request{W: w, Req: r} 15 | 16 | qname := state.QName() 17 | zone := plugin.Zones(k.Zones).Matches(qname) 18 | if zone == "" { 19 | return plugin.NextOrFailure(k.Name(), k.Next, ctx, w, r) 20 | } 21 | zone = qname[len(qname)-len(zone):] // maintain case of original query 22 | state.Zone = zone 23 | 24 | var ( 25 | records []dns.RR 26 | extra []dns.RR 27 | err error 28 | ) 29 | 30 | switch state.QType() { 31 | case dns.TypeA: 32 | records, err = plugin.A(ctx, &k, zone, state, nil, plugin.Options{}) 33 | case dns.TypeAAAA: 34 | records, err = plugin.AAAA(ctx, &k, zone, state, nil, plugin.Options{}) 35 | case dns.TypeTXT: 36 | records, err = plugin.TXT(ctx, &k, zone, state, nil, plugin.Options{}) 37 | case dns.TypeCNAME: 38 | records, err = plugin.CNAME(ctx, &k, zone, state, plugin.Options{}) 39 | case dns.TypePTR: 40 | records, err = plugin.PTR(ctx, &k, zone, state, plugin.Options{}) 41 | case dns.TypeMX: 42 | records, extra, err = plugin.MX(ctx, &k, zone, state, plugin.Options{}) 43 | case dns.TypeSRV: 44 | records, extra, err = plugin.SRV(ctx, &k, zone, state, plugin.Options{}) 45 | case dns.TypeSOA: 46 | records, err = plugin.SOA(ctx, &k, zone, state, plugin.Options{}) 47 | case dns.TypeNS: 48 | if state.Name() == zone { 49 | records, extra, err = plugin.NS(ctx, &k, zone, state, plugin.Options{}) 50 | break 51 | } 52 | fallthrough 53 | default: 54 | // Do a fake A lookup, so we can distinguish between NODATA and NXDOMAIN 55 | fake := state.NewWithQuestion(state.QName(), dns.TypeA) 56 | fake.Zone = state.Zone 57 | _, err = plugin.A(ctx, &k, zone, fake, nil, plugin.Options{}) 58 | } 59 | 60 | if k.IsNameError(err) { 61 | if k.Fall.Through(state.Name()) { 62 | return plugin.NextOrFailure(k.Name(), k.Next, ctx, w, r) 63 | } 64 | if !k.APIConn.HasSynced() { 65 | // If we haven't synchronized with the kubernetes cluster, return server failure 66 | return plugin.BackendError(ctx, &k, zone, dns.RcodeServerFailure, state, nil /* err */, plugin.Options{}) 67 | } 68 | return plugin.BackendError(ctx, &k, zone, dns.RcodeNameError, state, nil /* err */, plugin.Options{}) 69 | } 70 | if err != nil { 71 | return dns.RcodeServerFailure, err 72 | } 73 | 74 | if len(records) == 0 { 75 | return plugin.BackendError(ctx, &k, zone, dns.RcodeSuccess, state, nil, plugin.Options{}) 76 | } 77 | 78 | m := new(dns.Msg) 79 | m.SetReply(r) 80 | m.Authoritative = true 81 | m.Answer = append(m.Answer, records...) 82 | m.Extra = append(m.Extra, extra...) 83 | 84 | w.WriteMsg(m) 85 | return dns.RcodeSuccess, nil 86 | } 87 | 88 | // Name implements the Handler interface. 89 | func (k Kubernetes) Name() string { return "kubernetes" } 90 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/handler_case_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/coredns/coredns/plugin/pkg/dnstest" 8 | "github.com/coredns/coredns/plugin/test" 9 | 10 | "github.com/miekg/dns" 11 | ) 12 | 13 | var dnsPreserveCaseCases = []test.Case{ 14 | // Negative response 15 | { 16 | Qname: "not-a-service.testns.svc.ClUsTeR.lOcAl.", Qtype: dns.TypeA, 17 | Rcode: dns.RcodeNameError, 18 | Ns: []dns.RR{ 19 | test.SOA("ClUsTeR.lOcAl. 5 IN SOA ns.dns.ClUsTeR.lOcAl. hostmaster.ClUsTeR.lOcAl. 1499347823 7200 1800 86400 5"), 20 | }, 21 | }, 22 | // A Service 23 | { 24 | Qname: "SvC1.TeStNs.SvC.cLuStEr.LoCaL.", Qtype: dns.TypeA, 25 | Rcode: dns.RcodeSuccess, 26 | Answer: []dns.RR{ 27 | test.A("SvC1.TeStNs.SvC.cLuStEr.LoCaL. 5 IN A 10.0.0.1"), 28 | }, 29 | }, 30 | // SRV Service 31 | { 32 | Qname: "_HtTp._TcP.sVc1.TeStNs.SvC.cLuStEr.LoCaL.", Qtype: dns.TypeSRV, 33 | Rcode: dns.RcodeSuccess, 34 | Answer: []dns.RR{ 35 | test.SRV("_HtTp._TcP.sVc1.TeStNs.SvC.cLuStEr.LoCaL. 5 IN SRV 0 100 80 svc1.testns.svc.cLuStEr.LoCaL."), 36 | }, 37 | Extra: []dns.RR{ 38 | test.A("svc1.testns.svc.cLuStEr.LoCaL. 5 IN A 10.0.0.1"), 39 | }, 40 | }, 41 | } 42 | 43 | func TestPreserveCase(t *testing.T) { 44 | k := New([]string{"cluster.local."}) 45 | k.APIConn = &APIConnServeTest{} 46 | k.opts.ignoreEmptyService = true 47 | k.Next = test.NextHandler(dns.RcodeSuccess, nil) 48 | ctx := context.TODO() 49 | 50 | for i, tc := range dnsPreserveCaseCases { 51 | r := tc.Msg() 52 | 53 | w := dnstest.NewRecorder(&test.ResponseWriter{}) 54 | 55 | _, err := k.ServeDNS(ctx, w, r) 56 | if err != tc.Error { 57 | t.Errorf("Test %d expected no error, got %v", i, err) 58 | return 59 | } 60 | if tc.Error != nil { 61 | continue 62 | } 63 | 64 | resp := w.Msg 65 | if resp == nil { 66 | t.Fatalf("Test %d, got nil message and no error for %q", i, r.Question[0].Name) 67 | } 68 | 69 | if err := test.SortAndCheck(resp, tc); err != nil { 70 | t.Error(err) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/handler_ignore_emptyservice_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/coredns/coredns/plugin/pkg/dnstest" 8 | "github.com/coredns/coredns/plugin/test" 9 | 10 | "github.com/miekg/dns" 11 | ) 12 | 13 | var dnsEmptyServiceTestCases = []test.Case{ 14 | // A Service 15 | { 16 | Qname: "svcempty.testns.svc.cluster.local.", Qtype: dns.TypeA, 17 | Rcode: dns.RcodeNameError, 18 | Ns: []dns.RR{ 19 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 20 | }, 21 | }, 22 | // CNAME to external 23 | { 24 | Qname: "external.testns.svc.cluster.local.", Qtype: dns.TypeCNAME, 25 | Rcode: dns.RcodeSuccess, 26 | Answer: []dns.RR{ 27 | test.CNAME("external.testns.svc.cluster.local. 5 IN CNAME ext.interwebs.test."), 28 | }, 29 | }, 30 | } 31 | 32 | func TestServeDNSEmptyService(t *testing.T) { 33 | 34 | k := New([]string{"cluster.local."}) 35 | k.APIConn = &APIConnServeTest{} 36 | k.opts.ignoreEmptyService = true 37 | k.Next = test.NextHandler(dns.RcodeSuccess, nil) 38 | ctx := context.TODO() 39 | 40 | for i, tc := range dnsEmptyServiceTestCases { 41 | r := tc.Msg() 42 | 43 | w := dnstest.NewRecorder(&test.ResponseWriter{}) 44 | 45 | _, err := k.ServeDNS(ctx, w, r) 46 | if err != tc.Error { 47 | t.Errorf("Test %d expected no error, got %v", i, err) 48 | return 49 | } 50 | if tc.Error != nil { 51 | continue 52 | } 53 | 54 | resp := w.Msg 55 | if resp == nil { 56 | t.Fatalf("Test %d, got nil message and no error for %q", i, r.Question[0].Name) 57 | } 58 | 59 | // Before sorting, make sure that CNAMES do not appear after their target records 60 | if err := test.CNAMEOrder(resp); err != nil { 61 | t.Error(err) 62 | } 63 | 64 | if err := test.SortAndCheck(resp, tc); err != nil { 65 | t.Error(err) 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/handler_pod_disabled_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/coredns/coredns/plugin/pkg/dnstest" 8 | "github.com/coredns/coredns/plugin/test" 9 | 10 | "github.com/miekg/dns" 11 | ) 12 | 13 | var podModeDisabledCases = []test.Case{ 14 | { 15 | Qname: "10-240-0-1.podns.pod.cluster.local.", Qtype: dns.TypeA, 16 | Rcode: dns.RcodeNameError, 17 | Ns: []dns.RR{ 18 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 19 | }, 20 | }, 21 | { 22 | Qname: "172-0-0-2.podns.pod.cluster.local.", Qtype: dns.TypeA, 23 | Rcode: dns.RcodeNameError, 24 | Ns: []dns.RR{ 25 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 26 | }, 27 | }, 28 | } 29 | 30 | func TestServeDNSModeDisabled(t *testing.T) { 31 | 32 | k := New([]string{"cluster.local."}) 33 | k.APIConn = &APIConnServeTest{} 34 | k.Next = test.NextHandler(dns.RcodeSuccess, nil) 35 | k.podMode = podModeDisabled 36 | ctx := context.TODO() 37 | 38 | for i, tc := range podModeDisabledCases { 39 | r := tc.Msg() 40 | 41 | w := dnstest.NewRecorder(&test.ResponseWriter{}) 42 | 43 | _, err := k.ServeDNS(ctx, w, r) 44 | if err != tc.Error { 45 | t.Errorf("Test %d got unexpected error %v", i, err) 46 | return 47 | } 48 | if tc.Error != nil { 49 | continue 50 | } 51 | 52 | resp := w.Msg 53 | if resp == nil { 54 | t.Fatalf("Test %d, got nil message and no error for %q", i, r.Question[0].Name) 55 | } 56 | 57 | if err := test.SortAndCheck(resp, tc); err != nil { 58 | t.Error(err) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/handler_pod_insecure_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/coredns/coredns/plugin/pkg/dnstest" 8 | "github.com/coredns/coredns/plugin/test" 9 | 10 | "github.com/miekg/dns" 11 | ) 12 | 13 | var podModeInsecureCases = []test.Case{ 14 | { 15 | Qname: "10-240-0-1.podns.pod.cluster.local.", Qtype: dns.TypeA, 16 | Rcode: dns.RcodeSuccess, 17 | Answer: []dns.RR{ 18 | test.A("10-240-0-1.podns.pod.cluster.local. 5 IN A 10.240.0.1"), 19 | }, 20 | }, 21 | { 22 | Qname: "172-0-0-2.podns.pod.cluster.local.", Qtype: dns.TypeA, 23 | Rcode: dns.RcodeSuccess, 24 | Answer: []dns.RR{ 25 | test.A("172-0-0-2.podns.pod.cluster.local. 5 IN A 172.0.0.2"), 26 | }, 27 | }, 28 | { 29 | Qname: "blah.podns.pod.cluster.local.", Qtype: dns.TypeA, 30 | Rcode: dns.RcodeNameError, 31 | Ns: []dns.RR{ 32 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1515173576 7200 1800 86400 30"), 33 | }, 34 | }, 35 | { 36 | Qname: "blah.podns.pod.cluster.local.", Qtype: dns.TypeAAAA, 37 | Rcode: dns.RcodeNameError, 38 | Ns: []dns.RR{ 39 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1515173576 7200 1800 86400 30"), 40 | }, 41 | }, 42 | { 43 | Qname: "blah.podns.pod.cluster.local.", Qtype: dns.TypeHINFO, 44 | Rcode: dns.RcodeNameError, 45 | Ns: []dns.RR{ 46 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1515173576 7200 1800 86400 30"), 47 | }, 48 | }, 49 | { 50 | Qname: "blah.pod-nons.pod.cluster.local.", Qtype: dns.TypeA, 51 | Rcode: dns.RcodeNameError, 52 | Ns: []dns.RR{ 53 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1515173576 7200 1800 86400 30"), 54 | }, 55 | }, 56 | { 57 | Qname: "podns.pod.cluster.local.", Qtype: dns.TypeA, 58 | Rcode: dns.RcodeSuccess, 59 | Ns: []dns.RR{ 60 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1515173576 7200 1800 86400 30"), 61 | }, 62 | }, 63 | } 64 | 65 | func TestServeDNSModeInsecure(t *testing.T) { 66 | 67 | k := New([]string{"cluster.local."}) 68 | k.APIConn = &APIConnServeTest{} 69 | k.Next = test.NextHandler(dns.RcodeSuccess, nil) 70 | ctx := context.TODO() 71 | k.podMode = podModeInsecure 72 | 73 | for i, tc := range podModeInsecureCases { 74 | r := tc.Msg() 75 | 76 | w := dnstest.NewRecorder(&test.ResponseWriter{}) 77 | 78 | _, err := k.ServeDNS(ctx, w, r) 79 | if err != tc.Error { 80 | t.Errorf("Test %d expected no error, got %v", i, err) 81 | return 82 | } 83 | if tc.Error != nil { 84 | continue 85 | } 86 | 87 | resp := w.Msg 88 | if resp == nil { 89 | t.Fatalf("Test %d, got nil message and no error for %q", i, r.Question[0].Name) 90 | } 91 | 92 | if err := test.SortAndCheck(resp, tc); err != nil { 93 | t.Error(err) 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/handler_pod_verified_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/coredns/coredns/plugin/pkg/dnstest" 8 | "github.com/coredns/coredns/plugin/test" 9 | 10 | "github.com/miekg/dns" 11 | ) 12 | 13 | var podModeVerifiedCases = []test.Case{ 14 | { 15 | Qname: "10-240-0-1.podns.pod.cluster.local.", Qtype: dns.TypeA, 16 | Rcode: dns.RcodeSuccess, 17 | Answer: []dns.RR{ 18 | test.A("10-240-0-1.podns.pod.cluster.local. 5 IN A 10.240.0.1"), 19 | }, 20 | }, 21 | { 22 | Qname: "podns.pod.cluster.local.", Qtype: dns.TypeA, 23 | Rcode: dns.RcodeSuccess, 24 | Ns: []dns.RR{ 25 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 26 | }, 27 | }, 28 | { 29 | Qname: "svcns.svc.cluster.local.", Qtype: dns.TypeA, 30 | Rcode: dns.RcodeSuccess, 31 | Ns: []dns.RR{ 32 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 33 | }, 34 | }, 35 | { 36 | Qname: "pod-nons.pod.cluster.local.", Qtype: dns.TypeA, 37 | Rcode: dns.RcodeNameError, 38 | Ns: []dns.RR{ 39 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 40 | }, 41 | }, 42 | { 43 | Qname: "172-0-0-2.podns.pod.cluster.local.", Qtype: dns.TypeA, 44 | Rcode: dns.RcodeNameError, 45 | Ns: []dns.RR{ 46 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 47 | }, 48 | }, 49 | } 50 | 51 | func TestServeDNSModeVerified(t *testing.T) { 52 | 53 | k := New([]string{"cluster.local."}) 54 | k.APIConn = &APIConnServeTest{} 55 | k.Next = test.NextHandler(dns.RcodeSuccess, nil) 56 | ctx := context.TODO() 57 | k.podMode = podModeVerified 58 | 59 | for i, tc := range podModeVerifiedCases { 60 | r := tc.Msg() 61 | 62 | w := dnstest.NewRecorder(&test.ResponseWriter{}) 63 | 64 | _, err := k.ServeDNS(ctx, w, r) 65 | if err != tc.Error { 66 | t.Errorf("Test %d expected no error, got %v", i, err) 67 | return 68 | } 69 | if tc.Error != nil { 70 | continue 71 | } 72 | 73 | resp := w.Msg 74 | if resp == nil { 75 | t.Fatalf("Test %d, got nil message and no error for %q", i, r.Question[0].Name) 76 | } 77 | 78 | if err := test.SortAndCheck(resp, tc); err != nil { 79 | t.Error(err) 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/kubernetes_apex_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "testing" 7 | 8 | "github.com/coredns/coredns/plugin/pkg/dnstest" 9 | "github.com/coredns/coredns/plugin/test" 10 | 11 | "github.com/miekg/dns" 12 | ) 13 | 14 | var kubeApexCases = []test.Case{ 15 | { 16 | Qname: "cluster.local.", Qtype: dns.TypeSOA, 17 | Rcode: dns.RcodeSuccess, 18 | Answer: []dns.RR{ 19 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 20 | }, 21 | }, 22 | { 23 | Qname: "cluster.local.", Qtype: dns.TypeHINFO, 24 | Rcode: dns.RcodeSuccess, 25 | Ns: []dns.RR{ 26 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 27 | }, 28 | }, 29 | { 30 | Qname: "cluster.local.", Qtype: dns.TypeNS, 31 | Rcode: dns.RcodeSuccess, 32 | Answer: []dns.RR{ 33 | test.NS("cluster.local. 5 IN NS ns.dns.cluster.local."), 34 | }, 35 | Extra: []dns.RR{ 36 | test.A("ns.dns.cluster.local. 5 IN A 127.0.0.1"), 37 | }, 38 | }, 39 | { 40 | Qname: "cluster.local.", Qtype: dns.TypeA, 41 | Rcode: dns.RcodeSuccess, 42 | Ns: []dns.RR{ 43 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 44 | }, 45 | }, 46 | { 47 | Qname: "cluster.local.", Qtype: dns.TypeAAAA, 48 | Rcode: dns.RcodeSuccess, 49 | Ns: []dns.RR{ 50 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 51 | }, 52 | }, 53 | { 54 | Qname: "cluster.local.", Qtype: dns.TypeSRV, 55 | Rcode: dns.RcodeSuccess, 56 | Ns: []dns.RR{ 57 | test.SOA("cluster.local. 5 IN SOA ns.dns.cluster.local. hostmaster.cluster.local. 1499347823 7200 1800 86400 5"), 58 | }, 59 | }, 60 | } 61 | 62 | func TestServeDNSApex(t *testing.T) { 63 | 64 | k := New([]string{"cluster.local."}) 65 | k.APIConn = &APIConnServeTest{} 66 | k.Next = test.NextHandler(dns.RcodeSuccess, nil) 67 | k.localIPs = []net.IP{net.ParseIP("127.0.0.1")} 68 | ctx := context.TODO() 69 | 70 | for i, tc := range kubeApexCases { 71 | r := tc.Msg() 72 | 73 | w := dnstest.NewRecorder(&test.ResponseWriter{}) 74 | 75 | _, err := k.ServeDNS(ctx, w, r) 76 | if err != tc.Error { 77 | t.Errorf("Test %d, expected no error, got %v", i, err) 78 | return 79 | } 80 | if tc.Error != nil { 81 | continue 82 | } 83 | 84 | resp := w.Msg 85 | if resp == nil { 86 | t.Fatalf("Test %d, got nil message and no error ford", i) 87 | } 88 | 89 | if err := test.SortAndCheck(resp, tc); err != nil { 90 | t.Errorf("Test %d: %v", i, err) 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/local.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "net" 5 | 6 | "github.com/coredns/caddy" 7 | "github.com/coredns/coredns/core/dnsserver" 8 | ) 9 | 10 | // boundIPs returns the list of non-loopback IPs that CoreDNS is bound to 11 | func boundIPs(c *caddy.Controller) (ips []net.IP) { 12 | conf := dnsserver.GetConfig(c) 13 | hosts := conf.ListenHosts 14 | if hosts == nil || hosts[0] == "" { 15 | hosts = nil 16 | addrs, err := net.InterfaceAddrs() 17 | if err != nil { 18 | return nil 19 | } 20 | for _, addr := range addrs { 21 | hosts = append(hosts, addr.String()) 22 | } 23 | } 24 | for _, host := range hosts { 25 | ip, _, _ := net.ParseCIDR(host) 26 | ip4 := ip.To4() 27 | if ip4 != nil && !ip4.IsLoopback() { 28 | ips = append(ips, ip4) 29 | continue 30 | } 31 | ip6 := ip.To16() 32 | if ip6 != nil && !ip6.IsLoopback() { 33 | ips = append(ips, ip6) 34 | } 35 | } 36 | return ips 37 | } 38 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/log_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import clog "github.com/coredns/coredns/plugin/pkg/log" 4 | 5 | func init() { clog.Discard() } 6 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/metadata.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/coredns/coredns/plugin" 7 | "github.com/coredns/coredns/plugin/metadata" 8 | "github.com/coredns/coredns/request" 9 | ) 10 | 11 | // Metadata implements the metadata.Provider interface. 12 | func (k *Kubernetes) Metadata(ctx context.Context, state request.Request) context.Context { 13 | pod := k.podWithIP(state.IP()) 14 | if pod != nil { 15 | metadata.SetValueFunc(ctx, "kubernetes/client-namespace", func() string { 16 | return pod.Namespace 17 | }) 18 | 19 | metadata.SetValueFunc(ctx, "kubernetes/client-pod-name", func() string { 20 | return pod.Name 21 | }) 22 | } 23 | 24 | zone := plugin.Zones(k.Zones).Matches(state.Name()) 25 | if zone == "" { 26 | return ctx 27 | } 28 | // possible optimization: cache r so it doesn't need to be calculated again in ServeDNS 29 | r, err := parseRequest(state.Name(), zone) 30 | if err != nil { 31 | metadata.SetValueFunc(ctx, "kubernetes/parse-error", func() string { 32 | return err.Error() 33 | }) 34 | return ctx 35 | } 36 | 37 | metadata.SetValueFunc(ctx, "kubernetes/port-name", func() string { 38 | return r.port 39 | }) 40 | 41 | metadata.SetValueFunc(ctx, "kubernetes/protocol", func() string { 42 | return r.protocol 43 | }) 44 | 45 | metadata.SetValueFunc(ctx, "kubernetes/endpoint", func() string { 46 | return r.endpoint 47 | }) 48 | 49 | metadata.SetValueFunc(ctx, "kubernetes/service", func() string { 50 | return r.service 51 | }) 52 | 53 | metadata.SetValueFunc(ctx, "kubernetes/namespace", func() string { 54 | return r.namespace 55 | }) 56 | 57 | metadata.SetValueFunc(ctx, "kubernetes/kind", func() string { 58 | return r.podOrSvc 59 | }) 60 | 61 | return ctx 62 | } 63 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/namespace.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | // filteredNamespaceExists checks if namespace exists in this cluster 4 | // according to any `namespace_labels` plugin configuration specified. 5 | // Returns true even for namespaces not exposed by plugin configuration, 6 | // see namespaceExposed. 7 | func (k *Kubernetes) filteredNamespaceExists(namespace string) bool { 8 | ns, err := k.APIConn.GetNamespaceByName(namespace) 9 | if err != nil { 10 | return false 11 | } 12 | return ns.ObjectMeta.Name == namespace 13 | } 14 | 15 | // configuredNamespace returns true when the namespace is exposed through the plugin 16 | // `namespaces` configuration. 17 | func (k *Kubernetes) configuredNamespace(namespace string) bool { 18 | _, ok := k.Namespaces[namespace] 19 | if len(k.Namespaces) > 0 && !ok { 20 | return false 21 | } 22 | return true 23 | } 24 | 25 | func (k *Kubernetes) namespaceExposed(namespace string) bool { 26 | // return k.configuredNamespace(namespace) && k.filteredNamespaceExists(namespace) 27 | // This bug(https://github.com/kubeedge/kubeedge/issues/4582) caused coredns missing 28 | // namespace events and make k.filteredNamespaceExists(namespace) returns false for new created namespace 29 | return k.configuredNamespace(namespace) 30 | } 31 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/namespace_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestFilteredNamespaceExists(t *testing.T) { 8 | tests := []struct { 9 | expected bool 10 | kubernetesNamespaces map[string]struct{} 11 | testNamespace string 12 | }{ 13 | {true, map[string]struct{}{}, "foobar"}, 14 | {false, map[string]struct{}{}, "nsnoexist"}, 15 | } 16 | 17 | k := Kubernetes{} 18 | k.APIConn = &APIConnServeTest{} 19 | for i, test := range tests { 20 | k.Namespaces = test.kubernetesNamespaces 21 | actual := k.filteredNamespaceExists(test.testNamespace) 22 | if actual != test.expected { 23 | t.Errorf("Test %d failed. Filtered namespace %s was expected to exist", i, test.testNamespace) 24 | } 25 | } 26 | } 27 | 28 | func TestNamespaceExposed(t *testing.T) { 29 | tests := []struct { 30 | expected bool 31 | kubernetesNamespaces map[string]struct{} 32 | testNamespace string 33 | }{ 34 | {true, map[string]struct{}{"foobar": {}}, "foobar"}, 35 | {false, map[string]struct{}{"foobar": {}}, "nsnoexist"}, 36 | {true, map[string]struct{}{}, "foobar"}, 37 | {true, map[string]struct{}{}, "nsnoexist"}, 38 | } 39 | 40 | k := Kubernetes{} 41 | k.APIConn = &APIConnServeTest{} 42 | for i, test := range tests { 43 | k.Namespaces = test.kubernetesNamespaces 44 | actual := k.configuredNamespace(test.testNamespace) 45 | if actual != test.expected { 46 | t.Errorf("Test %d failed. Namespace %s was expected to be exposed", i, test.testNamespace) 47 | } 48 | } 49 | } 50 | 51 | func TestNamespaceValid(t *testing.T) { 52 | tests := []struct { 53 | expected bool 54 | kubernetesNamespaces map[string]struct{} 55 | testNamespace string 56 | }{ 57 | {true, map[string]struct{}{"foobar": {}}, "foobar"}, 58 | {false, map[string]struct{}{"foobar": {}}, "nsnoexist"}, 59 | {true, map[string]struct{}{}, "foobar"}, 60 | {false, map[string]struct{}{}, "nsnoexist"}, 61 | } 62 | 63 | k := Kubernetes{} 64 | k.APIConn = &APIConnServeTest{} 65 | for i, test := range tests { 66 | k.Namespaces = test.kubernetesNamespaces 67 | actual := k.namespaceExposed(test.testNamespace) 68 | if actual != test.expected { 69 | t.Errorf("Test %d failed. Namespace %s was expected to be valid", i, test.testNamespace) 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/ns.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "net" 5 | "strings" 6 | 7 | "github.com/coredns/coredns/plugin/kubernetes/object" 8 | "github.com/miekg/dns" 9 | api "k8s.io/api/core/v1" 10 | ) 11 | 12 | func isDefaultNS(name, zone string) bool { 13 | return strings.Index(name, defaultNSName) == 0 && strings.Index(name, zone) == len(defaultNSName) 14 | } 15 | 16 | // nsAddrs returns the A or AAAA records for the CoreDNS service in the cluster. If the service cannot be found, 17 | // it returns a record for the local address of the machine we're running on. 18 | func (k *Kubernetes) nsAddrs(external bool, zone string) []dns.RR { 19 | var ( 20 | svcNames []string 21 | svcIPs []net.IP 22 | ) 23 | 24 | // Find the CoreDNS Endpoints 25 | for _, localIP := range k.localIPs { 26 | endpoints := k.APIConn.EpIndexReverse(localIP.String()) 27 | 28 | // Collect IPs for all Services of the Endpoints 29 | for _, endpoint := range endpoints { 30 | svcs := k.APIConn.SvcIndex(object.ServiceKey(endpoint.Name, endpoint.Namespace)) 31 | for _, svc := range svcs { 32 | if external { 33 | svcName := strings.Join([]string{svc.Name, svc.Namespace, zone}, ".") 34 | for _, exIP := range svc.ExternalIPs { 35 | svcNames = append(svcNames, svcName) 36 | svcIPs = append(svcIPs, net.ParseIP(exIP)) 37 | } 38 | continue 39 | } 40 | svcName := strings.Join([]string{svc.Name, svc.Namespace, Svc, zone}, ".") 41 | if svc.ClusterIP == api.ClusterIPNone { 42 | // For a headless service, use the endpoints IPs 43 | for _, s := range endpoint.Subsets { 44 | for _, a := range s.Addresses { 45 | svcNames = append(svcNames, endpointHostname(a, k.endpointNameMode)+"."+svcName) 46 | svcIPs = append(svcIPs, net.ParseIP(a.IP)) 47 | } 48 | } 49 | } else { 50 | svcNames = append(svcNames, svcName) 51 | svcIPs = append(svcIPs, net.ParseIP(svc.ClusterIP)) 52 | } 53 | } 54 | } 55 | } 56 | 57 | // If no local IPs matched any endpoints, use the localIPs directly 58 | if len(svcIPs) == 0 { 59 | svcIPs = make([]net.IP, len(k.localIPs)) 60 | svcNames = make([]string, len(k.localIPs)) 61 | for i, localIP := range k.localIPs { 62 | svcNames[i] = defaultNSName + zone 63 | svcIPs[i] = localIP 64 | } 65 | } 66 | 67 | // Create an RR slice of collected IPs 68 | rrs := make([]dns.RR, len(svcIPs)) 69 | for i, ip := range svcIPs { 70 | if ip.To4() == nil { 71 | rr := new(dns.AAAA) 72 | rr.Hdr.Class = dns.ClassINET 73 | rr.Hdr.Rrtype = dns.TypeAAAA 74 | rr.Hdr.Name = svcNames[i] 75 | rr.AAAA = ip 76 | rrs[i] = rr 77 | continue 78 | } 79 | rr := new(dns.A) 80 | rr.Hdr.Class = dns.ClassINET 81 | rr.Hdr.Rrtype = dns.TypeA 82 | rr.Hdr.Name = svcNames[i] 83 | rr.A = ip 84 | rrs[i] = rr 85 | } 86 | 87 | return rrs 88 | } 89 | 90 | const defaultNSName = "ns.dns." 91 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/object/informer.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | meta "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | "k8s.io/client-go/tools/cache" 7 | ) 8 | 9 | // NewIndexerInformer is a copy of the cache.NewIndexerInformer function, but allows custom process function 10 | func NewIndexerInformer(lw cache.ListerWatcher, objType runtime.Object, h cache.ResourceEventHandler, indexers cache.Indexers, builder ProcessorBuilder) (cache.Indexer, cache.Controller) { 11 | clientState := cache.NewIndexer(cache.DeletionHandlingMetaNamespaceKeyFunc, indexers) 12 | 13 | cfg := &cache.Config{ 14 | Queue: cache.NewDeltaFIFO(cache.MetaNamespaceKeyFunc, clientState), 15 | ListerWatcher: lw, 16 | ObjectType: objType, 17 | FullResyncPeriod: defaultResyncPeriod, 18 | RetryOnError: false, 19 | Process: builder(clientState, h), 20 | } 21 | return clientState, cache.New(cfg) 22 | } 23 | 24 | type recordLatencyFunc func(meta.Object) 25 | 26 | // DefaultProcessor is based on the Process function from cache.NewIndexerInformer except it does a conversion. 27 | func DefaultProcessor(convert ToFunc, recordLatency recordLatencyFunc) ProcessorBuilder { 28 | return func(clientState cache.Indexer, h cache.ResourceEventHandler) cache.ProcessFunc { 29 | return func(obj interface{}) error { 30 | for _, d := range obj.(cache.Deltas) { 31 | switch d.Type { 32 | case cache.Sync, cache.Added, cache.Updated: 33 | obj, err := convert(d.Object) 34 | if err != nil { 35 | return err 36 | } 37 | if old, exists, err := clientState.Get(obj); err == nil && exists { 38 | if err := clientState.Update(obj); err != nil { 39 | return err 40 | } 41 | h.OnUpdate(old, obj) 42 | } else { 43 | if err := clientState.Add(obj); err != nil { 44 | return err 45 | } 46 | h.OnAdd(obj) 47 | } 48 | if recordLatency != nil { 49 | recordLatency(d.Object.(meta.Object)) 50 | } 51 | case cache.Deleted: 52 | var obj interface{} 53 | obj, ok := d.Object.(cache.DeletedFinalStateUnknown) 54 | if !ok { 55 | var err error 56 | obj, err = convert(d.Object) 57 | if err != nil && err != errPodTerminating { 58 | return err 59 | } 60 | } 61 | 62 | if err := clientState.Delete(obj); err != nil { 63 | return err 64 | } 65 | h.OnDelete(obj) 66 | if !ok && recordLatency != nil { 67 | recordLatency(d.Object.(meta.Object)) 68 | } 69 | } 70 | } 71 | return nil 72 | } 73 | } 74 | } 75 | 76 | const defaultResyncPeriod = 0 77 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/object/pod.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | api "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/runtime" 9 | ) 10 | 11 | // Pod is a stripped down api.Pod with only the items we need for CoreDNS. 12 | type Pod struct { 13 | // Don't add new fields to this struct without talking to the CoreDNS maintainers. 14 | Version string 15 | PodIP string 16 | Name string 17 | Namespace string 18 | 19 | *Empty 20 | } 21 | 22 | var errPodTerminating = errors.New("pod terminating") 23 | 24 | // ToPod returns a function that converts an api.Pod to a *Pod. 25 | func ToPod(skipCleanup bool) ToFunc { 26 | return func(obj interface{}) (interface{}, error) { 27 | apiPod, ok := obj.(*api.Pod) 28 | if !ok { 29 | return nil, fmt.Errorf("unexpected object %v", obj) 30 | } 31 | pod := toPod(skipCleanup, apiPod) 32 | t := apiPod.ObjectMeta.DeletionTimestamp 33 | if t != nil && !(*t).Time.IsZero() { 34 | // if the pod is in the process of termination, return an error so it can be ignored 35 | // during add/update event processing 36 | return pod, errPodTerminating 37 | } 38 | return pod, nil 39 | } 40 | } 41 | 42 | func toPod(skipCleanup bool, pod *api.Pod) *Pod { 43 | p := &Pod{ 44 | Version: pod.GetResourceVersion(), 45 | PodIP: pod.Status.PodIP, 46 | Namespace: pod.GetNamespace(), 47 | Name: pod.GetName(), 48 | } 49 | 50 | if !skipCleanup { 51 | *pod = api.Pod{} 52 | } 53 | 54 | return p 55 | } 56 | 57 | var _ runtime.Object = &Pod{} 58 | 59 | // DeepCopyObject implements the ObjectKind interface. 60 | func (p *Pod) DeepCopyObject() runtime.Object { 61 | p1 := &Pod{ 62 | Version: p.Version, 63 | PodIP: p.PodIP, 64 | Namespace: p.Namespace, 65 | Name: p.Name, 66 | } 67 | return p1 68 | } 69 | 70 | // GetNamespace implements the metav1.Object interface. 71 | func (p *Pod) GetNamespace() string { return p.Namespace } 72 | 73 | // SetNamespace implements the metav1.Object interface. 74 | func (p *Pod) SetNamespace(namespace string) {} 75 | 76 | // GetName implements the metav1.Object interface. 77 | func (p *Pod) GetName() string { return p.Name } 78 | 79 | // SetName implements the metav1.Object interface. 80 | func (p *Pod) SetName(name string) {} 81 | 82 | // GetResourceVersion implements the metav1.Object interface. 83 | func (p *Pod) GetResourceVersion() string { return p.Version } 84 | 85 | // SetResourceVersion implements the metav1.Object interface. 86 | func (p *Pod) SetResourceVersion(version string) {} 87 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/parse_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/coredns/coredns/request" 7 | 8 | "github.com/miekg/dns" 9 | ) 10 | 11 | func TestParseRequest(t *testing.T) { 12 | tests := []struct { 13 | query string 14 | expected string // output from r.String() 15 | }{ 16 | // valid SRV request 17 | {"_http._tcp.webs.mynamespace.svc.inter.webs.tests.", "http.tcp..webs.mynamespace.svc"}, 18 | // wildcard acceptance 19 | {"*.any.*.any.svc.inter.webs.tests.", "*.any..*.any.svc"}, 20 | // A request of endpoint 21 | {"1-2-3-4.webs.mynamespace.svc.inter.webs.tests.", "*.*.1-2-3-4.webs.mynamespace.svc"}, 22 | // bare zone 23 | {"inter.webs.tests.", "....."}, 24 | // bare svc type 25 | {"svc.inter.webs.tests.", "....."}, 26 | // bare pod type 27 | {"pod.inter.webs.tests.", "....."}, 28 | } 29 | for i, tc := range tests { 30 | m := new(dns.Msg) 31 | m.SetQuestion(tc.query, dns.TypeA) 32 | state := request.Request{Zone: zone, Req: m} 33 | 34 | r, e := parseRequest(state.Name(), state.Zone) 35 | if e != nil { 36 | t.Errorf("Test %d, expected no error, got '%v'.", i, e) 37 | } 38 | rs := r.String() 39 | if rs != tc.expected { 40 | t.Errorf("Test %d, expected (stringified) recordRequest: %s, got %s", i, tc.expected, rs) 41 | } 42 | } 43 | } 44 | 45 | func TestParseInvalidRequest(t *testing.T) { 46 | invalid := []string{ 47 | "webs.mynamespace.pood.inter.webs.test.", // Request must be for pod or svc subdomain. 48 | "too.long.for.what.I.am.trying.to.pod.inter.webs.tests.", // Too long. 49 | } 50 | 51 | for i, query := range invalid { 52 | m := new(dns.Msg) 53 | m.SetQuestion(query, dns.TypeA) 54 | state := request.Request{Zone: zone, Req: m} 55 | 56 | if _, e := parseRequest(state.Name(), state.Zone); e == nil { 57 | t.Errorf("Test %d: expected error from %s, got none", i, query) 58 | } 59 | } 60 | } 61 | 62 | const zone = "inter.webs.tests." 63 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/ready.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | // Ready implements the ready.Readiness interface. 4 | func (k *Kubernetes) Ready() bool { return k.APIConn.HasSynced() } 5 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/reverse.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "github.com/coredns/coredns/plugin" 8 | "github.com/coredns/coredns/plugin/etcd/msg" 9 | "github.com/coredns/coredns/plugin/pkg/dnsutil" 10 | "github.com/coredns/coredns/request" 11 | ) 12 | 13 | // Reverse implements the ServiceBackend interface. 14 | func (k *Kubernetes) Reverse(ctx context.Context, state request.Request, exact bool, opt plugin.Options) ([]msg.Service, error) { 15 | 16 | ip := dnsutil.ExtractAddressFromReverse(state.Name()) 17 | if ip == "" { 18 | _, e := k.Records(ctx, state, exact) 19 | return nil, e 20 | } 21 | 22 | records := k.serviceRecordForIP(ip, state.Name()) 23 | if len(records) == 0 { 24 | return records, errNoItems 25 | } 26 | return records, nil 27 | } 28 | 29 | // serviceRecordForIP gets a service record with a cluster ip matching the ip argument 30 | // If a service cluster ip does not match, it checks all endpoints 31 | func (k *Kubernetes) serviceRecordForIP(ip, name string) []msg.Service { 32 | // First check services with cluster ips 33 | for _, service := range k.APIConn.SvcIndexReverse(ip) { 34 | if len(k.Namespaces) > 0 && !k.namespaceExposed(service.Namespace) { 35 | continue 36 | } 37 | domain := strings.Join([]string{service.Name, service.Namespace, Svc, k.primaryZone()}, ".") 38 | return []msg.Service{{Host: domain, TTL: k.ttl}} 39 | } 40 | // If no cluster ips match, search endpoints 41 | var svcs []msg.Service 42 | for _, ep := range k.APIConn.EpIndexReverse(ip) { 43 | if len(k.Namespaces) > 0 && !k.namespaceExposed(ep.Namespace) { 44 | continue 45 | } 46 | for _, eps := range ep.Subsets { 47 | for _, addr := range eps.Addresses { 48 | if addr.IP == ip { 49 | domain := strings.Join([]string{endpointHostname(addr, k.endpointNameMode), ep.Name, ep.Namespace, Svc, k.primaryZone()}, ".") 50 | svcs = append(svcs, msg.Service{Host: domain, TTL: k.ttl}) 51 | } 52 | } 53 | } 54 | } 55 | return svcs 56 | } 57 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/setup_reverse_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/coredns/caddy" 7 | ) 8 | 9 | func TestKubernetesParseReverseZone(t *testing.T) { 10 | tests := []struct { 11 | input string // Corefile data as string 12 | expectedZones []string // expected count of defined zones. 13 | }{ 14 | {`kubernetes coredns.local 10.0.0.0/16`, []string{"coredns.local.", "0.10.in-addr.arpa."}}, 15 | {`kubernetes coredns.local 10.0.0.0/17`, []string{"coredns.local.", "0.10.in-addr.arpa."}}, 16 | {`kubernetes coredns.local fd00:77:30::0/110`, []string{"coredns.local.", "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.3.0.0.7.7.0.0.0.0.d.f.ip6.arpa."}}, 17 | } 18 | 19 | for i, tc := range tests { 20 | c := caddy.NewTestController("dns", tc.input) 21 | k, err := kubernetesParse(c) 22 | if err != nil { 23 | t.Fatalf("Test %d: Expected no error, got %q", i, err) 24 | } 25 | 26 | zl := len(k.Zones) 27 | if zl != len(tc.expectedZones) { 28 | t.Errorf("Test %d: Expected kubernetes to be initialized with %d zones, found %d zones", i, len(tc.expectedZones), zl) 29 | } 30 | for i, z := range tc.expectedZones { 31 | if k.Zones[i] != z { 32 | t.Errorf("Test %d: Expected zones to be %q, got %q", i, z, k.Zones[i]) 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/setup_ttl_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/coredns/caddy" 7 | ) 8 | 9 | func TestKubernetesParseTTL(t *testing.T) { 10 | tests := []struct { 11 | input string // Corefile data as string 12 | expectedTTL uint32 // expected count of defined zones. 13 | shouldErr bool 14 | }{ 15 | {`kubernetes cluster.local { 16 | ttl 56 17 | }`, 56, false}, 18 | {`kubernetes cluster.local`, defaultTTL, false}, 19 | {`kubernetes cluster.local { 20 | ttl -1 21 | }`, 0, true}, 22 | {`kubernetes cluster.local { 23 | ttl 3601 24 | }`, 0, true}, 25 | } 26 | 27 | for i, tc := range tests { 28 | c := caddy.NewTestController("dns", tc.input) 29 | k, err := kubernetesParse(c) 30 | if err != nil && !tc.shouldErr { 31 | t.Fatalf("Test %d: Expected no error, got %q", i, err) 32 | } 33 | if err == nil && tc.shouldErr { 34 | t.Fatalf("Test %d: Expected error, got none", i) 35 | } 36 | if err != nil && tc.shouldErr { 37 | // input should error 38 | continue 39 | } 40 | 41 | if k.ttl != tc.expectedTTL { 42 | t.Errorf("Test %d: Expected TTl to be %d, got %d", i, tc.expectedTTL, k.ttl) 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /coredns/plugin/kubernetes/watch.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | 6 | meta "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/apimachinery/pkg/labels" 8 | "k8s.io/apimachinery/pkg/watch" 9 | "k8s.io/client-go/kubernetes" 10 | ) 11 | 12 | func serviceWatchFunc(ctx context.Context, c kubernetes.Interface, ns string, s labels.Selector) func(options meta.ListOptions) (watch.Interface, error) { 13 | return func(options meta.ListOptions) (watch.Interface, error) { 14 | if s != nil { 15 | options.LabelSelector = s.String() 16 | } 17 | w, err := c.CoreV1().Services(ns).Watch(ctx, options) 18 | return w, err 19 | } 20 | } 21 | 22 | func podWatchFunc(ctx context.Context, c kubernetes.Interface, ns string, s labels.Selector) func(options meta.ListOptions) (watch.Interface, error) { 23 | return func(options meta.ListOptions) (watch.Interface, error) { 24 | if s != nil { 25 | options.LabelSelector = s.String() 26 | } 27 | if len(options.FieldSelector) > 0 { 28 | options.FieldSelector = options.FieldSelector + "," 29 | } 30 | options.FieldSelector = options.FieldSelector + "status.phase!=Succeeded,status.phase!=Failed,status.phase!=Unknown" 31 | w, err := c.CoreV1().Pods(ns).Watch(ctx, options) 32 | return w, err 33 | } 34 | } 35 | 36 | func endpointsWatchFunc(ctx context.Context, c kubernetes.Interface, ns string, s labels.Selector) func(options meta.ListOptions) (watch.Interface, error) { 37 | return func(options meta.ListOptions) (watch.Interface, error) { 38 | if s != nil { 39 | options.LabelSelector = s.String() 40 | } 41 | w, err := c.CoreV1().Endpoints(ns).Watch(ctx, options) 42 | return w, err 43 | } 44 | } 45 | 46 | func namespaceWatchFunc(ctx context.Context, c kubernetes.Interface, s labels.Selector) func(options meta.ListOptions) (watch.Interface, error) { 47 | return func(options meta.ListOptions) (watch.Interface, error) { 48 | if s != nil { 49 | options.LabelSelector = s.String() 50 | } 51 | w, err := c.CoreV1().Namespaces().Watch(ctx, options) 52 | return w, err 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /deploy/calico/ip-pool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: projectcalico.org/v3 2 | kind: IPPool 3 | metadata: 4 | name: fabedge 5 | spec: 6 | blockSize: 26 7 | # CIDR used by pods on edge nodes, modify it per your env. 8 | cidr: 10.10.0.0/16 9 | natOutgoing: false 10 | disabled: true 11 | ipipMode: Always -------------------------------------------------------------------------------- /deploy/cloud-agent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: fabedge-cloud-agent 5 | namespace: fabedge 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: fabedge-cloud-agent 10 | template: 11 | metadata: 12 | labels: 13 | app: fabedge-cloud-agent 14 | spec: 15 | affinity: 16 | nodeAffinity: 17 | requiredDuringSchedulingIgnoredDuringExecution: 18 | nodeSelectorTerms: 19 | - matchExpressions: 20 | - key: node-role.kubernetes.io/edge 21 | operator: DoesNotExist 22 | - key: node-role.kubernetes.io/connector 23 | operator: DoesNotExist 24 | containers: 25 | - args: 26 | - --connector-node-addresses=10.20.8.12 27 | - --component=cloud-agent 28 | - -v=5 29 | image: fabedge/cloud-agent 30 | imagePullPolicy: IfNotPresent 31 | name: fabedge-cloud-agent 32 | resources: 33 | limits: 34 | cpu: 100m 35 | memory: 64M 36 | requests: 37 | cpu: 100m 38 | memory: 64M 39 | securityContext: 40 | privileged: true 41 | hostNetwork: true -------------------------------------------------------------------------------- /deploy/connector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fabedge-connector 5 | namespace: fabedge 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: fabedge-connector 11 | strategy: 12 | type: Recreate 13 | template: 14 | metadata: 15 | labels: 16 | app: fabedge-connector 17 | spec: 18 | affinity: 19 | nodeAffinity: 20 | requiredDuringSchedulingIgnoredDuringExecution: 21 | nodeSelectorTerms: 22 | - matchExpressions: 23 | - key: node-role.kubernetes.io/connector 24 | operator: Exists 25 | podAntiAffinity: 26 | requiredDuringSchedulingIgnoredDuringExecution: 27 | - labelSelector: 28 | matchExpressions: 29 | - key: app 30 | operator: In 31 | values: 32 | - fabedge-connector 33 | topologyKey: kubernetes.io/hostname 34 | hostNetwork: true 35 | containers: 36 | - name: strongswan 37 | image: fabedge/strongswan 38 | imagePullPolicy: IfNotPresent 39 | readinessProbe: 40 | exec: 41 | command: 42 | - /usr/sbin/swanctl 43 | - --version 44 | initialDelaySeconds: 15 45 | periodSeconds: 10 46 | securityContext: 47 | capabilities: 48 | add: ["NET_ADMIN", "SYS_MODULE"] 49 | volumeMounts: 50 | - name: var-run 51 | mountPath: /var/run/ 52 | - name: ipsec-d 53 | mountPath: /etc/ipsec.d/ 54 | readOnly: true 55 | - name: ipsec-secrets 56 | mountPath: /etc/ipsec.secrets 57 | subPath: ipsec.secrets 58 | readOnly: true 59 | - name: connector 60 | securityContext: 61 | capabilities: 62 | add: ["NET_ADMIN"] 63 | image: fabedge/connector 64 | imagePullPolicy: IfNotPresent 65 | args: 66 | - --cni-type=calico 67 | - --sync-period=1m 68 | - --connector-node-addresses=10.20.8.28 69 | - --component=connector 70 | - -v=5 71 | volumeMounts: 72 | - name: var-run 73 | mountPath: /var/run/ 74 | - name: connector-config 75 | mountPath: /etc/fabedge/ 76 | - name: ipsec-d 77 | mountPath: /etc/ipsec.d/ 78 | readOnly: true 79 | volumes: 80 | - name: var-run 81 | emptyDir: {} 82 | - name: connector-config 83 | configMap: 84 | name: connector-config 85 | - name: ipsec-d 86 | secret: 87 | items: 88 | - key: ca.crt 89 | path: cacerts/ca.crt 90 | - key: tls.crt 91 | path: certs/tls.crt 92 | - key: tls.key 93 | path: private/tls.key 94 | secretName: connector-tls 95 | - name: ipsec-secrets 96 | secret: 97 | items: 98 | - key: ipsec.secrets 99 | path: ipsec.secrets 100 | secretName: connector-tls -------------------------------------------------------------------------------- /deploy/crds/fabedge.io_communities.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.7.0 8 | creationTimestamp: null 9 | name: communities.fabedge.io 10 | spec: 11 | group: fabedge.io 12 | names: 13 | kind: Community 14 | listKind: CommunityList 15 | plural: communities 16 | singular: community 17 | scope: Cluster 18 | versions: 19 | - additionalPrinterColumns: 20 | - description: community members 21 | jsonPath: .spec.members 22 | name: Members 23 | type: string 24 | - description: How long a community is created 25 | jsonPath: .metadata.creationTimestamp 26 | name: Age 27 | type: date 28 | name: v1alpha1 29 | schema: 30 | openAPIV3Schema: 31 | description: Community is used to manage a communication unit, it's members 32 | should be edge nodes 33 | properties: 34 | apiVersion: 35 | description: 'APIVersion defines the versioned schema of this representation 36 | of an object. Servers should convert recognized schemas to the latest 37 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 38 | type: string 39 | kind: 40 | description: 'Kind is a string value representing the REST resource this 41 | object represents. Servers may infer this from the endpoint the client 42 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 43 | type: string 44 | metadata: 45 | type: object 46 | spec: 47 | properties: 48 | members: 49 | items: 50 | type: string 51 | type: array 52 | type: object 53 | type: object 54 | served: true 55 | storage: true 56 | subresources: {} 57 | status: 58 | acceptedNames: 59 | kind: "" 60 | plural: "" 61 | conditions: [] 62 | storedVersions: [] 63 | -------------------------------------------------------------------------------- /deploy/operator-svc.yaml: -------------------------------------------------------------------------------- 1 | # 只有host集群才需要生成 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: fabedge-operator-api 6 | spec: 7 | selector: 8 | app: fabedge-operator 9 | type: NodePort 10 | ports: 11 | - protocol: TCP 12 | port: 3030 13 | targetPort: 3030 14 | nodePort: 30303 15 | -------------------------------------------------------------------------------- /deploy/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: fabedge-operator 5 | rules: 6 | - apiGroups: 7 | - fabedge.io 8 | resources: 9 | - communities 10 | - clusters 11 | verbs: 12 | - "*" 13 | - apiGroups: 14 | - "" 15 | resources: 16 | - nodes 17 | - services 18 | verbs: 19 | - get 20 | - list 21 | - watch 22 | - apiGroups: 23 | - "" 24 | resources: 25 | - nodes 26 | verbs: 27 | - update 28 | - apiGroups: 29 | - "" 30 | resources: 31 | - pods 32 | - configmaps 33 | - secrets 34 | verbs: 35 | - get 36 | - list 37 | - watch 38 | - create 39 | - update 40 | - patch 41 | - delete 42 | - apiGroups: 43 | - "" 44 | resources: 45 | - pods/status 46 | - configmaps/status 47 | - secrets/status 48 | verbs: 49 | - get 50 | - update 51 | - patch 52 | - apiGroups: 53 | - "discovery.k8s.io" 54 | resources: 55 | - endpointslices 56 | verbs: 57 | - get 58 | - list 59 | - watch 60 | - apiGroups: 61 | - "coordination.k8s.io" 62 | resources: 63 | - "leases" 64 | verbs: 65 | - "*" 66 | - apiGroups: 67 | - crd.projectcalico.org 68 | resources: 69 | - ipamblocks 70 | verbs: 71 | - get 72 | - list 73 | - watch 74 | 75 | --- 76 | 77 | apiVersion: v1 78 | kind: ServiceAccount 79 | metadata: 80 | name: fabedge-operator 81 | namespace: fabedge 82 | 83 | --- 84 | 85 | apiVersion: rbac.authorization.k8s.io/v1 86 | kind: ClusterRoleBinding 87 | metadata: 88 | name: fabedge-operator 89 | roleRef: 90 | apiGroup: rbac.authorization.k8s.io 91 | kind: ClusterRole 92 | name: fabedge-operator 93 | subjects: 94 | - kind: ServiceAccount 95 | name: fabedge-operator 96 | namespace: fabedge 97 | -------------------------------------------------------------------------------- /docs/adopters/bocloud.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/adopters/bocloud.png -------------------------------------------------------------------------------- /docs/adopters/linklogis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/adopters/linklogis.png -------------------------------------------------------------------------------- /docs/design/fab-dns-show3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/design/fab-dns-show3.png -------------------------------------------------------------------------------- /docs/design/images/fabedge-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/design/images/fabedge-arch.png -------------------------------------------------------------------------------- /docs/design/images/multi-cluster-commuication.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/design/images/multi-cluster-commuication.png -------------------------------------------------------------------------------- /docs/design/images/network-topology.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/design/images/network-topology.png -------------------------------------------------------------------------------- /docs/design/multi-cluster-communication.md: -------------------------------------------------------------------------------- 1 | # 多集群通信设计 2 | 3 | ## 概述 4 | 5 | 多集群通信是为了让多个异构的，分布在不同网络的集群可以相互访问彼此的节点和服务(目前仅支持集群的云端节点相互通信) 6 | 7 | 多个通信的集群中必须且只能有一个主(host)集群，其他集群是主集群的成员(member)集群, 每个成员集群在主集群注册自己需要外部通信的端点信息（目前仅限于Connector)。 8 | 9 | 集群间不能直接通信, 用户需要在主集群通过社区来组织需要通信的集群，在同一个社区的集群才能相互访问。 10 | 11 | 各个集群间的节点，Pod，服务的地址不能重复。 12 | 13 | ## 集群分类 14 | 15 | ### 按系统分类 16 | 17 | * 常规Kubernetes集群 18 | * 边缘计算集群(KubeEdge/OpenYurt/SuperEdge). 19 | 20 | ### 按拓扑结构分类 21 | 22 | * 所有节点都在一个区域，网络也在一个局域网，通常是企业内部或云上的一个Kubernetes集群，运行各种服务，也可能是某种设施的小型集群。 23 | * 管理节点在云端，边缘节点分布在多个位置，使用不同的网络. 24 | 25 | ### 按角色分类 26 | 27 | 通信的集群可以有多个，但角色分为两类：主集群和成员集群，主集群必须且只能有一个 28 | 29 | 主集群的功能如下： 30 | 31 | * 证书派发。所有集群的根证书存储在主机群，成员集群可以从主集群为Connector和边缘节点Agent申请证书。 32 | * 集中存储每个集群暴露的端点信息，目前仅限于Connector 33 | * 社区管理，需要通信的集群必须在同一社区 34 | * 向其他集群下发需要通信的集群的端点信息 35 | 36 | 成员集群的功能如下： 37 | * 向主集群申请本集群的端点证书 38 | * 向主集群提供自身对外暴露的端点信息 39 | * 通过主集群获取其他集群的端点信息，并跟其他集群建立通信 40 | 41 | ## 自定义资源 42 | 43 | 为了管理多集群间的通信，需要添加或修改一些CRD。 44 | 45 | #### Community 46 | 47 | Community原先用于管理一个集群内部边缘节点间的通信，现在可以用来管理需要通信的多个集群，但暂时不支持边缘节点跨集群通信。目前一个社区内的成员类型必须统一，要么是本集群的边缘节点，要么是各个集群的connector. 48 | 49 | #### Cluster 50 | 51 | Cluster是主集群用来记录其他集群端点信息的数据结构，有以下字段： 52 | 53 | * name. 集群名称，每个成员集群访问主集群时声明自己的身份。 54 | 55 | * token. 用于成员集群初始化，token由主集群的operator生成。 56 | 57 | * endpoints. 该集群内部所有需要跟其他集群通信的端点信息，其中Connector是必须上传的。 58 | * Name. 必须唯一，建议需要通信的集群在配置cluster domain时也保持唯一。 59 | * PublicAddresses. 集群用于对外通信的公网地址，该地址必须可以被其他集群和本集群的边缘节点(如果有)访问 60 | * Subnets. 主要是集群的PodCIDRs数据，但也含有提供集群的ServiceCIDR。 61 | * NodeCIDRs. 集群内部云端节点的所有节点的IP地址 62 | * Type. 表明端点类型： Connector和EdgeNode 63 | 64 | 65 | 66 | ## 名称管理 67 | 68 | 各个集群都有Connector，节点名称也可能重复，但使用社区时，需要保证成员名称唯一，为了达到这个目标，在每个集群上报本集群的端点信息时需要将名称修改一下，加上集群前缀，比如 cluster1的Connector名称要改为: cluster1.connector。 69 | 70 | -------------------------------------------------------------------------------- /docs/design/跨集群服务访问介绍.md: -------------------------------------------------------------------------------- 1 | 2 | # FabEdge跨集群服务访问 3 | 4 | FabEdge在0.4.0时已经支持多边缘集群通信，但集群间的相互访问只能通过IP来访问，即便访问目标是一个服务也会如此，这与日常中使用Kubernetes的习惯极不相符。事实上，自多集群通信的需求存在以来，跨集群的服务发现和访问的需求就一直存在，开源社区也一直在努力解决这个问题： 5 | 6 | * [Multi-cluster Service APIs](https://github.com/kubernetes-sigs/mcs-api) 7 | * [Lighthouse](https://submariner.io/getting-started/architecture/service-discovery/) 8 | * [Cilium Load-balancing & Service Discovery](https://docs.cilium.io/en/stable/gettingstarted/clustermesh/services/) 9 | 10 | 11 | 12 | 既然已经存在这些解决方案，为什么FabEdge要提出自己的解决方案呢？有如下原因： 13 | 14 | * mcs-api只是一套API，需要其他实现者解决各个集群间服务信息的导出导入。 15 | * Lighthouse依赖于submariner，而submariner并不是面向边缘场景的。 16 | * Cilium是一套整体解决方案，不能跟其他CNI共存，此外它也不是面向边缘场景。 17 | 18 | 19 | 20 | 为FabEdge提供跨集群服务访问的组件叫[FabDNS](https://github.com/FabEdge/fab-dns)，它尝试达成以下目标： 21 | 22 | * 它允许一个集群访问其他集群提供的服务，服务类型仅限于ClusterIP，Headless两种。 23 | 24 | * 一个服务可以部署于一个集群内部，也可以分散在多个集群里。 25 | 26 | * 提供一定的具备拓扑感知的DNS解析，访问者可以就近访问最近的服务节点。 27 | 28 | 29 | 30 | 31 | FabDNS有两个组件: service-hub与fab-dns。还提供了一个CRD: GlobalService。一个集群若想将一个服务提供给其他集群，首先要将该服务标注为全局服务。 service-hub负责各个集群间全局服务的导出与导入，fab-dns负责在集群内部提供全局服务的地址解析。每个集群部署FabDNS时要标注拓扑信息，即region和zone信息，FabDNS的拓扑感知就是基于这些拓扑信息来进行的。 32 | 33 | 34 | 35 | ![fab-dns-show3](fab-dns-show3.png) 36 | 37 | 38 | 39 | 以上图为例，共有三个集群，北京集群是主集群，上海集群和苏州集群的service-hub都要通过北京集群的service-hub交换全局服务信息。北京和上海集群同时暴露了一个nginx服务和一个mysql服务，假设这些服务都是在default命名空间下。如果上海或北京的一个pod去访问nginx.default.global，那么响应的pod只会是各自集群的pod，因为zone是匹配的。如果苏州集群的pod去访问nginx.default.global，那么它会被上海集群的nginx背后的pod响应，为什么呢？因为苏州和上海的region都是south 40 | -------------------------------------------------------------------------------- /docs/images/FabEdge-Arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/images/FabEdge-Arch.png -------------------------------------------------------------------------------- /docs/images/wechat-group-qr-code.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/images/wechat-group-qr-code.jpg -------------------------------------------------------------------------------- /docs/integration/loadbalancer/fabedge-with-lb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FabEdge/fabedge/4d44cdf846ba6dbd48fad9b9179030642c51858f/docs/integration/loadbalancer/fabedge-with-lb.png -------------------------------------------------------------------------------- /docs/obsoleted-doc/install-k8s-and-kubeedge.md: -------------------------------------------------------------------------------- 1 | # 部署k8s集群 2 | 3 | ## 安装条件 4 | 5 | - 遵循 [kubeadm的最低要求](https://kubernetes.io/zh/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin) ，Master && Node 最低2C2G，磁盘空间不小于10G； 6 | 7 | > ⚠️注意：尽可能使用干净的系统，避免其他因素引起安装错误。 8 | 9 | ## 支持的操作系统 10 | 11 | - **Ubuntu 18.04 （推荐使用）** 12 | - Ubuntu 20.04 13 | - CentOS 7.9 14 | - CentOS 7.8 15 | 16 | ## 部署k8s集群 17 | 18 | ### 安装k8s Master 节点 19 | 20 | 以Ubuntu 18.04.5 系统为例子，运行以下指令： 21 | 22 | ```shell 23 | sudo curl http://116.62.127.76/FabEdge/fabedge/main/deploy/cluster/install-k8s.sh | bash - 24 | ``` 25 | 26 | > ⚠️注意：如果加载时间过长，有可能网速较慢，请耐心等待 27 | 28 | 如果出现以下信息，表示安装成功： 29 | 30 | ``` 31 | PLAY RECAP ********************************************************************* 32 | master : ok=15 changed=13 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 33 | ``` 34 | 35 | ### 添加k8s边缘节点 36 | 37 | ```shell 38 | sudo curl http://116.62.127.76/FabEdge/fabedge/main/deploy/cluster/add-edge-node.sh | bash -s -- --host-vars ansible_hostname={hostname} ansible_user={username} ansible_password={password} ansible_host={edge-node-IP} 39 | ``` 40 | 41 | 参数说明： 42 | 43 | * ansible_hostname 指定边缘节点的主机名 44 | 45 | * ansible_user 配置边缘节点的用户名 46 | 47 | * ansible_password 配置边缘节点的密码 48 | 49 | * ansible_host 配置边缘节点的IP地址 50 | 51 | 例如：设置边缘节点的主机名为edge1、用户名是root、密码是pwd111、IP为10.22.45.26，指令如下： 52 | 53 | ```shell 54 | sudo curl http://116.62.127.76/FabEdge/fabedge/main/deploy/cluster/add-edge-node.sh | bash -s -- --host-vars ansible_hostname=edge1 ansible_user=root ansible_password=pwd111 ansible_host=10.22.45.26 55 | ``` 56 | 57 | 如果出现以下信息，表示安装成功： 58 | 59 | ``` 60 | PLAY RECAP ********************************************************************* 61 | edge1 : ok=13 changed=10 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 62 | ``` 63 | 64 | ### 确认节点添加成功 65 | 66 | ```shell 67 | sudo kubectl get node 68 | NAME STATUS ROLES AGE VERSION 69 | edge1 Ready agent,edge 22m v1.19.3-kubeedge-v1.5.0 70 | master Ready master,node 32m v1.19.7 71 | ``` 72 | 73 | > ⚠️注意：如果边缘节点没有配置密码，需要配置ssh证书。 74 | > 75 | > master节点配置ssh证书： 76 | > 77 | > ```shell 78 | > sudo docker exec -it installer bash 79 | > sudo ssh-copy-id {edge-node-IP} 80 | > ``` 81 | 82 | -------------------------------------------------------------------------------- /docs/roadmap.md: -------------------------------------------------------------------------------- 1 | # FabEdge Roadmap 2 | 3 | ## Q3 2021 4 | 5 | - Support KubeEdge/SuperEdge/Openyurt 6 | - Automatic management of node certificate 7 | - Air-gap installation 8 | - Support Flannel/Calico 9 | - Support IPV4 10 | - Support IPSec Tunnel 11 | 12 | ## Q4 2021 13 | 14 | - Support Edge Cluster 15 | - Support topology-aware service discovery 16 | 17 | ## v0.6.0 18 | 19 | - Support IPV6 20 | - Implement a flexiable way to configure fabedge-agent 21 | - Support auto networking of edge nodes in LAN 22 | 23 | ## v0.7.0 24 | 25 | - Change the naming strategy of fabedge-agent pods 26 | - Add commonName validation for fabedge-agent certificates 27 | - Implement node-specific configuration of fabedge-agent arguments 28 | - Let fabedge-agent configure sysctl parameters needed 29 | - Let fabedge-operator manage calico ippools for CIDRs 30 | 31 | ## v0.8.0 32 | 33 | * Support settings strongswan's port 34 | * Support strongswan hole punching 35 | * Release fabctl which is a CLI to help diagnosing networking problems; 36 | * Integerate fabedge-agent with coredns and kube-proxy 37 | 38 | ## v0.9.0 39 | 40 | * Implement connector high availability 41 | 42 | * Improve dual stack implementation 43 | 44 | * Improve iptables rule configuring (ensure the order of rules) 45 | 46 | 47 | -------------------------------------------------------------------------------- /docs/uninstall.md: -------------------------------------------------------------------------------- 1 | # Uninstall FabEdge 2 | 3 | English | [中文](uninstall_zh.md) 4 | 5 | 1. Delete helm release 6 | 7 | ``` 8 | $ helm uninstall fabedge -n fabedge 9 | ``` 10 | 11 | 2. Delete other resources 12 | 13 | ``` 14 | $ kubectl -n fabedge delete cm --all 15 | $ kubectl -n fabedge delete pods --all 16 | $ kubectl -n fabedge delete secret --all 17 | $ kubectl -n fabedge delete job.batch --all 18 | ``` 19 | 20 | 3. Delete namespace 21 | 22 | ``` 23 | $ kubectl delete namespace fabedge 24 | ``` 25 | 26 | 4. Delete all FabeEdge configuration file from all edge nodes 27 | 28 | ``` 29 | $ rm -f /etc/cni/net.d/fabedge.* 30 | ``` 31 | 32 | 5. Delete all fabedge images on all nodes 33 | 34 | ``` 35 | $ docker images | grep fabedge | awk '{print $3}' | xargs -I{} docker rmi {} 36 | ``` 37 | 38 | 6. Delete CustomResourceDefinition 39 | 40 | ``` 41 | $ kubectl delete CustomResourceDefinition "clusters.fabedge.io" 42 | $ kubectl delete CustomResourceDefinition "communities.fabedge.io" 43 | $ kubectl delete ClusterRole "fabedge-operator" 44 | $ kubectl delete ClusterRoleBinding "fabedge-operator" 45 | ``` 46 | 47 | -------------------------------------------------------------------------------- /docs/uninstall_zh.md: -------------------------------------------------------------------------------- 1 | ## 卸载FabEdge 2 | 3 | 4 | 5 | 1. 使用helm删除主要资源 6 | 7 | ```shell 8 | $ helm uninstall fabedge -n fabedge 9 | ``` 10 | 11 | 2. 删除其它资源 12 | 13 | ```shell 14 | $ kubectl -n fabedge delete cm --all 15 | $ kubectl -n fabedge delete pods --all 16 | $ kubectl -n fabedge delete secret --all 17 | $ kubectl -n fabedge delete job.batch --all 18 | ``` 19 | 20 | 3. 删除namespace 21 | 22 | ```shell 23 | $ kubectl delete namespace fabedge 24 | ``` 25 | 26 | 4. 删除所有边缘节点的`fabedge.conf` 27 | 28 | ```shell 29 | $ rm -f /etc/cni/net.d/fabedge.* 30 | ``` 31 | 32 | ​ 5. 删除所有节点的上fabedge相关的镜像 33 | 34 | ```shell 35 | $ docker images | grep fabedge | awk '{print $3}' | xargs -I{} docker rmi {} 36 | ``` 37 | 38 | 6.删除CustomResourceDefinition 39 | 40 | ```shell 41 | $ kubectl delete CustomResourceDefinition "clusters.fabedge.io" 42 | $ kubectl delete CustomResourceDefinition "communities.fabedge.io" 43 | $ kubectl delete ClusterRole "fabedge-operator" 44 | $ kubectl delete ClusterRoleBinding "fabedge-operator" 45 | ``` 46 | 47 | -------------------------------------------------------------------------------- /examples/communities.yaml: -------------------------------------------------------------------------------- 1 | # allow all edges of beijing cluster to communicate 2 | apiVersion: fabedge.io/v1alpha1 3 | kind: Community 4 | metadata: 5 | name: all-edges 6 | spec: 7 | members: 8 | - beijing.edge1 9 | - beijing.edge2 10 | 11 | --- 12 | 13 | # allow all cloud node of clusters to communicate 14 | apiVersion: fabedge.io/v1alpha1 15 | kind: Community 16 | metadata: 17 | name: all-clusters 18 | spec: 19 | members: 20 | - beijing.connector 21 | - hangzhou.connector 22 | - shenzhen.connector 23 | -------------------------------------------------------------------------------- /examples/mysql.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: mysql 5 | labels: 6 | app: mysql 7 | spec: 8 | selector: 9 | matchLabels: 10 | app: mysql 11 | serviceName: "mysql" 12 | replicas: 2 13 | template: 14 | metadata: 15 | labels: 16 | app: mysql 17 | spec: 18 | affinity: 19 | nodeAffinity: 20 | requiredDuringSchedulingIgnoredDuringExecution: 21 | nodeSelectorTerms: 22 | - matchExpressions: 23 | - key: kubernetes.io/os 24 | operator: In 25 | values: 26 | - linux 27 | - key: node-role.kubernetes.io/edge 28 | operator: DoesNotExist 29 | containers: 30 | - name: nginx 31 | image: fabedge/net-tool:v0.1.0 32 | ports: 33 | - containerPort: 80 34 | name: http 35 | protocol: TCP 36 | - containerPort: 443 37 | name: https 38 | protocol: TCP 39 | env: 40 | - name: HTTP_PORT 41 | value: "80" 42 | - name: HTTPS_PORT 43 | value: "443" 44 | 45 | --- 46 | 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | name: mysql 51 | labels: 52 | fabedge.io/global-service: "true" 53 | spec: 54 | clusterIP: None 55 | selector: 56 | app: mysql 57 | ports: 58 | - name: http 59 | protocol: TCP 60 | port: 80 61 | targetPort: 80 62 | - name: https 63 | protocol: TCP 64 | port: 443 65 | targetPort: 443 -------------------------------------------------------------------------------- /examples/nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nginx 5 | labels: 6 | app: nginx 7 | spec: 8 | selector: 9 | matchLabels: 10 | app: nginx 11 | replicas: 2 12 | template: 13 | metadata: 14 | labels: 15 | app: nginx 16 | spec: 17 | containers: 18 | - name: nginx 19 | image: fabedge/net-tool:v0.1.0 20 | ports: 21 | - containerPort: 80 22 | name: http 23 | protocol: TCP 24 | - containerPort: 443 25 | name: https 26 | protocol: TCP 27 | env: 28 | - name: HTTP_PORT 29 | value: "80" 30 | - name: HTTPS_PORT 31 | value: "443" 32 | 33 | --- 34 | 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | name: nginx 39 | labels: 40 | fabedge.io/global-service: "true" 41 | spec: 42 | selector: 43 | app: nginx 44 | ports: 45 | - name: http 46 | protocol: TCP 47 | port: 80 48 | targetPort: 80 49 | - name: https 50 | protocol: TCP 51 | port: 443 52 | targetPort: 443 -------------------------------------------------------------------------------- /pkg/agent/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package agent 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/coredns/coredns/coremain" 22 | "github.com/fsnotify/fsnotify" 23 | flag "github.com/spf13/pflag" 24 | "k8s.io/klog/v2" 25 | "k8s.io/klog/v2/klogr" 26 | 27 | "github.com/fabedge/fabedge/pkg/common/about" 28 | ) 29 | 30 | func Execute(cfg *Config) error { 31 | defer klog.Flush() 32 | 33 | // the version flag is added by importing kube-proxy packages, 34 | // but I don't know how that happened 35 | if flag.Lookup("version").Value.String() == "true" { 36 | about.DisplayVersion() 37 | fmt.Println("----------------") 38 | fmt.Println("kube-proxy: 1.22.5") 39 | fmt.Println("----------------") 40 | coremain.Run() 41 | } 42 | 43 | log := klogr.New().WithName("manager") 44 | if err := cfg.Validate(); err != nil { 45 | log.Error(err, "validation failed") 46 | return err 47 | } 48 | 49 | manager, err := cfg.Manager() 50 | if err != nil { 51 | log.Error(err, "failed to create manager") 52 | return err 53 | } 54 | 55 | if err := os.MkdirAll(cfg.CNI.ConfDir, 0777); err != nil { 56 | log.Error(err, "failed to create cni conf dir") 57 | return err 58 | } 59 | 60 | if err := os.MkdirAll(cfg.Workdir, 0777); err != nil { 61 | log.Error(err, "failed to create cni conf dir") 62 | return err 63 | } 64 | 65 | go manager.start() 66 | 67 | err = watchTunnelConfigFile(cfg.TunnelsConfPath, func(event fsnotify.Event) { 68 | log.V(5).Info("tunnels or services config may change", "file", event.Name, "event", event.Op.String()) 69 | manager.notify() 70 | }) 71 | if err != nil { 72 | log.Error(err, "failed to watch tunnels config file", "file", cfg.TunnelsConfPath) 73 | return err 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func watchTunnelConfigFile(tunnelsConfpath string, handleFn func(event fsnotify.Event)) error { 80 | watcher, err := fsnotify.NewWatcher() 81 | if err != nil { 82 | return err 83 | } 84 | defer watcher.Close() 85 | 86 | if err = watcher.Add(tunnelsConfpath); err != nil { 87 | return err 88 | } 89 | 90 | for { 91 | select { 92 | case event, ok := <-watcher.Events: 93 | if !ok { 94 | return nil 95 | } 96 | handleFn(event) 97 | case err, ok := <-watcher.Errors: 98 | if !ok { 99 | return nil 100 | } 101 | return err 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /pkg/agent/route.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package agent 16 | 17 | import ( 18 | "net" 19 | 20 | "k8s.io/apimachinery/pkg/util/sets" 21 | 22 | routeutil "github.com/fabedge/fabedge/pkg/util/route" 23 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 24 | ) 25 | 26 | func addRoutesToPeerViaGateway(gw net.IP, peer Endpoint) error { 27 | return routeutil.EnsureStrongswanRoutes(peer.Subnets, gw) 28 | } 29 | 30 | func addRoutesToPeer(peer Endpoint) error { 31 | var errors []error 32 | for _, nodeSubnet := range peer.NodeSubnets { 33 | gw := net.ParseIP(nodeSubnet) 34 | if gw == nil { 35 | continue 36 | } 37 | 38 | if err := addRoutesToPeerViaGateway(gw, peer); err != nil { 39 | errors = append(errors, err) 40 | } 41 | } 42 | 43 | return utilerrors.NewAggregate(errors) 44 | } 45 | 46 | func delStaleRoutes(peers []Endpoint) error { 47 | dstSet := sets.NewString() 48 | for _, peer := range peers { 49 | dstSet.Insert(peer.Subnets...) 50 | } 51 | 52 | return routeutil.PurgeStrongSwanRoutes(routeutil.NewDstWhitelist(dstSet)) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/agent/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package agent 16 | 17 | import ( 18 | "time" 19 | 20 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 21 | ) 22 | 23 | // CNINetConf describes a network. 24 | type CNINetConf struct { 25 | CNIVersion string `json:"cniVersion,omitempty"` 26 | Name string `json:"name,omitempty"` 27 | Plugins []interface{} `json:"plugins"` 28 | } 29 | 30 | type IPAMConfig struct { 31 | Type string `json:"type"` 32 | Ranges []RangeSet `json:"ranges"` 33 | } 34 | 35 | type RangeSet []Range 36 | type Range struct { 37 | Subnet string `json:"subnet"` 38 | } 39 | 40 | type BridgeConfig struct { 41 | Type string `json:"type"` 42 | Bridge string `json:"bridge"` 43 | IsGateway bool `json:"isGateway,omitempty"` 44 | IsDefaultGateway bool `json:"isDefaultGateway,omitempty"` 45 | ForceAddress bool `json:"forceAddress,omitempty"` 46 | HairpinMode bool `json:"hairpinMode,omitempty"` 47 | MTU int `json:"mtu,omitempty"` 48 | IPAM IPAMConfig `json:"ipam"` 49 | } 50 | 51 | type CapbilitiesConfig struct { 52 | Type string `json:"type"` 53 | Capabilities map[string]bool `json:"capabilities,omitempty"` 54 | } 55 | 56 | type Endpoint struct { 57 | apis.Endpoint 58 | 59 | // IsLocal mark an endpoint from LAN 60 | IsLocal bool 61 | 62 | // ExpireTime works only on local endpoint 63 | ExpireTime time.Time `json:"-"` 64 | } 65 | 66 | type Message struct { 67 | apis.Endpoint 68 | 69 | Token string 70 | } 71 | -------------------------------------------------------------------------------- /pkg/agent/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package agent 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "math" 21 | "time" 22 | 23 | "github.com/avast/retry-go" 24 | 25 | sysctlutil "github.com/fabedge/fabedge/third_party/sysctl" 26 | ) 27 | 28 | var sysctl = sysctlutil.New() 29 | 30 | // retryForever retry fn until it succeed 31 | func retryForever(ctx context.Context, retryableFunc retry.RetryableFunc, onRetryFunc retry.OnRetryFunc) { 32 | _ = retry.Do( 33 | retryableFunc, 34 | retry.Context(ctx), 35 | retry.Attempts(math.MaxUint32), 36 | retry.Delay(5*time.Second), 37 | retry.DelayType(retry.FixedDelay), 38 | retry.LastErrorOnly(true), 39 | retry.OnRetry(onRetryFunc), 40 | ) 41 | } 42 | 43 | // ensureSysctl sets a kernel sysctl to a given numeric value. 44 | func ensureSysctl(name string, newVal int) error { 45 | if oldVal, _ := sysctl.GetSysctl(name); oldVal != newVal { 46 | if err := sysctl.SetSysctl(name, newVal); err != nil { 47 | return fmt.Errorf("can't set sysctl %s to %d: %v", name, newVal, err) 48 | } 49 | } 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /pkg/apis/v1alpha1/cluster_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | type EndpointType string 8 | 9 | const ( 10 | Connector EndpointType = "Connector" 11 | EdgeNode EndpointType = "EdgeNode" 12 | ) 13 | 14 | type Endpoint struct { 15 | ID string `yaml:"id,omitempty" json:"id,omitempty"` 16 | Name string `yaml:"name,omitempty" json:"name,omitempty"` 17 | // public addresses can be IP, DNS 18 | PublicAddresses []string `yaml:"publicAddresses,omitempty" json:"publicAddresses,omitempty"` 19 | // pod subnets 20 | Subnets []string `yaml:"subnets,omitempty" json:"subnets,omitempty"` 21 | // internal IPs of kubernetes node 22 | NodeSubnets []string `yaml:"nodeSubnets,omitempty" json:"nodeSubnets,omitempty"` 23 | // Type of endpoints: Connector or EdgeNode 24 | Type EndpointType `yaml:"type,omitempty" json:"type,omitempty"` 25 | // public UDP port for IKE communication, only used to configure remote_port. Default: 500 26 | Port *uint `yaml:"port,omitempty" json:"port,omitempty"` 27 | } 28 | 29 | type ClusterSpec struct { 30 | // Token is used by child cluster to access root cluster's apiserver 31 | Token string `json:"token,omitempty"` 32 | // CIDRs is supposed to contain cluster-cidr and cluster-service-ip-range of a cluster, 33 | // these are mainly used to create ippools to avoid SNAT in calico environment 34 | CIDRs []string `json:"cidrs,omitempty"` 35 | // Endpoints of connector and exported edge nodes of a cluster 36 | EndPoints []Endpoint `json:"endpoints,omitempty"` 37 | } 38 | 39 | // Cluster is used to represent a cluster's endpoints of connector and edge nodes 40 | // +kubebuilder:object:root=true 41 | // +kubebuilder:resource:scope=Cluster 42 | // +kubebuilder:printcolumn:name="CIDRs",type="string",JSONPath=".spec.cidrs",description="pod and service cidr list of cluster" 43 | // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="How long a community is created" 44 | type Cluster struct { 45 | metav1.TypeMeta `json:",inline"` 46 | metav1.ObjectMeta `json:"metadata,omitempty"` 47 | 48 | Spec ClusterSpec `json:"spec,omitempty"` 49 | } 50 | 51 | // ClusterList contains a list of clusters 52 | // +kubebuilder:object:root=true 53 | type ClusterList struct { 54 | metav1.TypeMeta `json:",inline"` 55 | metav1.ListMeta `json:"metadata,omitempty"` 56 | Items []Cluster `json:"items"` 57 | } 58 | -------------------------------------------------------------------------------- /pkg/apis/v1alpha1/community_types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package v1alpha1 16 | 17 | import ( 18 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 19 | ) 20 | 21 | type CommunitySpec struct { 22 | Members []string `json:"members,omitempty"` 23 | } 24 | 25 | // Community is used to manage a communication unit, it's members 26 | // should be edge nodes 27 | // +kubebuilder:object:root=true 28 | // +kubebuilder:resource:scope=Cluster 29 | // +kubebuilder:printcolumn:name="Members",type="string",JSONPath=".spec.members",description="community members" 30 | // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="How long a community is created" 31 | type Community struct { 32 | metav1.TypeMeta `json:",inline"` 33 | metav1.ObjectMeta `json:"metadata,omitempty"` 34 | 35 | Spec CommunitySpec `json:"spec,omitempty"` 36 | } 37 | 38 | // CommunityList contains a list of Community 39 | // +kubebuilder:object:root=true 40 | type CommunityList struct { 41 | metav1.TypeMeta `json:",inline"` 42 | metav1.ListMeta `json:"metadata,omitempty"` 43 | Items []Community `json:"items"` 44 | } 45 | -------------------------------------------------------------------------------- /pkg/apis/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package v1alpha1 contains API Schema definitions for the community v1alpha1 API group 16 | // +kubebuilder:object:generate:=true 17 | // +groupName=fabedge.io 18 | package v1alpha1 19 | -------------------------------------------------------------------------------- /pkg/apis/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package v1alpha1 16 | 17 | import ( 18 | "k8s.io/apimachinery/pkg/runtime/schema" 19 | "sigs.k8s.io/controller-runtime/pkg/scheme" 20 | ) 21 | 22 | var ( 23 | // SchemeGroupVersion is group version used to register these objects 24 | SchemeGroupVersion = schema.GroupVersion{Group: "fabedge.io", Version: "v1alpha1"} 25 | 26 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 27 | SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} 28 | 29 | // AddToScheme adds the types in this group-version to the given scheme. 30 | AddToScheme = SchemeBuilder.AddToScheme 31 | ) 32 | 33 | func init() { 34 | // We only register manually written functions here. The registration of the 35 | // generated functions takes place in the generated files. The separation 36 | // makes the code compile even when the generated files are missing. 37 | SchemeBuilder.Register( 38 | &Community{}, 39 | &CommunityList{}, 40 | &Cluster{}, 41 | &ClusterList{}, 42 | ) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/common/about/about.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package about 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/spf13/pflag" 22 | ) 23 | 24 | var ( 25 | version = "0.0.0" // semantic version X.Y.Z 26 | gitCommit = "00000000" // sha1 from git 27 | buildTime = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') 28 | showVersion *bool 29 | ) 30 | 31 | func AddFlags(fs *pflag.FlagSet) { 32 | showVersion = fs.Bool("show-version", false, "Display version info") 33 | } 34 | 35 | func DisplayAndExitIfRequested() { 36 | if *showVersion { 37 | DisplayVersion() 38 | os.Exit(0) 39 | } 40 | } 41 | 42 | func DisplayVersion() { 43 | fmt.Printf("Version: %s\nBuildTime: %s\nGitCommit: %s\n", version, buildTime, gitCommit) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/common/constants/default.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package constants 16 | 17 | const ( 18 | KeyPodSubnets = "fabedge.io/subnets" 19 | KeyFabEdgeAPP = "fabedge.io/app" 20 | KeyFabEdgeName = "fabedge.io/name" 21 | KeyCreatedBy = "fabedge.io/created-by" 22 | KeyNode = "fabedge.io/node" 23 | KeyCluster = "fabedge.io/cluster" 24 | KeyNodePublicAddresses = "fabedge.io/node-public-addresses" 25 | KeyPodHash = "fabedge.io/pod-spec-hash" 26 | AppAgent = "fabedge-agent" 27 | AppOperator = "fabedge-operator" 28 | 29 | ConnectorConfigFileName = "tunnels.yaml" 30 | ConnectorConfigName = "connector-config" 31 | ConnectorTLSName = "connector-tls" 32 | ) 33 | 34 | const ( 35 | CNIFlannel = "flannel" 36 | CNICalico = "calico" 37 | ) 38 | 39 | const ( 40 | TableStrongswan = 220 41 | 42 | DefaultMediatorName = "mediator" 43 | ) 44 | -------------------------------------------------------------------------------- /pkg/common/netconf/ipvs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package netconf 16 | 17 | import ( 18 | "fmt" 19 | 20 | corev1 "k8s.io/api/core/v1" 21 | ) 22 | 23 | type VirtualServer struct { 24 | IP string `yaml:"ip,omitempty"` 25 | Port int32 `yaml:"port,omitempty"` 26 | Protocol corev1.Protocol `yaml:"protocol,omitempty"` 27 | 28 | Scheduler string `yaml:"scheduler,omitempty"` 29 | SessionAffinity corev1.ServiceAffinity `yaml:"sessionAffinity,omitempty"` 30 | StickyMaxAgeSeconds int32 `yaml:"stickyMaxAgeSeconds,omitempty"` 31 | 32 | RealServers RealServers `yaml:"realServers,omitempty"` 33 | } 34 | 35 | type RealServer struct { 36 | IP string `yaml:"ip,omitempty"` 37 | Port int32 `yaml:"port,omitempty"` 38 | } 39 | 40 | func (s RealServer) String() string { 41 | return fmt.Sprintf("%s:%d", s.IP, s.Port) 42 | } 43 | 44 | type VirtualServers []VirtualServer 45 | 46 | func (s VirtualServers) Len() int { 47 | return len(s) 48 | } 49 | 50 | func (s VirtualServers) Swap(i, j int) { 51 | s[i], s[j] = s[j], s[i] 52 | } 53 | 54 | func (s VirtualServers) Less(i, j int) bool { 55 | if s[i].IP < s[j].IP { 56 | return true 57 | } 58 | 59 | if s[i].IP == s[j].IP { 60 | if s[i].Port < s[j].Port { 61 | return true 62 | } 63 | } 64 | 65 | return false 66 | } 67 | 68 | type RealServers []RealServer 69 | 70 | func (s RealServers) Len() int { 71 | return len(s) 72 | } 73 | 74 | func (s RealServers) Swap(i, j int) { 75 | s[i], s[j] = s[j], s[i] 76 | } 77 | 78 | func (s RealServers) Less(i, j int) bool { 79 | if s[i].IP < s[j].IP { 80 | return true 81 | } 82 | 83 | if s[i].IP == s[j].IP { 84 | if s[i].Port < s[j].Port { 85 | return true 86 | } 87 | } 88 | 89 | return false 90 | } 91 | -------------------------------------------------------------------------------- /pkg/common/netconf/tunnels.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package netconf 16 | 17 | import ( 18 | "io/ioutil" 19 | 20 | "gopkg.in/yaml.v3" 21 | 22 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 23 | ) 24 | 25 | type NetworkConf struct { 26 | apis.Endpoint `yaml:"-,inline"` 27 | Peers []apis.Endpoint `yaml:"peers,omitempty" json:"peers,omitempty"` 28 | Mediator *apis.Endpoint `yaml:"mediator,omitempty" json:"mediator,omitempty"` 29 | } 30 | 31 | func LoadNetworkConf(path string) (NetworkConf, error) { 32 | var conf NetworkConf 33 | 34 | data, err := ioutil.ReadFile(path) 35 | if err != nil { 36 | return conf, err 37 | } 38 | 39 | return conf, yaml.Unmarshal(data, &conf) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/connector/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package connector 16 | 17 | import ( 18 | "k8s.io/klog/v2" 19 | 20 | "github.com/fabedge/fabedge/pkg/common/about" 21 | ) 22 | 23 | func Execute(cfg *Config) { 24 | defer klog.Flush() 25 | 26 | about.DisplayAndExitIfRequested() 27 | 28 | manger, err := cfg.Manager() 29 | if err != nil { 30 | klog.Fatalf("failed to create Manager: %s", err) 31 | } 32 | 33 | manger.Start() 34 | } 35 | -------------------------------------------------------------------------------- /pkg/connector/watcher.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package connector 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/fsnotify/fsnotify" 21 | "k8s.io/klog/v2" 22 | ) 23 | 24 | func eventOpIs(ent fsnotify.Event, Op fsnotify.Op) bool { 25 | return ent.Op&Op == Op 26 | } 27 | 28 | func (m *Manager) onConfigFileChange(fileToWatch string) { 29 | watcher, err := fsnotify.NewWatcher() 30 | if err != nil { 31 | m.log.Error(err, "failed to initialize fsnotify") 32 | return 33 | } 34 | 35 | defer func() { 36 | if err = watcher.Close(); err != nil { 37 | m.log.Error(err, "failed to close fsnotify watcher") 38 | } 39 | }() 40 | 41 | if err = watcher.Add(fileToWatch); err != nil { 42 | klog.Errorf("failed to monitor %s. Error: %s", fileToWatch, err) 43 | return 44 | } 45 | 46 | for { 47 | select { 48 | case event, _ := <-watcher.Events: 49 | switch { 50 | case eventOpIs(event, fsnotify.Remove): 51 | m.log.Info("file removed, add it back", "event", event) 52 | if err = watcher.Add(fileToWatch); err != nil { 53 | m.log.Error(err, "failed to watch file", "file", fileToWatch) 54 | } 55 | default: 56 | m.log.Info("file changed, start to sync", "event", event) 57 | m.notify() 58 | } 59 | 60 | case err, _ = <-watcher.Errors: 61 | m.log.Error(err, "fsnotify has an error") 62 | // not encounter it so far, hope it can be recovered after some time 63 | time.Sleep(5 * time.Minute) 64 | if err = watcher.Add(fileToWatch); err != nil { 65 | m.log.Error(err, "failed to monitor file", "file", fileToWatch) 66 | return 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /pkg/operator/allocator/allocator_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package allocator_test 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestAllocator(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Allocator Suite") 27 | } 28 | -------------------------------------------------------------------------------- /pkg/operator/apiserver/apiserver_suite_test.go: -------------------------------------------------------------------------------- 1 | package apiserver_test 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | "k8s.io/client-go/kubernetes/scheme" 10 | "k8s.io/client-go/rest" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | "sigs.k8s.io/controller-runtime/pkg/envtest" 13 | 14 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 15 | testutil "github.com/fabedge/fabedge/pkg/util/test" 16 | ) 17 | 18 | var cfg *rest.Config 19 | var k8sClient client.Client 20 | 21 | // envtest provide a api server which has some differences from real environments, 22 | // read https://book.kubebuilder.io/reference/envtest.html#testing-considerations 23 | var testEnv *envtest.Environment 24 | 25 | var _ = BeforeSuite(func(done Done) { 26 | testutil.SetupLogger() 27 | 28 | By("starting test environment") 29 | var err error 30 | testEnv, cfg, k8sClient, err = testutil.StartTestEnvWithCRD( 31 | []string{filepath.Join("..", "..", "..", "deploy", "crds")}, 32 | ) 33 | Expect(err).ToNot(HaveOccurred()) 34 | 35 | _ = apis.AddToScheme(scheme.Scheme) 36 | 37 | close(done) 38 | }, 60) 39 | 40 | var _ = AfterSuite(func() { 41 | By("tearing down the test environment") 42 | err := testEnv.Stop() 43 | Expect(err).ShouldNot(HaveOccurred()) 44 | }) 45 | 46 | func TestAPIServer(t *testing.T) { 47 | RegisterFailHandler(Fail) 48 | RunSpecs(t, "APIServer Suite") 49 | } 50 | -------------------------------------------------------------------------------- /pkg/operator/client/errors.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | type HttpError struct { 9 | Response *http.Response 10 | Message string 11 | } 12 | 13 | func (e HttpError) Error() string { 14 | return fmt.Sprintf("Status Code: %d. Message: %s", e.Response.StatusCode, e.Message) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/operator/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package operator 16 | 17 | import ( 18 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 19 | "k8s.io/client-go/kubernetes/scheme" 20 | "k8s.io/klog/v2" 21 | "k8s.io/klog/v2/klogr" 22 | 23 | "github.com/fabedge/fabedge/pkg/common/about" 24 | "github.com/fabedge/fabedge/third_party/calicoapi" 25 | ) 26 | 27 | var log = klogr.New().WithName("agent") 28 | 29 | func init() { 30 | _ = apis.AddToScheme(scheme.Scheme) 31 | _ = calicoapi.AddToScheme(scheme.Scheme) 32 | } 33 | 34 | func Execute(opts *Options) error { 35 | defer klog.Flush() 36 | 37 | about.DisplayAndExitIfRequested() 38 | 39 | opts.ExtractAgentArgumentMap() 40 | 41 | if err := opts.Validate(); err != nil { 42 | log.Error(err, "invalid arguments found") 43 | return err 44 | } 45 | 46 | if err := opts.Complete(); err != nil { 47 | return err 48 | } 49 | 50 | return opts.RunManager() 51 | } 52 | -------------------------------------------------------------------------------- /pkg/operator/controllers/cluster/cluster_suite_test.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | "k8s.io/client-go/kubernetes/scheme" 10 | "k8s.io/client-go/rest" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | "sigs.k8s.io/controller-runtime/pkg/envtest" 13 | 14 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 15 | testutil "github.com/fabedge/fabedge/pkg/util/test" 16 | ) 17 | 18 | var cfg *rest.Config 19 | var k8sClient client.Client 20 | 21 | // envtest provide a api server which has some differences from real environments, 22 | // read https://book.kubebuilder.io/reference/envtest.html#testing-considerations 23 | var testEnv *envtest.Environment 24 | 25 | func TestCluster(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecs(t, "Cluster Suite") 28 | } 29 | 30 | var _ = BeforeSuite(func(done Done) { 31 | testutil.SetupLogger() 32 | 33 | By("starting test environment") 34 | var err error 35 | testEnv, cfg, k8sClient, err = testutil.StartTestEnvWithCRD( 36 | []string{filepath.Join("..", "..", "..", "..", "deploy", "crds")}, 37 | ) 38 | Expect(err).ToNot(HaveOccurred()) 39 | 40 | _ = apis.AddToScheme(scheme.Scheme) 41 | 42 | close(done) 43 | }, 60) 44 | 45 | var _ = AfterSuite(func() { 46 | By("tearing down the test environment") 47 | err := testEnv.Stop() 48 | Expect(err).ShouldNot(HaveOccurred()) 49 | }) 50 | -------------------------------------------------------------------------------- /pkg/operator/controllers/community/community_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package community 16 | 17 | import ( 18 | "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 19 | "path/filepath" 20 | "testing" 21 | 22 | . "github.com/onsi/ginkgo" 23 | . "github.com/onsi/gomega" 24 | "k8s.io/client-go/kubernetes/scheme" 25 | "k8s.io/client-go/rest" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | "sigs.k8s.io/controller-runtime/pkg/envtest" 28 | 29 | testutil "github.com/fabedge/fabedge/pkg/util/test" 30 | ) 31 | 32 | var cfg *rest.Config 33 | var k8sClient client.Client 34 | 35 | // envtest provide a api server which has some differences from real environments, 36 | // read https://book.kubebuilder.io/reference/envtest.html#testing-considerations 37 | var testEnv *envtest.Environment 38 | 39 | func TestCommunity(t *testing.T) { 40 | RegisterFailHandler(Fail) 41 | RunSpecs(t, "Community Suite") 42 | } 43 | 44 | var _ = BeforeSuite(func(done Done) { 45 | testutil.SetupLogger() 46 | 47 | By("starting test environment") 48 | var err error 49 | testEnv, cfg, k8sClient, err = testutil.StartTestEnvWithCRD( 50 | []string{filepath.Join("..", "..", "..", "..", "deploy", "crds")}, 51 | ) 52 | Expect(err).ToNot(HaveOccurred()) 53 | 54 | _ = v1alpha1.AddToScheme(scheme.Scheme) 55 | 56 | close(done) 57 | }, 60) 58 | 59 | var _ = AfterSuite(func() { 60 | By("tearing down the test environment") 61 | err := testEnv.Stop() 62 | Expect(err).ShouldNot(HaveOccurred()) 63 | }) 64 | -------------------------------------------------------------------------------- /pkg/operator/controllers/connector/connector_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package connector 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | "k8s.io/client-go/rest" 23 | "sigs.k8s.io/controller-runtime/pkg/client" 24 | "sigs.k8s.io/controller-runtime/pkg/envtest" 25 | 26 | nodeutil "github.com/fabedge/fabedge/pkg/util/node" 27 | testutil "github.com/fabedge/fabedge/pkg/util/test" 28 | ) 29 | 30 | var cfg *rest.Config 31 | var k8sClient client.Client 32 | var testEnv *envtest.Environment 33 | var getNodeName = testutil.GenerateGetNameFunc("node") 34 | var getEdgeName = testutil.GenerateGetNameFunc("edge-node") 35 | var edgeLabels = map[string]string{ 36 | "edge": "", 37 | } 38 | 39 | func TestConnector(t *testing.T) { 40 | RegisterFailHandler(Fail) 41 | RunSpecs(t, "Endpoint Suite") 42 | } 43 | 44 | var _ = BeforeSuite(func(done Done) { 45 | testutil.SetupLogger() 46 | nodeutil.SetEdgeNodeLabels(edgeLabels) 47 | 48 | By("starting test environment") 49 | var err error 50 | testEnv, cfg, k8sClient, err = testutil.StartTestEnv() 51 | Expect(err).NotTo(HaveOccurred()) 52 | 53 | close(done) 54 | }, 60) 55 | 56 | var _ = AfterSuite(func() { 57 | By("tearing down the test environment") 58 | Expect(testEnv.Stop()).ShouldNot(HaveOccurred()) 59 | }) 60 | -------------------------------------------------------------------------------- /pkg/operator/controllers/ipamblockmonitor/ipam_block_monitor_test.go: -------------------------------------------------------------------------------- 1 | package ipamblockmonitor 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/klog/v2/klogr" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 13 | 14 | "github.com/fabedge/fabedge/pkg/operator/types" 15 | "github.com/fabedge/fabedge/third_party/calicoapi" 16 | ) 17 | 18 | var _ = Describe("IPAMBlockMonitor", func() { 19 | var ( 20 | monitor *ipamBlockMonitor 21 | ) 22 | 23 | BeforeEach(func() { 24 | monitor = &ipamBlockMonitor{ 25 | Config: Config{ 26 | Store: types.NewPodCIDRStore(), 27 | }, 28 | client: k8sClient, 29 | log: klogr.New(), 30 | } 31 | }) 32 | 33 | Context("A IPAM block is bound to a node", func() { 34 | var ( 35 | nodeName string 36 | block calicoapi.IPAMBlock 37 | request reconcile.Request 38 | ) 39 | 40 | BeforeEach(func() { 41 | nodeName = "node1" 42 | affinity := fmt.Sprintf("host:%s", nodeName) 43 | 44 | block = calicoapi.IPAMBlock{ 45 | ObjectMeta: metav1.ObjectMeta{ 46 | Name: "10-233-70-0-24", 47 | }, 48 | Spec: calicoapi.IPAMBlockSpec{ 49 | CIDR: "10.233.70.0/24", 50 | Affinity: &affinity, 51 | Unallocated: []int{}, 52 | Allocations: []*int{}, 53 | Attributes: []calicoapi.AllocationAttribute{}, 54 | }, 55 | } 56 | 57 | request = reconcile.Request{ 58 | NamespacedName: client.ObjectKey{ 59 | Name: block.Name, 60 | }, 61 | } 62 | 63 | Expect(k8sClient.Create(context.Background(), &block)).To(Succeed()) 64 | 65 | _, err := monitor.Reconcile(context.Background(), request) 66 | Expect(err).ShouldNot(HaveOccurred()) 67 | }) 68 | 69 | AfterEach(func() { 70 | _ = k8sClient.Delete(context.Background(), &block) 71 | }) 72 | 73 | It("should record it to store", func() { 74 | Expect(monitor.Store.Get(nodeName)).To(ConsistOf(block.Spec.CIDR)) 75 | }) 76 | 77 | It("delete record when IPAMBlock's delete field is true", func() { 78 | block.Spec.Deleted = true 79 | Expect(k8sClient.Update(context.Background(), &block)).To(Succeed()) 80 | 81 | _, err := monitor.Reconcile(context.Background(), request) 82 | Expect(err).ShouldNot(HaveOccurred()) 83 | 84 | Expect(monitor.Store.Get(nodeName)).To(ConsistOf()) 85 | }) 86 | 87 | It("delete record when IPAMBlock is deleted", func() { 88 | Expect(k8sClient.Delete(context.Background(), &block)).To(Succeed()) 89 | 90 | _, err := monitor.Reconcile(context.Background(), request) 91 | Expect(err).ShouldNot(HaveOccurred()) 92 | 93 | Expect(monitor.Store.Get(nodeName)).To(ConsistOf()) 94 | }) 95 | }) 96 | }) 97 | -------------------------------------------------------------------------------- /pkg/operator/controllers/ipamblockmonitor/ipamblockmonitor_suite_test.go: -------------------------------------------------------------------------------- 1 | package ipamblockmonitor 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | "k8s.io/client-go/kubernetes/scheme" 10 | "k8s.io/client-go/rest" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | "sigs.k8s.io/controller-runtime/pkg/envtest" 13 | 14 | testutil "github.com/fabedge/fabedge/pkg/util/test" 15 | "github.com/fabedge/fabedge/third_party/calicoapi" 16 | ) 17 | 18 | var cfg *rest.Config 19 | var k8sClient client.Client 20 | var testEnv *envtest.Environment 21 | 22 | func TestIpamblockmonitor(t *testing.T) { 23 | RegisterFailHandler(Fail) 24 | RunSpecs(t, "Ipamblockmonitor Suite") 25 | } 26 | 27 | var _ = BeforeSuite(func(done Done) { 28 | testutil.SetupLogger() 29 | 30 | By("starting test environment") 31 | var err error 32 | testEnv, cfg, k8sClient, err = testutil.StartTestEnvWithCRD( 33 | []string{filepath.Join("..", "..", "..", "..", "third_party", "calicoapi", "crd")}, 34 | ) 35 | Expect(err).NotTo(HaveOccurred()) 36 | 37 | _ = calicoapi.AddToScheme(scheme.Scheme) 38 | 39 | close(done) 40 | }, 60) 41 | 42 | var _ = AfterSuite(func() { 43 | By("tearing down the test environment") 44 | Expect(testEnv.Stop()).ShouldNot(HaveOccurred()) 45 | }) 46 | -------------------------------------------------------------------------------- /pkg/operator/routines/endpoints_test.go: -------------------------------------------------------------------------------- 1 | package routines 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | . "github.com/onsi/ginkgo" 9 | . "github.com/onsi/gomega" 10 | 11 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 12 | "github.com/fabedge/fabedge/pkg/operator/apiserver" 13 | storepkg "github.com/fabedge/fabedge/pkg/operator/store" 14 | ) 15 | 16 | var _ = Describe("LoadEndpointsAndCommunities", func() { 17 | It("can load endpoints and communities", func() { 18 | e1 := apis.Endpoint{ 19 | Name: "cluster1.connector", 20 | PublicAddresses: []string{"cluster1"}, 21 | Subnets: []string{"2.2.2.0/24"}, 22 | NodeSubnets: []string{"10.10.0.1/32"}, 23 | } 24 | e2 := apis.Endpoint{ 25 | Name: "cluster2.connector", 26 | PublicAddresses: []string{"cluster2.connector"}, 27 | Subnets: []string{"192.168.1.0/24"}, 28 | NodeSubnets: []string{"192.168.1.1/32"}, 29 | } 30 | e3 := apis.Endpoint{ 31 | Name: "cluster2.edge", 32 | PublicAddresses: []string{"cluster2"}, 33 | Subnets: []string{"192.168.2.0/24"}, 34 | NodeSubnets: []string{"192.168.1.2/32"}, 35 | } 36 | 37 | var ec = apiserver.EndpointsAndCommunity{ 38 | Communities: map[string][]string{ 39 | "connectors": {e1.Name, e2.Name}, 40 | }, 41 | Endpoints: []apis.Endpoint{e1, e2}, 42 | } 43 | 44 | store := storepkg.NewStore() 45 | var lock sync.RWMutex 46 | getEndpointsAndCommunities := func() (apiserver.EndpointsAndCommunity, error) { 47 | lock.RLock() 48 | defer lock.RUnlock() 49 | 50 | return ec, nil 51 | } 52 | 53 | loader := LoadEndpointsAndCommunities(10*time.Millisecond, store, getEndpointsAndCommunities) 54 | ctx, cancel := context.WithCancel(context.Background()) 55 | defer cancel() 56 | 57 | go loader.Start(ctx) 58 | 59 | time.Sleep(50 * time.Millisecond) 60 | 61 | community, _ := store.GetCommunity("connectors") 62 | Expect(community.Members.List()).Should(ConsistOf(e1.Name, e2.Name)) 63 | 64 | endpoint, _ := store.GetEndpoint(e1.Name) 65 | Expect(endpoint).Should(Equal(e1)) 66 | 67 | endpoint, _ = store.GetEndpoint(e2.Name) 68 | Expect(endpoint).Should(Equal(e2)) 69 | 70 | By("change endpoints and communities") 71 | lock.Lock() 72 | ec.Communities = map[string][]string{ 73 | "mixed": {e1.Name, e3.Name}, 74 | "void": {"edge1", "edge2"}, 75 | } 76 | ec.Endpoints = []apis.Endpoint{e1, e3} 77 | lock.Unlock() 78 | 79 | time.Sleep(50 * time.Millisecond) 80 | 81 | community, ok := store.GetCommunity("connectors") 82 | Expect(ok).Should(BeFalse()) 83 | 84 | community, ok = store.GetCommunity("void") 85 | Expect(community.Members.List()).Should(ConsistOf("edge1", "edge2")) 86 | 87 | community, ok = store.GetCommunity("mixed") 88 | Expect(ok).Should(BeTrue()) 89 | Expect(community.Members.List()).Should(ConsistOf(e1.Name, e3.Name)) 90 | 91 | endpoint, _ = store.GetEndpoint(e1.Name) 92 | Expect(endpoint).Should(Equal(e1)) 93 | 94 | endpoint, _ = store.GetEndpoint(e3.Name) 95 | Expect(endpoint).Should(Equal(e3)) 96 | 97 | _, ok = store.GetEndpoint(e2.Name) 98 | Expect(ok).Should(BeFalse()) 99 | }) 100 | }) 101 | -------------------------------------------------------------------------------- /pkg/operator/routines/local_cluster_reporter.go: -------------------------------------------------------------------------------- 1 | package routines 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "time" 7 | 8 | "github.com/go-logr/logr" 9 | "k8s.io/apimachinery/pkg/api/errors" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | 13 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 14 | "github.com/fabedge/fabedge/pkg/operator/types" 15 | ) 16 | 17 | // LocalClusterReporter create or update cluster data in the cluster where 18 | // controller is running 19 | type LocalClusterReporter struct { 20 | Cluster string 21 | ClusterCIDRs []string 22 | GetConnector types.EndpointGetter 23 | SyncInterval time.Duration 24 | Client client.Client 25 | Log logr.Logger 26 | } 27 | 28 | func (ctl *LocalClusterReporter) Start(ctx context.Context) error { 29 | tick := time.NewTicker(ctl.SyncInterval) 30 | 31 | ctl.report(ctx) 32 | for { 33 | select { 34 | case <-tick.C: 35 | ctl.report(ctx) 36 | case <-ctx.Done(): 37 | return nil 38 | } 39 | } 40 | } 41 | 42 | func (ctl *LocalClusterReporter) report(ctx context.Context) { 43 | connector := ctl.GetConnector() 44 | 45 | cluster := apis.Cluster{} 46 | err := ctl.Client.Get(ctx, client.ObjectKey{Name: ctl.Cluster}, &cluster) 47 | if err != nil { 48 | if !errors.IsNotFound(err) { 49 | ctl.Log.Error(err, "failed to get cluster") 50 | return 51 | } 52 | 53 | cluster = apis.Cluster{ 54 | ObjectMeta: metav1.ObjectMeta{ 55 | Name: ctl.Cluster, 56 | }, 57 | Spec: apis.ClusterSpec{ 58 | Token: "", 59 | CIDRs: ctl.ClusterCIDRs, 60 | EndPoints: []apis.Endpoint{ 61 | connector, 62 | }, 63 | }, 64 | } 65 | 66 | if err = ctl.Client.Create(ctx, &cluster); err != nil { 67 | ctl.Log.Error(err, "failed to create cluster") 68 | } 69 | return 70 | } 71 | 72 | endpoints := []apis.Endpoint{ 73 | connector, 74 | } 75 | 76 | if reflect.DeepEqual(endpoints, cluster.Spec.EndPoints) && reflect.DeepEqual(ctl.ClusterCIDRs, cluster.Spec.CIDRs) { 77 | return 78 | } 79 | 80 | cluster.Spec.EndPoints = endpoints 81 | cluster.Spec.CIDRs = ctl.ClusterCIDRs 82 | if err = ctl.Client.Update(ctx, &cluster); err != nil { 83 | ctl.Log.Error(err, "failed to update cluster") 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pkg/operator/routines/local_cluster_reporter_test.go: -------------------------------------------------------------------------------- 1 | package routines 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | "k8s.io/klog/v2/klogr" 10 | "sigs.k8s.io/controller-runtime/pkg/client" 11 | 12 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 13 | ) 14 | 15 | var _ = Describe("LocalClusterReporter", func() { 16 | It("should create or update cluster", func() { 17 | connector := apis.Endpoint{ 18 | ID: "connector", 19 | Name: "connector", 20 | PublicAddresses: []string{"10.10.10.10"}, 21 | Subnets: []string{"2.2.0.0/2"}, 22 | NodeSubnets: []string{"192.168.1.1"}, 23 | } 24 | 25 | reporter := &LocalClusterReporter{ 26 | Cluster: "test", 27 | ClusterCIDRs: []string{"10.100.0.0/16"}, 28 | Client: k8sClient, 29 | SyncInterval: time.Second, 30 | Log: klogr.New(), 31 | GetConnector: func() apis.Endpoint { 32 | return connector 33 | }, 34 | } 35 | 36 | By("first report") 37 | reporter.report(context.Background()) 38 | 39 | By("check if cluster is created") 40 | var cluster apis.Cluster 41 | err := k8sClient.Get(context.Background(), client.ObjectKey{Name: reporter.Cluster}, &cluster) 42 | Expect(err).Should(BeNil()) 43 | Expect(cluster.Spec.EndPoints[0]).Should(Equal(connector)) 44 | Expect(cluster.Spec.CIDRs).Should(Equal(reporter.ClusterCIDRs)) 45 | 46 | By("update connector and report again") 47 | connector.PublicAddresses = []string{"10.10.1.1"} 48 | reporter.report(context.Background()) 49 | 50 | By("check if cluster's endpoints is updated") 51 | err = k8sClient.Get(context.Background(), client.ObjectKey{Name: reporter.Cluster}, &cluster) 52 | Expect(err).Should(BeNil()) 53 | Expect(cluster.Spec.EndPoints[0]).Should(Equal(connector)) 54 | Expect(cluster.Spec.CIDRs).Should(Equal(reporter.ClusterCIDRs)) 55 | 56 | By("update cluster cidrs and report again") 57 | reporter.ClusterCIDRs = []string{"10.100.0.0/18"} 58 | reporter.report(context.Background()) 59 | 60 | By("check if cluster's CIDRs is updated") 61 | err = k8sClient.Get(context.Background(), client.ObjectKey{Name: reporter.Cluster}, &cluster) 62 | Expect(err).Should(BeNil()) 63 | Expect(cluster.Spec.EndPoints[0]).Should(Equal(connector)) 64 | Expect(cluster.Spec.CIDRs).Should(Equal(reporter.ClusterCIDRs)) 65 | }) 66 | }) 67 | -------------------------------------------------------------------------------- /pkg/operator/routines/periodic_runnable.go: -------------------------------------------------------------------------------- 1 | package routines 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "sigs.k8s.io/controller-runtime/pkg/manager" 8 | ) 9 | 10 | func Periodic(interval time.Duration, fn func(ctx context.Context)) manager.Runnable { 11 | return manager.RunnableFunc(func(ctx context.Context) error { 12 | tick := time.NewTicker(interval) 13 | 14 | fn(ctx) 15 | for { 16 | select { 17 | case <-tick.C: 18 | fn(ctx) 19 | case <-ctx.Done(): 20 | return nil 21 | } 22 | } 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/operator/routines/periodic_runnable_test.go: -------------------------------------------------------------------------------- 1 | package routines 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | "time" 7 | 8 | . "github.com/onsi/ginkgo" 9 | . "github.com/onsi/gomega" 10 | ) 11 | 12 | var _ = Describe("PeriodicRunnable", func() { 13 | It("should be able to execute specified function periodically", func() { 14 | counter := int32(0) 15 | fn := func(ctx context.Context) { 16 | counter += 0 17 | atomic.AddInt32(&counter, 1) 18 | } 19 | 20 | runnable := Periodic(10*time.Millisecond, fn) 21 | ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) 22 | defer cancel() 23 | 24 | Expect(runnable.Start(ctx)).Should(Succeed()) 25 | 26 | Expect(counter).Should(BeNumerically(">=", 10)) 27 | }) 28 | }) 29 | -------------------------------------------------------------------------------- /pkg/operator/routines/routines_suite_test.go: -------------------------------------------------------------------------------- 1 | package routines 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | "k8s.io/client-go/kubernetes/scheme" 10 | "k8s.io/client-go/rest" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | "sigs.k8s.io/controller-runtime/pkg/envtest" 13 | 14 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 15 | testutil "github.com/fabedge/fabedge/pkg/util/test" 16 | "github.com/fabedge/fabedge/third_party/calicoapi" 17 | ) 18 | 19 | var cfg *rest.Config 20 | var k8sClient client.Client 21 | 22 | // envtest provide an api server which has some differences from real environments, 23 | // read https://book.kubebuilder.io/reference/envtest.html#testing-considerations 24 | var testEnv *envtest.Environment 25 | 26 | func TestRoutines(t *testing.T) { 27 | RegisterFailHandler(Fail) 28 | RunSpecs(t, "Routines Suite") 29 | } 30 | 31 | var _ = BeforeSuite(func(done Done) { 32 | testutil.SetupLogger() 33 | 34 | By("starting test environment") 35 | var err error 36 | testEnv, cfg, k8sClient, err = testutil.StartTestEnvWithCRD( 37 | []string{ 38 | filepath.Join("..", "..", "..", "deploy", "crds"), 39 | filepath.Join("..", "..", "..", "third_party", "calicoapi", "crd"), 40 | }, 41 | ) 42 | Expect(err).ToNot(HaveOccurred()) 43 | 44 | Expect(apis.AddToScheme(scheme.Scheme)).Should(Succeed()) 45 | Expect(calicoapi.AddToScheme(scheme.Scheme)).Should(Succeed()) 46 | 47 | close(done) 48 | }, 60) 49 | 50 | var _ = AfterSuite(func() { 51 | By("tearing down the test environment") 52 | err := testEnv.Stop() 53 | Expect(err).ShouldNot(HaveOccurred()) 54 | }) 55 | -------------------------------------------------------------------------------- /pkg/operator/store/store_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package store_test 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestStore(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Store Suite") 27 | } 28 | -------------------------------------------------------------------------------- /pkg/operator/types/agent_argument_map.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "k8s.io/apimachinery/pkg/util/sets" 9 | ) 10 | 11 | // AgentArgumentMap is used to manage arguments of agent pod 12 | type AgentArgumentMap map[string]string 13 | 14 | func NewAgentArgumentMap() AgentArgumentMap { 15 | return make(AgentArgumentMap) 16 | } 17 | 18 | // NewAgentArgumentMapFromEnv extract arguments of agent pod 19 | // from ENV, each agent argument should be configured like: 20 | // 21 | // AGENT_ARG_ENABLE_IPAM=true 22 | // 23 | // The return value is a map, each key is the ENV variable name but with 24 | // prefix 'AGENT_ARG_' stripped and the key is also lowered. 25 | func NewAgentArgumentMapFromEnv() AgentArgumentMap { 26 | const prefix = "agent-arg-" 27 | 28 | argMap := make(AgentArgumentMap) 29 | for _, line := range os.Environ() { 30 | parts := strings.SplitN(line, "=", 2) 31 | 32 | // lower variable name and replace '_' with '-' 33 | name := strings.ToLower(parts[0]) 34 | name = strings.ReplaceAll(name, "_", "-") 35 | 36 | if !strings.HasPrefix(name, prefix) { 37 | continue 38 | } 39 | 40 | name = name[len(prefix):] 41 | value := "" 42 | if len(parts) > 1 { 43 | value = parts[1] 44 | } 45 | 46 | argMap[name] = value 47 | } 48 | 49 | return argMap 50 | } 51 | 52 | func (argMap AgentArgumentMap) Set(name, value string) { 53 | argMap[name] = value 54 | } 55 | 56 | func (argMap AgentArgumentMap) Get(name string) string { 57 | return argMap[name] 58 | } 59 | 60 | func (argMap AgentArgumentMap) Delete(name string) { 61 | delete(argMap, name) 62 | } 63 | 64 | func (argMap AgentArgumentMap) HasKey(name string) bool { 65 | _, ok := argMap[name] 66 | return ok 67 | } 68 | 69 | func (argMap AgentArgumentMap) IsProxyEnabled() bool { 70 | return argMap.isTrue("enable-proxy") 71 | } 72 | 73 | func (argMap AgentArgumentMap) IsDNSEnabled() bool { 74 | return argMap.isTrue("enable-dns") 75 | } 76 | 77 | func (argMap AgentArgumentMap) IsDNSProbeEnabled() bool { 78 | return argMap.isTrue("dns-probe") 79 | } 80 | 81 | func (argMap AgentArgumentMap) isTrue(name string) bool { 82 | return argMap[name] == "true" 83 | } 84 | 85 | // ArgumentArray translate argument map into sorted argument array 86 | // All arguments are sorted except log level, `agent` doesn't have 87 | // an option named log-level, instead it has a 'v' option which is used 88 | // to configure log level. ArgumentArray will put 'v' option at the end of argument array. 89 | func (argMap AgentArgumentMap) ArgumentArray() []string { 90 | const nameLogLevel = "log-level" 91 | 92 | nameSet := sets.NewString() 93 | for name := range argMap { 94 | nameSet.Insert(name) 95 | } 96 | 97 | args := make([]string, 0, len(argMap)+1) 98 | for _, name := range nameSet.List() { 99 | if name != nameLogLevel { 100 | args = append(args, fmt.Sprintf("--%s=%s", name, argMap[name])) 101 | } 102 | } 103 | 104 | if value, ok := argMap[nameLogLevel]; ok { 105 | args = append(args, fmt.Sprintf("--v=%s", value)) 106 | } 107 | 108 | return args 109 | } 110 | -------------------------------------------------------------------------------- /pkg/operator/types/cluster_cidrs_map.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "sync" 4 | 5 | type ClusterCIDRsMap struct { 6 | lock sync.RWMutex 7 | cidrsByName map[string][]string 8 | 9 | readonlyCopy map[string][]string 10 | } 11 | 12 | func NewClusterCIDRsMap() *ClusterCIDRsMap { 13 | return &ClusterCIDRsMap{ 14 | cidrsByName: make(map[string][]string), 15 | } 16 | } 17 | 18 | func (m *ClusterCIDRsMap) Set(name string, cidrs []string) { 19 | m.lock.Lock() 20 | defer m.lock.Unlock() 21 | 22 | m.cidrsByName[name] = cidrs 23 | m.readonlyCopy = nil 24 | } 25 | 26 | func (m *ClusterCIDRsMap) Get(name string) ([]string, bool) { 27 | m.lock.RLock() 28 | defer m.lock.RUnlock() 29 | 30 | cidrs, found := m.cidrsByName[name] 31 | return cidrs, found 32 | } 33 | 34 | func (m *ClusterCIDRsMap) Delete(name string) { 35 | m.lock.Lock() 36 | defer m.lock.Unlock() 37 | 38 | if _, found := m.cidrsByName[name]; !found { 39 | return 40 | } 41 | 42 | m.readonlyCopy = nil 43 | delete(m.cidrsByName, name) 44 | } 45 | 46 | // GetCopy return a copy of inner data, the returned data should not be changed 47 | func (m *ClusterCIDRsMap) GetCopy() map[string][]string { 48 | m.lock.RLock() 49 | cp := m.readonlyCopy 50 | m.lock.RUnlock() 51 | 52 | if cp != nil { 53 | return cp 54 | } 55 | 56 | m.lock.Lock() 57 | defer m.lock.Unlock() 58 | 59 | if m.readonlyCopy != nil { 60 | return m.readonlyCopy 61 | } 62 | 63 | cp = make(map[string][]string) 64 | for key, value := range m.cidrsByName { 65 | cp[key] = value 66 | } 67 | m.readonlyCopy = cp 68 | 69 | return cp 70 | } 71 | -------------------------------------------------------------------------------- /pkg/operator/types/cluster_cidrs_map_test.go: -------------------------------------------------------------------------------- 1 | package types_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "github.com/fabedge/fabedge/pkg/operator/types" 8 | ) 9 | 10 | var _ = Describe("ClusterCIDRsMap", func() { 11 | It("can set, get and delete CIDRs by cluster name", func() { 12 | cidrMap := types.NewClusterCIDRsMap() 13 | 14 | cidrMap.Set("beijing", []string{"192.168.0.0/18"}) 15 | cidrs, found := cidrMap.Get("beijing") 16 | Expect(found).To(BeTrue()) 17 | Expect(cidrs).To(ConsistOf("192.168.0.0/18")) 18 | 19 | cidrMap.Delete("beijing") 20 | _, found = cidrMap.Get("beijing") 21 | Expect(found).To(BeFalse()) 22 | }) 23 | 24 | It("GetCopy can get return an copy from inner data", func() { 25 | cidrMap := types.NewClusterCIDRsMap() 26 | 27 | cidrMap.Set("beijing", []string{"192.168.0.0/18"}) 28 | cp := cidrMap.GetCopy() 29 | 30 | Expect(len(cp)).To(Equal(1)) 31 | Expect(cp).To(HaveKeyWithValue("beijing", []string{"192.168.0.0/18"})) 32 | 33 | cidrMap.Set("shanghai", []string{"10.10.0.0/18"}) 34 | Expect(len(cp)).To(Equal(1)) 35 | 36 | cp2 := cidrMap.GetCopy() 37 | Expect(cp).NotTo(Equal(cp2)) 38 | Expect(cp2).To(HaveKeyWithValue("beijing", []string{"192.168.0.0/18"})) 39 | Expect(cp2).To(HaveKeyWithValue("shanghai", []string{"10.10.0.0/18"})) 40 | 41 | cp3 := cidrMap.GetCopy() 42 | Expect(cp2).To(Equal(cp3)) 43 | }) 44 | }) 45 | -------------------------------------------------------------------------------- /pkg/operator/types/community.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "k8s.io/apimachinery/pkg/util/sets" 19 | ) 20 | 21 | type Community struct { 22 | Name string 23 | Members sets.String 24 | } 25 | -------------------------------------------------------------------------------- /pkg/operator/types/funcs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | corev1 "k8s.io/api/core/v1" 22 | 23 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 24 | "github.com/fabedge/fabedge/pkg/common/constants" 25 | ) 26 | 27 | type GetIDFunc func(nodeName string) string 28 | type GetNameFunc func(nodeName string) string 29 | type NewEndpointFunc func(node corev1.Node) apis.Endpoint 30 | type PodCIDRsGetter func(node corev1.Node) []string 31 | type EndpointGetter func() apis.Endpoint 32 | type GetClusterCIDRInfo func() (map[string][]string, error) 33 | 34 | func NewEndpointFuncs(namePrefix, idFormat string, getPodCIDRs PodCIDRsGetter) (GetNameFunc, GetIDFunc, NewEndpointFunc) { 35 | getName := func(name string) string { 36 | return fmt.Sprintf("%s.%s", namePrefix, name) 37 | } 38 | 39 | getID := func(name string) string { 40 | return strings.ReplaceAll(idFormat, "{node}", getName(name)) 41 | } 42 | 43 | newEndpoint := func(node corev1.Node) apis.Endpoint { 44 | var nodeSubnets []string 45 | for _, addr := range node.Status.Addresses { 46 | if addr.Type == corev1.NodeInternalIP { 47 | nodeSubnets = append(nodeSubnets, addr.Address) 48 | } 49 | } 50 | 51 | publicAddresses := getPublicAddressesFromAnnotations(node) 52 | if len(publicAddresses) == 0 { 53 | publicAddresses = nodeSubnets 54 | } 55 | 56 | if node.Name == "" { 57 | return apis.Endpoint{} 58 | } 59 | 60 | return apis.Endpoint{ 61 | ID: getID(node.Name), 62 | Name: getName(node.Name), 63 | PublicAddresses: publicAddresses, 64 | Subnets: getPodCIDRs(node), 65 | NodeSubnets: nodeSubnets, 66 | Type: apis.EdgeNode, 67 | } 68 | } 69 | 70 | return getName, getID, newEndpoint 71 | } 72 | 73 | func getPublicAddressesFromAnnotations(node corev1.Node) []string { 74 | if len(node.Annotations) == 0 { 75 | return nil 76 | } 77 | 78 | publicAddresses := node.Annotations[constants.KeyNodePublicAddresses] 79 | if len(publicAddresses) == 0 { 80 | return nil 81 | } 82 | 83 | return strings.Split(publicAddresses, ",") 84 | } 85 | -------------------------------------------------------------------------------- /pkg/operator/types/funcs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types_test 16 | 17 | import ( 18 | . "github.com/onsi/ginkgo" 19 | . "github.com/onsi/gomega" 20 | corev1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | 23 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 24 | "github.com/fabedge/fabedge/pkg/common/constants" 25 | "github.com/fabedge/fabedge/pkg/operator/types" 26 | nodeutil "github.com/fabedge/fabedge/pkg/util/node" 27 | ) 28 | 29 | var _ = Describe("EndpointFuncs", func() { 30 | _, _, newEndpoint := types.NewEndpointFuncs("cluster", "C=CN, O=StrongSwan, CN={node}", nodeutil.GetPodCIDRsFromAnnotation) 31 | 32 | node := corev1.Node{ 33 | ObjectMeta: metav1.ObjectMeta{ 34 | Name: "edge1", 35 | Annotations: map[string]string{ 36 | constants.KeyPodSubnets: "2.2.0.1/26,2.2.0.128/26", 37 | }, 38 | }, 39 | Status: corev1.NodeStatus{ 40 | Addresses: []corev1.NodeAddress{ 41 | { 42 | Type: corev1.NodeInternalIP, 43 | Address: "192.168.1.1", 44 | }, 45 | }, 46 | }, 47 | } 48 | endpoint := newEndpoint(node) 49 | 50 | It("should mark endpoint as EdgeNode", func() { 51 | Expect(endpoint.Type).Should(Equal(apis.EdgeNode)) 52 | }) 53 | 54 | It("should replace {node} in id format", func() { 55 | Expect(endpoint.ID).Should(Equal("C=CN, O=StrongSwan, CN=cluster.edge1")) 56 | }) 57 | 58 | It("should extract subnets from annotations", func() { 59 | Expect(endpoint.Subnets).Should(ContainElement("2.2.0.1/26")) 60 | Expect(endpoint.Subnets).Should(ContainElement("2.2.0.128/26")) 61 | }) 62 | 63 | It("should read node subnets from node.status.address", func() { 64 | Expect(endpoint.NodeSubnets).Should(ConsistOf("192.168.1.1")) 65 | }) 66 | 67 | It("should ues internal IP as public addresses if no public addresses in annotation", func() { 68 | Expect(endpoint.NodeSubnets).Should(ConsistOf("192.168.1.1")) 69 | Expect(endpoint.PublicAddresses).Should(ConsistOf("192.168.1.1")) 70 | }) 71 | 72 | It("should read public addresses from annotation if it exists", func() { 73 | node = corev1.Node{ 74 | ObjectMeta: metav1.ObjectMeta{ 75 | Name: "edge1", 76 | Annotations: map[string]string{ 77 | constants.KeyPodSubnets: "2.2.0.1/26,2.2.0.128/26", 78 | constants.KeyNodePublicAddresses: "www.example.com,10.0.0.1", 79 | }, 80 | }, 81 | Status: corev1.NodeStatus{ 82 | Addresses: []corev1.NodeAddress{ 83 | { 84 | Type: corev1.NodeInternalIP, 85 | Address: "192.168.1.1", 86 | }, 87 | }, 88 | }, 89 | } 90 | 91 | endpoint = newEndpoint(node) 92 | 93 | Expect(endpoint.PublicAddresses).Should(ConsistOf("www.example.com", "10.0.0.1")) 94 | }) 95 | }) 96 | -------------------------------------------------------------------------------- /pkg/operator/types/podcidrstore.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "sync" 4 | 5 | type PodCIDRStore interface { 6 | Append(nodeName string, cidr ...string) 7 | Remove(nodeName string, cidr ...string) 8 | RemoveAll(nodeName string) 9 | RemoveByPodCIDR(podCIDR string) 10 | Get(nodeName string) []string 11 | GetNodeNameByPodCIDR(cidr string) (string, bool) 12 | } 13 | 14 | var _ PodCIDRStore = &podCIDRStore{} 15 | 16 | type podCIDRStore struct { 17 | // key is nodeName, value is pod CIDR list 18 | nodeToPodCIDRs map[string][]string 19 | podCIDRToNode map[string]string 20 | mux sync.RWMutex 21 | } 22 | 23 | func NewPodCIDRStore() PodCIDRStore { 24 | return &podCIDRStore{ 25 | nodeToPodCIDRs: make(map[string][]string), 26 | podCIDRToNode: make(map[string]string), 27 | } 28 | } 29 | 30 | func (s *podCIDRStore) Append(nodeName string, podCIDRs ...string) { 31 | s.mux.Lock() 32 | defer s.mux.Unlock() 33 | 34 | nodePodCIDRs := s.nodeToPodCIDRs[nodeName] 35 | for _, cidr := range podCIDRs { 36 | if !findPodCIDR(cidr, nodePodCIDRs) { 37 | nodePodCIDRs = append(nodePodCIDRs, cidr) 38 | s.podCIDRToNode[cidr] = nodeName 39 | } 40 | } 41 | 42 | s.nodeToPodCIDRs[nodeName] = nodePodCIDRs 43 | } 44 | 45 | func (s *podCIDRStore) Remove(nodeName string, podCIDRs ...string) { 46 | s.mux.Lock() 47 | defer s.mux.Unlock() 48 | 49 | nodePodCIDRs := s.nodeToPodCIDRs[nodeName] 50 | for _, value := range podCIDRs { 51 | nodePodCIDRs = deletePodCIDR(value, nodePodCIDRs) 52 | delete(s.podCIDRToNode, value) 53 | } 54 | 55 | if len(nodePodCIDRs) > 0 { 56 | s.nodeToPodCIDRs[nodeName] = nodePodCIDRs 57 | } else { 58 | delete(s.nodeToPodCIDRs, nodeName) 59 | } 60 | } 61 | 62 | func (s *podCIDRStore) RemoveByPodCIDR(podCIDR string) { 63 | s.mux.Lock() 64 | defer s.mux.Unlock() 65 | 66 | nodeName := s.podCIDRToNode[podCIDR] 67 | 68 | s.nodeToPodCIDRs[nodeName] = deletePodCIDR(podCIDR, s.nodeToPodCIDRs[nodeName]) 69 | delete(s.podCIDRToNode, podCIDR) 70 | } 71 | 72 | func (s *podCIDRStore) RemoveAll(nodeName string) { 73 | s.mux.Lock() 74 | defer s.mux.Unlock() 75 | 76 | for _, cidr := range s.nodeToPodCIDRs[nodeName] { 77 | delete(s.podCIDRToNode, cidr) 78 | } 79 | delete(s.nodeToPodCIDRs, nodeName) 80 | } 81 | 82 | func (s *podCIDRStore) Get(nodeName string) []string { 83 | s.mux.RLock() 84 | defer s.mux.RUnlock() 85 | return s.nodeToPodCIDRs[nodeName] 86 | } 87 | 88 | func (s *podCIDRStore) GetNodeNameByPodCIDR(cidr string) (string, bool) { 89 | s.mux.RLock() 90 | defer s.mux.RUnlock() 91 | nodeName, ok := s.podCIDRToNode[cidr] 92 | 93 | return nodeName, ok 94 | } 95 | 96 | func findPodCIDR(value string, cidrs []string) bool { 97 | for _, cidr := range cidrs { 98 | if cidr == value { 99 | return true 100 | } 101 | } 102 | 103 | return false 104 | } 105 | 106 | func deletePodCIDR(value string, cidrs []string) []string { 107 | for i, cidr := range cidrs { 108 | if cidr == value { 109 | return append(cidrs[0:i], cidrs[i+1:]...) 110 | } 111 | } 112 | 113 | return cidrs 114 | } 115 | -------------------------------------------------------------------------------- /pkg/operator/types/podcidrstore_test.go: -------------------------------------------------------------------------------- 1 | package types_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "github.com/fabedge/fabedge/pkg/operator/types" 8 | ) 9 | 10 | var _ = Describe("PodCIDRStore", func() { 11 | It("Should support append, remove and get", func() { 12 | store := types.NewPodCIDRStore() 13 | nodeName := "node1" 14 | 15 | store.Append(nodeName, "10.10.10.10/26", "10.10.10.20/26", "10.10.10.30/26", "10.10.10.40/26") 16 | Expect(store.Get(nodeName)).To(ConsistOf("10.10.10.10/26", "10.10.10.20/26", "10.10.10.30/26", "10.10.10.40/26")) 17 | 18 | store.Remove(nodeName, "10.10.10.10/26", "10.10.10.40/26") 19 | Expect(store.Get(nodeName)).To(ConsistOf("10.10.10.20/26", "10.10.10.30/26")) 20 | 21 | name, ok := store.GetNodeNameByPodCIDR("10.10.10.20/26") 22 | Expect(ok).Should(BeTrue()) 23 | Expect(name).Should(Equal(nodeName)) 24 | 25 | store.RemoveByPodCIDR("10.10.10.30/26") 26 | Expect(store.Get(nodeName)).To(ConsistOf("10.10.10.20/26")) 27 | 28 | store.RemoveAll(nodeName) 29 | Expect(store.Get(nodeName)).To(BeNil()) 30 | }) 31 | }) 32 | -------------------------------------------------------------------------------- /pkg/operator/types/safe_string_set.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "sync" 5 | 6 | "k8s.io/apimachinery/pkg/util/sets" 7 | ) 8 | 9 | type SafeStringSet struct { 10 | set sets.String 11 | mux sync.RWMutex 12 | } 13 | 14 | func NewSafeStringSet(v ...string) *SafeStringSet { 15 | set := &SafeStringSet{ 16 | set: make(sets.String), 17 | } 18 | 19 | set.Insert(v...) 20 | 21 | return set 22 | } 23 | 24 | func (s *SafeStringSet) Insert(value ...string) { 25 | s.mux.Lock() 26 | defer s.mux.Unlock() 27 | s.set.Insert(value...) 28 | } 29 | 30 | func (s *SafeStringSet) Delete(value ...string) { 31 | s.mux.Lock() 32 | defer s.mux.Unlock() 33 | 34 | s.set.Delete(value...) 35 | } 36 | 37 | func (s *SafeStringSet) Len() int { 38 | s.mux.RLock() 39 | defer s.mux.RUnlock() 40 | 41 | return s.set.Len() 42 | } 43 | 44 | func (s *SafeStringSet) Has(v string) bool { 45 | s.mux.RLock() 46 | defer s.mux.RUnlock() 47 | 48 | return s.set.Has(v) 49 | } 50 | 51 | func (s *SafeStringSet) Equal(o *SafeStringSet) bool { 52 | s.mux.RLock() 53 | defer s.mux.RUnlock() 54 | 55 | return s.set.Equal(o.set) 56 | } 57 | 58 | func (s *SafeStringSet) List() []string { 59 | s.mux.RLock() 60 | defer s.mux.RUnlock() 61 | 62 | return s.set.List() 63 | } 64 | -------------------------------------------------------------------------------- /pkg/operator/types/safe_string_set_test.go: -------------------------------------------------------------------------------- 1 | package types_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "github.com/fabedge/fabedge/pkg/operator/types" 8 | ) 9 | 10 | var _ = Describe("SafeStringSet", func() { 11 | It("Should support add, remove and contains", func() { 12 | set := types.NewSafeStringSet("edge") 13 | Expect(set.Has("edge")).Should(BeTrue()) 14 | 15 | set.Insert("edge3", "edge2", "edge1") 16 | Expect(set.Has("edge1")).Should(BeTrue()) 17 | Expect(set.Len()).Should(Equal(4)) 18 | 19 | set.Delete("edge") 20 | Expect(set.Has("edge")).Should(BeFalse()) 21 | Expect(set.Len()).Should(Equal(3)) 22 | 23 | Expect(set.List()).Should(ConsistOf("edge1", "edge2", "edge3")) 24 | 25 | set2 := types.NewSafeStringSet("edge1", "edge2", "edge3") 26 | Expect(set.Equal(set2)).Should(BeTrue()) 27 | }) 28 | }) 29 | -------------------------------------------------------------------------------- /pkg/operator/types/types_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types_test 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestTypes(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Types Suite") 27 | } 28 | -------------------------------------------------------------------------------- /pkg/tunnel/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tunnel 16 | 17 | import ( 18 | apis "github.com/fabedge/fabedge/pkg/apis/v1alpha1" 19 | ) 20 | 21 | type Manager interface { 22 | IsRunning() bool 23 | ListConnNames() ([]string, error) 24 | LoadConn(conn ConnConfig) error 25 | InitiateConn(name string) error 26 | UnloadConn(name string) error 27 | IsActive() (bool, error) 28 | } 29 | 30 | type ConnConfig struct { 31 | Name string // must be unique 32 | 33 | LocalID string 34 | LocalAddress []string 35 | LocalSubnets []string 36 | LocalNodeSubnets []string 37 | LocalCerts []string 38 | LocalType apis.EndpointType 39 | 40 | RemoteID string 41 | RemoteAddress []string 42 | RemoteSubnets []string 43 | RemoteNodeSubnets []string 44 | RemoteType apis.EndpointType 45 | RemotePort *uint 46 | 47 | // Whether this connection is used for mediation 48 | Mediation bool 49 | 50 | // whether is connection need mediation 51 | NeedMediation bool 52 | // check https://docs.strongswan.org/docs/5.9/swanctl/swanctlConf.html 53 | // for detailed explanation for MediatedBy and MediationPeer 54 | MediatedBy string 55 | MediationPeer string 56 | } 57 | -------------------------------------------------------------------------------- /pkg/tunnel/strongswan/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package strongswan 16 | 17 | type Options []option 18 | type option func(manager *StrongSwanManager) 19 | 20 | func SocketFile(path string) option { 21 | return func(m *StrongSwanManager) { 22 | m.socketPath = path 23 | } 24 | } 25 | 26 | func CertsDir(path string) option { 27 | return func(m *StrongSwanManager) { 28 | m.certsPath = path 29 | } 30 | } 31 | 32 | func StartAction(startAction string) option { 33 | return func(m *StrongSwanManager) { 34 | m.startAction = startAction 35 | } 36 | } 37 | 38 | func DpdAction(action string) option { 39 | return func(m *StrongSwanManager) { 40 | m.dpdAction = action 41 | } 42 | } 43 | 44 | func DpdDelay(delay string) option { 45 | return func(m *StrongSwanManager) { 46 | m.dpdDelay = delay 47 | } 48 | } 49 | 50 | func InterfaceID(id *uint) option { 51 | return func(m *StrongSwanManager) { 52 | m.interfaceID = id 53 | } 54 | } 55 | 56 | // InitTimeout set timeout for SA/child-SA initiation. 0 means blocking initiation 57 | // unit: second. 58 | func InitTimeout(timeout uint) option { 59 | return func(m *StrongSwanManager) { 60 | m.initTimeout = timeout * 1000 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pkg/util/cert/cert_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cert_test 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestCert(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Cert Suite") 27 | } 28 | -------------------------------------------------------------------------------- /pkg/util/cert/remote_manager.go: -------------------------------------------------------------------------------- 1 | package cert 2 | 3 | import ( 4 | "crypto/x509" 5 | "encoding/pem" 6 | "fmt" 7 | ) 8 | 9 | // SignCertFunc receive csr and return a cert bytes 10 | type SignCertFunc func(csr []byte) ([]byte, error) 11 | 12 | type remoteManager struct { 13 | caCert *x509.Certificate 14 | signCert SignCertFunc 15 | 16 | caCertPEM []byte 17 | certPool *x509.CertPool 18 | } 19 | 20 | func NewRemoteManager(caCertDER []byte, signCert SignCertFunc) (Manager, error) { 21 | caCert, err := x509.ParseCertificate(caCertDER) 22 | if err != nil { 23 | return nil, fmt.Errorf("failed to parse a caCert. err: %v", err) 24 | } 25 | 26 | if signCert == nil { 27 | return nil, fmt.Errorf("a signCert function is required") 28 | } 29 | 30 | pool := x509.NewCertPool() 31 | pool.AddCert(caCert) 32 | 33 | return &remoteManager{ 34 | caCertPEM: EncodeCertPEM(caCertDER), 35 | caCert: caCert, 36 | certPool: pool, 37 | signCert: signCert, 38 | }, nil 39 | } 40 | 41 | func (m remoteManager) GetCACertPEM() []byte { 42 | return m.caCertPEM 43 | } 44 | 45 | func (m remoteManager) GetCACert() *x509.Certificate { 46 | return m.caCert 47 | } 48 | 49 | func (m remoteManager) NewCertKey(cfg Config) ([]byte, []byte, error) { 50 | keyDER, csr, err := NewCertRequest(Request{ 51 | CommonName: cfg.CommonName, 52 | Organization: cfg.Organization, 53 | IPs: cfg.IPs, 54 | DNSNames: cfg.DNSNames, 55 | }) 56 | 57 | certDER, err := m.signCert(csr) 58 | 59 | return certDER, keyDER, err 60 | } 61 | 62 | func (m remoteManager) SignCert(csr []byte) ([]byte, error) { 63 | return m.signCert(csr) 64 | } 65 | 66 | func (m remoteManager) VerifyCert(cert *x509.Certificate, usages []x509.ExtKeyUsage) error { 67 | opts := x509.VerifyOptions{ 68 | Roots: m.certPool, 69 | KeyUsages: usages, 70 | } 71 | 72 | _, err := cert.Verify(opts) 73 | return err 74 | } 75 | 76 | func (m remoteManager) VerifyCertInPEM(certPEM []byte, usages []x509.ExtKeyUsage) error { 77 | block, _ := pem.Decode(certPEM) 78 | 79 | cert, err := x509.ParseCertificate(block.Bytes) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | return m.VerifyCert(cert, usages) 85 | } 86 | -------------------------------------------------------------------------------- /pkg/util/ginkgoext/ginkgo_extension.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ginkgoext 16 | 17 | import ( 18 | . "github.com/onsi/gomega" 19 | "sigs.k8s.io/controller-runtime/pkg/client" 20 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 21 | ) 22 | 23 | func ReceiveKey(key client.ObjectKey) OmegaMatcher { 24 | return Receive(Equal(reconcile.Request{ 25 | NamespacedName: key, 26 | })) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/util/log/flags.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "flag" 5 | 6 | "github.com/spf13/pflag" 7 | "k8s.io/klog/v2" 8 | ) 9 | 10 | func AddFlags(fs *pflag.FlagSet) { 11 | local := flag.NewFlagSet("klog", flag.ExitOnError) 12 | klog.InitFlags(local) 13 | 14 | fs.AddGoFlag(local.Lookup("v")) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/util/net/net.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "net" 5 | "strings" 6 | ) 7 | 8 | // IsIPv6 returns if netIP is IPv6. 9 | func IsIPv6(netIP net.IP) bool { 10 | return netIP != nil && netIP.To4() == nil 11 | } 12 | 13 | // IsIPv6String returns if ip is IPv6. 14 | func IsIPv6String(ip string) bool { 15 | netIP := net.ParseIP(ip) 16 | return IsIPv6(netIP) 17 | } 18 | 19 | // IsIPv6CIDRString returns if cidr is IPv6. 20 | // This assumes cidr is a valid CIDR. 21 | func IsIPv6CIDRString(cidr string) bool { 22 | ip, _, _ := net.ParseCIDR(cidr) 23 | return IsIPv6(ip) 24 | } 25 | 26 | // IsIPv6OrIPv6CIDRString returns if netIP is IPv6 family. 27 | func IsIPv6OrIPv6CIDRString(ipOrCIDR string) bool { 28 | if strings.IndexByte(ipOrCIDR, ':') == -1 { 29 | return false 30 | } 31 | 32 | return IsIPv6String(ipOrCIDR) || IsIPv6CIDRString(ipOrCIDR) 33 | } 34 | 35 | // IsIPv6CIDR returns if a cidr is ipv6 36 | func IsIPv6CIDR(cidr *net.IPNet) bool { 37 | ip := cidr.IP 38 | return IsIPv6(ip) 39 | } 40 | 41 | // IsIPv4 returns if netIP is IPv4. 42 | func IsIPv4(netIP net.IP) bool { 43 | return netIP != nil && netIP.To4() != nil 44 | } 45 | 46 | // IsIPv4String returns if ip is IPv4. 47 | func IsIPv4String(ip string) bool { 48 | netIP := net.ParseIP(ip) 49 | return IsIPv4(netIP) 50 | } 51 | 52 | // IsIPv4CIDR returns if a cidr is ipv4 53 | func IsIPv4CIDR(cidr *net.IPNet) bool { 54 | ip := cidr.IP 55 | return IsIPv4(ip) 56 | } 57 | 58 | // IsIPv4CIDRString returns if cidr is IPv4. 59 | // This assumes cidr is a valid CIDR. 60 | func IsIPv4CIDRString(cidr string) bool { 61 | ip, _, _ := net.ParseCIDR(cidr) 62 | return IsIPv4(ip) 63 | } 64 | 65 | // IsIPv4OrIPv4CIDRString returns if netIP is IPv4 family. 66 | func IsIPv4OrIPv4CIDRString(ipOrCIDR string) bool { 67 | if strings.IndexByte(ipOrCIDR, ':') == -1 { 68 | return false 69 | } 70 | 71 | return IsIPv6String(ipOrCIDR) || IsIPv6CIDRString(ipOrCIDR) 72 | } 73 | 74 | func IsCompatible(ip *net.IPNet, ipNet net.IP) bool { 75 | return (IsIPv4CIDR(ip) && IsIPv4(ipNet)) || (IsIPv6CIDR(ip) && IsIPv6(ipNet)) 76 | } 77 | 78 | func HasIPv6CIDRString(cidrs []string) bool { 79 | for _, cidr := range cidrs { 80 | if IsIPv6CIDRString(cidr) { 81 | return true 82 | } 83 | } 84 | 85 | return false 86 | } 87 | -------------------------------------------------------------------------------- /pkg/util/node/nodeutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package node 16 | 17 | import ( 18 | "strings" 19 | "sync" 20 | 21 | corev1 "k8s.io/api/core/v1" 22 | 23 | "github.com/fabedge/fabedge/pkg/common/constants" 24 | ) 25 | 26 | var once sync.Once 27 | var edgeNodeLabels map[string]string 28 | 29 | func SetEdgeNodeLabels(labels map[string]string) { 30 | once.Do(func() { 31 | edgeNodeLabels = labels 32 | }) 33 | } 34 | 35 | func GetEdgeNodeLabels() map[string]string { 36 | return edgeNodeLabels 37 | } 38 | 39 | func GetInternalIPs(node corev1.Node) []string { 40 | var ips []string 41 | for _, addr := range node.Status.Addresses { 42 | if addr.Type == corev1.NodeInternalIP { 43 | ips = append(ips, addr.Address) 44 | } 45 | } 46 | 47 | return ips 48 | } 49 | 50 | func GetPodCIDRs(node corev1.Node) []string { 51 | switch { 52 | case len(node.Spec.PodCIDRs) > 0: 53 | return node.Spec.PodCIDRs 54 | case len(node.Spec.PodCIDR) > 0: 55 | return []string{node.Spec.PodCIDR} 56 | default: 57 | return nil 58 | } 59 | } 60 | 61 | func GetPodCIDRsFromAnnotation(node corev1.Node) []string { 62 | annotations := node.Annotations 63 | if annotations == nil { 64 | return nil 65 | } 66 | 67 | return strings.Split(annotations[constants.KeyPodSubnets], ",") 68 | } 69 | 70 | func IsEdgeNode(node corev1.Node) bool { 71 | if len(edgeNodeLabels) == 0 { 72 | return false 73 | } 74 | 75 | labels := node.GetLabels() 76 | if len(labels) == 0 { 77 | return false 78 | } 79 | 80 | for key, value := range edgeNodeLabels { 81 | if v, exist := labels[key]; !exist || v != value { 82 | return false 83 | } 84 | } 85 | 86 | return true 87 | } 88 | -------------------------------------------------------------------------------- /pkg/util/node/nodeutil_test.go: -------------------------------------------------------------------------------- 1 | package node_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/gomega" 7 | corev1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | 10 | nodeutil "github.com/fabedge/fabedge/pkg/util/node" 11 | ) 12 | 13 | func TestIsEdgeNode(t *testing.T) { 14 | g := NewGomegaWithT(t) 15 | 16 | nodeutil.SetEdgeNodeLabels(map[string]string{ 17 | "edge": "", 18 | "managed": "true", 19 | }) 20 | 21 | node := corev1.Node{ 22 | ObjectMeta: metav1.ObjectMeta{ 23 | Labels: map[string]string{ 24 | "edge": "", 25 | "managed": "true", 26 | }, 27 | }, 28 | } 29 | 30 | g.Expect(nodeutil.IsEdgeNode(node)).To(BeTrue()) 31 | 32 | node.Labels["edge"] = "not-blank" 33 | g.Expect(nodeutil.IsEdgeNode(node)).To(BeFalse()) 34 | 35 | node.Labels["edge"] = "" 36 | node.Labels["managed"] = "false" 37 | g.Expect(nodeutil.IsEdgeNode(node)).To(BeFalse()) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/util/route/route.go: -------------------------------------------------------------------------------- 1 | package route 2 | 3 | import ( 4 | "net" 5 | "strings" 6 | 7 | "github.com/vishvananda/netlink" 8 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 9 | "k8s.io/apimachinery/pkg/util/sets" 10 | 11 | "github.com/fabedge/fabedge/pkg/common/constants" 12 | netutil "github.com/fabedge/fabedge/pkg/util/net" 13 | ) 14 | 15 | type CheckRouteFunc func(route netlink.Route) bool 16 | 17 | // NewDstWhitelist return a CheckRouteFunc which 18 | // return true if specified route's Dst is not in set 19 | func NewDstWhitelist(set sets.String) CheckRouteFunc { 20 | return func(route netlink.Route) bool { 21 | return !set.Has(route.Dst.String()) 22 | } 23 | } 24 | 25 | // NewDstBlacklist return a CheckRouteFunc which 26 | // return true if specified route's Dst is in set 27 | func NewDstBlacklist(set sets.String) CheckRouteFunc { 28 | return func(route netlink.Route) bool { 29 | return set.Has(route.Dst.String()) 30 | } 31 | } 32 | 33 | func FileExistsError(err error) bool { 34 | msg := err.Error() 35 | return strings.Contains(msg, "file exists") 36 | } 37 | 38 | func NoSuchProcessError(err error) bool { 39 | msg := err.Error() 40 | return strings.Contains(msg, "no such process") 41 | } 42 | 43 | func GetDefaultGateway() (net.IP, error) { 44 | defaultRoute, err := netlink.RouteGet(net.ParseIP("8.8.8.8")) 45 | if len(defaultRoute) != 1 || err != nil { 46 | return nil, err 47 | } 48 | return defaultRoute[0].Gw, nil 49 | } 50 | 51 | func GetDefaultGateway6() (net.IP, error) { 52 | defaultRoute, err := netlink.RouteGet(net.ParseIP("2001:4860:4860::8888")) 53 | if len(defaultRoute) != 1 || err != nil { 54 | return nil, err 55 | } 56 | return defaultRoute[0].Gw, nil 57 | } 58 | 59 | // PurgeStrongSwanRoutes will delete any route in strongswan table which satisfy checkRoute 60 | func PurgeStrongSwanRoutes(checkRoute CheckRouteFunc) error { 61 | var routeFilter = &netlink.Route{ 62 | Table: 220, 63 | } 64 | 65 | routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, routeFilter, netlink.RT_FILTER_TABLE) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | var errors []error 71 | for _, route := range routes { 72 | if !checkRoute(route) { 73 | continue 74 | } 75 | 76 | if err = netlink.RouteDel(&route); err != nil { 77 | errors = append(errors, err) 78 | } 79 | } 80 | 81 | return utilerrors.NewAggregate(errors) 82 | } 83 | 84 | func EnsureStrongswanRoutes(prefixes []string, gw net.IP) error { 85 | var errors []error 86 | 87 | for _, prefix := range prefixes { 88 | dst, err := netlink.ParseIPNet(prefix) 89 | if err != nil { 90 | errors = append(errors, err) 91 | continue 92 | } 93 | 94 | if !netutil.IsCompatible(dst, gw) { 95 | continue 96 | } 97 | 98 | err = netlink.RouteReplace(&netlink.Route{Dst: dst, Gw: gw, Table: constants.TableStrongswan}) 99 | if err != nil { 100 | errors = append(errors, err) 101 | } 102 | } 103 | 104 | return utilerrors.NewAggregate(errors) 105 | } 106 | -------------------------------------------------------------------------------- /pkg/util/secret/secretutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package secret 16 | 17 | import corev1 "k8s.io/api/core/v1" 18 | 19 | // GetCACert get the ca cert from the secret by the key ca.crt 20 | func GetCACert(secret corev1.Secret) []byte { 21 | return secret.Data[KeyCACert] 22 | } 23 | 24 | // GetCAKey get the ca cert from the secret by the key ca.key 25 | func GetCAKey(secret corev1.Secret) []byte { 26 | return secret.Data[KeyCAKey] 27 | } 28 | 29 | // GetCA get the ca cert/key from the secret by the key ca.crt and ca.key 30 | func GetCA(secret corev1.Secret) ([]byte, []byte) { 31 | return secret.Data[KeyCACert], secret.Data[KeyCAKey] 32 | } 33 | 34 | // GetCert get the cert from the secret by the key tls.crt 35 | func GetCert(secret corev1.Secret) []byte { 36 | return secret.Data[corev1.TLSCertKey] 37 | } 38 | 39 | // GetCertAndKey get the cert and Key from the secret by the key tls.crt and tls.key 40 | func GetCertAndKey(secret corev1.Secret) ([]byte, []byte) { 41 | return secret.Data[corev1.TLSCertKey], secret.Data[corev1.TLSPrivateKeyKey] 42 | } 43 | -------------------------------------------------------------------------------- /pkg/util/test/purge.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package test 16 | 17 | import ( 18 | "context" 19 | 20 | corev1 "k8s.io/api/core/v1" 21 | "sigs.k8s.io/controller-runtime/pkg/client" 22 | ) 23 | 24 | func PurgeAllSecrets(cli client.Client, opts ...client.ListOption) error { 25 | var secrets corev1.SecretList 26 | err := cli.List(context.TODO(), &secrets) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | for _, secret := range secrets.Items { 32 | if err := cli.Delete(context.TODO(), &secret); err != nil { 33 | return err 34 | } 35 | } 36 | 37 | return nil 38 | } 39 | 40 | func PurgeAllConfigMaps(cli client.Client, opts ...client.ListOption) error { 41 | var configs corev1.ConfigMapList 42 | err := cli.List(context.TODO(), &configs) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | for _, secret := range configs.Items { 48 | if err := cli.Delete(context.TODO(), &secret); err != nil { 49 | return err 50 | } 51 | } 52 | 53 | return nil 54 | } 55 | 56 | func PurgeAllPods(cli client.Client, opts ...client.ListOption) error { 57 | var pods corev1.PodList 58 | err := cli.List(context.TODO(), &pods) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | for _, secret := range pods.Items { 64 | if err := cli.Delete(context.TODO(), &secret); err != nil { 65 | return err 66 | } 67 | } 68 | 69 | return nil 70 | } 71 | 72 | func PurgeAllNodes(cli client.Client, opts ...client.ListOption) error { 73 | var nodes corev1.NodeList 74 | err := cli.List(context.TODO(), &nodes) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | for _, secret := range nodes.Items { 80 | if err := cli.Delete(context.TODO(), &secret); err != nil { 81 | return err 82 | } 83 | } 84 | 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /pkg/util/time/time.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package time 16 | 17 | import "time" 18 | 19 | func Days(days int64) time.Duration { 20 | return time.Duration(days) * 24 * time.Hour 21 | } 22 | 23 | func Hours(value int64) time.Duration { 24 | return time.Duration(value) * time.Hour 25 | } 26 | 27 | func Minutes(value int64) time.Duration { 28 | return time.Duration(value) * time.Minute 29 | } 30 | 31 | func Seconds(value int64) time.Duration { 32 | return time.Duration(value) * time.Second 33 | } 34 | -------------------------------------------------------------------------------- /test/e2e/e2e_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 FabEdge Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package e2e 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/onsi/ginkgo" 21 | 22 | "github.com/fabedge/fabedge/test/e2e/framework" 23 | ) 24 | 25 | func init() { 26 | defer ginkgo.GinkgoRecover() 27 | framework.RegisterAndHandleFlags() 28 | } 29 | 30 | func TestE2E(t *testing.T) { 31 | RunE2ETests(t) 32 | } 33 | -------------------------------------------------------------------------------- /test/e2e/framework/README.md: -------------------------------------------------------------------------------- 1 | Some source files were originally from: 2 | [k8s.io/kubernetes@/v1.21.0](https://github.com/kubernetes/kubernetes/tree/v1.21.0) 3 | -------------------------------------------------------------------------------- /test/e2e/framework/cleanup.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package framework 18 | 19 | import "sync" 20 | 21 | // CleanupActionHandle is an integer pointer type for handling cleanup action 22 | type CleanupActionHandle *int 23 | 24 | var cleanupActionsLock sync.Mutex 25 | var cleanupActions = map[CleanupActionHandle]func(){} 26 | 27 | // AddCleanupAction installs a function that will be called in the event of the 28 | // whole test being terminated. This allows arbitrary pieces of the overall 29 | // test to hook into SynchronizedAfterSuite(). 30 | func AddCleanupAction(fn func()) CleanupActionHandle { 31 | p := CleanupActionHandle(new(int)) 32 | cleanupActionsLock.Lock() 33 | defer cleanupActionsLock.Unlock() 34 | cleanupActions[p] = fn 35 | return p 36 | } 37 | 38 | // RemoveCleanupAction removes a function that was installed by 39 | // AddCleanupAction. 40 | func RemoveCleanupAction(p CleanupActionHandle) { 41 | cleanupActionsLock.Lock() 42 | defer cleanupActionsLock.Unlock() 43 | delete(cleanupActions, p) 44 | } 45 | 46 | // RunCleanupActions runs all functions installed by AddCleanupAction. It does 47 | // not remove them (see RemoveCleanupAction) but it does run unlocked, so they 48 | // may remove themselves. 49 | func RunCleanupActions() { 50 | list := []func(){} 51 | func() { 52 | cleanupActionsLock.Lock() 53 | defer cleanupActionsLock.Unlock() 54 | for _, fn := range cleanupActions { 55 | list = append(list, fn) 56 | } 57 | }() 58 | // Run unlocked. 59 | for _, fn := range list { 60 | fn() 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /test/e2e/framework/expect.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package framework 18 | 19 | import ( 20 | "github.com/onsi/gomega" 21 | ) 22 | 23 | // ExpectEqual expects the specified two are the same, otherwise an exception raises 24 | func ExpectEqual(actual interface{}, extra interface{}, explain ...interface{}) { 25 | gomega.ExpectWithOffset(1, actual).To(gomega.Equal(extra), explain...) 26 | } 27 | 28 | // ExpectNotEqual expects the specified two are not the same, otherwise an exception raises 29 | func ExpectNotEqual(actual interface{}, extra interface{}, explain ...interface{}) { 30 | gomega.ExpectWithOffset(1, actual).NotTo(gomega.Equal(extra), explain...) 31 | } 32 | 33 | // ExpectError expects an error happens, otherwise an exception raises 34 | func ExpectError(err error, explain ...interface{}) { 35 | gomega.ExpectWithOffset(1, err).To(gomega.HaveOccurred(), explain...) 36 | } 37 | 38 | // ExpectNoError checks if "err" is set, and if so, fails assertion while logging the error. 39 | func ExpectNoError(err error, explain ...interface{}) { 40 | ExpectNoErrorWithOffset(1, err, explain...) 41 | } 42 | 43 | // ExpectNoErrorWithOffset checks if "err" is set, and if so, fails assertion while logging the error at "offset" levels above its caller 44 | // (for example, for call chain f -> g -> ExpectNoErrorWithOffset(1, ...) error would be logged for "f"). 45 | func ExpectNoErrorWithOffset(offset int, err error, explain ...interface{}) { 46 | gomega.ExpectWithOffset(1+offset, err).NotTo(gomega.HaveOccurred(), explain...) 47 | } 48 | 49 | // ExpectConsistOf expects actual contains precisely the extra elements. The ordering of the elements does not matter. 50 | func ExpectConsistOf(actual interface{}, extra interface{}, explain ...interface{}) { 51 | gomega.ExpectWithOffset(1, actual).To(gomega.ConsistOf(extra), explain...) 52 | } 53 | 54 | // ExpectHaveKey expects the actual map has the key in the keyset 55 | func ExpectHaveKey(actual interface{}, key interface{}, explain ...interface{}) { 56 | gomega.ExpectWithOffset(1, actual).To(gomega.HaveKey(key), explain...) 57 | } 58 | 59 | // ExpectEmpty expects actual is empty 60 | func ExpectEmpty(actual interface{}, explain ...interface{}) { 61 | gomega.ExpectWithOffset(1, actual).To(gomega.BeEmpty(), explain...) 62 | } 63 | -------------------------------------------------------------------------------- /third_party/calicoapi/constants.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-2020 Tigera, Inc. All rights reserved. 2 | 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package calicoapi 16 | 17 | const ( 18 | // API group details for the Calico v3 API. 19 | Group = "projectcalico.org" 20 | VersionCurrent = "v3" 21 | GroupVersionCurrent = Group + "/" + VersionCurrent 22 | 23 | // AllNamepaces is used for client instantiation, either for when the namespace 24 | // will be specified in the resource request, or for List or Watch queries across 25 | // all namespaces. 26 | AllNamespaces = "" 27 | 28 | // AllNames is used for List or Watch queries to wildcard the name. 29 | AllNames = "" 30 | 31 | // Label used to denote the Namespace. This is added to workload endpoints and network sets by Calico 32 | // and may be used for label matches by Policy selectors. 33 | LabelNamespace = "projectcalico.org/namespace" 34 | 35 | // Label used to denote the ServiceAccount. This is added to the workload endpoints by Calico 36 | // and may be used for label matches by Policy selectors. 37 | LabelServiceAccount = "projectcalico.org/serviceaccount" 38 | 39 | // Label used to denote the Orchestrator. This is added to the workload endpoints by an 40 | // orchestrator. 41 | LabelOrchestrator = "projectcalico.org/orchestrator" 42 | 43 | // Known orchestrators. Orchestrators are not limited to this list. 44 | OrchestratorKubernetes = "k8s" 45 | OrchestratorCNI = "cni" 46 | OrchestratorDocker = "libnetwork" 47 | OrchestratorOpenStack = "openstack" 48 | 49 | // Enum options for enable/disable fields 50 | Enabled = "Enabled" 51 | Disabled = "Disabled" 52 | ) 53 | -------------------------------------------------------------------------------- /third_party/calicoapi/crd/crd.projectcalico.org_ipamblocks.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: ipamblocks.crd.projectcalico.org 6 | spec: 7 | group: crd.projectcalico.org 8 | names: 9 | kind: IPAMBlock 10 | listKind: IPAMBlockList 11 | plural: ipamblocks 12 | singular: ipamblock 13 | scope: Cluster 14 | versions: 15 | - name: v1 16 | schema: 17 | openAPIV3Schema: 18 | properties: 19 | apiVersion: 20 | description: 'APIVersion defines the versioned schema of this representation 21 | of an object. Servers should convert recognized schemas to the latest 22 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 23 | type: string 24 | kind: 25 | description: 'Kind is a string value representing the REST resource this 26 | object represents. Servers may infer this from the endpoint the client 27 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 28 | type: string 29 | metadata: 30 | type: object 31 | spec: 32 | description: IPAMBlockSpec contains the specification for an IPAMBlock 33 | resource. 34 | properties: 35 | affinity: 36 | type: string 37 | allocations: 38 | items: 39 | type: integer 40 | # TODO: This nullable is manually added in. We should update controller-gen 41 | # to handle []*int properly itself. 42 | nullable: true 43 | type: array 44 | attributes: 45 | items: 46 | properties: 47 | handle_id: 48 | type: string 49 | secondary: 50 | additionalProperties: 51 | type: string 52 | type: object 53 | type: object 54 | type: array 55 | cidr: 56 | type: string 57 | deleted: 58 | type: boolean 59 | strictAffinity: 60 | type: boolean 61 | unallocated: 62 | items: 63 | type: integer 64 | type: array 65 | required: 66 | - allocations 67 | - attributes 68 | - cidr 69 | - strictAffinity 70 | - unallocated 71 | type: object 72 | type: object 73 | served: true 74 | storage: true 75 | status: 76 | acceptedNames: 77 | kind: "" 78 | plural: "" 79 | conditions: [] 80 | storedVersions: [] 81 | -------------------------------------------------------------------------------- /third_party/calicoapi/ipam_block.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Tigera, Inc. All rights reserved. 2 | 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package calicoapi 16 | 17 | import ( 18 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 19 | ) 20 | 21 | const ( 22 | KindIPAMBlock = "IPAMBlock" 23 | KindIPAMBlockList = "IPAMBlockList" 24 | ) 25 | 26 | // +genclient 27 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 28 | 29 | // IPAMBlock contains information about a block for IP address assignment. 30 | type IPAMBlock struct { 31 | metav1.TypeMeta `json:",inline"` 32 | // Standard object's metadata. 33 | metav1.ObjectMeta `json:"metadata,omitempty"` 34 | // Specification of the IPAMBlock. 35 | Spec IPAMBlockSpec `json:"spec,omitempty"` 36 | } 37 | 38 | // IPAMBlockSpec contains the specification for an IPAMBlock resource. 39 | type IPAMBlockSpec struct { 40 | CIDR string `json:"cidr"` 41 | Affinity *string `json:"affinity,omitempty"` 42 | StrictAffinity bool `json:"strictAffinity"` 43 | Allocations []*int `json:"allocations"` 44 | Unallocated []int `json:"unallocated"` 45 | Attributes []AllocationAttribute `json:"attributes"` 46 | 47 | // +optional 48 | Deleted bool `json:"deleted"` 49 | } 50 | 51 | type AllocationAttribute struct { 52 | AttrPrimary *string `json:"handle_id,omitempty"` 53 | AttrSecondary map[string]string `json:"secondary,omitempty"` 54 | } 55 | 56 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 57 | 58 | // IPAMBlockList contains a list of IPAMBlock resources. 59 | type IPAMBlockList struct { 60 | metav1.TypeMeta `json:",inline"` 61 | metav1.ListMeta `json:"metadata"` 62 | Items []IPAMBlock `json:"items"` 63 | } 64 | 65 | // NewIPAMBlock creates a new (zeroed) IPAMBlock struct with the TypeMetadata initialised to the current 66 | // version. 67 | func NewIPAMBlock() *IPAMBlock { 68 | return &IPAMBlock{ 69 | TypeMeta: metav1.TypeMeta{ 70 | Kind: KindIPAMBlock, 71 | APIVersion: GroupVersionCurrent, 72 | }, 73 | } 74 | } 75 | 76 | // NewIPAMBlockList creates a new (zeroed) IPAMBlockList struct with the TypeMetadata initialised to the current 77 | // version. 78 | func NewIPAMBlockList() *IPAMBlockList { 79 | return &IPAMBlockList{ 80 | TypeMeta: metav1.TypeMeta{ 81 | Kind: KindIPAMBlockList, 82 | APIVersion: GroupVersionCurrent, 83 | }, 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /third_party/calicoapi/register.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016-2017 Tigera, Inc. All rights reserved. 2 | 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package calicoapi 16 | 17 | import ( 18 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 19 | "k8s.io/apimachinery/pkg/runtime" 20 | "k8s.io/apimachinery/pkg/runtime/schema" 21 | ) 22 | 23 | var SchemeGroupVersion = schema.GroupVersion{Group: "crd.projectcalico.org", Version: "v1"} 24 | 25 | var ( 26 | SchemeBuilder runtime.SchemeBuilder 27 | localSchemeBuilder = &SchemeBuilder 28 | AddToScheme = localSchemeBuilder.AddToScheme 29 | ) 30 | 31 | func init() { 32 | // We only register manually written functions here. The registration of the 33 | // generated functions takes place in the generated files. The separation 34 | // makes the code compile even when the generated files are missing. 35 | localSchemeBuilder.Register(addKnownTypes) 36 | } 37 | 38 | // Resource takes an unqualified resource and returns a Group qualified GroupResource 39 | func Resource(resource string) schema.GroupResource { 40 | return SchemeGroupVersion.WithResource(resource).GroupResource() 41 | } 42 | 43 | // Adds the list of known types to api.Scheme. 44 | func addKnownTypes(scheme *runtime.Scheme) error { 45 | scheme.AddKnownTypes(SchemeGroupVersion, 46 | &IPAMBlock{}, 47 | &IPAMBlockList{}, 48 | &IPPool{}, 49 | &IPPoolList{}, 50 | ) 51 | metav1.AddToGroupVersion(scheme, SchemeGroupVersion) 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /third_party/ipset/README.md: -------------------------------------------------------------------------------- 1 | This source file was originally from: 2 | [k8s.io/kubernetes@/v1.21.0](https://github.com/kubernetes/kubernetes/tree/v1.21.0) 3 | 4 | We added two ipset types: `hash:ip` and `hash:net` -------------------------------------------------------------------------------- /third_party/ipset/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package ipset 18 | 19 | // Type represents the ipset type 20 | type Type string 21 | 22 | const ( 23 | // HashIPPort represents the `hash:ip,port` type ipset. The hash:ip,port is similar to hash:ip but 24 | // you can store IP address and protocol-port pairs in it. TCP, SCTP, UDP, UDPLITE, ICMP and ICMPv6 are supported 25 | // with port numbers/ICMP(v6) types and other protocol numbers without port information. 26 | HashIPPort Type = "hash:ip,port" 27 | // HashIPPortIP represents the `hash:ip,port,ip` type ipset. The hash:ip,port,ip set type uses a hash to store 28 | // IP address, port number and a second IP address triples. The port number is interpreted together with a 29 | // protocol (default TCP) and zero protocol number cannot be used. 30 | HashIPPortIP Type = "hash:ip,port,ip" 31 | // HashIPPortNet represents the `hash:ip,port,net` type ipset. The hash:ip,port,net set type uses a hash to store IP address, port number and IP network address triples. The port 32 | // number is interpreted together with a protocol (default TCP) and zero protocol number cannot be used. Network address 33 | // with zero prefix size cannot be stored either. 34 | HashIPPortNet Type = "hash:ip,port,net" 35 | // BitmapPort represents the `bitmap:port` type ipset. The bitmap:port set type uses a memory range, where each bit 36 | // represents one TCP/UDP port. A bitmap:port type of set can store up to 65535 ports. 37 | BitmapPort Type = "bitmap:port" 38 | // HashIP represents the `hash:ip` type ipset. The hash:ip set type uses a hash to store IP host addresses (default) or network addresses. 39 | // Zero valued IP address cannot be stored in a hash:ip type of set. 40 | HashIP Type = "hash:ip" 41 | // HashNet represents the `hash:Net` type ipset. The hash:net set type uses a hash to store different sized IP network addresses. Network ad‐ 42 | // dress with zero prefix size cannot be stored in this type of sets. 43 | HashNet Type = "hash:net" 44 | ) 45 | 46 | // DefaultPortRange defines the default bitmap:port valid port range. 47 | const DefaultPortRange string = "0-65535" 48 | 49 | const ( 50 | // ProtocolFamilyIPV4 represents IPv4 protocol. 51 | ProtocolFamilyIPV4 = "inet" 52 | // ProtocolFamilyIPV6 represents IPv6 protocol. 53 | ProtocolFamilyIPV6 = "inet6" 54 | // ProtocolTCP represents TCP protocol. 55 | ProtocolTCP = "tcp" 56 | // ProtocolUDP represents UDP protocol. 57 | ProtocolUDP = "udp" 58 | // ProtocolSCTP represents SCTP protocol. 59 | ProtocolSCTP = "sctp" 60 | ) 61 | 62 | // ValidIPSetTypes defines the supported ip set type. 63 | var ValidIPSetTypes = []Type{ 64 | HashIPPort, 65 | HashIPPortIP, 66 | BitmapPort, 67 | HashIPPortNet, 68 | HashIP, 69 | HashNet, 70 | } 71 | -------------------------------------------------------------------------------- /third_party/ipvs/README.md: -------------------------------------------------------------------------------- 1 | This source file was originally from: 2 | [k8s.io/kubernetes@/v1.21.0](https://github.com/kubernetes/kubernetes/tree/v1.21.0) 3 | 4 | We've changed it for handle xfrm interface and route -------------------------------------------------------------------------------- /third_party/ipvs/netlink.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package ipvs 18 | 19 | import ( 20 | "github.com/vishvananda/netlink" 21 | "k8s.io/apimachinery/pkg/util/sets" 22 | ) 23 | 24 | // NetLinkHandle for revoke netlink interface 25 | type NetLinkHandle interface { 26 | // EnsureAddressBind checks if address is bound to the interface and, if not, binds it. If the address is already bound, return true. 27 | EnsureAddressBind(address, devName string) (exist bool, err error) 28 | // UnbindAddress unbind address from the interface 29 | UnbindAddress(address, devName string) error 30 | // EnsureDummyDevice checks if dummy device is exist and, if not, create one. If the dummy device is already exist, return true. 31 | EnsureDummyDevice(devName string) (exist bool, err error) 32 | // DeleteDummyDevice deletes the given dummy device by name. 33 | DeleteDummyDevice(devName string) error 34 | // ListBindAddress will list all IP addresses which are bound in a given interface 35 | ListBindAddress(devName string) ([]string, error) 36 | // GetLocalAddresses returns all unique local type IP addresses based on specified device and filter device 37 | // If device is not specified, it will list all unique local type addresses except filter device addresses 38 | GetLocalAddresses(dev, filterDev string) (sets.String, error) 39 | // EnsureXfrmInterface checks if xfrm interface is exist and, if not, create one and up one 40 | EnsureXfrmInterface(devName string, ifid uint32) error 41 | // DeleteXfrmInterface deletes the given xfrm interface by name. 42 | DeleteXfrmInterface(devName string) error 43 | // EnsureRouteAdd checks if the route is exist and, if not, adds it 44 | EnsureRouteAdd(subnet, devName string) error 45 | // DeleteRoute deletes the route 46 | DeleteRoute(subnet, devName string) error 47 | // GetRoute get route by subnet and devName 48 | GetRoute(subnet, devName string) (*netlink.Route, error) 49 | } 50 | --------------------------------------------------------------------------------