├── ci ├── playbook │ ├── roles │ │ ├── worker │ │ │ ├── meta │ │ │ │ └── main.yaml │ │ │ └── tasks │ │ │ │ └── main.yaml │ │ ├── control-plane │ │ │ ├── meta │ │ │ │ └── main.yaml │ │ │ ├── templates │ │ │ │ └── kubeadm.conf.j2 │ │ │ └── tasks │ │ │ │ └── main.yaml │ │ └── common │ │ │ ├── handlers │ │ │ └── main.yaml │ │ │ └── tasks │ │ │ ├── main.yaml │ │ │ ├── kube.yaml │ │ │ └── containerd.yaml │ ├── k8s.yaml │ └── push_image.yaml ├── push_image.sh ├── tf │ ├── values.tftpl │ ├── example.tfvars │ └── hosts.tftpl ├── provision.sh ├── README.md ├── destroy.sh └── conformance_test.sh ├── docs ├── images │ ├── eni.png │ ├── feishu.png │ ├── eniip_ipvlan.png │ └── eni_allocation.png ├── host-stack-forward.md ├── iam-policy.md ├── traffic-ctrl.md ├── metrics.md └── hubble-intergration.md ├── chart ├── Chart.yaml ├── templates │ ├── serviceaccount.yaml │ ├── cilium-configmap.yaml │ ├── clusterrolebinding.yaml │ ├── cello-configmap.yaml │ └── clusterrole.yaml ├── .helmignore └── values.yaml ├── codecov.yml ├── tests ├── test.sh ├── cni_ready.bats ├── templates │ └── testcases │ │ ├── cilium_network_policy │ │ ├── check.yaml │ │ ├── l3_service_stage2.yaml │ │ ├── l4_cidr_dependent_stage1.yaml │ │ ├── l4_cidr_dependent_stage2.yaml │ │ ├── l4_port.yaml │ │ ├── l3_cidr.yaml │ │ ├── l3_service_stage1.yaml │ │ ├── l3_identify.yaml │ │ ├── l3_service_stage3.yaml │ │ ├── deny.yaml │ │ └── l3_label_additional_require.yaml │ │ ├── stress │ │ └── scale.yaml │ │ ├── service │ │ ├── loadblancer.yaml │ │ ├── hostnetwork │ │ │ ├── loadblancer.yaml │ │ │ ├── nodeport.yaml │ │ │ └── clusterip.yaml │ │ ├── nodeport.yaml │ │ └── clusterip.yaml │ │ └── pod_connection │ │ ├── samenode.yaml │ │ └── crossnode.yaml ├── network_policy.bats ├── stress │ └── pod_scale.sh ├── utils │ └── utils.bash └── pod_connection.bats ├── .gitignore ├── images ├── Dockerfile.builder └── Dockerfile.base ├── copyright.txt ├── pkg ├── cni │ ├── plugins │ │ └── cello │ │ │ └── doc.go │ ├── doc.go │ ├── driver │ │ └── doc.go │ ├── log │ │ └── log.go │ ├── types │ │ └── kubelet.go │ ├── utils │ │ └── utils.go │ ├── client │ │ └── daemon.go │ └── device │ │ ├── veth.go │ │ ├── ipvlan.go │ │ ├── vlan.go │ │ └── generic.go ├── utils │ ├── runtime │ │ └── runtime.go │ ├── sysctl │ │ ├── doc.go │ │ ├── sysctl_test.go │ │ ├── sysctl.go │ │ └── sysctl_linux_privileged_test.go │ ├── math │ │ └── math.go │ ├── kernel │ │ ├── kernel_unsupport.go │ │ ├── kernel_linux.go │ │ └── kernel.go │ ├── netns │ │ └── netns.go │ ├── logger │ │ ├── config.go │ │ └── logger.go │ ├── ip │ │ └── ip.go │ └── datatype │ │ └── types.go ├── deviceplugin │ ├── doc.go │ ├── mock │ │ └── mock_manager.go │ └── types.go ├── pbrpc │ ├── generate.go │ └── endpoint.proto ├── provider │ └── volcengine │ │ ├── ec2 │ │ └── generate_mock.go │ │ ├── metadata │ │ ├── generate_mock.go │ │ ├── ec2metadatawrapper_test.go │ │ ├── ec2metadata.go │ │ └── mock │ │ │ └── ec2wrapper_mocks.go │ │ ├── cellohelper │ │ ├── generate_mock.go │ │ ├── security_group_manager.go │ │ └── eni_tag.go │ │ └── credential │ │ ├── static.go │ │ └── credential.go ├── version │ └── version.go ├── daemon │ ├── log.go │ ├── netcontext.go │ ├── ecsmeta.go │ ├── daemon_main.go │ └── resource_manager.go ├── tracing │ ├── mock │ │ └── tracing.go │ ├── event.go │ └── tracing.go ├── metrics │ ├── daemon.go │ ├── resource_pool.go │ └── metric.go ├── signal │ ├── signal_test.go │ └── signal.go ├── store │ ├── store.go │ └── disk_test.go ├── backoff │ └── backoff.go └── k8s │ └── k8s_test.go ├── cmd ├── cello-agent │ └── cello-agent.go ├── cello-cni │ ├── handler_cello.go │ └── cello-cni.go └── cello-cli │ ├── main.go │ └── util.go ├── script └── bootstrap │ ├── cello-probe.sh │ └── install_env.sh ├── types ├── k8s.go ├── volcengine.go └── pod.go ├── .golangci.yaml ├── patch └── cilium │ ├── 003-adapt-to-ipvlan-datapath.patch │ ├── 006-support-pod-egress-qos.patch │ └── 001-fib-host-gateway.patch ├── opstools └── tag │ └── main.go ├── .github └── workflows │ └── go.yaml ├── Makefile └── CONTRIBUTING.md /ci/playbook/roles/worker/meta/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: common 4 | -------------------------------------------------------------------------------- /ci/playbook/roles/control-plane/meta/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | dependencies: 3 | - role: common 4 | -------------------------------------------------------------------------------- /docs/images/eni.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/volcengine/cello/HEAD/docs/images/eni.png -------------------------------------------------------------------------------- /docs/images/feishu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/volcengine/cello/HEAD/docs/images/feishu.png -------------------------------------------------------------------------------- /docs/images/eniip_ipvlan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/volcengine/cello/HEAD/docs/images/eniip_ipvlan.png -------------------------------------------------------------------------------- /docs/images/eni_allocation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/volcengine/cello/HEAD/docs/images/eni_allocation.png -------------------------------------------------------------------------------- /ci/playbook/roles/common/handlers/main.yaml: -------------------------------------------------------------------------------- 1 | - name: kubelet status 2 | service: 3 | name: kubelet 4 | state: started 5 | -------------------------------------------------------------------------------- /chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: vpc-cni 3 | description: cello 4 | type: application 5 | version: 1.6.6 6 | appVersion: v1.6.6 7 | -------------------------------------------------------------------------------- /chart/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: cello-service-account 5 | namespace: kube-system 6 | -------------------------------------------------------------------------------- /chart/templates/cilium-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: cilium-config 5 | namespace: kube-system 6 | data: 7 | debug: {{ .Values.cilium.debug | quote }} 8 | -------------------------------------------------------------------------------- /ci/playbook/k8s.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: control-plane 3 | gather_facts: no 4 | roles: 5 | - control-plane 6 | - hosts: workers 7 | gather_facts: no 8 | roles: 9 | - worker 10 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | # https://docs.codecov.io/docs/ignoring-paths 2 | ignore: 3 | - "types/*" 4 | - "tests/*" 5 | - "scripts/*" 6 | - "patch/*" 7 | - "images/*" 8 | - "docs/*" 9 | - "chart/*" 10 | 11 | coverage: 12 | status: 13 | project: on 14 | patch: off -------------------------------------------------------------------------------- /ci/playbook/roles/common/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | - name: Wait ssh port ready 2 | wait_for: 3 | host: "{{ k8s_api_server_ip }}" 4 | port: 22 5 | timeout: 300 6 | state: started 7 | connection: local 8 | - import_tasks: containerd.yaml 9 | - import_tasks: kube.yaml 10 | -------------------------------------------------------------------------------- /ci/playbook/push_image.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | gather_facts: no 4 | tasks: 5 | - name: Copy image to worker 6 | copy: 7 | src: /tmp/cello.tar 8 | dest: /tmp/cello.tar 9 | - name: Load image 10 | command: ctr -n k8s.io images import /tmp/cello.tar 11 | -------------------------------------------------------------------------------- /ci/push_image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 5 | pushd $THIS_DIR > /dev/null 6 | docker save -o /tmp/cello.tar volcstack/cello:latest 7 | 8 | pushd playbook > /dev/null 9 | ansible-playbook -i ../tf/hosts.yaml push_image.yaml 10 | popd > /dev/null 11 | popd > /dev/null 12 | -------------------------------------------------------------------------------- /tests/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # test cello pod ready 6 | bats cni_ready.bats 7 | 8 | # test pod connection 9 | bats pod_connection.bats 10 | 11 | # test service of kinds of 12 | bats service.bats 13 | 14 | # test network policy 15 | bats network_policy.bats 16 | 17 | # test cilium network policy 18 | bats cilium_network_policy.bats 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /.vscode 3 | /output 4 | *.swp 5 | *.swo 6 | *.db 7 | 8 | /pkg/utils/logger/log_test.log 9 | 10 | # ci files 11 | ci/playbook/kube 12 | ci/playbook/inventory.yaml 13 | ci/output 14 | ci/tf/.terraform 15 | ci/tf/.terraform.lock.hcl 16 | ci/tf/*.tfstate 17 | ci/tf/*.tfstate.backup 18 | ci/tf/hosts.yaml 19 | ci/tf/values.yaml 20 | 21 | *.DS_Store -------------------------------------------------------------------------------- /ci/playbook/roles/control-plane/templates/kubeadm.conf.j2: -------------------------------------------------------------------------------- 1 | apiVersion: kubeadm.k8s.io/v1beta3 2 | kind: InitConfiguration 3 | skipPhases: 4 | - addon/kube-proxy 5 | --- 6 | apiVersion: kubeadm.k8s.io/v1beta3 7 | kind: ClusterConfiguration 8 | apiServer: 9 | certSANs: 10 | - "{{ k8s_api_server_ip }}" 11 | kubernetesVersion: "v1.24.10" 12 | imageRepository: "cr-cn-beijing.volces.com/cello-k8s" 13 | -------------------------------------------------------------------------------- /images/Dockerfile.builder: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/golang:1.20-bullseye as cni-plugins-builder 2 | 3 | RUN PROTOC_ZIP=protoc-3.14.0-linux-x86_64.zip && \ 4 | curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/$PROTOC_ZIP && \ 5 | sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \ 6 | sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \ 7 | rm -f $PROTOC_ZIP -------------------------------------------------------------------------------- /tests/cni_ready.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load utils/utils 4 | 5 | @test "cello ds ready test" { 6 | cello_ready_count="$(kubectl get ds cello -n kube-system -o jsonpath='{.status.numberReady}')" 7 | node_count="$(kubectl get node -o name |grep -o -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' |wc -l)" 8 | echo $cello_ready_count " " $node_count 9 | [ "$cello_ready_count" -eq "$node_count" ] 10 | } -------------------------------------------------------------------------------- /chart/templates/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: cello-cluster-role-binding 5 | namespace: kube-system 6 | subjects: 7 | - kind: ServiceAccount 8 | name: cello-service-account 9 | namespace: kube-system 10 | roleRef: 11 | apiGroup: rbac.authorization.k8s.io 12 | kind: ClusterRole 13 | name: cello-cluster-role 14 | -------------------------------------------------------------------------------- /chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /ci/tf/values.tftpl: -------------------------------------------------------------------------------- 1 | image: 2 | repository: "volcstack/cello" 3 | tag: "latest" 4 | pullPolicy: "IfNotPresent" 5 | cello: 6 | subnetIDs: 7 | - "${pod_subnet_id}" 8 | securityGroupIDs: ["${pod_sg_id}"] 9 | accessKey: "${access_key}" 10 | secretKey: "${secret_key}" 11 | ipFamily: "ipv4" 12 | redirectToHostCIDRs: ["169.254.0.0/16"] 13 | openAPIAddress: "open.volcengineapi.com" 14 | k8sServiceHost: "${k8s_api_server_ip}" 15 | k8sServicePort: "6443" 16 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/check.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: test-check 5 | 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: test-check 11 | namespace: test-check 12 | labels: 13 | role: test-check 14 | spec: 15 | restartPolicy: Never 16 | containers: 17 | - name: allow 18 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:netshoot-latest 19 | command: ["sh", "-c", "sleep 360000"] -------------------------------------------------------------------------------- /ci/provision.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | extra_args="" 5 | if [ -n "$TF_PLUGIN_DIR" ]; then 6 | extra_args="$extra_args -plugin-dir $TF_PLUGIN_DIR" 7 | fi 8 | 9 | THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 10 | pushd $THIS_DIR > /dev/null 11 | 12 | pushd tf > /dev/null 13 | terraform init $extra_args 14 | terraform apply -auto-approve -var-file=example.tfvars 15 | popd > /dev/null 16 | pushd playbook > /dev/null 17 | ansible-playbook -i ../tf/hosts.yaml k8s.yaml 18 | popd > /dev/null 19 | -------------------------------------------------------------------------------- /ci/tf/example.tfvars: -------------------------------------------------------------------------------- 1 | access_key="" 2 | secret_key="" 3 | key_pair_name="" 4 | key_pair_path="" 5 | zone_id="" 6 | region="" 7 | instance_type="ecs.g3i.xlarge" 8 | vpc_name="my-ci-test" 9 | vpc_cidr="172.16.0.0/16" 10 | node_subnet_name="my-ci-node-subnet" 11 | node_subnet_cidr="172.16.1.0/24" 12 | pod_subnet_name="my-ci-pod-subnet" 13 | pod_subnet_cidr="172.16.2.0/24" 14 | control_plane_node_name="my-ci-control-plane" 15 | worker_node_name="my-ci-worker" 16 | worker2_node_name="my-ci-worker2" 17 | control_eip_name="my-ci-eip" 18 | project_name="default" 19 | -------------------------------------------------------------------------------- /ci/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Create kubernetes cluster 3 | 4 | Edit cluster variables in file `ci/tf/example.tfvars` and run: 5 | 6 | ``` bash 7 | ./provision.sh 8 | ``` 9 | kubectl config file locates at `ci/playbook/kube/config-public`. 10 | 11 | ```bash 12 | export KUBECONFIG=ci/playbook/kube/config-public 13 | kubectl get nodes -A -o wide 14 | ``` 15 | 16 | # Destroy kubernetes cluster 17 | 18 | Uninstall Cello and then run: 19 | 20 | ``` bash 21 | ./destroy.sh 22 | ``` 23 | 24 | # Run conformance tests 25 | 26 | ``` bash 27 | ./conformance_test.sh 28 | ``` 29 | -------------------------------------------------------------------------------- /tests/templates/testcases/stress/scale.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: stress-scale 5 | 6 | --- 7 | apiVersion: apps/v1 8 | kind: Deployment 9 | metadata: 10 | name: nginx-deployment 11 | namespace: stress-scale 12 | spec: 13 | replicas: 3 14 | selector: 15 | matchLabels: 16 | app: nginx 17 | template: 18 | metadata: 19 | labels: 20 | app: nginx 21 | spec: 22 | containers: 23 | - name: nginx 24 | # image: docker.io/library/nginx:latest 25 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx -------------------------------------------------------------------------------- /copyright.txt: -------------------------------------------------------------------------------- 1 | Copyright 2023 The Cello Authors 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /ci/tf/hosts.tftpl: -------------------------------------------------------------------------------- 1 | control-plane: 2 | hosts: 3 | %{ for addr in control_plane_nodes ~} 4 | ${addr}: 5 | %{ endfor ~} 6 | workers: 7 | hosts: 8 | %{ for addr in worker_nodes ~} 9 | ${addr}: 10 | %{ endfor ~} 11 | all: 12 | vars: 13 | ansible_ssh_common_args: -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q -i ${key_pair_path} root@${jump_host}" 14 | ansible_ssh_private_key_file: ${key_pair_path} 15 | host_key_checking: false 16 | ansible_python_interpreter: python3 17 | ansible_user: root 18 | k8s_api_server_ip: ${jump_host} 19 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l3_service_stage2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Endpoints 3 | metadata: 4 | name: l3-policy-service1 5 | namespace: cilium-policy-l3-service 6 | labels: 7 | case: l3-rule-service 8 | subsets: 9 | - addresses: 10 | - ip: BACKEND_IP_0 11 | ports: 12 | - port: 11414 13 | protocol: TCP 14 | 15 | --- 16 | 17 | apiVersion: v1 18 | kind: Endpoints 19 | metadata: 20 | name: l3-policy-service2 21 | namespace: cilium-policy-l3-service 22 | labels: 23 | case: l3-rule-service 24 | subsets: 25 | - addresses: 26 | - ip: BACKEND_IP_1 27 | ports: 28 | - port: 11414 29 | protocol: TCP 30 | 31 | -------------------------------------------------------------------------------- /pkg/cni/plugins/cello/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | // Package cello is the VPC-CNI command handler. 17 | package cello 18 | -------------------------------------------------------------------------------- /pkg/utils/runtime/runtime.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "github.com/volcengine/cello/pkg/utils/logger" 5 | "runtime" 6 | ) 7 | 8 | func logPanic(r interface{}, log logger.Logger) { 9 | // Same as stdlib http server code. Manually allocate stack trace buffer size 10 | // to prevent excessively large logs 11 | const size = 64 << 10 12 | stacktrace := make([]byte, size) 13 | stacktrace = stacktrace[:runtime.Stack(stacktrace, false)] 14 | if _, ok := r.(string); ok { 15 | log.Errorf("Observed a panic: %s\n%s\n", r, stacktrace) 16 | } else { 17 | log.Errorf("Observed a panic: %#v (%v)\n%s", r, r, stacktrace) 18 | } 19 | } 20 | 21 | func HandleCrash(log logger.Logger) { 22 | if r := recover(); r != nil { 23 | logPanic(r, log) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pkg/utils/sysctl/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | // Package sysctl allows to change kernel parameters at runtime. 17 | package sysctl 18 | -------------------------------------------------------------------------------- /pkg/deviceplugin/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | // Package deviceplugin implement utility k8s device plugin of VPC-ENI resources. 17 | package deviceplugin 18 | -------------------------------------------------------------------------------- /ci/playbook/roles/worker/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | - name: Copy the join command from host 2 | copy: 3 | src: kube/join-command 4 | dest: /tmp/join-command.sh 5 | mode: 0777 6 | register: join_command 7 | 8 | - name: Copy kubeconfig from host 9 | copy: 10 | src: kube/config 11 | dest: /tmp/kubeconfig 12 | 13 | - name: Compare kubeconfig with existing one 14 | command: diff /tmp/kubeconfig ~/.kube/config 15 | failed_when: False 16 | register: kubeconfig_diff 17 | changed_when: kubeconfig_diff.rc != 0 18 | 19 | - name: Reset kubeadm if kubeconfig has changed 20 | command: kubeadm reset -f 21 | when: kubeconfig_diff.changed 22 | failed_when: False 23 | 24 | - name: Join the node to cluster 25 | command: sh /tmp/join-command.sh 26 | when: kubeconfig_diff.changed 27 | -------------------------------------------------------------------------------- /pkg/pbrpc/generate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | //go:generate protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative endpoint.proto 17 | package pbrpc 18 | -------------------------------------------------------------------------------- /pkg/cni/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | // Package cni contains all the method and types used by cello-cni, 17 | // which handles CNI calls from runtime, setup/teardown networks for pod sandboxes. 18 | package cni 19 | -------------------------------------------------------------------------------- /cmd/cello-agent/cello-agent.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package main 17 | 18 | import ( 19 | "github.com/volcengine/cello/pkg/daemon" 20 | 21 | _ "go.uber.org/automaxprocs" 22 | ) 23 | 24 | func main() { 25 | daemon.Execute() 26 | } 27 | -------------------------------------------------------------------------------- /pkg/provider/volcengine/ec2/generate_mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package ec2 17 | 18 | //go:generate go run github.com/golang/mock/mockgen -copyright_file "../../../../copyright.txt" -destination mock/ec2_mocks.go -package mock . EC2 19 | -------------------------------------------------------------------------------- /script/bootstrap/cello-probe.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function retry() { 4 | local attempts=$1 5 | local delay=$2 6 | local i 7 | 8 | for ((i=0; i < attempts; i++)); do 9 | "${@:3}" 10 | # shellcheck disable=SC2181 11 | if [[ $? -eq 0 ]] ; then 12 | # shellcheck disable=SC2145 13 | echo "Command \"${@:3}\" success $i times." 14 | return 0 15 | fi 16 | sleep "$delay" 17 | done 18 | 19 | # shellcheck disable=SC2145 20 | echo "Command \"${@:3}\" failed $attempts times." 21 | false 22 | } 23 | 24 | function probe_cello_ready() { 25 | # shellcheck disable=SC2006 26 | result=`curl -s --unix-socket /var/run/cello/cello_debug.socket http://localhost/ |grep ok` 27 | if [[ "$result" != "" ]] 28 | then 29 | return 0 30 | else 31 | return 1 32 | fi 33 | } 34 | 35 | retry 30 2 probe_cello_ready 36 | -------------------------------------------------------------------------------- /pkg/provider/volcengine/metadata/generate_mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package metadata 17 | 18 | //go:generate go run github.com/golang/mock/mockgen -copyright_file "../../../../copyright.txt" -destination mock/ec2wrapper_mocks.go . EC2MetadataIface 19 | -------------------------------------------------------------------------------- /chart/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | repository: "cr-cn-beijing.volces.com/cello/cello" 3 | tag: v1.6.8 4 | pullPolicy: "Always" 5 | 6 | cello: 7 | # IDs of the subnets to be used by Pods. 8 | subnetIDs: [] 9 | # IDs of the security groups to be used by Pods. 10 | securityGroupIDs: [] 11 | # IPFamily represents the IP Family: "ipv4", "ipv6" or "dual". 12 | ipFamily: "ipv4" 13 | # All traffic targeting these CIDRs will be redirected to host. 14 | redirectToHostCIDRs: ["169.254.0.0/16"] 15 | # VolcEngine OpenAPI address. 16 | openAPIAddress: "open.volcengineapi.com" 17 | # VolcEngine iamRole policy authorization. 18 | #iamRole: "KubernetesNodeRoleForECS" 19 | # VolcEngine OpenAPI secrets. 20 | accessKey: "" 21 | secretKey: "" 22 | # Kubernetes service host. 23 | k8sServiceHost: "" 24 | # Kubernetes service port. 25 | k8sServicePort: "" 26 | cilium: 27 | debug: false -------------------------------------------------------------------------------- /pkg/provider/volcengine/cellohelper/generate_mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package cellohelper 17 | 18 | //go:generate go run github.com/golang/mock/mockgen -copyright_file "../../../../copyright.txt" -destination mock/instance_mocks.go -package mock . InstanceMetadataGetter 19 | -------------------------------------------------------------------------------- /cmd/cello-cni/handler_cello.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | //go:build !meta 17 | 18 | package main 19 | 20 | import ( 21 | "github.com/volcengine/cello/pkg/cni/plugins/cello" 22 | ) 23 | 24 | var ( 25 | about = "Cello CNI" 26 | Add = cello.CmdAdd 27 | Del = cello.CmdDel 28 | Check = cello.CmdCheck 29 | ) 30 | -------------------------------------------------------------------------------- /pkg/cni/driver/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | // Package driver include all the data path driver supported by cello, 17 | // currently cello uses plain ENI driver for exclusive ENI mode, 18 | // IPVlan driver for shared ENI with multi IP, veth driver for the fastpath 19 | // of branch-trunk ENI mode. 20 | package driver 21 | -------------------------------------------------------------------------------- /tests/templates/testcases/service/loadblancer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: spod 5 | labels: 6 | app: spod 7 | spec: 8 | containers: 9 | - name: nginx 10 | # image: docker.io/library/nginx:latest 11 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 12 | 13 | --- 14 | # Cluster Loadbalancer 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: loadbalancer-cluster 19 | labels: 20 | test: lbsvc 21 | spec: 22 | selector: 23 | app: spod 24 | ports: 25 | - port: 80 26 | targetPort: 80 27 | protocol: TCP 28 | type: LoadBalancer 29 | 30 | --- 31 | # Local Traffic Loadbalancer 32 | apiVersion: v1 33 | kind: Service 34 | metadata: 35 | name: loadbalancer-local 36 | labels: 37 | test: lbsvc 38 | spec: 39 | selector: 40 | app: spod 41 | ports: 42 | - port: 80 43 | targetPort: 80 44 | protocol: TCP 45 | type: LoadBalancer 46 | externalTrafficPolicy: Local -------------------------------------------------------------------------------- /pkg/utils/math/math.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package math 17 | 18 | import "math" 19 | 20 | func Max(x, y int) int { 21 | if x < y { 22 | return y 23 | } 24 | return x 25 | } 26 | 27 | func Min(x, y int) int { 28 | if y < x { 29 | return y 30 | } 31 | return x 32 | } 33 | 34 | func Floor(x float64) int { 35 | y := math.Floor(x) 36 | return int(y) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package version 17 | 18 | import ( 19 | "fmt" 20 | "runtime" 21 | ) 22 | 23 | var Version = "v1.6.8" 24 | var ComponentName = "cello-agent" 25 | var GitCommit = "unset" 26 | 27 | func UserAgent() string { 28 | return fmt.Sprintf("%s/%s (%s/%s)/%s", ComponentName, Version, runtime.GOOS, runtime.GOARCH, GitCommit) 29 | } 30 | -------------------------------------------------------------------------------- /types/k8s.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package types 17 | 18 | const ( 19 | AnnotationK8sPrefix = "k8s.volcengine.com/" 20 | 21 | AnnotationTrunkENI = AnnotationK8sPrefix + "trunk-eni" 22 | AnnotationPodNetworksDefinition = AnnotationK8sPrefix + "pod-networks-definition" 23 | AnnotationVKEPodNetworks = AnnotationK8sPrefix + "pod-networks" 24 | ) 25 | -------------------------------------------------------------------------------- /tests/templates/testcases/service/hostnetwork/loadblancer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: spod-hn 5 | labels: 6 | app: spod-hn 7 | spec: 8 | hostNetwork: true 9 | containers: 10 | - name: nginx 11 | # image: docker.io/library/nginx:latest 12 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 13 | 14 | --- 15 | # Cluster Loadbalancer 16 | apiVersion: v1 17 | kind: Service 18 | metadata: 19 | name: loadbalancer-cluster-hn 20 | labels: 21 | test: lbsvc-hn 22 | spec: 23 | selector: 24 | app: spod-hn 25 | ports: 26 | - port: 80 27 | targetPort: 80 28 | protocol: TCP 29 | type: LoadBalancer 30 | 31 | --- 32 | # Local Traffic Loadbalancer 33 | apiVersion: v1 34 | kind: Service 35 | metadata: 36 | name: loadbalancer-local-hn 37 | labels: 38 | test: lbsvc-hn 39 | spec: 40 | selector: 41 | app: spod-hn 42 | ports: 43 | - port: 80 44 | targetPort: 80 45 | protocol: TCP 46 | type: LoadBalancer 47 | externalTrafficPolicy: Local -------------------------------------------------------------------------------- /types/volcengine.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package types 17 | 18 | const ( 19 | AnnotationVolcEnginePrefix = "vke.volcengine.com/" 20 | LabelNodeDynamicConfigKey = AnnotationVolcEnginePrefix + "vpc-cni-config" 21 | EvictionPolicyAnnotation = AnnotationVolcEnginePrefix + "cello-pod-evict-policy" 22 | AnnotationTrue = "true" 23 | AllowEviction = "allow" 24 | ) 25 | -------------------------------------------------------------------------------- /docs/host-stack-forward.md: -------------------------------------------------------------------------------- 1 | # 主机网络栈路由 2 | 3 | ## 背景 4 | 在一些特殊的通信场景下，需要将目的地址是特定cidr的流量转发给主机网络栈进行处理。常见的如DNS缓存方案 5 | node-local-dns 中，Local DNS 缓存作为 DaemonSet 部署在每个集群节点上，通过 Link-Local Address 暴露缓存服务。 6 | 如需使用本地DNS缓存，可以将 Link-Local Address 设置到主机网络栈路由中。这样节点上的所有Pod就可以通过Link-Local Address访问到本地DNS缓存。 7 | 8 | ## 配置流程 9 | 1. 在kube-system命名空间下，修改configmap `cello-config`，修改`redirectToHostCIDRs`字段，添加需要转发到主机网络栈的cidr 10 | ```yaml 11 | 10-cello.conflist: | 12 | { 13 | "cniVersion": "0.3.1", 14 | "name": "cello-chainer", 15 | "plugins": [ 16 | { 17 | "type": "cello", 18 | ... 19 | "redirectToHostCIDRs": ["169.254.0.0/16"] 20 | } 21 | ] 22 | } 23 | ``` 24 | 25 | 2. 重启所有Cello Pod 26 | ```bash 27 | kubectl delete pod -l app=cello -n kube-system 28 | ``` 29 | 30 | 3. Cello pod重启后登录任意节点执行以下命令确认配置是否生效 31 | ```bash 32 | cat /etc/cni/net.d/10-cello.conflist 33 | ``` 34 | 4. 重建Pod后，Pod即可访问该新网段。 -------------------------------------------------------------------------------- /pkg/utils/kernel/kernel_unsupport.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | // +build !linux 3 | 4 | // Copyright 2023 The Cello Authors 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | package kernel 20 | 21 | import ( 22 | "errors" 23 | ) 24 | 25 | func GetKernelVersion() (*VersionInfo, error) { 26 | return nil, errors.New("GetKernelVersion is available only on linux") 27 | } 28 | 29 | func CheckKernelVersion(k, major, minor int) bool { 30 | return false 31 | } 32 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l4_cidr_dependent_stage1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-policy-l4-cidr-dep 5 | 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: l4-cidr-dep-source1 11 | namespace: cilium-policy-l4-cidr-dep 12 | labels: 13 | case: l4-cidr-dep 14 | role: l4-cidr-dep-source1 15 | spec: 16 | restartPolicy: Never 17 | containers: 18 | - name: fail 19 | # image: docker.io/library/busybox:1.28.4 20 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 21 | command: ["sh", "-c", "wget -T 1 100.96.0.96/volcstack"] 22 | 23 | --- 24 | apiVersion: "cilium.io/v2" 25 | kind: CiliumNetworkPolicy 26 | metadata: 27 | name: l4-rule-cidr-dep1 28 | namespace: cilium-policy-l4-cidr-dep 29 | spec: 30 | endpointSelector: 31 | matchLabels: 32 | role: l4-cidr-dep-source1 33 | egress: 34 | - toCIDR: 35 | - 100.96.0.0/16 36 | toPorts: 37 | - ports: 38 | - port: "70" 39 | protocol: TCP -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l4_cidr_dependent_stage2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-policy-l4-cidr-dep 5 | 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: l4-cidr-dep-source2 11 | namespace: cilium-policy-l4-cidr-dep 12 | labels: 13 | case: l4-cidr-dep 14 | role: l4-cidr-dep-source2 15 | spec: 16 | restartPolicy: Never 17 | containers: 18 | - name: success 19 | # image: docker.io/library/busybox:1.28.4 20 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 21 | command: ["sh", "-c", "wget -T 1 100.96.0.96/volcstack"] 22 | 23 | --- 24 | apiVersion: "cilium.io/v2" 25 | kind: CiliumNetworkPolicy 26 | metadata: 27 | name: l4-rule-cidr-dep2 28 | namespace: cilium-policy-l4-cidr-dep 29 | spec: 30 | endpointSelector: 31 | matchLabels: 32 | role: l4-cidr-dep-source2 33 | egress: 34 | - toCIDR: 35 | - 100.96.0.0/16 36 | toPorts: 37 | - ports: 38 | - port: "80" 39 | protocol: TCP -------------------------------------------------------------------------------- /pkg/provider/volcengine/credential/static.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package credential 17 | 18 | // StaticProvider provide static Credential. 19 | type StaticProvider struct { 20 | static *Credential 21 | } 22 | 23 | func (p *StaticProvider) Get() *Credential { 24 | return p.static 25 | } 26 | 27 | func NewStaticProvider(credential *Credential) Provider { 28 | return &StaticProvider{static: credential} 29 | } 30 | -------------------------------------------------------------------------------- /pkg/provider/volcengine/credential/credential.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package credential 17 | 18 | import "time" 19 | 20 | // Credential for access volcengine service. 21 | type Credential struct { 22 | ExpiredTime time.Time 23 | CurrentTime time.Time 24 | AccessKeyId string 25 | SecretAccessKey string 26 | SessionToken string 27 | } 28 | 29 | // Provider of Credential. 30 | type Provider interface { 31 | // Get Credential from Provider 32 | Get() *Credential 33 | } 34 | -------------------------------------------------------------------------------- /pkg/cni/log/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package log 17 | 18 | import ( 19 | "k8s.io/apimachinery/pkg/util/uuid" 20 | 21 | "github.com/volcengine/cello/pkg/utils/logger" 22 | ) 23 | 24 | var conf = logger.Configuration{ 25 | LogLocation: "/var/log/cello/cni.log", 26 | LogLevel: "info", 27 | } 28 | 29 | // Log uses logger with traceID print log. 30 | var Log = logger.New(&conf).WithFields(logger.Fields{"component": "celloCNI", "subsys": "cniBin", "traceID": string(uuid.NewUUID())[:8]}) 31 | -------------------------------------------------------------------------------- /pkg/cni/types/kubelet.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package types 17 | 18 | // ResourceInfo is struct to hold Pod device allocation information. 19 | type ResourceInfo struct { 20 | Index int 21 | DeviceIDs []string 22 | } 23 | 24 | // ResourceClient provides a kubelet Pod resource handle. 25 | type ResourceClient interface { 26 | // GetPodResourceMap returns an instance of a map of Pod ResourceInfo given a (Pod name, namespace) tuple. 27 | GetPodResourceMap(podNamespace, podName string) (map[string]*ResourceInfo, error) 28 | } 29 | -------------------------------------------------------------------------------- /ci/playbook/roles/common/tasks/kube.yaml: -------------------------------------------------------------------------------- 1 | # https://developer.volcengine.com/articles/7132385421970915358 2 | - name: Add an apt signing key for Kubernetes 3 | apt_key: 4 | url: http://mirrors.ivolces.com/kubernetes/apt/doc/apt-key.gpg 5 | state: present 6 | 7 | - name: Adding apt repository for Kubernetes 8 | apt_repository: 9 | repo: deb http://mirrors.ivolces.com/kubernetes/apt/ kubernetes-xenial main 10 | state: present 11 | filename: /etc/apt/sources.list.d/kubernetes.list 12 | 13 | - name: Install Kubernetes binaries 14 | apt: 15 | name: "{{ packages }}" 16 | state: present 17 | update_cache: yes 18 | force_apt_get: yes 19 | vars: 20 | packages: 21 | - kubelet=1.24.10-00 22 | - kubeadm=1.24.10-00 23 | - kubectl=1.24.10-00 24 | notify: 25 | - kubelet status 26 | 27 | - name: Configure node ip 28 | lineinfile: 29 | path: /etc/default/kubelet 30 | line: KUBELET_EXTRA_ARGS=--node-ip={{ inventory_hostname }} 31 | state: present 32 | create: yes 33 | register: kubelet_config 34 | 35 | - name: Restart kubelet 36 | service: 37 | name: kubelet 38 | daemon_reload: yes 39 | state: restarted 40 | when: kubelet_config.changed 41 | -------------------------------------------------------------------------------- /pkg/deviceplugin/mock/mock_manager.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package mock 17 | 18 | import "github.com/volcengine/cello/pkg/deviceplugin" 19 | 20 | type PluginManager struct{} 21 | 22 | func (p PluginManager) Plugin(resourceName string) deviceplugin.Plugin { 23 | return nil 24 | } 25 | 26 | func (p PluginManager) AddPlugin(plugin deviceplugin.Plugin) { 27 | return 28 | } 29 | 30 | func (p PluginManager) Update(resourceName string, count int) error { 31 | return nil 32 | } 33 | 34 | func (p PluginManager) Serve(_ chan struct{}) error { 35 | return nil 36 | } 37 | 38 | func (p PluginManager) Stop() { 39 | } 40 | -------------------------------------------------------------------------------- /docs/iam-policy.md: -------------------------------------------------------------------------------- 1 | # IAM Policy 2 | Cello需要将以下策略附加到特定的IAM Role（比如`KubernetesNodeRoleForECS`）以获得访问相应OpenAPI的权限。 3 | 最终附加了以下策略的IAM Role需要传递给Cello。 4 | 5 | 仅开启IPv4时： 6 | ```json 7 | { 8 | "Version": "1", 9 | "Statement": [ 10 | { 11 | "Effect": "Allow", 12 | "Action": [ 13 | "ecs:DescribeInstances", 14 | "ecs:DescribeInstanceTypes" 15 | ], 16 | "Resource": [ 17 | "*" 18 | ] 19 | }, 20 | { 21 | "Effect": "Allow", 22 | "Action": [ 23 | "vpc:DescribeNetworkInterfaces", 24 | "vpc:CreateNetworkInterface", 25 | "vpc:AttachNetworkInterface", 26 | "vpc:DescribeNetworkInterfaceAttributes", 27 | "vpc:DetachNetworkInterface", 28 | "vpc:DeleteNetworkInterface", 29 | "vpc:AssignPrivateIpAddresses", 30 | "vpc:UnassignPrivateIpAddresses", 31 | "vpc:DescribeSubnets", 32 | "vpc:DescribeSubnetAttributes" 33 | ], 34 | "Resource": [ 35 | "*" 36 | ] 37 | } 38 | ] 39 | } 40 | ``` 41 | 开启IPv6时还需增加以下策略： 42 | ```json 43 | { 44 | "Effect": "Allow", 45 | "Action": [ 46 | "vpc:AssignIpv6Addresses", 47 | "vpc:UnassignIpv6Addresses" 48 | ], 49 | "Resource": [ 50 | "*" 51 | ] 52 | } 53 | ``` -------------------------------------------------------------------------------- /pkg/daemon/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package daemon 17 | 18 | import ( 19 | "fmt" 20 | "net/http" 21 | 22 | "github.com/gin-gonic/gin" 23 | 24 | "github.com/volcengine/cello/pkg/utils/logger" 25 | ) 26 | 27 | var ( 28 | log = logger.GetLogger().WithFields(logger.Fields{"subsys": "daemon"}) 29 | ) 30 | 31 | type setLogLevel struct{} 32 | 33 | func (l *setLogLevel) Handle(c *gin.Context) { 34 | logLevel := c.Query("logLevel") 35 | log.SetLogLevel(logLevel) 36 | log.Infof("Set logLevel to %s", logLevel) 37 | c.JSON(http.StatusOK, fmt.Sprintf("set log to level %s\n", logLevel)) 38 | } 39 | 40 | func newSetLogLevelHandler() Handler { 41 | return &setLogLevel{} 42 | } 43 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | # golangci-lint configuration used for CI 2 | run: 3 | tests: true 4 | timeout: 10m 5 | skip-files: 6 | - ".*\\.pb\\.go" 7 | skip-dirs-use-default: true 8 | 9 | # all available settings of specific linters 10 | linters-settings: 11 | goimports: 12 | local-prefixes: github.com/volcengine/cello 13 | goheader: 14 | template-path: ./copyright.txt 15 | 16 | issues: 17 | # Excluding configuration per-path, per-linter, per-text and per-source 18 | exclude-rules: 19 | - linters: [ staticcheck ] 20 | text: "SA1019" # this is rule for deprecated method 21 | - linters: [ staticcheck ] 22 | text: "SA9003: empty branch" 23 | - linters: [ staticcheck ] 24 | text: "SA2001: empty critical section" 25 | - linters: [ goerr113 ] 26 | text: "do not define dynamic errors, use wrapped static errors instead" # This rule to avoid opinionated check fmt.Errorf("text") 27 | # Skip goimports check on generated files 28 | - path: \\.(generated\\.deepcopy|pb)\\.go$ 29 | linters: 30 | - goimports 31 | 32 | linters: 33 | disable-all: true 34 | enable: 35 | - goerr113 36 | - gofmt 37 | - goimports 38 | - govet 39 | - ineffassign 40 | - misspell 41 | - staticcheck 42 | - unused 43 | - goheader -------------------------------------------------------------------------------- /cmd/cello-cli/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package main 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | 22 | "github.com/urfave/cli/v2" 23 | 24 | "github.com/volcengine/cello/pkg/version" 25 | ) 26 | 27 | var BuildInfo = "master" 28 | 29 | const title = "cello-ctl" 30 | const usage = "show cello information" 31 | 32 | const baseUrl = "http://cello_debug.socket" 33 | 34 | func main() { 35 | app := &cli.App{ 36 | Name: title, 37 | Usage: usage, 38 | Version: fmt.Sprintf("%s\n%s", version.Version, BuildInfo), 39 | Commands: buildCommand(), 40 | } 41 | 42 | err := app.Run(os.Args) 43 | if err != nil { 44 | fmt.Printf("%s", err) 45 | os.Exit(1) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l4_port.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-policy-l4-port 5 | 6 | --- 7 | apiVersion: "cilium.io/v2" 8 | kind: CiliumNetworkPolicy 9 | metadata: 10 | name: l4-rule-port 11 | namespace: cilium-policy-l4-port 12 | spec: 13 | endpointSelector: 14 | matchLabels: 15 | role: l4-rule-port-source 16 | egress: 17 | - toPorts: 18 | - ports: 19 | - port: "11414" 20 | protocol: TCP 21 | --- 22 | apiVersion: v1 23 | kind: Pod 24 | metadata: 25 | name: l4-rule-port-source 26 | namespace: cilium-policy-l4-port 27 | labels: 28 | case: l4-rule-port 29 | role: l4-rule-port-source 30 | spec: 31 | restartPolicy: Never 32 | containers: 33 | - name: fail 34 | # image: docker.io/library/busybox:1.28.4 35 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 36 | command: ["sh", "-c", "wget -T 1 100.96.0.96/volcstack"] 37 | - name: success 38 | # image: docker.io/library/busybox:1.28.4 39 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 40 | command: ["sh", "-c", "wget -T 1 $MY_NODE_IP:11414/healthz"] 41 | env: 42 | - name: MY_NODE_IP 43 | valueFrom: 44 | fieldRef: 45 | fieldPath: status.hostIP 46 | -------------------------------------------------------------------------------- /pkg/daemon/netcontext.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package daemon 17 | 18 | import ( 19 | "context" 20 | 21 | "github.com/volcengine/cello/pkg/utils/logger" 22 | "github.com/volcengine/cello/types" 23 | ) 24 | 25 | // netContext is the context for network operation. 26 | type netContext struct { 27 | context.Context 28 | pod *types.Pod 29 | res []types.VPCResource // record resource allocate for rollback 30 | log logger.Logger 31 | } 32 | 33 | func (ctx *netContext) Log() logger.Logger { 34 | return ctx.log.WithFields(logger.Fields{ 35 | "Namespace": ctx.pod.Namespace, 36 | "Name": ctx.pod.Name, 37 | "SandboxContainerId": ctx.pod.SandboxContainerId, 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /patch/cilium/003-adapt-to-ipvlan-datapath.patch: -------------------------------------------------------------------------------- 1 | From f8a6c547d44444389e05eb8ce5b580a67db7b4f9 Mon Sep 17 00:00:00 2001 2 | From: xinwenqiang 3 | Date: Wed, 26 Jan 2022 17:25:15 +0800 4 | Subject: [PATCH 2/2] adapt to ipvlan datapath 5 | 6 | --- 7 | bpf/lib/nodeport.h | 9 ++++++--- 8 | 1 file changed, 6 insertions(+), 3 deletions(-) 9 | 10 | diff --git a/bpf/lib/nodeport.h b/bpf/lib/nodeport.h 11 | index 28848f4..1d98e97 100644 12 | --- a/bpf/lib/nodeport.h 13 | +++ b/bpf/lib/nodeport.h 14 | @@ -2059,7 +2059,7 @@ __section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_NODEPORT_REVNAT) 15 | int tail_rev_nodeport_lb4(struct __ctx_buff *ctx) 16 | { 17 | int ifindex = 0; 18 | - int ret = 0; 19 | + int ret, dir = 0; 20 | #if defined(ENABLE_HOST_FIREWALL) && defined(IS_BPF_HOST) 21 | /* We only enforce the host policies if nodeport.h is included from 22 | * bpf_host. 23 | @@ -2081,8 +2081,11 @@ int tail_rev_nodeport_lb4(struct __ctx_buff *ctx) 24 | 25 | edt_set_aggregate(ctx, 0); 26 | cilium_capture_out(ctx); 27 | - 28 | - return ctx_redirect(ctx, ifindex, 0); 29 | + dir = ctx_load_meta(ctx, CB_NAT); 30 | + if (dir == NAT_DIR_INGRESS){ 31 | + return ctx_redirect(ctx, ifindex, 0); 32 | + } 33 | + return CTX_ACT_OK; 34 | } 35 | 36 | declare_tailcall_if(__or3(__and(is_defined(ENABLE_IPV4), 37 | -- 38 | 2.32.0 (Apple Git-132) 39 | 40 | -------------------------------------------------------------------------------- /pkg/tracing/mock/tracing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package mock 17 | 18 | import ( 19 | "github.com/volcengine/cello/pkg/tracing" 20 | "github.com/volcengine/cello/pkg/utils/logger" 21 | ) 22 | 23 | var log = logger.GetLogger().WithFields(logger.Fields{"subsys": "fake_tracing"}) 24 | 25 | func NewFakeTracker() *tracing.Tracer { 26 | t := tracing.NewTracer() 27 | podEventRecord := func(podName, podNamespace, eventType, reason, message string) error { 28 | log.Infof("%s %s From %s/%s %s", eventType, reason, podNamespace, podName, message) 29 | return nil 30 | } 31 | nodeEventRecord := func(eventType, reason, message string) { 32 | log.Infof("%s %s %s", eventType, reason, message) 33 | } 34 | t.RegisterEventRecorder(nodeEventRecord, podEventRecord) 35 | return t 36 | } 37 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l3_cidr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-policy-l3-cidr 5 | 6 | --- 7 | apiVersion: "cilium.io/v2" 8 | kind: CiliumNetworkPolicy 9 | metadata: 10 | name: l3-rule-cidr 11 | namespace: cilium-policy-l3-cidr 12 | spec: 13 | endpointSelector: 14 | matchLabels: 15 | role: l3-rule-cidr-source 16 | egress: 17 | - toCIDR: 18 | - 100.96.0.2/32 19 | 20 | --- 21 | apiVersion: v1 22 | kind: Pod 23 | metadata: 24 | name: l3-rule-cidr-source 25 | namespace: cilium-policy-l3-cidr 26 | labels: 27 | case: l3-rule-cidr 28 | role: l3-rule-cidr-source 29 | spec: 30 | affinity: 31 | podAntiAffinity: 32 | requiredDuringSchedulingIgnoredDuringExecution: 33 | - labelSelector: 34 | matchExpressions: 35 | - key: "case" 36 | operator: In 37 | values: 38 | - l3-rule-cidr 39 | topologyKey: kubernetes.io/hostname 40 | restartPolicy: Never 41 | containers: 42 | - name: fail 43 | # image: docker.io/library/busybox:1.28.4 44 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 45 | command: ["sh", "-c", "wget -T 1 100.96.0.96/volcstack"] 46 | - name: success 47 | # image: docker.io/library/busybox:1.28.4 48 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 49 | command: ["sh", "-c", "ping -c 2 100.96.0.2"] 50 | 51 | -------------------------------------------------------------------------------- /pkg/metrics/daemon.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package metrics 17 | 18 | import "github.com/prometheus/client_golang/prometheus" 19 | 20 | var ( 21 | // RpcLatency the latency of rpc call. 22 | RpcLatency = prometheus.NewSummaryVec( 23 | prometheus.SummaryOpts{ 24 | Name: "rpc_latency_ms", 25 | Help: "cello rpc call latency in ms", 26 | }, 27 | []string{"rpc_api", "error"}, 28 | ) 29 | 30 | // ResourceManagerErr error counter of resource manager. 31 | ResourceManagerErr = prometheus.NewCounterVec( 32 | prometheus.CounterOpts{ 33 | Name: "resource_manager_error_count", 34 | Help: "The number of errors encountered in eni manager", 35 | }, 36 | []string{"fn", "error"}, 37 | ) 38 | ) 39 | 40 | func ResourceManagerErrInc(fn string, err error) { 41 | ResourceManagerErr.WithLabelValues(fn, err.Error()).Inc() 42 | } 43 | -------------------------------------------------------------------------------- /pkg/signal/signal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package signal 17 | 18 | import ( 19 | "testing" 20 | "time" 21 | 22 | "github.com/stretchr/testify/assert" 23 | "k8s.io/apimachinery/pkg/util/wait" 24 | ) 25 | 26 | func TestSignal(t *testing.T) { 27 | stopCh := make(chan struct{}) 28 | sig := make(chan SigData) 29 | 30 | err := RegisterChannel(WakeGC, sig) 31 | assert.NoError(t, err) 32 | go wait.JitterUntil(func() { 33 | NotifySignal(WakeGC, SigWakeGC) 34 | }, time.Second, 0.2, true, stopCh) 35 | 36 | go func() { 37 | for { 38 | select { 39 | case <-sig: 40 | t.Logf("Recevice signal") 41 | time.Sleep(2 * time.Second) 42 | case <-stopCh: 43 | return 44 | } 45 | } 46 | }() 47 | 48 | time.Sleep(5 * time.Second) 49 | MuteChannel(WakeGC) 50 | close(sig) 51 | time.Sleep(2 * time.Second) 52 | close(stopCh) 53 | time.Sleep(2 * time.Second) 54 | } 55 | -------------------------------------------------------------------------------- /docs/traffic-ctrl.md: -------------------------------------------------------------------------------- 1 | # Traffic Shaping 2 | 3 | ## Support 4 | Kernel >= 5.1 needed 5 | 6 | | Mode | Egress Shaping | Ingress Shaping | 7 | |-----------------------------------|-----------------|-----------------| 8 | | eni_shared(eni-multi-ip + ipvlan) | ☑️ | - | 9 | | eni_exclusive | ☑️ | - | 10 | | trunk(branch eni + vlan) (WIP) | ☑️ | - | 11 | 12 | ## Config 13 | to enable traffic shaping, follow config need to add in cello-config configmap 14 | 15 | ```yaml 16 | kind: ConfigMap 17 | apiVersion: v1 18 | metadata: 19 | name: cello-config 20 | namespace: kube-system 21 | data: 22 | 10-cello.conflist: | 23 | { 24 | "cniVersion": "0.3.1", 25 | "name": "cello-chainer", 26 | "plugins": [ 27 | { 28 | "type": "cello", 29 | "capabilities": { 30 | "bandwidth": true, #Add 31 | }, 32 | } 33 | ] 34 | } 35 | ``` 36 | follow config need to add in cilium-config configmap 37 | ```yaml 38 | kind: ConfigMap 39 | apiVersion: v1 40 | metadata: 41 | name: cilium-config 42 | namespace: kube-system 43 | data: 44 | enable-bandwidth-manager: "true" 45 | ``` 46 | 47 | ## Usage 48 | Add follow annotation to Pod 49 | 50 | | Annotation | Mean | 51 | |---------------------------------------| ---------------- | 52 | | `kubernetes.io/egress-bandwidth: 10M` | egress banwidth | 53 | -------------------------------------------------------------------------------- /script/bootstrap/install_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -x 5 | set -o nounset 6 | 7 | CURDIR=$( 8 | cd $(dirname $0) || exit 9 | pwd 10 | ) 11 | 12 | cd "${CURDIR}" || exit 13 | 14 | init_node_bpf() { 15 | nsenter -t 1 -m -- bash -c ' 16 | mount | grep "/sys/fs/bpf type bpf" || { 17 | # Mount the filesystem until next reboot 18 | echo "Mounting BPF filesystem..." 19 | mount bpffs /sys/fs/bpf -t bpf 20 | 21 | echo "Link information:" 22 | ip link 23 | 24 | echo "Routing table:" 25 | ip -4 route 26 | ip -6 route 27 | 28 | echo "Addressing:" 29 | ip -4 addr 30 | ip -6 addr 31 | # date > /tmp/cilium-bootstrap-time 32 | echo "Node initialization complete" 33 | }' 34 | } 35 | 36 | # check kernel version & enable cilium 37 | read KERNEL_MAJOR_VERSION KERNEL_MINOR_VERSION < <(uname -r | awk -F . '{print $1,$2}') 38 | # kernel version equal and above 4.19 39 | if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } || 40 | [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then 41 | echo "Init node BPF" 42 | init_node_bpf 43 | else 44 | echo "Linux kernel version <= 4.19, cant install cilium" 45 | exit 1 46 | fi 47 | 48 | echo "modprobe ipvlan" 49 | modprobe ipvlan || echo "modprobe ipvlan failed" 50 | echo "modprobe sch_htb" 51 | modprobe sch_htb || echo "modprobe sch_htb failed" 52 | 53 | # install CNIs 54 | /bin/cp -f /etc/cello/net.d/* /etc/cni/net.d 55 | /bin/cp -f /cello/cello-cni /opt/cni/bin 56 | /bin/cp -f /containernetworking/plugins/* /opt/cni/bin/ 57 | -------------------------------------------------------------------------------- /tests/network_policy.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load utils/utils 4 | 5 | function clean() { 6 | kubectl delete pods -n policy-test --all || true 7 | retry 30 3 object_not_exist pod -l app=policy-spod -n policy-test 8 | retry 30 3 object_not_exist pod -l app=non-policy-spod -n policy-test 9 | retry 30 3 object_not_exist pod -l app=non-policy-cli -n policy-test 10 | retry 30 3 object_not_exist pod -l app=policy-cli -n policy-test 11 | 12 | kubectl delete svc -n policy-test --all || true 13 | retry 30 3 object_not_exist svc -n policy-test 14 | 15 | kubectl delete networkpolicies -n policy-test --all || true 16 | retry 30 3 object_not_exist networkpolicies -n policy-test 17 | 18 | kubectl delete ns policy-test || true 19 | retry 30 3 object_not_exist ns policy-test 20 | } 21 | 22 | function setup() { 23 | clean 24 | } 25 | 26 | @test "test network policy" { 27 | kubectl apply -f templates/testcases/network_policy/policy.yaml 28 | retry 5 20 bash -c "kubectl get pod -n policy-test policy-cli | grep Completed" 29 | retry 5 20 bash -c "kubectl get pod -n policy-test non-policy-cli | grep Completed" 30 | result=`kubectl get pod -n policy-test -o jsonpath='{range .status.containerStatuses[*]}{.state.terminated.reason}{end}' policy-cli` 31 | [ "$result" = "CompletedCompleted" ] 32 | result=`kubectl get pod -n policy-test -o jsonpath='{range .status.containerStatuses[*]}{.state.terminated.reason}{end}' non-policy-cli` 33 | [ "$result" = "CompletedError" ] 34 | clean 35 | } -------------------------------------------------------------------------------- /pkg/store/store.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package store 17 | 18 | import "errors" 19 | 20 | // ErrNotFound key not found in store. 21 | var ErrNotFound = errors.New("not found") 22 | 23 | // Interface offers a generic interface for data persistence operations. 24 | type Interface interface { 25 | // Put save the value for the given key. 26 | Put(key string, value interface{}) error 27 | // Get retrieves the value for the given key. 28 | Get(key string) (interface{}, error) 29 | // List returns all values in storage. 30 | List() []interface{} 31 | // Delete deletes the value for the given key. 32 | Delete(key string) error 33 | // Close closes the storage. 34 | Close() 35 | } 36 | 37 | // Serializer transforms the given object into serialized format. 38 | type Serializer func(interface{}) ([]byte, error) 39 | 40 | // Deserializer transforms the given serialized data into the original object. 41 | type Deserializer func([]byte) (interface{}, error) 42 | -------------------------------------------------------------------------------- /tests/templates/testcases/pod_connection/samenode.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: nginx-configmap 5 | data: 6 | nginx.conf: | 7 | worker_processes auto; 8 | events { 9 | } 10 | http { 11 | server { 12 | server_name nginx; 13 | listen 80 default_server; 14 | location /healthz { 15 | add_header Content-Type text/plain; 16 | return 200 'ok'; 17 | } 18 | } 19 | } 20 | 21 | --- 22 | 23 | apiVersion: apps/v1 24 | kind: Deployment 25 | metadata: 26 | name: samenode-nginx 27 | labels: 28 | app: samenode-nginx 29 | spec: 30 | replicas: 2 31 | selector: 32 | matchLabels: 33 | app: samenode-nginx 34 | template: 35 | metadata: 36 | labels: 37 | app: samenode-nginx 38 | spec: 39 | volumes: 40 | - name: nginx 41 | configMap: 42 | name: nginx-configmap 43 | containers: 44 | - name: nginx 45 | # image: docker.io/library/nginx:latest 46 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 47 | volumeMounts: 48 | - name: nginx 49 | mountPath: /etc/nginx 50 | readOnly: true 51 | affinity: 52 | podAffinity: 53 | requiredDuringSchedulingIgnoredDuringExecution: 54 | - labelSelector: 55 | matchExpressions: 56 | - key: app 57 | operator: In 58 | values: 59 | - samenode-nginx 60 | topologyKey: kubernetes.io/hostname 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /tests/templates/testcases/pod_connection/crossnode.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: nginx-configmap 5 | data: 6 | nginx.conf: | 7 | worker_processes auto; 8 | events { 9 | } 10 | http { 11 | server { 12 | server_name nginx; 13 | listen 80 default_server; 14 | location /healthz { 15 | add_header Content-Type text/plain; 16 | return 200 'ok'; 17 | } 18 | } 19 | } 20 | 21 | --- 22 | 23 | apiVersion: apps/v1 24 | kind: Deployment 25 | metadata: 26 | name: crossnode-nginx 27 | labels: 28 | app: crossnode-nginx 29 | spec: 30 | replicas: 2 31 | selector: 32 | matchLabels: 33 | app: crossnode-nginx 34 | template: 35 | metadata: 36 | labels: 37 | app: crossnode-nginx 38 | spec: 39 | volumes: 40 | - name: nginx 41 | configMap: 42 | name: nginx-configmap 43 | containers: 44 | - name: nginx 45 | # image: docker.io/library/nginx:latest 46 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 47 | volumeMounts: 48 | - name: nginx 49 | mountPath: /etc/nginx 50 | readOnly: true 51 | affinity: 52 | podAntiAffinity: 53 | requiredDuringSchedulingIgnoredDuringExecution: 54 | - labelSelector: 55 | matchExpressions: 56 | - key: app 57 | operator: In 58 | values: 59 | - crossnode-nginx 60 | topologyKey: kubernetes.io/hostname 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /pkg/tracing/event.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package tracing 17 | 18 | const ( 19 | EventConfigMapUpdateFailed = "ConfigMapUpdateFailed" 20 | EventUpdateSubnetFailed = "UpdateSubnetFailed" 21 | EventUpdateSecurityGroupFailed = "UpdateSecurityGroupFailed" 22 | 23 | EventNoAvailableSubnet = "NoAvailableSubnet" 24 | EventSubnetAvailableIPBelowThreshold = "SubnetAvailableIPBelowThreshold" 25 | 26 | EventVpcResourceQuotaExceeded = "VpcResourceQuotaExceeded" 27 | EventOpenApiFlowLimit = "OpenApiFlowLimit" 28 | EventInsufficientIpInSubnet = "InsufficientIpInSubnet" 29 | EventGetInstanceQuotaFailed = "GetInstanceQuotaFailed" 30 | EventInstanceQuotaUpdated = "InstanceQuotaUpdated" 31 | 32 | EventMetadataServiceAbnormal = "MetadataServiceAbnormal" 33 | EventCredentialServiceAbnormal = "CredentialServiceAbnormal" 34 | 35 | EventAllocateResourceFailed = "AllocateResourceFailed" 36 | EventReleaseResourceFailed = "ReleaseResourceFailed" 37 | ) 38 | -------------------------------------------------------------------------------- /images/Dockerfile.base: -------------------------------------------------------------------------------- 1 | ARG CILIUM_LLVM_IMAGE=quay.io/cilium/cilium-llvm:0147a23fdada32bd51b4f313c645bcb5fbe188d6@sha256:24fd3ad32471d0e45844c856c38f1b2d4ac8bd0a2d4edf64cffaaa3fd0b21202 2 | ARG CILIUM_BPFTOOL_IMAGE=quay.io/cilium/cilium-bpftool:b5ba881d2a7ec68d88ecd72efd60ac551c720701@sha256:458282e59657b8f779d52ae2be2cdbeecfe68c3d807ff87c97c8d5c6f97820a9 3 | ARG CILIUM_IPROUTE2_IMAGE=quay.io/cilium/cilium-iproute2:4db2c4bdf00ce461406e1c82aada461356fac935@sha256:e4c9ba92996a07964c1b7cd97c4aac950754ec75d7ac8c626a99c79acd0479ab 4 | 5 | FROM ${CILIUM_LLVM_IMAGE} as llvm-dist 6 | FROM ${CILIUM_BPFTOOL_IMAGE} as bpftool-dist 7 | FROM ${CILIUM_IPROUTE2_IMAGE} as iproute2-dist 8 | 9 | FROM ubuntu:20.04 10 | 11 | COPY --from=llvm-dist /usr/local/bin/clang /usr/local/bin/llc /bin/ 12 | COPY --from=bpftool-dist /usr/local/bin/bpftool /usr/local/bin/ 13 | COPY --from=iproute2-dist /usr/local/bin/ip /usr/local/bin/ss /usr/local/bin/tc /usr/local/bin/ 14 | 15 | ENV TZ=Asia/Shanghai DEBIAN_FRONTEND=noninteractive 16 | RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ 17 | apt-get update &&\ 18 | apt-get install -y apt-utils kmod libelf1 libmnl0 iptables nftables wget \ 19 | tzdata && \ 20 | ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \ 21 | echo ${TZ} > /etc/timezone && \ 22 | dpkg-reconfigure --frontend ${DEBIAN_FRONTEND} tzdata && \ 23 | apt-get purge --auto-remove && apt-get clean && rm -rf /var/lib/apt/lists/* 24 | 25 | RUN useradd -d /home/tiger -m -s /bin/bash tiger && mkdir -p /home/tiger/.service/ /home/tiger/.ssh/ && chown -R tiger:tiger /home/tiger 26 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l3_service_stage1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-policy-l3-service 5 | 6 | --- 7 | apiVersion: "cilium.io/v2" 8 | kind: CiliumNetworkPolicy 9 | metadata: 10 | name: "l3-service-policy" 11 | namespace: cilium-policy-l3-service 12 | spec: 13 | endpointSelector: 14 | matchLabels: 15 | role: l3-service-client 16 | egress: 17 | - toServices: 18 | - k8sService: 19 | serviceName: l3-policy-service1 20 | namespace: cilium-policy-l3-service 21 | - toPorts: 22 | - ports: 23 | - port: "80" 24 | protocol: TCP 25 | --- 26 | # allow dns 27 | apiVersion: "cilium.io/v2" 28 | kind: CiliumNetworkPolicy 29 | metadata: 30 | name: "l3-service-policy-coredns" 31 | namespace: cilium-policy-l3-service 32 | spec: 33 | endpointSelector: 34 | matchLabels: 35 | role: l3-service-client 36 | egress: 37 | - toEndpoints: 38 | - matchLabels: 39 | k8s:io.kubernetes.pod.namespace: kube-system 40 | k8s-app: kube-dns 41 | toPorts: 42 | - ports: 43 | - port: '53' 44 | protocol: UDP 45 | --- 46 | 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | name: l3-policy-service1 51 | namespace: cilium-policy-l3-service 52 | spec: 53 | ports: 54 | - protocol: TCP 55 | port: 80 56 | targetPort: 11414 57 | 58 | --- 59 | 60 | apiVersion: v1 61 | kind: Service 62 | metadata: 63 | name: l3-policy-service2 64 | namespace: cilium-policy-l3-service 65 | spec: 66 | ports: 67 | - protocol: TCP 68 | port: 80 69 | targetPort: 11414 70 | 71 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l3_identify.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-policy-l3-identify 5 | 6 | --- 7 | apiVersion: "cilium.io/v2" 8 | kind: CiliumNetworkPolicy 9 | metadata: 10 | name: l3-rule-identify 11 | namespace: cilium-policy-l3-identify 12 | spec: 13 | endpointSelector: 14 | matchLabels: 15 | role: l3-rule-identify-source 16 | egress: 17 | - toEntities: 18 | - cluster 19 | 20 | --- 21 | apiVersion: v1 22 | kind: Pod 23 | metadata: 24 | name: l3-rule-identify-source 25 | namespace: cilium-policy-l3-identify 26 | labels: 27 | case: l3-rule-identify 28 | role: l3-rule-identify-source 29 | spec: 30 | affinity: 31 | podAntiAffinity: 32 | requiredDuringSchedulingIgnoredDuringExecution: 33 | - labelSelector: 34 | matchExpressions: 35 | - key: "case" 36 | operator: In 37 | values: 38 | - l3-rule-identify 39 | topologyKey: kubernetes.io/hostname 40 | restartPolicy: Never 41 | containers: 42 | - name: fail 43 | # image: docker.io/library/busybox:1.28.4 44 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 45 | command: ["sh", "-c", "wget -T 1 100.96.0.96/volcstack"] 46 | - name: success 47 | # image: docker.io/library/busybox:1.28.4 48 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 49 | command: ["sh", "-c", "wget -T 1 $MY_NODE_IP:11414/healthz"] 50 | env: 51 | - name: MY_NODE_IP 52 | valueFrom: 53 | fieldRef: 54 | fieldPath: status.hostIP 55 | 56 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l3_service_stage3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: l3-service-client 5 | namespace: cilium-policy-l3-service 6 | labels: 7 | role: l3-service-client 8 | case: l3-rule-service 9 | spec: 10 | affinity: 11 | podAntiAffinity: 12 | requiredDuringSchedulingIgnoredDuringExecution: 13 | - labelSelector: 14 | matchExpressions: 15 | - key: "role1" 16 | operator: In 17 | values: 18 | - l3-policy-service1 19 | topologyKey: kubernetes.io/hostname 20 | restartPolicy: Never 21 | initContainers: 22 | - name: waiting 23 | # After version 1.28.4, the following errors may occur in dns parsing: server can't find policy-svc.policy-test.svc.cluster.local: NXDOMAIN 24 | # image: docker.io/library/busybox:1.28.4 25 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 26 | command: ["sh", "-c", "for i in `seq 1 120`; do nslookup l3-policy-service1.cilium-policy-l3-service && nslookup l3-policy-service2.cilium-policy-l3-service && sleep 5 && exit 0; done"] 27 | containers: 28 | - name: success 29 | # image: docker.io/library/busybox:1.28.4 30 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 31 | command: ["sh", "-c", "wget -T 1 l3-policy-service1.cilium-policy-l3-service/healthz"] 32 | - name: fail 33 | # image: docker.io/library/busybox:1.28.4 34 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 35 | command: ["sh", "-c", "wget -T 1 l3-policy-service2.cilium-policy-l3-service/healthz"] 36 | --- -------------------------------------------------------------------------------- /pkg/utils/netns/netns.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package netns 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | 22 | "github.com/vishvananda/netns" 23 | ) 24 | 25 | const nsRootPath = "/var/run/netns" 26 | 27 | // CheckNetNsExist check if netns exists, 28 | // if there is an error, return false and error. 29 | func CheckNetNsExist(nsPath string) (bool, error) { 30 | h, err := netns.GetFromPath(nsPath) 31 | if err != nil { 32 | if os.IsNotExist(err) { 33 | return false, nil 34 | } 35 | return false, err 36 | } 37 | defer func(h *netns.NsHandle) { 38 | _ = h.Close() 39 | }(&h) 40 | return true, nil 41 | } 42 | 43 | func ListAllNetNs() ([]string, error) { 44 | var namespaces []string 45 | files, err := os.ReadDir(nsRootPath) 46 | if err != nil { 47 | return namespaces, fmt.Errorf("read dir %s failed: %v", nsRootPath, err) 48 | } 49 | for _, f := range files { 50 | if f.IsDir() { 51 | continue 52 | } 53 | namespaces = append(namespaces, fmt.Sprintf("%s/%s", nsRootPath, f.Name())) 54 | } 55 | return namespaces, nil 56 | } 57 | -------------------------------------------------------------------------------- /tests/stress/pod_scale.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source ../utils/utils.bash 4 | 5 | stress_ns=stress-scale 6 | deploymet_yaml=../templates/testcases/stress/scale.yaml 7 | max_scale=20 8 | total=10 9 | test_succeed=0 10 | test_count=0 11 | 12 | trap 'teardown' 2 13 | 14 | setup_env() { 15 | kubectl apply -f ${deploymet_yaml} 16 | } 17 | 18 | teardown() { 19 | kubectl delete -f ${deploymet_yaml} 20 | echo tested $test_count, succeed $test_succeed 21 | exit 22 | } 23 | 24 | scale_period=60 #secs 25 | scale_jitter=30 #secs 26 | test_scale() { 27 | deploy=$1 28 | scale_num=$((RANDOM%max_scale)) 29 | echo "scale dep to $scale_num" 30 | kubectl -n ${stress_ns} scale --replicas ${scale_num} deploy "${deploy}" 31 | jitter_time=$((RANDOM%(scale_jitter*2)-scale_jitter)) 32 | sleep $((scale_period+jitter_time)) 33 | running=`kubectl get pods -A -owide |grep nginx-deployment |grep Running |wc -l` 34 | pending=`kubectl get pods -A -owide |grep nginx-deployment |grep Pending |wc -l` 35 | terminat=`kubectl get pods -A -owide |grep nginx-deployment |grep Terminating |wc -l` 36 | creating=`kubectl get pods -A -owide |grep nginx-deployment |grep ContainerCreating |wc -l` 37 | echo "running: $running pending: $pending terminat: $terminat creating: $creating" 38 | if [ $scale_num -ne $running ]; then 39 | sleep $((scale_period+jitter_time)) 40 | fi 41 | running=`kubectl get pods -A -owide |grep nginx-deployment |grep Running |wc -l` 42 | ((test_count++)) 43 | if [ $scale_num -ne $running ]; then 44 | return 45 | fi 46 | ((test_succeed++)) 47 | } 48 | 49 | setup_env 50 | for (( i=0; i /dev/null 27 | apt update -y 28 | when: '"containerd" not in services' 29 | 30 | - name: Install containerd 31 | apt: 32 | name: 33 | - containerd.io 34 | state: present 35 | register: containerd 36 | 37 | - name: Wait containerd started 38 | service: 39 | name: containerd 40 | state: started 41 | 42 | - name: Create containerd daemon configuration from template 43 | template: 44 | src: templates/containerd.conf.j2 45 | dest: /etc/containerd/config.toml 46 | register: containerd_config 47 | 48 | - name: Restart containerd 49 | service: 50 | name: containerd 51 | state: restarted 52 | when: containerd_config.changed 53 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/deny.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-policy-deny 5 | 6 | --- 7 | apiVersion: "cilium.io/v2" 8 | kind: CiliumNetworkPolicy 9 | metadata: 10 | name: deny-rule 11 | namespace: cilium-policy-deny 12 | spec: 13 | endpointSelector: 14 | matchLabels: 15 | env: prod 16 | ingressDeny: 17 | - fromEndpoints: 18 | - matchLabels: 19 | role: test 20 | ingress: 21 | - fromEntities: 22 | - "all" 23 | 24 | --- 25 | apiVersion: v1 26 | kind: Pod 27 | metadata: 28 | name: deny-rule-prod-front 29 | namespace: cilium-policy-deny 30 | labels: 31 | case: deny-rule 32 | env: prod 33 | role: frontend 34 | spec: 35 | containers: 36 | - name: busybox 37 | # image: docker.io/library/busybox:1.28.4 38 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 39 | command: ["sh", "-c", "sleep 360000"] 40 | --- 41 | apiVersion: v1 42 | kind: Pod 43 | metadata: 44 | name: deny-rule-prod-backend 45 | namespace: cilium-policy-deny 46 | labels: 47 | case: deny-rule 48 | env: prod 49 | role: backend 50 | spec: 51 | containers: 52 | - name: busybox 53 | # image: docker.io/library/busybox:1.28.4 54 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 55 | command: ["sh", "-c", "sleep 360000"] 56 | 57 | --- 58 | apiVersion: v1 59 | kind: Pod 60 | metadata: 61 | name: deny-rule-prod-test 62 | namespace: cilium-policy-deny 63 | labels: 64 | case: deny-rule 65 | env: prod 66 | role: test 67 | spec: 68 | containers: 69 | - name: busybox 70 | # image: docker.io/library/busybox:1.28.4 71 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 72 | command: ["sh", "-c", "sleep 360000"] -------------------------------------------------------------------------------- /pkg/provider/volcengine/cellohelper/security_group_manager.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package cellohelper 17 | 18 | import ( 19 | "fmt" 20 | "sync" 21 | ) 22 | 23 | type SecurityGroupManager interface { 24 | // GetSecurityGroups get security groups used by pods 25 | GetSecurityGroups() []string 26 | // UpdateSecurityGroups update security groups used by pods 27 | UpdateSecurityGroups(sec []string) error 28 | } 29 | 30 | type securityGroupManager struct { 31 | lock sync.RWMutex 32 | securityGroups []string 33 | } 34 | 35 | func (m *securityGroupManager) GetSecurityGroups() []string { 36 | m.lock.RLock() 37 | defer m.lock.RUnlock() 38 | return m.securityGroups 39 | } 40 | 41 | func (m *securityGroupManager) UpdateSecurityGroups(sec []string) error { 42 | if len(sec) == 0 { 43 | return fmt.Errorf("security groups is empty") 44 | } 45 | m.lock.Lock() 46 | defer m.lock.Unlock() 47 | m.securityGroups = sec 48 | log.Infof("SecurityGroups update to %v", m.securityGroups) 49 | return nil 50 | } 51 | 52 | func NewSecurityGroupManager() SecurityGroupManager { 53 | return &securityGroupManager{ 54 | lock: sync.RWMutex{}, 55 | securityGroups: []string{}, 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /opstools/tag/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package main 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | 22 | "github.com/urfave/cli/v2" 23 | ) 24 | 25 | var ( 26 | endpoint string 27 | ramRole string 28 | ) 29 | 30 | func main() { 31 | app := &cli.App{ 32 | Name: "tag-tool", 33 | Usage: "convert eni description to tags", 34 | Version: "1.0.0", 35 | Commands: buildCommand(), 36 | Flags: []cli.Flag{ 37 | &cli.StringFlag{Name: "endpoint", Value: "", Required: true, Destination: &endpoint}, 38 | &cli.StringFlag{Name: "ramRole", Value: "KubernetesNodeRoleForECS", Destination: &ramRole}, 39 | }, 40 | } 41 | 42 | err := app.Run(os.Args) 43 | if err != nil { 44 | fmt.Printf("%s", err) 45 | os.Exit(1) 46 | } 47 | } 48 | 49 | func buildCommand() []*cli.Command { 50 | return []*cli.Command{ 51 | { 52 | Name: "tagEni", 53 | Usage: "convert eni description to tags", 54 | Flags: []cli.Flag{ 55 | &cli.BoolFlag{Name: "exec", Value: false}, 56 | &cli.BoolFlag{Name: "vpc-all", Aliases: []string{"v"}, Value: false}, 57 | }, 58 | Action: tagEni, 59 | }, 60 | { 61 | Name: "listTaggedEni", 62 | Usage: "list tagged eni", 63 | Flags: []cli.Flag{ 64 | &cli.BoolFlag{Name: "vpc-all", Aliases: []string{"v"}, Value: false}, 65 | }, 66 | Action: listTaggedEni, 67 | }, 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /pkg/cni/utils/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package utils 17 | 18 | import ( 19 | "crypto/sha1" 20 | "encoding/hex" 21 | "fmt" 22 | 23 | "github.com/containernetworking/plugins/pkg/utils/sysctl" 24 | "github.com/vishvananda/netlink" 25 | ) 26 | 27 | // routePreferenceStart is the start table ID for policy route table. 28 | const routePreferenceStart = 1000 29 | 30 | // EnsureNetConfSet calls sysctl to ensure system network config. 31 | func EnsureNetConfSet(link netlink.Link, item, conf string) error { 32 | _, err := sysctl.Sysctl(fmt.Sprintf(item, link.Attrs().Name), conf) 33 | if err != nil { 34 | return fmt.Errorf("ensure net config %s to %s failed: %s", item, conf, err.Error()) 35 | } 36 | return nil 37 | } 38 | 39 | // GetPolicyRouteTableID returns routePreferenceStart(1000) + linkIndex as route table index. 40 | func GetPolicyRouteTableID(linkIndex int) int { 41 | return routePreferenceStart + linkIndex 42 | } 43 | 44 | // VethNameForPod returns host-side veth name for pod. 45 | func VethNameForPod(name, namespace, ifName, prefix string) (string, error) { 46 | h := sha1.New() 47 | if ifName == "eth0" { 48 | ifName = "" 49 | } 50 | _, err := h.Write([]byte(namespace + "." + name + ifName)) 51 | if err != nil { 52 | return "", err 53 | } 54 | return fmt.Sprintf("%s%s", prefix, hex.EncodeToString(h.Sum(nil))[:11]), nil 55 | } 56 | -------------------------------------------------------------------------------- /tests/utils/utils.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Retry a command $1 times until it succeeds. Wait $2 seconds between retries. 4 | function retry() { 5 | local attempts=$1 6 | local delay=$2 7 | local i 8 | 9 | for ((i=0; i < attempts; i++)); do 10 | run "${@:3}" 11 | if [[ "$status" -eq 0 ]] ; then 12 | return 0 13 | fi 14 | sleep $delay 15 | done 16 | 17 | echo "Command \"${@:3}\" failed $attempts times. Output: $output" 18 | false 19 | } 20 | 21 | # Repeat a command $1 times. Wait $2 seconds between retries. 22 | function repeat() { 23 | local attempts=$1 24 | local delay=$2 25 | local i 26 | 27 | for ((i=0; i < attempts; i++)); do 28 | run "${@:3}" 29 | if [[ "$status" -ne 0 ]] ; then 30 | return false 31 | fi 32 | sleep $delay 33 | done 34 | return 0 35 | } 36 | 37 | function object_not_exist() { 38 | run kubectl get $@ 39 | if [[ "$status" -gt 0 ]] || [[ ${#lines[@]} -eq 1 ]]; then 40 | return 0 41 | fi 42 | echo "object $@ exist, status: $status, lines: ${#lines[@]} output $output" 43 | false 44 | } 45 | 46 | function object_exist() { 47 | run kubectl get $@ 48 | if [[ "$status" -eq 0 ]] && [[ ${#lines[@]} -gt 1 ]]; then 49 | return 0 50 | fi 51 | echo "object $@ not ready, status: $status, lines: ${#lines[@]} output $output" 52 | false 53 | } 54 | 55 | 56 | function pod_running() { 57 | run kubectl get $@ 58 | if [[ "$status" -eq 0 ]] && [[ ${#lines[@]} -gt 1 ]] && echo $output | grep -q "Running"; then 59 | return 0 60 | fi 61 | echo "object $@ not ready, status: $status, lines: ${#lines[@]} output $output" 62 | false 63 | } 64 | 65 | 66 | function svc_ready() { 67 | run kubectl get $@ 68 | if [[ "$status" -eq 0 ]] && [[ ${#lines[@]} -gt 1 ]]; then 69 | if echo $output | grep -q "pending"; then 70 | false 71 | echo "object $@ pending, status: $status, lines: ${#lines[@]} output $output" 72 | return 1 73 | fi 74 | return 0 75 | fi 76 | echo "object $@ exist, status: $status, lines: ${#lines[@]} output $output" 77 | false 78 | } -------------------------------------------------------------------------------- /pkg/cni/client/daemon.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package client 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "net" 22 | 23 | "google.golang.org/grpc" 24 | "google.golang.org/grpc/credentials/insecure" 25 | 26 | "github.com/volcengine/cello/pkg/cni/log" 27 | "github.com/volcengine/cello/pkg/daemon" 28 | "github.com/volcengine/cello/pkg/pbrpc" 29 | ) 30 | 31 | // NewCelloClient creates a client with default parameters connecting to UNIX domain socket and waits for cello-agent availability. 32 | func NewCelloClient(ctx context.Context) (pbrpc.CelloClient, *grpc.ClientConn, error) { 33 | grpcConn, err := grpc.DialContext(ctx, daemon.DefaultSocketPath, grpc.WithBlock(), 34 | grpc.WithTransportCredentials(insecure.NewCredentials()), 35 | grpc.WithContextDialer( 36 | func(ctx context.Context, s string) (net.Conn, error) { 37 | unixAddr, err := net.ResolveUnixAddr("unix", daemon.DefaultSocketPath) 38 | if err != nil { 39 | return nil, fmt.Errorf("error while resolve unix addr:%w", err) 40 | } 41 | d := net.Dialer{} 42 | return d.DialContext(ctx, "unix", unixAddr.String()) 43 | })) 44 | if err != nil { 45 | log.Log.Errorf("dial to grpc server failed: %v", err) 46 | return nil, nil, fmt.Errorf("error dial cello grpc server: %w", err) 47 | } 48 | 49 | celloClient := pbrpc.NewCelloClient(grpcConn) 50 | return celloClient, grpcConn, nil 51 | } 52 | -------------------------------------------------------------------------------- /pkg/utils/logger/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package logger 17 | 18 | import "os" 19 | 20 | const ( 21 | envLogLevel = "CELLO_CNI_LOGLEVEL" 22 | defaultLogLevel = "info" 23 | envLogLocation = "CELLO_CNI_LOCATION" 24 | 25 | envReportCaller = "CELLO_CNI_LOG_CALLER" 26 | defaultReportCaller = false 27 | ) 28 | 29 | // Configuration stores the config of the logger. 30 | type Configuration struct { 31 | LogLevel string 32 | LogLocation string 33 | ReportCaller bool 34 | } 35 | 36 | // LoadLogConfig returns the log configuration. 37 | func LoadLogConfig() *Configuration { 38 | return &Configuration{ 39 | LogLevel: GetLogLevel(), 40 | LogLocation: GetLogLocation(), 41 | ReportCaller: GetLogReportCaller(), 42 | } 43 | } 44 | 45 | // GetLogLocation get the log location from env. 46 | func GetLogLocation() string { 47 | logLocation := os.Getenv(envLogLocation) 48 | return logLocation 49 | } 50 | 51 | // GetLogLevel get the log level from env. 52 | func GetLogLevel() string { 53 | logLevel := os.Getenv(envLogLevel) 54 | if logLevel == "" { 55 | logLevel = defaultLogLevel 56 | } 57 | 58 | return logLevel 59 | } 60 | 61 | func GetLogReportCaller() bool { 62 | reportCaller := os.Getenv(envReportCaller) 63 | switch reportCaller { 64 | case "true": 65 | return true 66 | case "false": 67 | return false 68 | default: 69 | return defaultReportCaller 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/cni/device/veth.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package device 17 | 18 | import ( 19 | "github.com/containernetworking/plugins/pkg/ip" 20 | "github.com/containernetworking/plugins/pkg/ns" 21 | "github.com/vishvananda/netlink" 22 | ) 23 | 24 | // VethConf is interface config for veth pair. 25 | type VethConf struct { 26 | IfName string 27 | PeerName string 28 | MTU int 29 | } 30 | 31 | // Setup veth pair interface for netns. 32 | func (vethConf *VethConf) Setup(netNS ns.NetNS) error { 33 | peer, err := netlink.LinkByName(vethConf.PeerName) 34 | if err == nil { 35 | err = netlink.LinkDel(peer) 36 | if err != nil { 37 | if _, ok := err.(netlink.LinkNotFoundError); !ok { 38 | return err 39 | } 40 | } 41 | } 42 | 43 | tempIfName, err := ip.RandomVethName() 44 | if err != nil { 45 | return err 46 | } 47 | link := &netlink.Veth{ 48 | LinkAttrs: netlink.LinkAttrs{ 49 | MTU: vethConf.MTU, 50 | Name: tempIfName, 51 | Namespace: netlink.NsFd(int(netNS.Fd())), 52 | }, 53 | PeerName: vethConf.PeerName, 54 | } 55 | err = netlink.LinkAdd(link) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | return netNS.Do(func(netNS ns.NetNS) error { 61 | link, inErr := netlink.LinkByName(tempIfName) 62 | if inErr != nil { 63 | return inErr 64 | } 65 | if link.Attrs().Name == vethConf.IfName { 66 | return nil 67 | } 68 | return netlink.LinkSetName(link, vethConf.IfName) 69 | }) 70 | } 71 | -------------------------------------------------------------------------------- /pkg/backoff/backoff.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package backoff 17 | 18 | import ( 19 | "time" 20 | 21 | "k8s.io/apimachinery/pkg/util/wait" 22 | ) 23 | 24 | const ( 25 | DefaultKey = "" 26 | APIWriteOps = "api_write_ops" 27 | APIStatusWait = "api_status_wait" 28 | APIFastRetry = "api_fast_retry" 29 | WaitCRDStatus = "wait_crd_status" 30 | MetaStatusWait = "meta_status_wait" 31 | ) 32 | 33 | var backoffMap = map[string]wait.Backoff{ 34 | DefaultKey: { 35 | Duration: time.Second * 2, 36 | Factor: 1.5, 37 | Jitter: 0.3, 38 | Steps: 6, 39 | }, 40 | APIWriteOps: { 41 | Duration: time.Second * 4, 42 | Factor: 1.5, 43 | Jitter: 0.5, 44 | Steps: 6, 45 | }, 46 | APIStatusWait: { 47 | Duration: time.Second * 5, 48 | Factor: 1.5, 49 | Jitter: 0.5, 50 | Steps: 8, 51 | }, 52 | APIFastRetry: { 53 | Duration: time.Millisecond * 500, 54 | Factor: 1.7, 55 | Jitter: 0.3, 56 | Steps: 6, 57 | }, 58 | WaitCRDStatus: { 59 | Duration: time.Second * 5, 60 | Factor: 2, 61 | Jitter: 0.3, 62 | Steps: 3, 63 | }, 64 | MetaStatusWait: { 65 | Duration: time.Millisecond * 1100, 66 | Factor: 1, 67 | Jitter: 0.2, 68 | Steps: 10, 69 | }, 70 | } 71 | 72 | // BackOff return a specific wait.Backoff according to the key. 73 | func BackOff(key string) wait.Backoff { 74 | if b, exist := backoffMap[key]; exist { 75 | return b 76 | } 77 | 78 | return backoffMap[DefaultKey] 79 | } 80 | -------------------------------------------------------------------------------- /pkg/deviceplugin/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package deviceplugin 17 | 18 | import ( 19 | "time" 20 | 21 | "golang.org/x/net/context" 22 | "google.golang.org/grpc" 23 | pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" 24 | ) 25 | 26 | // ResourcesName. 27 | const ( 28 | // VolcNameSpace is the resource namespace for VKE volcengine. 29 | VolcNameSpace = "vke.volcengine.com/" 30 | // ENIResourceName indicates exclusive ENI resource resourceName. 31 | ENIResourceName = "eni" 32 | // ENIIPResourceName indicates shared ENI ip addresses resource resourceName. 33 | ENIIPResourceName = "eni-ip" 34 | // BranchENIResourceName indicates truck ENI. 35 | BranchENIResourceName = "branch-eni" 36 | ) 37 | 38 | const reportPeriod = time.Minute * 3 39 | 40 | var ( 41 | DevicePluginPath = pluginapi.DevicePluginPath 42 | KubeletSocket = DevicePluginPath + "kubelet.sock" 43 | ) 44 | 45 | // Manager is the interface of device plugin manager. 46 | type Manager interface { 47 | AddPlugin(plugin Plugin) 48 | Plugin(resourceName string) Plugin 49 | Serve(stopCh chan struct{}) error 50 | Stop() 51 | Update(resourceName string, count int) error 52 | } 53 | 54 | // Plugin is the interface for generic device plugin which can be managed by Manager. 55 | type Plugin interface { 56 | pluginapi.DevicePluginServer 57 | Endpoint() string 58 | ResourceName() string 59 | Server() *grpc.Server 60 | ResetServer() 61 | SetContext(ctx context.Context) 62 | Update(count int) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/provider/volcengine/metadata/ec2metadatawrapper_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package metadata 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "testing" 22 | 23 | "github.com/golang/mock/gomock" 24 | "github.com/stretchr/testify/assert" 25 | 26 | mockmetadata "github.com/volcengine/cello/pkg/provider/volcengine/metadata/mock" 27 | ) 28 | 29 | const ( 30 | az = "cn-beijing-a" 31 | eniMac = "82:94:32:47:34:01" 32 | eniId = "eni-test123" 33 | ) 34 | 35 | var ( 36 | mockMetadataIface *mockmetadata.MockEC2MetadataIface 37 | testClient *EC2MetadataWrapper 38 | ctx context.Context 39 | ) 40 | 41 | func setup(t *testing.T) { 42 | ctrl := gomock.NewController(t) 43 | defer ctrl.Finish() 44 | mockMetadataIface = mockmetadata.NewMockEC2MetadataIface(ctrl) 45 | testClient = &EC2MetadataWrapper{mockMetadataIface} 46 | ctx = context.TODO() 47 | } 48 | 49 | func TestEC2MetadataWrapper_GetAvailabilityZone(t *testing.T) { 50 | setup(t) 51 | 52 | mockMetadataIface.EXPECT().GetMetadata(gomock.Any(), gomock.Eq(azPath)).Return(az, nil) 53 | azG, err := testClient.GetAvailabilityZone(ctx) 54 | assert.NoError(t, err) 55 | assert.Equal(t, az, azG) 56 | } 57 | 58 | func TestEC2MetadataWrapper_GetENIID(t *testing.T) { 59 | setup(t) 60 | 61 | mockMetadataIface.EXPECT().GetMetadata(gomock.Any(), gomock.Eq(fmt.Sprintf(eniIDPath, eniMac))).Return(eniId, nil) 62 | eniIdG, err := testClient.GetENIID(ctx, eniMac) 63 | assert.NoError(t, err) 64 | assert.Equal(t, eniId, eniIdG) 65 | } 66 | -------------------------------------------------------------------------------- /ci/destroy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 5 | 6 | if ! volcengine-cli -h &> /dev/null; then 7 | echo "volcengine-cli is not installed, please install it first." 8 | echo "See: https://github.com/volcengine/volcengine-cli" 9 | exit 1 10 | fi 11 | 12 | pushd $THIS_DIR/tf > /dev/null 13 | # Remove secondary interfaces created by Cello 14 | source example.tfvars 15 | export VOLCENGINE_ACCESS_KEY="$access_key" 16 | export VOLCENGINE_SECRET_KEY="$secret_key" 17 | export VOLCENGINE_REGION=cn-bejing 18 | vpc_id=$(terraform show -json | jq -r '.values.root_module.resources[] | select(.address=="volcengine_vpc.vpc_cello") | .values.vpc_id') 19 | interfaces=$(volcengine-cli vpc DescribeNetworkInterfaces --Type "secondary" --VpcId $vpc_id --TagFilters.1.Key "k8s:cello:created-by" --TagFilters.1.Values.1 "cello") 20 | echo "$interfaces" | jq -r '.Result.NetworkInterfaceSets[].NetworkInterfaceId' | while read -r interface_id; 21 | do 22 | instance_id=$(volcengine-cli vpc DescribeNetworkInterfaceAttributes --NetworkInterfaceId $interface_id | jq -r '.Result.DeviceId') 23 | set +e 24 | max_retries=10 25 | count=0 26 | if [ -n "$instance_id" ]; then 27 | while [ "$count" -lt "$max_retries" ]; do 28 | volcengine-cli vpc DetachNetworkInterface --NetworkInterfaceId $interface_id --InstanceId $instance_id 29 | if [ "$?" -eq "0" ]; then 30 | break 31 | fi 32 | count=$((counter+1)) 33 | sleep 2 34 | done 35 | if [ "$count" -eq "$max_retries" ]; then 36 | exit 1 37 | fi 38 | fi 39 | # Retry multiple times since interface may at detaching status 40 | count=0 41 | while [ "$count" -lt "$max_retries" ]; do 42 | volcengine-cli vpc DeleteNetworkInterface --NetworkInterfaceId $interface_id 43 | if [ "$?" -eq "0" ]; then 44 | break 45 | fi 46 | count=$((counter+1)) 47 | sleep 2 48 | done 49 | if [ "$count" -eq "$max_retries" ]; then 50 | exit 1 51 | fi 52 | set -e 53 | done 54 | 55 | terraform destroy -auto-approve -var-file=example.tfvars 56 | popd > /dev/null 57 | -------------------------------------------------------------------------------- /pkg/utils/logger/logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | // Package logger is the CNI logger interface, using logrus 17 | package logger 18 | 19 | // log is a global var so that log function can be directly accessed. 20 | var log Logger 21 | 22 | // Fields Used when we want to call WithFields for structured logging. 23 | type Fields map[string]interface{} 24 | 25 | type Logger interface { 26 | Debugf(format string, args ...interface{}) 27 | 28 | Debug(args ...interface{}) 29 | 30 | DebugWithFields(fields Fields, args ...interface{}) 31 | 32 | Infof(format string, args ...interface{}) 33 | 34 | Info(args ...interface{}) 35 | 36 | InfoWithFields(fields Fields, args ...interface{}) 37 | 38 | Warnf(format string, args ...interface{}) 39 | 40 | Warn(args ...interface{}) 41 | 42 | WarnWithFields(fields Fields, args ...interface{}) 43 | 44 | Errorf(format string, args ...interface{}) 45 | 46 | Error(args ...interface{}) 47 | 48 | ErrorWithFields(fields Fields, args ...interface{}) 49 | 50 | Fatalf(format string, args ...interface{}) 51 | 52 | Panicf(format string, args ...interface{}) 53 | 54 | WithFields(fields Fields) Logger 55 | 56 | SetLogLevel(level string) 57 | } 58 | 59 | // GetLogger return the default instance. 60 | func GetLogger() Logger { 61 | if log == nil { 62 | logConfig := LoadLogConfig() 63 | log = New(logConfig) 64 | } 65 | 66 | return log 67 | } 68 | 69 | // New return new initializes logger. 70 | func New(config *Configuration) Logger { 71 | return config.newLogrusLogger() 72 | } 73 | -------------------------------------------------------------------------------- /chart/templates/cello-configmap.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: cello-config 5 | namespace: kube-system 6 | data: 7 | conf: | 8 | { 9 | {{- $configmap := (lookup "v1" "ConfigMap" "kube-system" "cello-config") }} 10 | {{- if $configmap }} 11 | {{- $conf := $configmap.data.conf | fromJson }} 12 | {{- else }} 13 | "subnets": [ 14 | {{- $last := sub (len .Values.cello.subnetIDs) 1}} 15 | {{- range $idx,$item := .Values.cello.subnetIDs }} 16 | {{ quote $item }}{{ if ne $idx $last }},{{end}} 17 | {{- end}} 18 | ], 19 | "securityGroups": [ 20 | {{- $last := sub (len .Values.cello.securityGroupIDs) 1}} 21 | {{- range $idx,$item := .Values.cello.securityGroupIDs }} 22 | {{ quote $item }}{{ if ne $idx $last }},{{end}} 23 | {{- end}} 24 | ], 25 | {{- end -}} 26 | {{- if .Values.cello.iamRole }} 27 | "ramRole": "{{.Values.cello.iamRole | quote }}", 28 | {{- end }} 29 | "credentialAccessKeyId": "{{ .Values.cello.accessKey }}", 30 | "credentialAccessKeySecret": "{{ .Values.cello.secretKey }}", 31 | "openApiAddress": "{{ .Values.cello.openAPIAddress }}", 32 | "poolTarget": 3, 33 | "poolTargetMin": 5, 34 | "poolMaxCapProbe": true, 35 | "poolMonitorIntervalSec": 120, 36 | "networkMode": "eni_shared", 37 | "ipFamily": "{{ .Values.cello.ipFamily }}", 38 | "platform": "kubernetes" 39 | } 40 | 10-cello.conflist: | 41 | { 42 | "cniVersion": "0.3.1", 43 | "name": "cello-chainer", 44 | "plugins": [ 45 | { 46 | "type": "cello-cni", 47 | "capabilities": { 48 | "bandwidth": true 49 | }, 50 | "redirectToHostCIDRs": [ 51 | {{- $last := sub (len .Values.cello.redirectToHostCIDRs) 1}} 52 | {{- range $idx,$item := .Values.cello.redirectToHostCIDRs }} 53 | {{ quote $item }}{{ if ne $idx $last }},{{end}} 54 | {{- end}} 55 | ] 56 | }, 57 | { 58 | "type": "cilium-cni" 59 | } 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /pkg/utils/sysctl/sysctl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package sysctl 17 | 18 | import ( 19 | "fmt" 20 | "io" 21 | "io/ioutil" 22 | "os" 23 | "path/filepath" 24 | "strings" 25 | ) 26 | 27 | const ( 28 | prefixDir = "/proc/sys" 29 | ) 30 | 31 | func fullPath(name string) string { 32 | return filepath.Join(prefixDir, strings.ReplaceAll(name, ".", "/")) 33 | } 34 | 35 | func writeSysctl(name string, value string) error { 36 | fPath := fullPath(name) 37 | f, err := os.OpenFile(fPath, os.O_RDWR, 0644) 38 | if err != nil { 39 | return fmt.Errorf("could not open the sysctl file %s: %s", 40 | fPath, err) 41 | } 42 | defer f.Close() 43 | if _, err := io.WriteString(f, value); err != nil { 44 | return fmt.Errorf("could not write to the systctl file %s: %s", 45 | fPath, err) 46 | } 47 | return nil 48 | } 49 | 50 | // Disable disables the given sysctl parameter. 51 | func Disable(name string) error { 52 | return writeSysctl(name, "0") 53 | } 54 | 55 | // Enable enables the given sysctl parameter. 56 | func Enable(name string) error { 57 | return writeSysctl(name, "1") 58 | } 59 | 60 | // Write writes the given sysctl parameter. 61 | func Write(name string, val string) error { 62 | return writeSysctl(name, val) 63 | } 64 | 65 | // Read reads the given sysctl parameter. 66 | func Read(name string) (string, error) { 67 | fPath := fullPath(name) 68 | val, err := ioutil.ReadFile(fPath) 69 | if err != nil { 70 | return "", fmt.Errorf("failed to read %s: %s", fPath, val) 71 | } 72 | 73 | return strings.TrimRight(string(val), "\n"), nil 74 | } 75 | -------------------------------------------------------------------------------- /tests/templates/testcases/service/nodeport.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: nginx-configmap 5 | data: 6 | nginx.conf: | 7 | worker_processes auto; 8 | events { 9 | } 10 | http { 11 | server { 12 | server_name nginx; 13 | listen 80 default_server; 14 | location /healthz { 15 | add_header Content-Type text/plain; 16 | return 200 'ok'; 17 | } 18 | } 19 | } 20 | 21 | --- 22 | 23 | apiVersion: apps/v1 24 | kind: Deployment 25 | metadata: 26 | name: nodeport-nginx 27 | labels: 28 | app: nodeport-nginx 29 | spec: 30 | replicas: 5 31 | selector: 32 | matchLabels: 33 | app: nodeport-nginx 34 | template: 35 | metadata: 36 | labels: 37 | app: nodeport-nginx 38 | spec: 39 | volumes: 40 | - name: nginx 41 | configMap: 42 | name: nginx-configmap 43 | containers: 44 | - name: nginx 45 | # image: docker.io/library/nginx:latest 46 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 47 | volumeMounts: 48 | - name: nginx 49 | mountPath: /etc/nginx 50 | readOnly: true 51 | --- 52 | 53 | apiVersion: v1 54 | kind: Service 55 | metadata: 56 | name: nodeport-service 57 | spec: 58 | type: NodePort 59 | selector: 60 | app: nodeport-nginx 61 | ports: 62 | - protocol: TCP 63 | nodePort: 30080 64 | port: 80 65 | 66 | --- 67 | 68 | apiVersion: apps/v1 69 | kind: Deployment 70 | metadata: 71 | name: nodeport-tool 72 | labels: 73 | app: nodeport-tool 74 | spec: 75 | replicas: 3 76 | selector: 77 | matchLabels: 78 | app: nodeport-tool 79 | template: 80 | metadata: 81 | labels: 82 | app: nodeport-tool 83 | spec: 84 | volumes: 85 | - name: nginx 86 | configMap: 87 | name: nginx-configmap 88 | containers: 89 | - name: nginx 90 | # image: docker.io/library/nginx:latest 91 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 92 | volumeMounts: 93 | - name: nginx 94 | mountPath: /etc/nginx 95 | readOnly: true -------------------------------------------------------------------------------- /pkg/metrics/resource_pool.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package metrics 17 | 18 | import "github.com/prometheus/client_golang/prometheus" 19 | 20 | var ( 21 | // ResourcePoolMaxCap Gauge of resource pool maximum capacity. 22 | ResourcePoolMaxCap = prometheus.NewGaugeVec( 23 | prometheus.GaugeOpts{ 24 | Name: "resource_pool_max_cap", 25 | Help: "The max capacity of resource pool"}, 26 | []string{"name", "type"}, 27 | ) 28 | 29 | // ResourcePoolTarget Gauge of resource pool target. 30 | ResourcePoolTarget = prometheus.NewGaugeVec( 31 | prometheus.GaugeOpts{ 32 | Name: "resource_pool_target", 33 | Help: "The cache target of resource pool"}, 34 | []string{"name", "type"}, 35 | ) 36 | 37 | // ResourcePoolTargetMin Gauge of resource pool targetMin. 38 | ResourcePoolTargetMin = prometheus.NewGaugeVec( 39 | prometheus.GaugeOpts{ 40 | Name: "resource_pool_target_min", 41 | Help: "The min cache target of resource pool"}, 42 | []string{"name", "type"}, 43 | ) 44 | 45 | // ResourcePoolTotal Gauge of total resources in resource pool. 46 | ResourcePoolTotal = prometheus.NewGaugeVec( 47 | prometheus.GaugeOpts{ 48 | Name: "resource_pool_total", 49 | Help: "The total number of resource in pool"}, 50 | []string{"name", "type"}, 51 | ) 52 | 53 | // ResourcePoolAvailable Gauge of available resources in resource pool. 54 | ResourcePoolAvailable = prometheus.NewGaugeVec( 55 | prometheus.GaugeOpts{ 56 | Name: "resource_pool_available", 57 | Help: "The available number of resource in pool"}, 58 | []string{"name", "type"}, 59 | ) 60 | ) 61 | -------------------------------------------------------------------------------- /tests/templates/testcases/service/clusterip.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: nginx-configmap 5 | data: 6 | nginx.conf: | 7 | worker_processes auto; 8 | events { 9 | } 10 | http { 11 | server { 12 | server_name nginx; 13 | listen 80 default_server; 14 | location /healthz { 15 | add_header Content-Type text/plain; 16 | return 200 'ok'; 17 | } 18 | } 19 | } 20 | 21 | --- 22 | 23 | apiVersion: apps/v1 24 | kind: Deployment 25 | metadata: 26 | name: clusterip-nginx 27 | labels: 28 | app: clusterip-nginx 29 | spec: 30 | replicas: 5 31 | selector: 32 | matchLabels: 33 | app: clusterip-nginx 34 | template: 35 | metadata: 36 | labels: 37 | app: clusterip-nginx 38 | spec: 39 | volumes: 40 | - name: nginx 41 | configMap: 42 | name: nginx-configmap 43 | containers: 44 | - name: nginx 45 | # image: docker.io/library/nginx:latest 46 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 47 | volumeMounts: 48 | - name: nginx 49 | mountPath: /etc/nginx 50 | readOnly: true 51 | --- 52 | 53 | apiVersion: v1 54 | kind: Service 55 | metadata: 56 | name: clusterip-service 57 | spec: 58 | type: ClusterIP 59 | selector: 60 | app: clusterip-nginx 61 | ports: 62 | - protocol: TCP 63 | port: 80 64 | targetPort: 80 65 | 66 | --- 67 | 68 | apiVersion: apps/v1 69 | kind: Deployment 70 | metadata: 71 | name: clusterip-tool 72 | labels: 73 | app: clusterip-tool 74 | spec: 75 | replicas: 3 76 | selector: 77 | matchLabels: 78 | app: clusterip-tool 79 | template: 80 | metadata: 81 | labels: 82 | app: clusterip-tool 83 | spec: 84 | volumes: 85 | - name: nginx 86 | configMap: 87 | name: nginx-configmap 88 | containers: 89 | - name: nginx 90 | # image: docker.io/library/nginx:latest 91 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 92 | volumeMounts: 93 | - name: nginx 94 | mountPath: /etc/nginx 95 | readOnly: true -------------------------------------------------------------------------------- /.github/workflows/go.yaml: -------------------------------------------------------------------------------- 1 | name: PreCommit Check 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | tags: 7 | - v* 8 | pull_request: 9 | branches: [ "main" ] 10 | 11 | jobs: 12 | golangci-lint: 13 | name: ci-lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - name: Set up Go 19 | uses: actions/setup-go@v4 20 | with: 21 | go-version-file: ./go.mod 22 | 23 | - name: Run golangci-lint 24 | uses: golangci/golangci-lint-action@v3.4.0 25 | with: 26 | version: latest 27 | 28 | unit-tests: 29 | name: Run unit unit-tests 30 | runs-on: ubuntu-latest 31 | steps: 32 | - name: Checkout code 33 | uses: actions/checkout@v3 34 | - name: Prepare environment 35 | run: docker run --rm --name cello-test -d --privileged -v $(pwd):/cello -v /go/pkg/mod:/go/pkg/mod -v /go/cache:/go/cache -w /cello docker.io/library/golang:latest sleep infinity 36 | - run: docker exec cello-test bash -c "apt-get update && apt-get install -y iproute2" 37 | - run: docker exec cello-test go install github.com/containernetworking/plugins/plugins/ipam/host-local@v1.2.0 38 | - run: docker exec cello-test sysctl -w net.ipv6.conf.all.disable_ipv6=0 39 | - run: docker exec cello-test ip addr add FEF6:BDFE:7654:593F:9721:20B0:C3C3:1A66/10 dev eth0 40 | - run: docker exec cello-test ip route add default via FEF6:BDFE:7654:593F::1 41 | 42 | - name: Run unit tests 43 | run: docker exec cello-test make test 44 | 45 | 46 | build: 47 | name: Build binaries 48 | runs-on: ubuntu-latest 49 | steps: 50 | - uses: actions/checkout@v3 51 | 52 | - name: Set up Go 53 | uses: actions/setup-go@v4 54 | with: 55 | go-version-file: ./go.mod 56 | 57 | - name: Mod tidy 58 | run: go mod tidy 59 | 60 | - name: Build cello-agent 61 | run: make cello-agent 62 | 63 | - name: Build cello-cni 64 | run: make cello-cni 65 | 66 | - name: Build cello-cli 67 | run: make cello-ctl 68 | 69 | - name: Build cilium-launcher 70 | run: make cilium-launcher -------------------------------------------------------------------------------- /pkg/pbrpc/endpoint.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option go_package = "./;pbrpc"; 3 | package endpoint; 4 | 5 | service Cello { 6 | rpc CreateEndpoint (CreateEndpointRequest) returns (CreateEndpointResponse) { 7 | } 8 | rpc DeleteEndpoint (DeleteEndpointRequest) returns (DeleteEndpointResponse) { 9 | } 10 | rpc GetPodMetaInfo (GetPodMetaRequest) returns (GetPodMetaResponse) { 11 | } 12 | rpc PatchPodAnnotation (PatchPodAnnotationRequest) returns (PatchPodAnnotationResponse) { 13 | } 14 | } 15 | 16 | // 详细请求 17 | message CreateEndpointRequest { 18 | string Name = 1; 19 | string Namespace = 2; 20 | string InfraContainerId = 3; 21 | string IfName = 4; 22 | string NetNs = 5; 23 | } 24 | 25 | message IPSet { 26 | string IPv4 = 1; 27 | string IPv6 = 2; 28 | } 29 | 30 | message Route { 31 | string Dst = 1; 32 | } 33 | 34 | message NetworkInterface { 35 | ENI ENI = 1; 36 | string IPv4Addr = 2; // ipNet 37 | string IPv6Addr = 3; // ipNet 38 | string IfName = 4; 39 | repeated Route ExtraRoutes = 5; 40 | bool DefaultRoute = 6; 41 | } 42 | 43 | enum IfType { 44 | TypeENIShare = 0; 45 | TypeENIExclusive = 1; 46 | } 47 | 48 | message CreateEndpointResponse { 49 | IfType IfType = 1; 50 | repeated NetworkInterface Interfaces = 2; 51 | } 52 | 53 | message ENI { 54 | string ID = 1; 55 | string Mac = 2; 56 | string IPv4Gateway = 3; 57 | string IPv6Gateway = 4; 58 | // Deprecated 59 | string GatewayMac = 5; 60 | // Deprecated 61 | IPSet Subnet = 6; // IPNet 62 | bool Trunk = 7; // use trunk ? 63 | uint32 Vid = 8; 64 | string SlaveMac = 9; // for vlan slave device 65 | } 66 | 67 | message DeleteEndpointRequest { 68 | string Name = 1; 69 | string Namespace = 2; 70 | string InfraContainerId = 3; 71 | } 72 | 73 | message DeleteEndpointResponse { 74 | } 75 | 76 | message GetPodMetaRequest { 77 | string Name = 1; 78 | string Namespace = 2; 79 | string InfraContainerId = 3; 80 | } 81 | 82 | message GetPodMetaResponse { 83 | map Annotations = 1; 84 | } 85 | 86 | message PatchPodAnnotationRequest { 87 | string Name = 1; 88 | string Namespace = 2; 89 | map Annotations = 3; 90 | } 91 | 92 | message PatchPodAnnotationResponse { 93 | } 94 | -------------------------------------------------------------------------------- /pkg/utils/kernel/kernel_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | //go:build linux 17 | // +build linux 18 | 19 | package kernel 20 | 21 | import ( 22 | "bytes" 23 | "syscall" 24 | 25 | "github.com/golang/glog" 26 | ) 27 | 28 | // GetKernelVersion gets the current kernel version. 29 | func GetKernelVersion() (*VersionInfo, error) { 30 | uts, err := uname() 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | release := make([]byte, len(uts.Release)) 36 | 37 | i := 0 38 | for _, c := range uts.Release { 39 | release[i] = byte(c) 40 | i++ 41 | } 42 | 43 | // Remove the \x00 from the release for Atoi to parse correctly 44 | release = release[:bytes.IndexByte(release, 0)] 45 | 46 | return ParseRelease(string(release)) 47 | } 48 | 49 | // CheckKernelVersion checks if current kernel is newer than (or equal to) 50 | // the given version. 51 | func CheckKernelVersion(k, major, minor int) bool { 52 | if v, err := GetKernelVersion(); err != nil { 53 | glog.Errorf("Error getting kernel version: %s", err) 54 | } else if CompareKernelVersion(*v, VersionInfo{Kernel: k, Major: major, Minor: minor}) < 0 { 55 | return false 56 | } 57 | return true 58 | } 59 | 60 | // Copy from docker-ce 61 | // Utsname represents the system name structure. 62 | // It is passthrough for syscall.Utsname in order to make it portable with 63 | // other platforms where it is not available. 64 | type Utsname syscall.Utsname 65 | 66 | func uname() (*syscall.Utsname, error) { 67 | uts := &syscall.Utsname{} 68 | 69 | if err := syscall.Uname(uts); err != nil { 70 | return nil, err 71 | } 72 | return uts, nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/daemon/ecsmeta.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package daemon 17 | 18 | import ( 19 | "fmt" 20 | "net/http" 21 | 22 | "github.com/gin-gonic/gin" 23 | 24 | "github.com/volcengine/volcengine-go-sdk/service/ecs" 25 | "github.com/volcengine/volcengine-go-sdk/volcengine" 26 | 27 | helper "github.com/volcengine/cello/pkg/provider/volcengine/cellohelper" 28 | "github.com/volcengine/cello/pkg/provider/volcengine/ec2" 29 | ) 30 | 31 | // getInstanceMeta is the entity to handle ECS meta info request. 32 | type getInstanceMeta struct { 33 | meta helper.InstanceMetadataGetter 34 | api ec2.APIGroupECS 35 | } 36 | 37 | func (m *getInstanceMeta) Handle(c *gin.Context) { 38 | api := c.Query("api") 39 | switch api { 40 | case "DescribeInstances": 41 | output, err := m.api.DescribeInstances(&ecs.DescribeInstancesInput{ 42 | VpcId: volcengine.String(m.meta.GetVpcId()), 43 | InstanceIds: []*string{volcengine.String(m.meta.GetInstanceId())}, 44 | }) 45 | if err != nil { 46 | _ = c.Error(err) 47 | return 48 | } 49 | c.JSON(http.StatusOK, output) 50 | case "DescribeInstanceTypes": 51 | output, err := m.api.DescribeInstanceTypes(&ecs.DescribeInstanceTypesInput{ 52 | InstanceTypes: []*string{volcengine.String(m.meta.GetInstanceType())}, 53 | }) 54 | if err != nil { 55 | _ = c.Error(err) 56 | return 57 | } 58 | c.JSON(http.StatusOK, output) 59 | default: 60 | _ = c.Error(fmt.Errorf("%s not support", api)) 61 | } 62 | } 63 | 64 | func newGetInstanceMetaHandler(api ec2.APIGroupECS, getter helper.InstanceMetadataGetter) Handler { 65 | return &getInstanceMeta{api: api, meta: getter} 66 | } 67 | -------------------------------------------------------------------------------- /pkg/signal/signal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package signal 17 | 18 | import ( 19 | "fmt" 20 | 21 | "github.com/volcengine/cello/pkg/utils/logger" 22 | ) 23 | 24 | type SigData uint32 25 | 26 | var log = logger.GetLogger().WithFields(logger.Fields{"subSys": "signal"}) 27 | 28 | const ( 29 | WakeGC = "wakeGC" 30 | ) 31 | 32 | const ( 33 | SigWakeGC = iota 34 | ) 35 | 36 | var ( 37 | signalChannels map[string]chan<- SigData 38 | muteSignal map[string]struct{} 39 | ) 40 | 41 | // RegisterChannel register a notify channel for event. 42 | func RegisterChannel(signal string, ch chan<- SigData) error { 43 | if _, exist := signalChannels[signal]; exist { 44 | return fmt.Errorf("register a exist signal") 45 | } 46 | signalChannels[signal] = ch 47 | return nil 48 | } 49 | 50 | // NotifySignal non-blocking notification. 51 | func NotifySignal(signal string, data SigData) { 52 | if _, exist := muteSignal[signal]; exist { 53 | log.Infof("Signal %s muted", signal) 54 | return 55 | } 56 | if ch, exist := signalChannels[signal]; exist { 57 | select { 58 | case ch <- data: 59 | default: 60 | log.Infof("Signal [%v %v] processing", signal, data) 61 | } 62 | } 63 | } 64 | 65 | // MuteChannel tells to not send any signal to a particular channel. 66 | func MuteChannel(signal string) { 67 | muteSignal[signal] = struct{}{} 68 | } 69 | 70 | // UnmuteChannel tells to allow sending new signal to a particular channel. 71 | func UnmuteChannel(signal string) { 72 | delete(muteSignal, signal) 73 | } 74 | 75 | func init() { 76 | signalChannels = map[string]chan<- SigData{} 77 | muteSignal = map[string]struct{}{} 78 | } 79 | -------------------------------------------------------------------------------- /tests/templates/testcases/service/hostnetwork/nodeport.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: nginx-configmap 5 | data: 6 | nginx.conf: | 7 | worker_processes auto; 8 | events { 9 | } 10 | http { 11 | server { 12 | server_name nginx; 13 | listen 80 default_server; 14 | location /healthz { 15 | add_header Content-Type text/plain; 16 | return 200 'ok'; 17 | } 18 | } 19 | } 20 | 21 | --- 22 | 23 | apiVersion: apps/v1 24 | kind: Deployment 25 | metadata: 26 | name: nodeport-nginx-hn 27 | labels: 28 | app: nodeport-nginx-hn 29 | spec: 30 | replicas: 2 31 | selector: 32 | matchLabels: 33 | app: nodeport-nginx-hn 34 | template: 35 | metadata: 36 | labels: 37 | app: nodeport-nginx-hn 38 | spec: 39 | hostNetwork: true 40 | volumes: 41 | - name: nginx 42 | configMap: 43 | name: nginx-configmap 44 | containers: 45 | - name: nginx 46 | # image: docker.io/library/nginx:latest 47 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 48 | volumeMounts: 49 | - name: nginx 50 | mountPath: /etc/nginx 51 | readOnly: true 52 | --- 53 | 54 | apiVersion: v1 55 | kind: Service 56 | metadata: 57 | name: nodeport-service-hn 58 | spec: 59 | type: NodePort 60 | selector: 61 | app: nodeport-nginx-hn 62 | ports: 63 | - protocol: TCP 64 | nodePort: 30080 65 | port: 80 66 | 67 | --- 68 | 69 | apiVersion: apps/v1 70 | kind: Deployment 71 | metadata: 72 | name: nodeport-tool-hn 73 | labels: 74 | app: nodeport-tool-hn 75 | spec: 76 | replicas: 3 77 | selector: 78 | matchLabels: 79 | app: nodeport-tool-hn 80 | template: 81 | metadata: 82 | labels: 83 | app: nodeport-tool-hn 84 | spec: 85 | volumes: 86 | - name: nginx 87 | configMap: 88 | name: nginx-configmap 89 | containers: 90 | - name: nginx 91 | # image: docker.io/library/nginx:latest 92 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 93 | volumeMounts: 94 | - name: nginx 95 | mountPath: /etc/nginx 96 | readOnly: true -------------------------------------------------------------------------------- /pkg/utils/ip/ip.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package ip 17 | 18 | import ( 19 | "fmt" 20 | "net" 21 | 22 | "k8s.io/apimachinery/pkg/util/sets" 23 | ) 24 | 25 | // ParseIP parse string to net.IP. 26 | func ParseIP(ip string) (net.IP, error) { 27 | netIP := net.ParseIP(ip) 28 | if netIP == nil { 29 | return nil, fmt.Errorf("invalid ip [%s]", ip) 30 | } 31 | return netIP, nil 32 | } 33 | 34 | // ParseIPs parse string slice to net.IP slice. 35 | func ParseIPs(ips []string) ([]net.IP, error) { 36 | var result []net.IP 37 | for _, ipStr := range ips { 38 | ip, err := ParseIP(ipStr) 39 | if err != nil { 40 | return nil, err 41 | } 42 | result = append(result, ip) 43 | } 44 | return result, nil 45 | } 46 | 47 | // ToStringSlice convert net.IP slice of ips to string slice. 48 | func ToStringSlice(ips []net.IP) []string { 49 | var result []string 50 | for _, ip := range ips { 51 | result = append(result, ip.String()) 52 | } 53 | return result 54 | } 55 | 56 | // NetIPContainAll return true if all items in b can be found in a. 57 | func NetIPContainAll(a, b []net.IP) bool { 58 | return sets.NewString(ToStringSlice(a)...).HasAll(ToStringSlice(b)...) 59 | } 60 | 61 | // NetIPContainAny return true if any items in b can be found in a. 62 | func NetIPContainAny(a, b []net.IP) bool { 63 | return sets.NewString(ToStringSlice(a)...).HasAny(ToStringSlice(b)...) 64 | } 65 | 66 | // NetIPToMap convert slice of net.IP to map that key is string. 67 | func NetIPToMap(ips []net.IP) map[string]net.IP { 68 | result := make(map[string]net.IP) 69 | for _, ip := range ips { 70 | result[ip.String()] = ip 71 | } 72 | return result 73 | } 74 | -------------------------------------------------------------------------------- /tests/templates/testcases/service/hostnetwork/clusterip.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: nginx-configmap 5 | data: 6 | nginx.conf: | 7 | worker_processes auto; 8 | events { 9 | } 10 | http { 11 | server { 12 | server_name nginx; 13 | listen 80 default_server; 14 | location /healthz { 15 | add_header Content-Type text/plain; 16 | return 200 'ok'; 17 | } 18 | } 19 | } 20 | 21 | --- 22 | 23 | apiVersion: apps/v1 24 | kind: Deployment 25 | metadata: 26 | name: clusterip-nginx-hn 27 | labels: 28 | app: clusterip-nginx-hn 29 | spec: 30 | replicas: 2 31 | selector: 32 | matchLabels: 33 | app: clusterip-nginx-hn 34 | template: 35 | metadata: 36 | labels: 37 | app: clusterip-nginx-hn 38 | spec: 39 | hostNetwork: true 40 | volumes: 41 | - name: nginx 42 | configMap: 43 | name: nginx-configmap 44 | containers: 45 | - name: nginx 46 | # image: docker.io/library/nginx:latest 47 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 48 | volumeMounts: 49 | - name: nginx 50 | mountPath: /etc/nginx 51 | readOnly: true 52 | --- 53 | 54 | apiVersion: v1 55 | kind: Service 56 | metadata: 57 | name: clusterip-service-hn 58 | spec: 59 | type: ClusterIP 60 | selector: 61 | app: clusterip-nginx-hn 62 | ports: 63 | - protocol: TCP 64 | port: 80 65 | targetPort: 80 66 | 67 | --- 68 | 69 | apiVersion: apps/v1 70 | kind: Deployment 71 | metadata: 72 | name: clusterip-tool-hn 73 | labels: 74 | app: clusterip-tool-hn 75 | spec: 76 | replicas: 3 77 | selector: 78 | matchLabels: 79 | app: clusterip-tool-hn 80 | template: 81 | metadata: 82 | labels: 83 | app: clusterip-tool-hn 84 | spec: 85 | volumes: 86 | - name: nginx 87 | configMap: 88 | name: nginx-configmap 89 | containers: 90 | - name: nginx 91 | # image: docker.io/library/nginx:latest 92 | image: cr-cn-guilin-boe.ivolces.com/vke/cello-tools:nginx 93 | volumeMounts: 94 | - name: nginx 95 | mountPath: /etc/nginx 96 | readOnly: true -------------------------------------------------------------------------------- /tests/pod_connection.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load utils/utils 4 | 5 | function delete_deploy() { 6 | retry 20 2 object_not_exist pod -l app=samenode-nginx 7 | retry 20 2 object_not_exist pod -l app=crossnode-nginx 8 | sleep 10 9 | } 10 | 11 | function setup() { 12 | kubectl delete -f templates/testcases/pod_connection/samenode.yaml || true 13 | kubectl delete -f templates/testcases/pod_connection/crossnode.yaml || true 14 | delete_deploy || true 15 | } 16 | 17 | function node_request_pod() { 18 | run "kubectl get pod -n kube-system -l app=cello -o name | cut -d '/' -f 2 | xargs -n1 -I {} kubectl exec -i {} -n kube-system -c cello -- curl $1:80/healthz" 19 | if [[ "$status" -eq 0 ]]; then 20 | return 0 21 | fi 22 | false 23 | echo "request $1:80/healthz result: "$result 24 | } 25 | 26 | # pods which app=$1 request ip list $2 27 | function pod_request_pod() { 28 | run "kubectl get pod -l app=$1 -o name | cut -d '/' -f 2 | xargs -n1 -I {} kubectl exec -i {} -- curl $2:80/healthz" 29 | if [[ "$status" -eq 0 ]]; then 30 | return 0 31 | fi 32 | false 33 | echo "request $1:80/healthz result: "$result 34 | } 35 | 36 | @test "pod connection same node" { 37 | retry 5 5 kubectl apply -f templates/testcases/pod_connection/samenode.yaml 38 | retry 20 5 object_exist pod -l app=samenode-nginx 39 | sleep 20 40 | 41 | podIPs=`kubectl get pod -l app=samenode-nginx -o wide | awk 'NR!=1 {print $6}' | tr "\n" " "|sed -e 's/,$/\n/'` 42 | for podIP in $podIPs 43 | do 44 | retry 5 5 node_request_pod $podIP 45 | retry 5 5 pod_request_pod samenode-nginx $podIP 46 | done 47 | [ "$status" -eq "0" ] 48 | kubectl delete -f templates/testcases/pod_connection/samenode.yaml || true 49 | delete_deploy 50 | } 51 | 52 | 53 | @test "pod connection cross node" { 54 | retry 5 5 kubectl apply -f templates/testcases/pod_connection/crossnode.yaml 55 | retry 20 5 object_exist pod -l app=crossnode-nginx 56 | sleep 20 57 | 58 | podIPs=`kubectl get pod -l app=crossnode-nginx -o wide | awk 'NR!=1 {print $6}' | tr "\n" " "|sed -e 's/,$/\n/'` 59 | for podIP in $podIPs 60 | do 61 | retry 5 5 node_request_pod $podIP 62 | retry 5 5 pod_request_pod crossnode-nginx $podIP 63 | done 64 | [ "$status" -eq "0" ] 65 | kubectl delete -f templates/testcases/pod_connection/crossnode.yaml || true 66 | delete_deploy 67 | } 68 | -------------------------------------------------------------------------------- /ci/conformance_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2023 The Cello Authors 3 | 4 | set -x 5 | 6 | THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 7 | TEST_OUTPUT=$THIS_DIR/"output" 8 | SONOBUOY_VRSION="v0.56.16" 9 | SONOBUOY="go run github.com/vmware-tanzu/sonobuoy@$SONOBUOY_VRSION" 10 | 11 | 12 | DEFAULT_E2E_SIG_NETWORK_CONFORMANCE="\[sig-network\].*Conformance" 13 | DEFAULT_E2E_SIG_NETWORK_SKIP="\[Slow\]|\[Serial\]|\[Disruptive\]|\[GCE\]|\[Feature:.+\]|\[Feature:IPv6DualStack\]|\[Feature:IPv6DualStackAlphaFeature\]|should create pod that uses dns|should provide Internet connection for containers|\ 14 | HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" 15 | 16 | extra_args="" 17 | if [ -n "$SONOBUOY_IMAGE" ]; then 18 | extra_args="$extra_args --sonobuoy-image $SONOBUOY_IMAGE" 19 | fi 20 | if [ -n "$CONFORMANCE_IMAGE" ]; then 21 | extra_args="$extra_args --kube-conformance-image $CONFORMANCE_IMAGE" 22 | fi 23 | if [ -n $SYSTEMD_LOGS_IMAGE ]; then 24 | extra_args="$extra_args --systemd-logs-image $SYSTEMD_LOGS_IMAGE" 25 | fi 26 | if [ -n $E2E_REPO_CONFIG ]; then 27 | extra_args="$extra_args --e2e-repo-config $E2E_REPO_CONFIG" 28 | fi 29 | pushd $THIS_DIR > /dev/null 30 | $SONOBUOY run \ 31 | --wait \ 32 | --e2e-focus "$DEFAULT_E2E_SIG_NETWORK_CONFORMANCE" \ 33 | --e2e-skip "$DEFAULT_E2E_SIG_NETWORK_SKIP" \ 34 | $extra_args 35 | 36 | mkdir -f $TEST_OUTPUT 37 | results_path=$($SONOBUOY retrieve $TEST_OUTPUT) 38 | results=$($SONOBUOY results $results_path --plugin e2e) 39 | echo "$results" > $TEST_OUTPUT/test_summary.log 40 | echo "$($SONOBUOY results $results_path --plugin e2e --mode=detailed | jq 'select(.status=="passed" or .status=="failed)')" > $TEST_OUTPUT/tests.log 41 | echo "$($SONOBUOY results $results_path --plugin e2e --mode=detailed | jq 'select(.status=="passed")')" > $TEST_OUTPUT/passed_tests.log 42 | echo "$($SONOBUOY results $results_path --plugin e2e --mode=detailed | jq 'select(.status=="failed")')" > $TEST_OUTPUT/failed_tests.log 43 | echo "$($SONOBUOY results $results_path --plugin e2e --mode=detailed | jq 'select(.status=="skipped")')" > $TEST_OUTPUT/skpped_tests.log 44 | if [[ ! $results == *"Failed: 0"* ]]; then 45 | echo "Test failed!" 46 | exit 1 47 | fi 48 | echo "Test successfully!" 49 | exit 0 50 | -------------------------------------------------------------------------------- /tests/templates/testcases/cilium_network_policy/l3_label_additional_require.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cilium-policy-l3-label-add-req 5 | --- 6 | 7 | apiVersion: "cilium.io/v2" 8 | kind: CiliumNetworkPolicy 9 | metadata: 10 | name: l3-label-add-req 11 | namespace: cilium-policy-l3-label-add-req 12 | specs: 13 | - description: "For endpoints with env=prod, only allow if source also has label env=prod" 14 | endpointSelector: 15 | matchLabels: 16 | env: prod 17 | ingress: 18 | - fromRequires: 19 | - matchLabels: 20 | env: prod 21 | 22 | --- 23 | apiVersion: "cilium.io/v2" 24 | kind: CiliumNetworkPolicy 25 | metadata: 26 | name: l3-label-add-rule 27 | namespace: cilium-policy-l3-label-add-req 28 | specs: 29 | - description: "For endpoints with env=prod, allow if source also has label role=frontend" 30 | endpointSelector: 31 | matchLabels: 32 | env: prod 33 | ingress: 34 | - fromEndpoints: 35 | - matchLabels: 36 | role: frontend 37 | --- 38 | 39 | apiVersion: v1 40 | kind: Pod 41 | metadata: 42 | name: l3-label-add-req-prod-front 43 | namespace: cilium-policy-l3-label-add-req 44 | labels: 45 | case: l3-label-add-req 46 | env: prod 47 | role: frontend 48 | spec: 49 | containers: 50 | - name: busybox 51 | # image: docker.io/library/busybox:1.28.4 52 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 53 | command: ["sh", "-c", "sleep 360000"] 54 | --- 55 | apiVersion: v1 56 | kind: Pod 57 | metadata: 58 | name: l3-label-add-req-prod-backend 59 | namespace: cilium-policy-l3-label-add-req 60 | labels: 61 | case: l3-label-add-req 62 | env: prod 63 | role: backend 64 | spec: 65 | containers: 66 | - name: busybox 67 | # image: docker.io/library/busybox:1.28.4 68 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 69 | command: ["sh", "-c", "sleep 360000"] 70 | 71 | --- 72 | apiVersion: v1 73 | kind: Pod 74 | metadata: 75 | name: l3-label-add-req-test 76 | namespace: cilium-policy-l3-label-add-req 77 | labels: 78 | case: l3-label-add-req 79 | env: test 80 | spec: 81 | containers: 82 | - name: busybox 83 | # image: docker.io/library/busybox:1.28.4 84 | image: cr-inner-cn-guilin-boe.ivolces.com/vke/cello-tools:busybox-1.28.4 85 | command: ["sh", "-c", "sleep 360000"] -------------------------------------------------------------------------------- /cmd/cello-cli/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package main 17 | 18 | import ( 19 | "bytes" 20 | "context" 21 | "encoding/json" 22 | "fmt" 23 | "io/ioutil" 24 | "net" 25 | "net/http" 26 | 27 | "github.com/volcengine/cello/pkg/daemon" 28 | ) 29 | 30 | func debugClient() *http.Client { 31 | httpc := http.Client{ 32 | Transport: &http.Transport{ 33 | DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { 34 | return net.Dial("unix", daemon.DefaultDebugSocketPath) 35 | }, 36 | }, 37 | } 38 | return &httpc 39 | } 40 | 41 | type AdditionArg struct { 42 | Key string 43 | Value string 44 | } 45 | 46 | func debugClientGet(url string, message interface{}, args ...AdditionArg) error { 47 | request, _ := http.NewRequest(http.MethodGet, url, nil) 48 | query := request.URL.Query() 49 | for _, arg := range args { 50 | query.Add(arg.Key, arg.Value) 51 | } 52 | request.URL.RawQuery = query.Encode() 53 | resp, err := debugClient().Do(request) 54 | if err != nil { 55 | return err 56 | } 57 | defer resp.Body.Close() 58 | 59 | if resp.StatusCode != http.StatusOK { 60 | return fmt.Errorf("response: %+v", resp) 61 | } 62 | if message == nil { 63 | return nil 64 | } 65 | bodyText, err := ioutil.ReadAll(resp.Body) 66 | if err != nil { 67 | return fmt.Errorf("read resp body err %s", err.Error()) 68 | } 69 | 70 | err = json.Unmarshal(bodyText, &message) 71 | if err != nil { 72 | return fmt.Errorf("unmarshal to %v err, %v", message, err.Error()) 73 | } 74 | return nil 75 | } 76 | 77 | func PrettyJson(obj interface{}) string { 78 | if obj == nil { 79 | return "" 80 | } 81 | 82 | bs, _ := json.Marshal(obj) 83 | var out bytes.Buffer 84 | err := json.Indent(&out, bs, "", "\t") 85 | if err != nil { 86 | return "" 87 | } 88 | return out.String() 89 | } 90 | -------------------------------------------------------------------------------- /pkg/daemon/daemon_main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package daemon 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | "os/signal" 22 | "runtime" 23 | "runtime/debug" 24 | "sync" 25 | "syscall" 26 | "time" 27 | ) 28 | 29 | func Execute() { 30 | if err := PreHookAction(); err != nil { 31 | log.Fatalf("PreHook action execute failed, %v", err) 32 | } 33 | 34 | d, err := NewDaemon() 35 | if err != nil { 36 | log.Fatalf("Create cello daemon failed, %v", err) 37 | } 38 | 39 | stopCh := make(chan struct{}) 40 | go signalHandler(stopCh) 41 | 42 | err = d.start(stopCh) 43 | if err != nil { 44 | log.Fatalf("Run Daemon failed, %v", err) 45 | } 46 | } 47 | 48 | func signalHandler(stopCh chan struct{}) { 49 | sig := make(chan os.Signal, 1) 50 | var closeOnce sync.Once 51 | signal.Notify(sig, 52 | syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGABRT, 53 | syscall.SIGUSR1, syscall.SIGUSR2) 54 | 55 | for s := range sig { 56 | log.Infof(fmt.Sprintf("Cello-agent received user signal %d", s)) 57 | switch s { 58 | case syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM: 59 | log.Infof(fmt.Sprintf("cello-agent received user signal %d, graceful exit now...", s)) 60 | closeOnce.Do(func() { 61 | close(stopCh) 62 | time.Sleep(1 * time.Second) 63 | }) 64 | case syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGQUIT, syscall.SIGABRT: 65 | log.Infof(fmt.Sprintf("Cello-agent received user signal %d, graceful exit now...", s)) 66 | log.Infof(string(debug.Stack())) 67 | closeOnce.Do(func() { 68 | close(stopCh) 69 | time.Sleep(1 * time.Second) 70 | }) 71 | default: 72 | log.Warnf(fmt.Sprintf("Cello-agent receive unknown os term signal %v. ignore...", s)) 73 | } 74 | } 75 | 76 | } 77 | 78 | func init() { 79 | runtime.SetBlockProfileRate(1) 80 | runtime.SetMutexProfileFraction(1) 81 | } 82 | -------------------------------------------------------------------------------- /pkg/provider/volcengine/metadata/ec2metadata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package metadata 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "io" 22 | "io/ioutil" 23 | "net/http" 24 | "time" 25 | 26 | "github.com/pkg/errors" 27 | v1 "k8s.io/api/core/v1" 28 | 29 | "github.com/volcengine/cello/pkg/tracing" 30 | "github.com/volcengine/cello/pkg/utils/logger" 31 | ) 32 | 33 | var log = logger.GetLogger().WithFields(logger.Fields{"subsys": "metadata"}) 34 | 35 | const ( 36 | metadataURL = "http://100.96.0.96/volcstack/latest" 37 | ) 38 | 39 | var metadataTimeout = time.Second * 5 40 | 41 | type EC2Metadata struct { 42 | *http.Client 43 | } 44 | 45 | func New() *EC2Metadata { 46 | return &EC2Metadata{ 47 | &http.Client{ 48 | Timeout: metadataTimeout, 49 | }, 50 | } 51 | } 52 | 53 | // GetMetadata get information from metadata by path. 54 | func (c *EC2Metadata) GetMetadata(ctx context.Context, path string) (info string, err error) { 55 | var req *http.Request 56 | var resp *http.Response 57 | info = "" 58 | url := fmt.Sprintf("%s/%s", metadataURL, path) 59 | 60 | defer func() { 61 | if err != nil { 62 | fmtErr := fmt.Sprintf("Call metadata failed, path: %s, err: %v", url, err) 63 | _ = tracing.RecordNodeEvent(v1.EventTypeWarning, tracing.EventMetadataServiceAbnormal, fmtErr) 64 | log.Errorf(fmtErr) 65 | } 66 | }() 67 | 68 | if req, err = http.NewRequestWithContext(ctx, http.MethodGet, url, nil); err != nil { 69 | return 70 | } 71 | 72 | if resp, err = c.Do(req); err != nil { 73 | return 74 | } 75 | defer func(Body io.ReadCloser) { 76 | _ = Body.Close() 77 | }(resp.Body) 78 | 79 | if resp.StatusCode != http.StatusOK { 80 | err = errors.New(fmt.Sprintf("HttpRequestStatus: %d", resp.StatusCode)) 81 | return 82 | } 83 | var respBytes []byte 84 | if respBytes, err = ioutil.ReadAll(resp.Body); err != nil { 85 | return 86 | } 87 | info = string(respBytes) 88 | return 89 | } 90 | -------------------------------------------------------------------------------- /pkg/provider/volcengine/metadata/mock/ec2wrapper_mocks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Code generated by MockGen. DO NOT EDIT. 16 | // Source: github.com/volcengine/cello/pkg/provider/volcengine/metadata (interfaces: EC2MetadataIface) 17 | 18 | // Package mock_metadata is a generated GoMock package. 19 | package mock_metadata 20 | 21 | import ( 22 | context "context" 23 | gomock "github.com/golang/mock/gomock" 24 | reflect "reflect" 25 | ) 26 | 27 | // MockEC2MetadataIface is a mock of EC2MetadataIface interface 28 | type MockEC2MetadataIface struct { 29 | ctrl *gomock.Controller 30 | recorder *MockEC2MetadataIfaceMockRecorder 31 | } 32 | 33 | // MockEC2MetadataIfaceMockRecorder is the mock recorder for MockEC2MetadataIface 34 | type MockEC2MetadataIfaceMockRecorder struct { 35 | mock *MockEC2MetadataIface 36 | } 37 | 38 | // NewMockEC2MetadataIface creates a new mock instance 39 | func NewMockEC2MetadataIface(ctrl *gomock.Controller) *MockEC2MetadataIface { 40 | mock := &MockEC2MetadataIface{ctrl: ctrl} 41 | mock.recorder = &MockEC2MetadataIfaceMockRecorder{mock} 42 | return mock 43 | } 44 | 45 | // EXPECT returns an object that allows the caller to indicate expected use 46 | func (m *MockEC2MetadataIface) EXPECT() *MockEC2MetadataIfaceMockRecorder { 47 | return m.recorder 48 | } 49 | 50 | // GetMetadata mocks base method 51 | func (m *MockEC2MetadataIface) GetMetadata(arg0 context.Context, arg1 string) (string, error) { 52 | m.ctrl.T.Helper() 53 | ret := m.ctrl.Call(m, "GetMetadata", arg0, arg1) 54 | ret0, _ := ret[0].(string) 55 | ret1, _ := ret[1].(error) 56 | return ret0, ret1 57 | } 58 | 59 | // GetMetadata indicates an expected call of GetMetadata 60 | func (mr *MockEC2MetadataIfaceMockRecorder) GetMetadata(arg0, arg1 interface{}) *gomock.Call { 61 | mr.mock.ctrl.T.Helper() 62 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetadata", reflect.TypeOf((*MockEC2MetadataIface)(nil).GetMetadata), arg0, arg1) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/utils/kernel/kernel.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package kernel 17 | 18 | import ( 19 | "errors" 20 | "fmt" 21 | ) 22 | 23 | // VersionInfo holds information about the kernel. 24 | type VersionInfo struct { 25 | Kernel int // Version of the kernel (e.g. 4.1.2-generic -> 4) 26 | Major int // Major part of the kernel version (e.g. 4.1.2-generic -> 1) 27 | Minor int // Minor part of the kernel version (e.g. 4.1.2-generic -> 2) 28 | Flavor string // Flavor of the kernel version (e.g. 4.1.2-generic -> generic) 29 | } 30 | 31 | // CompareKernelVersion compares two kernel.VersionInfo structs. 32 | // Returns -1 if a < b, 0 if a == b, 1 it a > b. 33 | func CompareKernelVersion(a, b VersionInfo) int { 34 | if a.Kernel < b.Kernel { 35 | return -1 36 | } else if a.Kernel > b.Kernel { 37 | return 1 38 | } 39 | 40 | if a.Major < b.Major { 41 | return -1 42 | } else if a.Major > b.Major { 43 | return 1 44 | } 45 | 46 | if a.Minor < b.Minor { 47 | return -1 48 | } else if a.Minor > b.Minor { 49 | return 1 50 | } 51 | 52 | return 0 53 | } 54 | 55 | // ParseRelease parses a string and creates a VersionInfo based on it. 56 | func ParseRelease(release string) (*VersionInfo, error) { 57 | var ( 58 | kernel, major, minor, parsed int 59 | flavor, partial string 60 | ) 61 | 62 | // Ignore error from Sscanf to allow an empty flavor. Instead, just 63 | // make sure we got all the version numbers. 64 | parsed, _ = fmt.Sscanf(release, "%d.%d%s", &kernel, &major, &partial) 65 | if parsed < 2 { 66 | return nil, errors.New("Can't parse kernel version " + release) 67 | } 68 | 69 | // sometimes we have 3.12.25-gentoo, but sometimes we just have 3.12-1-amd64 70 | parsed, _ = fmt.Sscanf(partial, ".%d%s", &minor, &flavor) 71 | if parsed < 1 { 72 | flavor = partial 73 | } 74 | 75 | return &VersionInfo{ 76 | Kernel: kernel, 77 | Major: major, 78 | Minor: minor, 79 | Flavor: flavor, 80 | }, nil 81 | } 82 | -------------------------------------------------------------------------------- /patch/cilium/006-support-pod-egress-qos.patch: -------------------------------------------------------------------------------- 1 | From bab28966546a8c7a78467b4296158ac17013fd46 Mon Sep 17 00:00:00 2001 2 | From: xinwenqiang 3 | Date: Mon, 13 Mar 2023 16:01:00 +0800 4 | Subject: [PATCH] Support pod egress qos 5 | 6 | --- 7 | bpf/bpf_lxc.c | 15 +++++++++++++-- 8 | pkg/datapath/linux/config/config.go | 6 ++++++ 9 | 2 files changed, 19 insertions(+), 2 deletions(-) 10 | 11 | diff --git a/bpf/bpf_lxc.c b/bpf/bpf_lxc.c 12 | index e4baa4d207..fe689b4fd0 100644 13 | --- a/bpf/bpf_lxc.c 14 | +++ b/bpf/bpf_lxc.c 15 | @@ -974,10 +974,21 @@ int handle_xgress(struct __ctx_buff *ctx) 16 | goto out; 17 | } 18 | 19 | +#if defined(ENABLE_BANDWIDTH_MANAGER) 20 | + edt_set_aggregate(ctx, LXC_ID); 21 | + ret = edt_sched_departure(ctx); 22 | + /* No send_drop_notify_error() here given we're rate-limiting. */ 23 | + if (ret == CTX_ACT_DROP) { 24 | + update_metrics(ctx_full_len(ctx), METRIC_EGRESS, 25 | + -DROP_EDT_HORIZON); 26 | + return ret; 27 | + } 28 | +#endif 29 | + 30 | switch (proto) { 31 | #ifdef ENABLE_IPV6 32 | case bpf_htons(ETH_P_IPV6): 33 | - edt_set_aggregate(ctx, LXC_ID); 34 | +// edt_set_aggregate(ctx, LXC_ID); 35 | invoke_tailcall_if(__or(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)), 36 | is_defined(DEBUG)), 37 | CILIUM_CALL_IPV6_FROM_LXC, tail_handle_ipv6); 38 | @@ -985,7 +996,7 @@ int handle_xgress(struct __ctx_buff *ctx) 39 | #endif /* ENABLE_IPV6 */ 40 | #ifdef ENABLE_IPV4 41 | case bpf_htons(ETH_P_IP): 42 | - edt_set_aggregate(ctx, LXC_ID); 43 | +// edt_set_aggregate(ctx, LXC_ID); 44 | invoke_tailcall_if(__or(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)), 45 | is_defined(DEBUG)), 46 | CILIUM_CALL_IPV4_FROM_LXC, tail_handle_ipv4); 47 | diff --git a/pkg/datapath/linux/config/config.go b/pkg/datapath/linux/config/config.go 48 | index 572d553f3f..70d173c507 100644 49 | --- a/pkg/datapath/linux/config/config.go 50 | +++ b/pkg/datapath/linux/config/config.go 51 | @@ -792,6 +792,12 @@ func (h *HeaderfileWriter) writeTemplateConfig(fw *bufio.Writer, e datapath.Endp 52 | fmt.Fprint(fw, "#define ENABLE_ARP_PASSTHROUGH 1\n") 53 | } 54 | 55 | + if option.Config.EnableBandwidthManager { 56 | + fmt.Fprint(fw, "#define ENABLE_BANDWIDTH_MANAGER 1\n") 57 | + fmt.Fprintf(fw, "#define THROTTLE_MAP %s\n", bwmap.MapName) 58 | + fmt.Fprintf(fw, "#define THROTTLE_MAP_SIZE %d\n", bwmap.MapSize) 59 | + } 60 | + 61 | if !e.HasIpvlanDataPath() { 62 | if !e.RequireARPPassthrough() { 63 | fmt.Fprint(fw, "#define ENABLE_ARP_RESPONDER 1\n") 64 | -- 65 | 2.34.1 66 | 67 | -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics 2 | 3 | Cello exposes metrics through the default port `11414`, path is `:11414/metrics` 4 | 5 | ## indicators 6 | 7 | | No. | Metrics | Labels | Description | 8 | | ---- | ------------------------ | --------------------------- | --------------------------------- | 9 | | 1 | openapi_latency_ms | api, error, code, requestId | latency in ms of openapi call | 10 | | 2 | openapi_error_count | api, error, code, requestId | count of openapi error | 11 | | 3 | metadata_latency_ms | metadata, error, status | latency in ms of metadata call | 12 | | 4 | metadata_error_count | metadata, error | count of metadata error | 13 | | 5 | rpc_latency_ms | rpc_api, error | latency in ms of rpc call | 14 | | 6 | resource_pool_max_cap | name, type | max capacity of resource pool | 15 | | 7 | resource_pool_target | name, type | cache target of resource pool | 16 | | 8 | resource_pool_target_min | name, type | cache target min of resource pool | 17 | | 9 | resource_pool_total | name, type | total resource in pool | 18 | | 10 | resource_pool_available | name, type | available resource in pool | 19 | 20 | 21 | 22 | ## Monitor 23 | 24 | Deploy monitor: 25 | 26 | * deploy prometheus 27 | 28 | ``` 29 | kubectl apply -f monitor/cello-monitor-prometheus.yaml 30 | ``` 31 | 32 | * deploy grafana 33 | 34 | * edit `prometheus_pod_ip` according to prometheus pod 35 | 36 | ```yaml 37 | apiVersion: v1 38 | kind: ConfigMap 39 | data: 40 | prometheus.yaml: |- 41 | apiVersion: 1 42 | datasources: 43 | - name: Prometheus 44 | type: prometheus 45 | # Access mode - proxy (server in the UI) or direct (browser in the UI). 46 | access: proxy 47 | editable: true 48 | url: http://{prometheus_pod_ip}:9090/ 49 | jsonData: 50 | httpMethod: GET 51 | 52 | metadata: 53 | name: grafana-datasources 54 | namespace: cello-monitor 55 | ``` 56 | 57 | * apply yaml 58 | 59 | ```bash 60 | kubectl apply -f monitor/cello-monitor-grafana.yaml 61 | ``` 62 | 63 | **Notic**: This method is only suitable for clusters which size is not too big. The bottleneck of prometheus will be reached if the cluster is too big, and other plan to collecte metrics of Cello are required. 64 | 65 | -------------------------------------------------------------------------------- /pkg/cni/device/ipvlan.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package device 17 | 18 | import ( 19 | "crypto/rand" 20 | "crypto/sha1" 21 | "encoding/hex" 22 | "fmt" 23 | 24 | "github.com/containernetworking/plugins/pkg/ns" 25 | "github.com/vishvananda/netlink" 26 | 27 | "github.com/volcengine/cello/pkg/cni/log" 28 | ) 29 | 30 | // IPVlanConf is interface config for IPVlan slaves. 31 | type IPVlanConf struct { 32 | MasterName string 33 | IfName string 34 | MTU int 35 | } 36 | 37 | func randomIPVlanIfName() (string, error) { 38 | entropy := make([]byte, 6) 39 | _, err := rand.Read(entropy) 40 | if err != nil { 41 | return "", fmt.Errorf("get random string failed: %s", err.Error()) 42 | } 43 | 44 | h := sha1.New() 45 | _, err = h.Write(entropy) 46 | if err != nil { 47 | return "", err 48 | } 49 | return fmt.Sprintf("%s%s", "ipvl", hex.EncodeToString(h.Sum(nil))[:11]), nil 50 | } 51 | 52 | // Setup IPVlan slave interface for netns. 53 | func (c *IPVlanConf) Setup(netNS ns.NetNS) error { 54 | masterLink, err := netlink.LinkByName(c.MasterName) 55 | if err != nil { 56 | log.Log.Errorf("LinkByName error, err:%s", err.Error()) 57 | return err 58 | } 59 | 60 | tempIfName, err := randomIPVlanIfName() 61 | if err != nil { 62 | return err 63 | } 64 | 65 | link := &netlink.IPVlan{ 66 | LinkAttrs: netlink.LinkAttrs{ 67 | MTU: c.MTU, 68 | Name: tempIfName, 69 | Namespace: netlink.NsFd(int(netNS.Fd())), 70 | ParentIndex: masterLink.Attrs().Index, 71 | }, 72 | Mode: netlink.IPVLAN_MODE_L2, 73 | } 74 | 75 | err = netlink.LinkAdd(link) 76 | if err != nil { 77 | log.Log.Errorf("LinkAdd error, err:%s", err.Error()) 78 | return err 79 | } 80 | 81 | err = netNS.Do(func(netNS ns.NetNS) error { 82 | link, inErr := netlink.LinkByName(tempIfName) 83 | if inErr != nil { 84 | return inErr 85 | } 86 | if link.Attrs().Name == c.IfName { 87 | return nil 88 | } 89 | return netlink.LinkSetName(link, c.IfName) 90 | }) 91 | if err != nil { 92 | log.Log.Errorf("LinkSetName error, err:%s", err.Error()) 93 | } 94 | return err 95 | } 96 | -------------------------------------------------------------------------------- /pkg/cni/device/vlan.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package device 17 | 18 | import ( 19 | "fmt" 20 | "net" 21 | 22 | "github.com/containernetworking/plugins/pkg/ns" 23 | "github.com/vishvananda/netlink" 24 | ) 25 | 26 | // VlanConfig is interface config of vlan slave. 27 | type VlanConfig struct { 28 | MasterName string 29 | IfName string 30 | Vid int 31 | MTU int 32 | HardwareAddr net.HardwareAddr 33 | } 34 | 35 | // Setup vlan slave for netns. 36 | func (vlanConfig *VlanConfig) Setup(netNS ns.NetNS) error { 37 | master, err := netlink.LinkByName(vlanConfig.MasterName) 38 | if err != nil { 39 | return fmt.Errorf("cannot found master link by name %s", vlanConfig.MasterName) 40 | } 41 | vlanName := generateVlanDeviceName(master.Attrs().Name, vlanConfig.Vid) 42 | vlan, err := netlink.LinkByName(vlanName) 43 | if err == nil { 44 | // del pre link 45 | err = netlink.LinkDel(vlan) 46 | if err != nil { 47 | return err 48 | } 49 | } 50 | if _, ok := err.(netlink.LinkNotFoundError); !ok { 51 | return err 52 | } 53 | 54 | vlan = &netlink.Vlan{ 55 | LinkAttrs: netlink.LinkAttrs{ 56 | MTU: vlanConfig.MTU, 57 | Name: vlanName, 58 | ParentIndex: master.Attrs().Index, 59 | Namespace: netlink.NsFd(int(netNS.Fd())), 60 | }, 61 | VlanId: vlanConfig.Vid, 62 | } 63 | if vlanConfig.HardwareAddr.String() != "" { 64 | vlan.Attrs().HardwareAddr = vlanConfig.HardwareAddr 65 | } 66 | 67 | err = netlink.LinkAdd(vlan) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | return netNS.Do(func(netNS ns.NetNS) error { 73 | contLink, innerErr := netlink.LinkByName(vlanName) 74 | if innerErr != nil { 75 | return innerErr 76 | } 77 | if contLink.Attrs().Name == vlanConfig.IfName { 78 | return nil 79 | } 80 | return netlink.LinkSetName(contLink, vlanConfig.IfName) 81 | 82 | }) 83 | } 84 | 85 | func generateVlanDeviceName(masterName string, vlanID int) string { 86 | vlanName := fmt.Sprintf("%s.%d", masterName, vlanID) 87 | if len(vlanName) > 15 { 88 | vlanName = vlanName[len(vlanName)-15:] 89 | } 90 | return vlanName 91 | } 92 | -------------------------------------------------------------------------------- /cmd/cello-cni/cello-cni.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package main 17 | 18 | import ( 19 | "time" 20 | 21 | "github.com/containernetworking/cni/pkg/skel" 22 | cniVersion "github.com/containernetworking/cni/pkg/version" 23 | 24 | cniLog "github.com/volcengine/cello/pkg/cni/log" 25 | "github.com/volcengine/cello/pkg/metrics" 26 | "github.com/volcengine/cello/pkg/utils/logger" 27 | ) 28 | 29 | func main() { 30 | skel.PluginMain(cmdAdd, cmdCheck, cmdDel, cniVersion.All, about) 31 | } 32 | 33 | func cmdAdd(args *skel.CmdArgs) error { 34 | start := time.Now() 35 | defer func() { 36 | duration := metrics.MsSince(start) 37 | cniLog.Log.Infof("CmdAdd time cost:%f Millisecond", duration) 38 | }() 39 | cniLog.Log.WithFields(logger.Fields{ 40 | "ContainerId": args.ContainerID, 41 | "Netns": args.Netns}, 42 | ).Infof("Handle cmd add") 43 | 44 | err := Add(args) 45 | if err != nil { 46 | cniLog.Log.WithFields(logger.Fields{ 47 | "Command": "Add", 48 | "ContainerId": args.ContainerID, 49 | "Netns": args.Netns}, 50 | ).Errorf("Handle cmd add failed: %v", err) 51 | } 52 | 53 | return err 54 | } 55 | 56 | func cmdDel(args *skel.CmdArgs) error { 57 | start := time.Now() 58 | defer func() { 59 | duration := metrics.MsSince(start) 60 | cniLog.Log.Infof("CmdDel time cost:%f Millisecond", duration) 61 | }() 62 | cniLog.Log.WithFields(logger.Fields{ 63 | "ContainerId": args.ContainerID, 64 | "Netns": args.Netns}, 65 | ).Infof("Handle cmd del") 66 | err := Del(args) 67 | if err != nil { 68 | cniLog.Log.WithFields( 69 | logger.Fields{ 70 | "Command": "Del", 71 | "ContainerId": args.ContainerID, 72 | "Netns": args.Netns, 73 | }).Infof("Handle cmd del failed: %s", err.Error()) 74 | } 75 | return err 76 | } 77 | 78 | func cmdCheck(args *skel.CmdArgs) error { 79 | err := Check(args) 80 | if err != nil { 81 | cniLog.Log.WithFields( 82 | logger.Fields{ 83 | "Command": "Check", 84 | "ContainerId": args.ContainerID, 85 | "Netns": args.Netns, 86 | }).Infof("Handle cmd del failed: %s", err.Error()) 87 | } 88 | return err 89 | } 90 | -------------------------------------------------------------------------------- /pkg/metrics/metric.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package metrics 17 | 18 | import ( 19 | "net/http" 20 | "os" 21 | "strconv" 22 | "time" 23 | 24 | "github.com/prometheus/client_golang/prometheus" 25 | "github.com/prometheus/client_golang/prometheus/promhttp" 26 | 27 | "github.com/volcengine/cello/pkg/utils/logger" 28 | ) 29 | 30 | var log = logger.GetLogger().WithFields(logger.Fields{"subsys": "metrics"}) 31 | 32 | const ( 33 | // Environment variable to disable the metrics endpoint on 61678. 34 | envDisableMetrics = "CELLO_DISABLE_METRICS" 35 | ) 36 | 37 | // ServeMetrics sets up cello metrics. 38 | func ServeMetrics(serveMux *http.ServeMux) { 39 | if disableMetrics() { 40 | log.Info("Metrics endpoint disabled") 41 | return 42 | } 43 | 44 | log.Infof("Register metrics server") 45 | serveMux.Handle("/metrics", promhttp.Handler()) 46 | } 47 | 48 | // disableMetrics returns true if we should disable metrics. 49 | func disableMetrics() bool { 50 | return getEnvBoolWithDefault(envDisableMetrics, false) 51 | } 52 | 53 | func getEnvBoolWithDefault(envName string, def bool) bool { 54 | if strValue := os.Getenv(envName); strValue != "" { 55 | parsedValue, err := strconv.ParseBool(strValue) 56 | if err == nil { 57 | return parsedValue 58 | } 59 | log.Errorf("Failed to parse %s, using default `%t`: %v", envName, def, err.Error()) 60 | } 61 | return def 62 | } 63 | 64 | var prometheusRegistered = false 65 | 66 | func PrometheusRegister() { 67 | if prometheusRegistered { 68 | return 69 | } 70 | 71 | prometheus.MustRegister(RpcLatency) 72 | prometheus.MustRegister(OpenAPILatency) 73 | prometheus.MustRegister(OpenAPIErr) 74 | prometheus.MustRegister(MetadataLatency) 75 | prometheus.MustRegister(MetadataErr) 76 | 77 | prometheus.MustRegister(ResourcePoolMaxCap) 78 | prometheus.MustRegister(ResourcePoolTarget) 79 | prometheus.MustRegister(ResourcePoolTargetMin) 80 | prometheus.MustRegister(ResourcePoolTotal) 81 | prometheus.MustRegister(ResourcePoolAvailable) 82 | 83 | prometheusRegistered = true 84 | } 85 | 86 | // MsSince returns milliseconds since start. 87 | func MsSince(start time.Time) float64 { 88 | return float64(time.Since(start) / time.Millisecond) 89 | } 90 | -------------------------------------------------------------------------------- /pkg/utils/datatype/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package datatype 17 | 18 | // String returns a pointer to the string value passed in. 19 | func String(v string) *string { 20 | return &v 21 | } 22 | 23 | // StringValue returns the value of the string pointer passed in or 24 | // "" if the pointer is nil. 25 | func StringValue(s *string) string { 26 | if s != nil { 27 | return *s 28 | } 29 | return "" 30 | } 31 | 32 | // Float64 returns a pointer to the float64 value passed in. 33 | func Float64(v float64) *float64 { 34 | return &v 35 | } 36 | 37 | // Float64Value returns the value of the float64 pointer passed in or 38 | // 0 if the pointer is nil. 39 | func Float64Value(f *float64) float64 { 40 | if f != nil { 41 | return *f 42 | } 43 | return 0 44 | } 45 | 46 | // Int returns a pointer to the int value passed in. 47 | func Int(v int) *int { 48 | return &v 49 | } 50 | 51 | // IntValue returns the value of the int pointer passed in or 52 | // 0 if the pointer is nil. 53 | func IntValue(v *int) int { 54 | if v != nil { 55 | return *v 56 | } 57 | return 0 58 | } 59 | 60 | // Uint16 returns a pointer to the uint16 value passed in. 61 | func Uint16(v uint16) *uint16 { 62 | return &v 63 | } 64 | 65 | // Uint16Value returns the value of the uint16 pointer passed in or 66 | // 0 if the pointer is nil. 67 | func Uint16Value(p *uint16) uint16 { 68 | if p == nil { 69 | return 0 70 | } 71 | return *p 72 | } 73 | 74 | // Uint32 returns a pointer to the uint32 value passed in. 75 | func Uint32(v uint32) *uint32 { 76 | return &v 77 | } 78 | 79 | // Uint32Value returns the value of the uint32 pointer passed in or 80 | // 0 if the pointer is nil. 81 | func Uint32Value(v *uint32) uint32 { 82 | if v != nil { 83 | return *v 84 | } 85 | return 0 86 | } 87 | 88 | // Bool returns a pointer to the bool value passed in. 89 | func Bool(v bool) *bool { 90 | return &v 91 | } 92 | 93 | // BoolValue returns the value of the bool pointer passed in or 94 | // false if the pointer is nil. 95 | func BoolValue(b *bool) bool { 96 | if b != nil { 97 | return *b 98 | } 99 | return false 100 | } 101 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # VERSION is the source revision for binary and image building. 2 | VERSION ?= $(shell git log -1 --date='format:%Y%m%d' --format='format:%ad').$(shell git describe --always --contains HEAD) 3 | BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) 4 | DATE = $(shell date +"%Y-%m-%d_%H:%M:%S") 5 | COMMIT = $(shell git rev-parse HEAD | head -c 8) 6 | 7 | # Information of OS and ARCH 8 | OS = $(shell uname -s) 9 | ARCH = $(shell uname -m) 10 | 11 | # Output Directory 12 | OUTPUT ?= output 13 | 14 | # Container runtime engine 15 | ENGINE ?= docker 16 | 17 | # IMAGE Name and Tag 18 | IMAGE_NAME ?= volcstack/cello 19 | IMAGE_TAG ?= $(VERSION) 20 | IMAGE_NAME_TAG ?= $(IMAGE_NAME):$(IMAGE_TAG) 21 | 22 | # GO FLAGS 23 | GOPROXY ?= 24 | GO_FLAGS=-ldflags="-s -w" 25 | CNI_VERSION_LD_FLAG=-ldflags="-X github.com/volcengine/cello/pkg/version.Version=$(VERSION)@$(BRANCH) -X github.com/volcengine/cello/pkg/version.GitCommit=$(COMMIT)" 26 | BUILD_INFO=-ldflags="-X main.BuildInfo=$(VERSION)@$(BRANCH)_$(DATE)" 27 | 28 | # BUILD FLAGS 29 | CELLO_META ?= 30 | 31 | BUILD_ARGS = --build-arg HTTPS_PROXY=$(HTTPS_PROXY) --build-arg GOPROXY=$(GOPROXY) 32 | ifdef GOPROXY 33 | BUILD_ARGS+=--build-arg GOPROXY=$(GOPROXY) 34 | endif 35 | ifdef CELLO_META 36 | BUILD_ARGS+=--build-arg CELLO_META=$(CELLO_META) 37 | endif 38 | 39 | tidy: 40 | go mod tidy 41 | 42 | cello-agent: 43 | CGO_ENABLED=0 GOOS=linux go build -o $(OUTPUT)/cello-agent $(GO_FLAGS) $(CNI_VERSION_LD_FLAG) \ 44 | ./cmd/cello-agent 45 | 46 | cello-ctl: 47 | CGO_ENABLED=0 GOOS=linux go build -o $(OUTPUT)/cello-ctl $(GO_FLAGS) $(CNI_VERSION_LD_FLAG) $(BUILD_INFO)\ 48 | ./cmd/cello-cli 49 | 50 | cello-cni: 51 | ifdef CELLO_META 52 | CGO_ENABLED=0 GOOS=linux go build -tags meta -o $(OUTPUT)/cello-cni $(GO_FLAGS) $(CNI_VERSION_LD_FLAG) \ 53 | ./cmd/cello-cni 54 | $(info with meta) 55 | else 56 | CGO_ENABLED=0 GOOS=linux go build -o $(OUTPUT)/cello-cni $(GO_FLAGS) $(CNI_VERSION_LD_FLAG) \ 57 | ./cmd/cello-cni 58 | endif 59 | 60 | cilium-launcher: 61 | CGO_ENABLED=0 GOOS=linux go build -o $(OUTPUT)/cilium-launcher $(GO_FLAGS) $(CNI_VERSION_LD_FLAG) \ 62 | ./cmd/launcher/cilium 63 | 64 | protobuf: tidy 65 | go generate ./pkg/pbrpc 66 | 67 | all: pkg image 68 | 69 | bin: tidy cello-cni cello-ctl cello-agent cilium-launcher 70 | 71 | pkg: bin 72 | cp ./script/bootstrap/* $(OUTPUT)/ 73 | chmod +x $(OUTPUT)/*.sh 74 | 75 | image: 76 | $(ENGINE) build -f ./images/Dockerfile -t $(IMAGE_NAME_TAG) ${BUILD_ARGS} . 77 | @echo "Built OCI image \"$(IMAGE_NAME)\"" 78 | 79 | test: 80 | # Skip race detection for now due to issue: https://github.com/etcd-io/bbolt/issues/391 81 | go test -v ./... -cover -coverprofile=coverage.out 82 | 83 | clean: 84 | rm -rf ./output 85 | 86 | .PHONY: clean protobuf cello-agent cello-cni cello-ctl bin pkg image all test 87 | 88 | .DEFAULT: bin 89 | -------------------------------------------------------------------------------- /chart/templates/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: cello-cluster-role 5 | namespace: kube-system 6 | rules: 7 | - apiGroups: [ "" ] 8 | resources: [ "nodes", "namespaces", "configmaps", "serviceaccounts" ] 9 | verbs: [ "get", "watch", "list", "update" ] 10 | - apiGroups: [ "" ] 11 | resources: 12 | - pods 13 | verbs: 14 | - get 15 | - watch 16 | - list 17 | - update 18 | - patch 19 | - apiGroups: [ "" ] 20 | resources: 21 | - events 22 | verbs: 23 | - create 24 | - patch 25 | - apiGroups: [ "networking.k8s.io" ] 26 | resources: 27 | - networkpolicies 28 | verbs: 29 | - get 30 | - list 31 | - watch 32 | - apiGroups: [ "coordination.k8s.io" ] 33 | resources: [ "leases" ] 34 | verbs: [ "get", "watch", "update", "create" ] 35 | - apiGroups: [ "extensions" ] 36 | resources: 37 | - networkpolicies 38 | verbs: 39 | - get 40 | - list 41 | - watch 42 | - apiGroups: [ "" ] 43 | resources: 44 | - pods/status 45 | verbs: 46 | - update 47 | - apiGroups: 48 | - "" 49 | resources: 50 | - "pods/eviction" 51 | verbs: 52 | - "create" 53 | attributeRestrictions: null 54 | - apiGroups: [ "discovery.k8s.io" ] 55 | resources: 56 | - endpointslices 57 | verbs: 58 | - get 59 | - list 60 | - watch 61 | - apiGroups: [ "" ] 62 | resources: 63 | - endpoints 64 | - services 65 | verbs: 66 | - get 67 | - list 68 | - watch 69 | - apiGroups: [ "" ] 70 | resources: 71 | - nodes 72 | - nodes/status 73 | verbs: 74 | - patch 75 | - apiGroups: 76 | - apiextensions.k8s.io 77 | resources: 78 | - customresourcedefinitions 79 | verbs: 80 | - create 81 | - get 82 | - list 83 | - watch 84 | - update 85 | - apiGroups: 86 | - cilium.io 87 | resources: 88 | - ciliumnodes 89 | - ciliumnodes/status 90 | - ciliumexternalworkloads 91 | - ciliumexternalworkloads/status 92 | - ciliumidentities 93 | - ciliumidentities/status 94 | - ciliumendpoints 95 | - ciliumendpoints/status 96 | - ciliumnetworkpolicies 97 | - ciliumnetworkpolicies/status 98 | - ciliumclusterwidenetworkpolicies 99 | - ciliumclusterwidenetworkpolicies/status 100 | - ciliumlocalredirectpolicies 101 | - ciliumlocalredirectpolicies/status 102 | - ciliumegressnatpolicies 103 | verbs: 104 | - delete 105 | - deletecollection 106 | - get 107 | - list 108 | - patch 109 | - create 110 | - update 111 | - watch 112 | -------------------------------------------------------------------------------- /pkg/provider/volcengine/cellohelper/eni_tag.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package cellohelper 17 | 18 | import ( 19 | "github.com/volcengine/volcengine-go-sdk/service/vpc" 20 | "github.com/volcengine/volcengine-go-sdk/volcengine" 21 | ) 22 | 23 | const ( 24 | VkeTagPrefix = "volc:vke:" 25 | VkePlatformTagKey = VkeTagPrefix + "createdby-vke-flag" 26 | VkePlatformTagValue = "true" 27 | VkeComponentTagKey = VkeTagPrefix + "created-by" 28 | VkeInstanceIdTagKey = VkeTagPrefix + "ecs-id" 29 | 30 | K8sTagPrefix = "k8s:cello:" 31 | K8sComponentTagKey = K8sTagPrefix + "created-by" 32 | K8sInstanceIdTagKey = K8sTagPrefix + "ecs-id" 33 | 34 | Component = "cello" 35 | eniDescription = "interface create by cello" 36 | ) 37 | 38 | func BuildTagsForCreateNetworkInterfaceInput(tags map[string]string) []*vpc.TagForCreateNetworkInterfaceInput { 39 | var tagsInput []*vpc.TagForCreateNetworkInterfaceInput 40 | for k, v := range tags { 41 | tagsInput = append(tagsInput, &vpc.TagForCreateNetworkInterfaceInput{ 42 | Key: volcengine.String(k), 43 | Value: volcengine.String(v), 44 | }) 45 | } 46 | return tagsInput 47 | } 48 | 49 | func BuildFilterForDescribeNetworkInterfacesInput(tags map[string]string) []*vpc.TagFilterForDescribeNetworkInterfacesInput { 50 | var tagsInput []*vpc.TagFilterForDescribeNetworkInterfacesInput 51 | for k, v := range tags { 52 | tagsInput = append(tagsInput, &vpc.TagFilterForDescribeNetworkInterfacesInput{ 53 | Key: volcengine.String(k), 54 | Values: volcengine.StringSlice([]string{v}), 55 | }) 56 | } 57 | return tagsInput 58 | } 59 | 60 | // ConvertTagForDescribeNetworkInterfacesOutput convert list of vpc.TagForDescribeNetworkInterfacesOutput to map[string]string 61 | func ConvertTagForDescribeNetworkInterfacesOutput(output []*vpc.TagForDescribeNetworkInterfacesOutput) map[string]string { 62 | tags := map[string]string{} 63 | for _, item := range output { 64 | if item == nil { 65 | continue 66 | } 67 | tags[volcengine.StringValue(item.Key)] = volcengine.StringValue(item.Value) 68 | } 69 | return tags 70 | } 71 | 72 | // AssertTag assert actual tags match the expected tags 73 | func AssertTag(expected, actual map[string]string) bool { 74 | for k, v := range expected { 75 | if value, exist := actual[k]; !exist || value != v { 76 | return false 77 | } 78 | } 79 | return true 80 | } 81 | -------------------------------------------------------------------------------- /docs/hubble-intergration.md: -------------------------------------------------------------------------------- 1 | # Hubble 集成 2 | ## Enable 3 | 集成hubble需要在kube-system命名空间下的configmap cilium-config中开启以下参数，可参考 [cilium-agent.md] 4 | ```yaml 5 | enable-hubble: "true" 6 | hubble-listen-address: ":4244" 7 | hubble-metrics: "drop,tcp,flow,port-distribution,icmp" # 开启metrics有性能损失 8 | hubble-metrics-server: ":9091" 9 | ``` 10 | 开启以上参数后，需要重启所有的Cello pod 11 | ```bash 12 | kubectl delete pod -l app=cello -n kube-system 13 | ``` 14 | 15 | ## 部署 Hubble UI + Hubble Relay 16 | 17 | 1. 执行以下命令部署，需要本地安装有 `Helm v3`： 18 | 19 | ```bash 20 | git clone https://github.com/cilium/cilium.git 21 | cd cilium 22 | git checkout v1.8.1 23 | helm install hubble-ui install/kubernetes/cilium/charts/hubble-ui --set global.hubble.ui.enabled=true --set global.hubble.enabled=true --set global.hubble.relay.enabled=true --set ingress.enabled=true --set ingress.hosts={hubble.local} --namespace kube-system 24 | helm install hubble-relay install/kubernetes/cilium/charts/hubble-relay --set global.hubble.enabled=true --set global.hubble.relay.enabled=true --set global.hubble.socketPath=/var/run/cilium/hubble.sock --set image.repository=quay.io/cilium/hubble-relay:v1.8.1 --namespace kube-system 25 | ``` 26 | 27 | 2. 删除 `hubble-relay` deployment yaml文件中的以下亲和性配置： 28 | 29 | ```yaml 30 | # kubectl -n kube-system edit deployment hubble-relay 31 | spec: 32 | template: 33 | spec: 34 | # Deletion Begin 35 | affinity: 36 | podAffinity: 37 | requiredDuringSchedulingIgnoredDuringExecution: 38 | - labelSelector: 39 | matchExpressions: 40 | - key: k8s-app 41 | operator: In 42 | values: 43 | - cilium 44 | topologyKey: kubernetes.io/hostname 45 | # Deletion End 46 | containers: 47 | # ... 48 | ``` 49 | 50 | 3. 访问 Hubble WEB UI 51 | 将 `hubble-service-address` 指向 service `hubble-ui` 的 IP 地址，访问 `http://hubble-service-address:80` 52 | 也可以自行通过其他方式暴露`hubble-ui`服务 53 | 54 | ## 采集 metrics 55 | 56 | 1. 创建以下 SVC，用于被 Prometheus 采集 Metrics： 57 | 58 | ```yaml 59 | --- 60 | kind: Service 61 | apiVersion: v1 62 | metadata: 63 | name: hubble-metrics 64 | namespace: kube-system 65 | annotations: 66 | prometheus.io/scrape: 'true' 67 | prometheus.io/port: '9091' # 需要和cilium-config中的配置一致 68 | labels: 69 | k8s-app: hubble 70 | spec: 71 | clusterIP: None 72 | type: ClusterIP 73 | ports: 74 | - name: hubble-metrics 75 | port: 9091 76 | protocol: TCP 77 | targetPort: 9091 78 | selector: 79 | app: cello 80 | ``` 81 | 82 | 2. 采集 Hubble Metrics 83 | 配置 Prometheus 采集 `kube-system` 下 `hubble-metrics` 即可采集到 Hubble 暴露的指标，可参考 [cilium-metrics] 进行 Dashboard 配置。 84 | 85 | [cilium-agent.md]: https://github.com/cilium/cilium/blob/master/Documentation/cmdref/cilium-agent.md 86 | [cilium-metrics]: https://docs.cilium.io/en/v1.8/operations/metrics/ -------------------------------------------------------------------------------- /pkg/tracing/tracing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package tracing 17 | 18 | import ( 19 | "errors" 20 | "sync" 21 | ) 22 | 23 | var ( 24 | defaultTracer Tracer 25 | ) 26 | 27 | // PodEventRecorder records event on pod. 28 | type PodEventRecorder func(podName, podNamespace, eventType, reason, message string) error 29 | 30 | // NodeEventRecorder records event on node. 31 | type NodeEventRecorder func(eventType, reason, message string) 32 | 33 | // Tracer manages tracing handlers registered from the system. 34 | type Tracer struct { 35 | mtx sync.Mutex 36 | 37 | podEvent PodEventRecorder 38 | nodeEvent NodeEventRecorder 39 | } 40 | 41 | // RegisterEventRecorder registers pod & node event recorder to a tracer. 42 | func (t *Tracer) RegisterEventRecorder(node NodeEventRecorder, pod PodEventRecorder) { 43 | t.nodeEvent = node 44 | t.podEvent = pod 45 | } 46 | 47 | // RecordPodEvent records pod event via PodEventRecorder. 48 | func (t *Tracer) RecordPodEvent(podName, podNamespace, eventType, reason, message string) error { 49 | if t.podEvent == nil { 50 | return errors.New("no pod event recorder registered") 51 | } 52 | 53 | return t.podEvent(podName, podNamespace, eventType, reason, message) 54 | } 55 | 56 | // RecordNodeEvent records node event via PodEventRecorder. 57 | func (t *Tracer) RecordNodeEvent(eventType, reason, message string) error { 58 | if t.nodeEvent == nil { 59 | return errors.New("no node event recorder registered") 60 | } 61 | 62 | t.nodeEvent(eventType, reason, message) 63 | return nil 64 | } 65 | 66 | // RegisterEventRecorder registers pod & node event recorder to a tracer. 67 | func RegisterEventRecorder(node NodeEventRecorder, pod PodEventRecorder) { 68 | defaultTracer.RegisterEventRecorder(node, pod) 69 | } 70 | 71 | // RecordPodEvent records pod event via PodEventRecorder. 72 | func RecordPodEvent(podName, podNamespace, eventType, reason, message string) error { 73 | return defaultTracer.RecordPodEvent(podName, podNamespace, eventType, reason, message) 74 | } 75 | 76 | // RecordNodeEvent records node event via PodEventRecorder. 77 | func RecordNodeEvent(eventType, reason, message string) error { 78 | return defaultTracer.RecordNodeEvent(eventType, reason, message) 79 | } 80 | 81 | // NewTracer creates a new tracer. 82 | func NewTracer() *Tracer { 83 | return &Tracer{ 84 | mtx: sync.Mutex{}, 85 | } 86 | } 87 | 88 | func DefaultGlobalTracer() *Tracer { 89 | return &defaultTracer 90 | } 91 | -------------------------------------------------------------------------------- /pkg/store/disk_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package store 17 | 18 | import ( 19 | "encoding/json" 20 | "os" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | type mockObj struct { 27 | Name string `json:"name"` 28 | Version string `json:"version"` 29 | } 30 | 31 | func newStorage() (Interface, error) { 32 | return NewDiskStorage("test1", "./test.db", json.Marshal, func(bytes []byte) (interface{}, error) { 33 | obj := &mockObj{} 34 | err := json.Unmarshal(bytes, obj) 35 | if err != nil { 36 | return nil, err 37 | } 38 | return *obj, nil 39 | }) 40 | } 41 | 42 | func TestDistStore(t *testing.T) { 43 | tests := []struct { 44 | description string 45 | useDiskBackend bool 46 | }{ 47 | {"Test disk storage", true}, 48 | {"Test cache storage", true}, 49 | } 50 | objs := []mockObj{ 51 | {"t0", "v0"}, 52 | {"t1", "v1"}, 53 | {"t2", "v2"}, 54 | {"t3", "v4"}, 55 | } 56 | 57 | for _, test := range tests { 58 | t.Run(test.description, func(t *testing.T) { 59 | defer func() { 60 | _ = os.Remove("./test.db") 61 | }() 62 | 63 | db, err := newStorage() 64 | if err != nil { 65 | t.Errorf("NewDiskStorage err %s", err) 66 | } 67 | 68 | // Put 69 | for i := range objs { 70 | err = db.Put(objs[i].Name, objs[i]) 71 | if err != nil { 72 | t.Errorf("db Put err %s", err) 73 | } 74 | } 75 | // Reload data from disk. 76 | if test.useDiskBackend { 77 | db.Close() 78 | db, err = newStorage() 79 | if err != nil { 80 | t.Errorf("NewDiskStorage err %s", err) 81 | } 82 | } 83 | 84 | // Get 85 | for i := range objs { 86 | gotObj, err := db.Get(objs[i].Name) 87 | if err != nil { 88 | t.Errorf("db Put err %s", err) 89 | } 90 | t.Logf("got obj %+v", gotObj) 91 | assert.Equal(t, objs[i], gotObj) 92 | } 93 | // List 94 | listObjs := db.List() 95 | assert.Equal(t, len(objs), len(listObjs)) 96 | // Delete 97 | err = db.Delete(objs[0].Name) 98 | if err != nil { 99 | t.Errorf("db Delete err %s", err) 100 | } 101 | 102 | // Reload data from disk. 103 | if test.useDiskBackend { 104 | db.Close() 105 | db, err = newStorage() 106 | if err != nil { 107 | t.Errorf("NewDiskStorage err %s", err) 108 | } 109 | } 110 | listObjs = db.List() 111 | assert.Equal(t, len(objs)-1, len(listObjs)) 112 | }) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /pkg/utils/sysctl/sysctl_linux_privileged_test.go: -------------------------------------------------------------------------------- 1 | //go:build linux && privileged_tests 2 | // +build linux,privileged_tests 3 | 4 | // Copyright 2023 The Cello Authors 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | package sysctl 20 | 21 | import ( 22 | "testing" 23 | 24 | . "gopkg.in/check.v1" 25 | ) 26 | 27 | // Hook up gocheck into the "go test" runner. 28 | func Test(t *testing.T) { 29 | TestingT(t) 30 | } 31 | 32 | type SysctlLinuxPrivilegedTestSuite struct{} 33 | 34 | var _ = Suite(&SysctlLinuxPrivilegedTestSuite{}) 35 | 36 | func (s *SysctlLinuxPrivilegedTestSuite) TestWriteSysctl(c *C) { 37 | testCases := []struct { 38 | name string 39 | value []byte 40 | oldValue []byte 41 | expectedErr bool 42 | }{ 43 | { 44 | name: "net.ipv4.ip_forward", 45 | value: "1", 46 | expectedErr: false, 47 | }, 48 | { 49 | name: "net.ipv4.conf.all.forwarding", 50 | value: "1", 51 | expectedErr: false, 52 | }, 53 | { 54 | name: "net.ipv6.conf.all.forwarding", 55 | value: "1", 56 | expectedErr: false, 57 | }, 58 | { 59 | name: "foo.bar", 60 | value: "1", 61 | expectedErr: true, 62 | }, 63 | } 64 | 65 | for _, tc := range testCases { 66 | err := writeSysctl(tc.name, tc.value) 67 | if tc.expectedErr { 68 | c.Assert(err, NotNil) 69 | } else { 70 | c.Assert(err, IsNil) 71 | } 72 | } 73 | } 74 | 75 | func (s *SysctlLinuxPrivilegedTestSuite) TestDisableEnable(c *C) { 76 | testCases := []struct { 77 | name string 78 | expectedErr bool 79 | }{ 80 | { 81 | name: "net.ipv4.ip_forward", 82 | expectedErr: false, 83 | }, 84 | { 85 | name: "net.ipv4.conf.all.forwarding", 86 | expectedErr: false, 87 | }, 88 | { 89 | name: "net.ipv6.conf.all.forwarding", 90 | expectedErr: false, 91 | }, 92 | { 93 | name: "foo.bar", 94 | expectedErr: true, 95 | }, 96 | } 97 | 98 | for _, tc := range testCases { 99 | err := Enable(tc.name) 100 | if tc.expectedErr { 101 | c.Assert(err, NotNil) 102 | } else { 103 | c.Assert(err, IsNil) 104 | 105 | val, err := Read(tc.name) 106 | c.Assert(err, IsNil) 107 | c.Assert(val, Equals, "1") 108 | } 109 | err = Disable(tc.name) 110 | if tc.expectedErr { 111 | c.Assert(err, NotNil) 112 | } else { 113 | c.Assert(err, IsNil) 114 | 115 | val, err := Read(tc.name) 116 | c.Assert(err, IsNil) 117 | c.Assert(val, Equals, "0") 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /pkg/daemon/resource_manager.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package daemon 17 | 18 | import ( 19 | "net/http" 20 | 21 | "github.com/gin-gonic/gin" 22 | 23 | "github.com/volcengine/cello/pkg/pool" 24 | "github.com/volcengine/cello/types" 25 | ) 26 | 27 | // ResourceManager is the interface to manage network resources. 28 | type ResourceManager interface { 29 | // Name returns the name of the resource manager. 30 | Name() string 31 | // Allocate allocates a NetResource. 32 | Allocate(ctx *netContext, prefer string) (types.NetResource, error) 33 | // Release releases the given NetResource. 34 | Release(ctx *netContext, resource *types.VPCResource) error 35 | // GetSnapshot returns the snapshot of the resource manager. 36 | GetSnapshot() (pool.ResourcePoolSnapshot, error) 37 | // GetPoolStatus returns the status of the resource pool. 38 | GetPoolStatus() pool.Status 39 | // GetResourceLimit returns the limit of the resource manager. 40 | GetResourceLimit() int 41 | } 42 | 43 | type getResourceManagerSnapshot struct { 44 | daemon *daemon 45 | } 46 | 47 | func (s *getResourceManagerSnapshot) Handle(c *gin.Context) { 48 | info := map[string]struct { 49 | Pool map[string]types.NetResourceSnapshot 50 | Meta map[string]types.NetResourceSnapshot 51 | }{} 52 | 53 | for resType, mgr := range s.daemon.managers { 54 | snapshot, err := mgr.GetSnapshot() 55 | if err != nil { 56 | _ = c.Error(err) 57 | return 58 | } 59 | poolRes := map[string]types.NetResourceSnapshot{} 60 | metaRes := map[string]types.NetResourceSnapshot{} 61 | for id, item := range snapshot.PoolSnapshot() { 62 | poolRes[id] = *item.(*types.NetResourceSnapshot) 63 | } 64 | for id, item := range snapshot.MetaSnapshot() { 65 | metaRes[id] = *item.(*types.NetResourceSnapshot) 66 | } 67 | info[resType] = struct { 68 | Pool map[string]types.NetResourceSnapshot 69 | Meta map[string]types.NetResourceSnapshot 70 | }{Pool: poolRes, Meta: metaRes} 71 | } 72 | c.JSON(http.StatusOK, info) 73 | } 74 | 75 | func newGetResourceSnapshotHandler(d *daemon) Handler { 76 | return &getResourceManagerSnapshot{ 77 | daemon: d, 78 | } 79 | } 80 | 81 | type getPoolStatus struct { 82 | daemon *daemon 83 | } 84 | 85 | func (s *getPoolStatus) Handle(c *gin.Context) { 86 | info := map[string]pool.Status{} 87 | 88 | for resType, mgr := range s.daemon.managers { 89 | info[resType] = mgr.GetPoolStatus() 90 | } 91 | c.JSON(http.StatusOK, info) 92 | } 93 | 94 | func newGetPoolStatusHandler(d *daemon) Handler { 95 | return &getPoolStatus{ 96 | daemon: d, 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /pkg/k8s/k8s_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package k8s 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | corev1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/client-go/kubernetes/fake" 27 | 28 | "github.com/volcengine/cello/types" 29 | ) 30 | 31 | func TestPatchTrunkInfo(t *testing.T) { 32 | nodeName := "fake-node" 33 | k8sClient := fake.NewSimpleClientset() 34 | 35 | _, err := k8sClient.CoreV1().Nodes().Create(context.Background(), &corev1.Node{ 36 | TypeMeta: metav1.TypeMeta{}, 37 | ObjectMeta: metav1.ObjectMeta{ 38 | Name: nodeName, 39 | }, 40 | Spec: corev1.NodeSpec{}, 41 | Status: corev1.NodeStatus{ 42 | Addresses: []corev1.NodeAddress{ 43 | { 44 | Type: corev1.NodeExternalIP, 45 | Address: "172.16.0.3", 46 | }, 47 | { 48 | Type: corev1.NodeHostName, 49 | Address: "172.16.0.3", 50 | }, 51 | }, 52 | }, 53 | }, metav1.CreateOptions{}) 54 | assert.NoError(t, err) 55 | 56 | k8sService, err := NewK8sService(nodeName, k8sClient) 57 | assert.NoError(t, err) 58 | 59 | trunkENI := "eni-saj223d2344s" 60 | 61 | t.Run("TestAdd", func(t *testing.T) { 62 | t.Parallel() 63 | trunkInfo := types.TrunkInfo{ 64 | EniID: trunkENI, 65 | Mac: "ef:ef:ef:ef:ef:ef", 66 | BranchLimit: 6, 67 | } 68 | err = k8sService.PatchTrunkInfo(&trunkInfo) 69 | assert.NoError(t, err) 70 | node, err := k8sService.GetLocalNode(context.Background()) 71 | assert.NoError(t, err) 72 | var trunkInfoGot types.TrunkInfo 73 | err = json.Unmarshal([]byte(node.Annotations[types.AnnotationTrunkENI]), &trunkInfoGot) 74 | assert.NoError(t, err) 75 | assert.Equal(t, trunkInfo, trunkInfoGot) 76 | }) 77 | 78 | t.Run("TestDelete", func(t *testing.T) { 79 | trunkInfo := types.TrunkInfo{ 80 | EniID: trunkENI, 81 | Mac: "ef:ef:ef:ef:ef:ef", 82 | BranchLimit: 6, 83 | } 84 | err = k8sService.PatchTrunkInfo(&trunkInfo) 85 | assert.NoError(t, err) 86 | node, err := k8sService.GetLocalNode(context.Background()) 87 | assert.NoError(t, err) 88 | _, exist := node.Annotations[types.AnnotationTrunkENI] 89 | assert.Equal(t, true, exist) 90 | err = k8sService.PatchTrunkInfo(nil) 91 | assert.NoError(t, err) 92 | node, err = k8sService.GetLocalNode(context.Background()) 93 | assert.NoError(t, err) 94 | _, exist = node.Annotations[types.AnnotationTrunkENI] 95 | t.Log(node.Annotations[types.AnnotationTrunkENI]) 96 | assert.Equal(t, false, exist) 97 | }) 98 | } 99 | -------------------------------------------------------------------------------- /types/pod.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package types 17 | 18 | import ( 19 | "fmt" 20 | "time" 21 | 22 | "github.com/gdexlab/go-render/render" 23 | 24 | "github.com/volcengine/cello/pkg/pbrpc" 25 | ) 26 | 27 | const ( 28 | PodNetworkModeENIShare = "eni_shared" 29 | PodNetworkModeENIExclusive = "eni_exclusive" 30 | ) 31 | 32 | type Pod struct { 33 | Namespace string `json:"nameSpace"` 34 | Name string `json:"name"` 35 | SandboxContainerId string `json:"sandboxContainerId"` 36 | CreateTime time.Time `json:"createTime,omitempty"` 37 | 38 | NetNs string `json:"netNs,omitempty"` 39 | 40 | AllowEviction bool `json:"allowEviction,omitempty"` 41 | VpcENI bool `json:"vpcENI,omitempty"` 42 | 43 | MainInterface *pbrpc.NetworkInterface `json:"mainInterface,omitempty"` // Deprecated 44 | IsMainInterfaceSharedMode bool `json:"isMainInterfaceSharedMode,omitempty"` // Deprecated 45 | ExtraInterfaces []*pbrpc.NetworkInterface `json:"extraInterfaces,omitempty"` // Deprecated 46 | PodNetworkMode string `json:"podNetworkMode,omitempty"` 47 | 48 | Resources []VPCResource `json:"resources,omitempty"` 49 | } 50 | 51 | func PodKey(podNameSpace, podName string) string { 52 | return fmt.Sprintf("%s/%s", podNameSpace, podName) 53 | } 54 | 55 | func (p *Pod) GetVPCResourceByType(rType string) []VPCResource { 56 | var ret []VPCResource 57 | if p == nil { 58 | return ret 59 | } 60 | for _, r := range p.Resources { 61 | if rType == r.Type { 62 | ret = append(ret, r) 63 | } 64 | } 65 | return ret 66 | } 67 | 68 | func (p *Pod) String() string { 69 | if p == nil { 70 | return "" 71 | } 72 | return render.AsCode(p) 73 | } 74 | 75 | type Route struct { 76 | // Dst means the destination address 77 | Dst string `json:"dst,omitempty"` 78 | } 79 | 80 | type PodNetwork struct { 81 | Name string `json:"name,omitempty"` 82 | Namespace string `json:"namespace,omitempty"` 83 | IfName string `json:"ifName,omitempty"` 84 | PodIP pbrpc.IPSet `json:"podIP"` 85 | ExtraIPs []IPSet `json:"extraIPs,omitempty"` 86 | Mac string `json:"mac,omitempty"` 87 | Gateway pbrpc.IPSet `json:"gateway,omitempty"` 88 | Cidr *pbrpc.IPSet `json:"cidr,omitempty"` 89 | Route *Route `json:"route,omitempty"` 90 | ENIId string `json:"eniId,omitempty"` 91 | TrunkENIId string `json:"trunkENIId,omitempty"` 92 | VlanID uint32 `json:"vlanID,omitempty"` 93 | } 94 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for investing your time in contributing to project! 4 | 5 | Read our [Code of Coduct](./CODE_OF_CONDUCT.md) to keep our community approachable and respectable. 6 | 7 | This guide details how to use issues and pull requests to improve project. 8 | 9 | ## General Guidelines 10 | 11 | ### Pull Requests 12 | 13 | Make sure to keep Pull Requests small and functional to make them easier to review, understand, and look up in commit history. This repository uses "Squash and Commit" to keep our history clean and make it easier to revert changes based on PR. 14 | 15 | Adding the appropriate documentation, unit tests and e2e tests as part of a feature is the responsibility of the feature owner, whether it is done in the same Pull Request or not. 16 | 17 | Pull Requests should follow the "subject: message" format, where the subject describes what part of the code is being modified. 18 | 19 | Refer to the template for more information on what goes into a PR description. 20 | 21 | ### Design Docs 22 | 23 | A contributor proposes a design with a PR on the repository to allow for revisions and discussions. If a design needs to be discussed before formulating a document for it, make use of Google doc and GitHub issue to involve the community on the discussion. 24 | 25 | ### GitHub Issues 26 | 27 | GitHub Issues are used to file bugs, work items, and feature requests with actionable items/issues (Please refer to the "Reporting Bugs/Feature Requests" section below for more information). 28 | 29 | ### Reporting Bugs/Feature Requests 30 | 31 | We welcome you to use the GitHub issue tracker to report bugs or suggest features that have actionable items/issues (as opposed to introducing a feature request on GitHub Discussions). 32 | 33 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 34 | 35 | - A reproducible test case or series of steps 36 | - The version of the code being used 37 | - Any modifications you've made relevant to the bug 38 | - Anything unusual about your environment or deployment 39 | 40 | ## Contributing via Pull Requests 41 | 42 | ### Find interesting issue 43 | 44 | TODO 45 | ### Solve an issue 46 | 47 | TODO 48 | 49 | ### Open a Pull request. 50 | 51 | When you're done making the changes, open a pull request and fill PR template so we can better review your PR. The template helps reviewers understand your changes and the purpose of your pull request. 52 | 53 | Don't forget to link PR to issue if you are solving one. 54 | 55 | If you run into any merge issues, checkout this [git tutorial](https://lab.github.com/githubtraining/managing-merge-conflicts) to help you resolve merge conflicts and other issues. 56 | 57 | 58 | ## Finding contributions to work on 59 | 60 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' and 'good first issue' issues are a great place to start. -------------------------------------------------------------------------------- /patch/cilium/001-fib-host-gateway.patch: -------------------------------------------------------------------------------- 1 | --- 2 | bpf/lib/nodeport.h | 27 +++++++++++++++++++++++++++ 3 | pkg/datapath/linux/config/config.go | 9 +++++++++ 4 | 2 files changed, 36 insertions(+) 5 | 6 | diff --git a/bpf/lib/nodeport.h b/bpf/lib/nodeport.h 7 | index 2159fec71..cf774266f 100644 8 | --- a/bpf/lib/nodeport.h 9 | +++ b/bpf/lib/nodeport.h 10 | @@ -1566,6 +1566,15 @@ int tail_nodeport_ipv4_dsr(struct __ctx_buff *ctx) 11 | 12 | ret = fib_lookup(ctx, &fib_params.l, sizeof(fib_params), 13 | BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT); 14 | + 15 | +#ifdef HOST_GATEWAY_IPV4 16 | + if(ret != 0) { 17 | + fib_params.l.ipv4_dst = HOST_GATEWAY_IPV4; 18 | + ret = fib_lookup(ctx, &fib_params.l, sizeof(fib_params), 19 | + BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT); 20 | + } 21 | +#endif/* HOST_GATEWAY_IPV4 */ 22 | + 23 | if (ret != 0) { 24 | ret = DROP_NO_FIB; 25 | goto drop_err; 26 | @@ -1709,6 +1718,15 @@ int tail_nodeport_nat_ipv4(struct __ctx_buff *ctx) 27 | 28 | ret = fib_lookup(ctx, &fib_params.l, sizeof(fib_params), 29 | BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT); 30 | + 31 | +#ifdef HOST_GATEWAY_IPV4 32 | + if(ret != 0) { 33 | + fib_params.l.ipv4_dst = HOST_GATEWAY_IPV4; 34 | + ret = fib_lookup(ctx, &fib_params.l, sizeof(fib_params), 35 | + BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT); 36 | + } 37 | +#endif/* HOST_GATEWAY_IPV4 */ 38 | + 39 | if (ret != 0) { 40 | ret = DROP_NO_FIB; 41 | goto drop_err; 42 | @@ -2009,6 +2027,15 @@ static __always_inline int rev_nodeport_lb4(struct __ctx_buff *ctx, int *ifindex 43 | ret = fib_lookup(ctx, &fib_params, sizeof(fib_params), 44 | BPF_FIB_LOOKUP_DIRECT | 45 | BPF_FIB_LOOKUP_OUTPUT); 46 | + 47 | +#ifdef HOST_GATEWAY_IPV4 48 | + if(ret != 0) { 49 | + fib_params.ipv4_dst = HOST_GATEWAY_IPV4; 50 | + ret = fib_lookup(ctx, &fib_params, sizeof(fib_params), 51 | + BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT); 52 | + } 53 | +#endif/* HOST_GATEWAY_IPV4 */ 54 | + 55 | if (ret != 0) 56 | return DROP_NO_FIB; 57 | 58 | diff --git a/pkg/datapath/linux/config/config.go b/pkg/datapath/linux/config/config.go 59 | index 3c0c2d1a4..53d3a9eb5 100644 60 | --- a/pkg/datapath/linux/config/config.go 61 | +++ b/pkg/datapath/linux/config/config.go 62 | @@ -280,6 +280,16 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC 63 | if option.Config.EnableHealthDatapath { 64 | cDefinesMap["LB4_HEALTH_MAP"] = lbmap.HealthProbe4MapName 65 | } 66 | + // add hostGatewayIPv4 67 | + defaultRoutes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: nil}, netlink.RT_FILTER_DST) 68 | + if err != nil { 69 | + log.Errorf("Get host gateway ip failed: %s", err.Error()) 70 | + } else if len(defaultRoutes) == 0 { 71 | + log.Errorf("Get host gateway ip failed: empty") 72 | + } else { 73 | + log.Infof("Set HOST_GATEWAY_IPV4 to %s", defaultRoutes[0].Gw.String()) 74 | + fmt.Fprint(fw, defineIPv4("HOST_GATEWAY_IPV4", defaultRoutes[0].Gw)) 75 | + } 76 | } 77 | if option.Config.EnableIPv6 { 78 | cDefinesMap["NODEPORT_NEIGH6"] = neighborsmap.Map6Name 79 | -- 80 | 2.30.1 (Apple Git-130) 81 | -------------------------------------------------------------------------------- /pkg/cni/device/generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Cello Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | package device 17 | 18 | import ( 19 | "fmt" 20 | 21 | "github.com/vishvananda/netlink" 22 | 23 | "github.com/volcengine/cello/pkg/cni/utils" 24 | "github.com/volcengine/cello/pkg/utils/iproute" 25 | ) 26 | 27 | // Conf indicates network devices configures. 28 | type Conf struct { 29 | IfName string 30 | MTU int 31 | 32 | Addresses []*netlink.Addr 33 | Routes []*netlink.Route 34 | Rules []*netlink.Rule 35 | Neighs []*netlink.Neigh 36 | SysCtl [][]string 37 | } 38 | 39 | // Setup interfaces for container netns. 40 | func Setup(link netlink.Link, conf *Conf) error { 41 | var err error 42 | if conf.IfName != "" && link.Attrs().Name != conf.IfName { 43 | err = netlink.LinkSetName(link, conf.IfName) 44 | if err != nil { 45 | return fmt.Errorf("link set name failed: %s", err.Error()) 46 | } 47 | link, err = netlink.LinkByName(conf.IfName) 48 | if err != nil { 49 | return fmt.Errorf("could not find interface %d inside netns after name changed", link.Attrs().Index) 50 | } 51 | } 52 | 53 | if link.Attrs().OperState != netlink.OperUp { 54 | err = netlink.LinkSetUp(link) 55 | if err != nil { 56 | return fmt.Errorf("failed to bring link %s up: %w", link.Attrs().Name, err) 57 | } 58 | } 59 | 60 | if conf.MTU > 0 && link.Attrs().MTU != conf.MTU { 61 | err = netlink.LinkSetMTU(link, conf.MTU) 62 | if err != nil { 63 | return fmt.Errorf("link %d set mtu failed: %s", link.Attrs().Index, err.Error()) 64 | } 65 | } 66 | 67 | for _, v := range conf.SysCtl { 68 | if len(v) != 2 { 69 | return fmt.Errorf("sysctl config err") 70 | } 71 | err = utils.EnsureNetConfSet(link, v[0], v[1]) 72 | if err != nil { 73 | return err 74 | } 75 | } 76 | 77 | for _, addr := range conf.Addresses { 78 | err = netlink.AddrReplace(link, addr) 79 | if err != nil { 80 | return fmt.Errorf("add address %s to link %s failed: %w", addr.String(), link.Attrs().Name, err) 81 | } 82 | } 83 | 84 | for _, neigh := range conf.Neighs { 85 | err = iproute.EnsureNeigh(neigh) 86 | if err != nil { 87 | return fmt.Errorf("ensure neigh failed: %w", err) 88 | } 89 | } 90 | 91 | for _, route := range conf.Routes { 92 | err = iproute.EnsureRoute(route) 93 | if err != nil { 94 | return fmt.Errorf("ensure route failed: %w", err) 95 | } 96 | } 97 | 98 | for _, rule := range conf.Rules { 99 | err = iproute.EnsureIPRule(rule) 100 | if err != nil { 101 | return fmt.Errorf("ensure rule failed: %w", err) 102 | } 103 | } 104 | 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /ci/playbook/roles/control-plane/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | - name: Create kubeadm configuration file from template 2 | template: 3 | src: templates/kubeadm.conf.j2 4 | dest: /tmp/kubeadm.conf 5 | 6 | - name: Compare kubeadmin configuration with existing one 7 | command: diff /tmp/kubeadm.conf ~/kubeadm.conf 8 | failed_when: False 9 | register: kubeadm_conf_diff 10 | changed_when: kubeadm_conf_diff.rc != 0 11 | 12 | - name: Check cluster status 13 | # specify kubeconfig since we are logged in as root 14 | command: kubectl --kubeconfig ~/.kube/config cluster-info 15 | register: cluster_status 16 | failed_when: False 17 | changed_when: False 18 | 19 | - debug: 20 | msg: 21 | - "Cluster status is {{ cluster_status.rc }}" 22 | 23 | # Reset cluster if an issue is detected or if the config has changed 24 | - name: Reset cluster 25 | command: kubeadm reset -f 26 | when: kubeadm_conf_diff.changed or (cluster_status.rc != 0) 27 | failed_when: False 28 | 29 | - name: Initialize the Kubernetes cluster using kubeadm 30 | command: kubeadm init --config /tmp/kubeadm.conf 31 | when: kubeadm_conf_diff.changed or (cluster_status.rc != 0) 32 | register: kubeadm_init 33 | 34 | - name: Setup kubeconfig for test user 35 | block: 36 | - name: Create ~/.kube directory 37 | file: 38 | path: ~/.kube 39 | state: directory 40 | - name: Copy kubeconfig file to ~/.kube 41 | copy: 42 | src: /etc/kubernetes/admin.conf 43 | remote_src: true 44 | dest: ~/.kube/config 45 | mode: '0600' 46 | 47 | # We currently copy the kube config and join command to the host so that it can 48 | # be copied to all the worker nodes when provisioning them. An alternative is to 49 | # create a new host with add_host and use hostvars. See 50 | # https://stackoverflow.com/a/47811099. 51 | 52 | - name: Generate join command and write to host 53 | block: 54 | - name: Generate join command 55 | command: kubeadm token create --print-join-command 56 | register: join_command 57 | - name: Write join command to temporary file 58 | copy: 59 | content: "{{ join_command.stdout_lines[0] }}" 60 | dest: /tmp/join-command 61 | - name: Copy join command file to host 62 | fetch: 63 | src: /tmp/join-command 64 | dest: kube/join-command 65 | flat: yes 66 | when: kubeadm_init.changed 67 | 68 | # We copy the config file last, as we use it to determine whether the cluster 69 | # was setup correctly and with the appropriate configguration in future runs. 70 | - name: Copy kubeadm configuration to user's home 71 | copy: 72 | src: /tmp/kubeadm.conf 73 | remote_src: true 74 | dest: ~/kubeadm.conf 75 | 76 | - name: Copy kubeconfig to host 77 | fetch: 78 | src: ~/.kube/config 79 | dest: kube/config 80 | flat: yes 81 | 82 | - name: Copy kubeconfig to host for accessing 83 | fetch: 84 | src: ~/.kube/config 85 | dest: kube/config-public 86 | flat: yes 87 | 88 | # TODO: Replace file content of local host 89 | - name: Update local kubeconfig 90 | lineinfile: 91 | path: kube/config-public 92 | search_string: "server: " 93 | line: " server: https://{{ k8s_api_server_ip }}:6443" 94 | register: containerd_config 95 | connection: local 96 | --------------------------------------------------------------------------------