├── olm ├── clickhouse │ └── README.md ├── picture │ ├── 1.jpg │ ├── ns.png │ ├── allcsv.png │ ├── deploy.png │ ├── install.png │ ├── packagemanifests.png │ └── targetnamespace.png ├── redis-operators │ ├── picture │ │ ├── csv.png │ │ ├── set.png │ │ ├── catsrc.png │ │ ├── ip-yaml.png │ │ ├── get-cluster.png │ │ ├── cluster-nodes.png │ │ ├── redis-operators.png │ │ ├── get-redis-cluster.png │ │ ├── create-redis-cluster.png │ │ ├── create-redis-cluster1.png │ │ └── redis-operator-yaml.png │ ├── yml │ │ └── create-redis-cluster.yaml │ └── README.md ├── mongodb-operators │ ├── picture │ │ ├── csv.png │ │ ├── create.png │ │ ├── local-pv.png │ │ ├── conn_mongo.png │ │ ├── validation.png │ │ ├── mongo-secret.png │ │ ├── storage-class.png │ │ ├── mongodb-operator.png │ │ ├── create-mongo-cluster.png │ │ ├── create-mongo-cluster1.png │ │ └── create-mongodb-cluster.png │ ├── yml │ │ └── create-mongodb-cluster.yml │ └── README.md ├── postgres-Operators │ ├── picture │ │ ├── po.png │ │ ├── pv.png │ │ ├── sc.png │ │ ├── catsrc.png │ │ ├── ip-yaml.png │ │ ├── operator.png │ │ ├── validation.png │ │ ├── my-postgres.png │ │ ├── validation2.png │ │ ├── postgres-Operator.png │ │ ├── create-postgres-cluster.png │ │ └── create-postgres-cluster1.png │ ├── yml │ │ ├── Postgres-cluster.yml │ │ └── create-postgres-cluster.yaml │ └── README.md ├── rabbitmq-operators │ ├── picture │ │ ├── csv.png │ │ ├── rabbitmq.png │ │ ├── validation.png │ │ ├── create-rabbitmq-cluster.png │ │ └── rabbitmq-cluster-operator.png │ ├── yml │ │ └── create-rabbitmq-cluster.yaml │ └── README.md ├── README.md └── yml │ └── olm.yaml ├── k8s架构图.jpeg ├── apiserver ├── img │ └── apiserver.png └── README.md ├── informer ├── picture │ ├── informer.png │ └── no-informer.png └── README.md ├── kubelet ├── picture │ ├── cmd-kubelet.png │ └── pkg-kubelet.png ├── kubelet-create.md ├── README.md └── kubelet-run.md ├── .gitignore ├── README.md ├── operator └── README.md ├── helm └── README.md └── controller-manager └── README.md /olm/clickhouse/README.md: -------------------------------------------------------------------------------- 1 | # Clickhouse-Operators -------------------------------------------------------------------------------- /k8s架构图.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/k8s架构图.jpeg -------------------------------------------------------------------------------- /olm/picture/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/picture/1.jpg -------------------------------------------------------------------------------- /olm/picture/ns.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/picture/ns.png -------------------------------------------------------------------------------- /olm/picture/allcsv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/picture/allcsv.png -------------------------------------------------------------------------------- /olm/picture/deploy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/picture/deploy.png -------------------------------------------------------------------------------- /olm/picture/install.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/picture/install.png -------------------------------------------------------------------------------- /apiserver/img/apiserver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/apiserver/img/apiserver.png -------------------------------------------------------------------------------- /informer/picture/informer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/informer/picture/informer.png -------------------------------------------------------------------------------- /informer/picture/no-informer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/informer/picture/no-informer.png -------------------------------------------------------------------------------- /kubelet/picture/cmd-kubelet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/kubelet/picture/cmd-kubelet.png -------------------------------------------------------------------------------- /kubelet/picture/pkg-kubelet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/kubelet/picture/pkg-kubelet.png -------------------------------------------------------------------------------- /olm/picture/packagemanifests.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/picture/packagemanifests.png -------------------------------------------------------------------------------- /olm/picture/targetnamespace.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/picture/targetnamespace.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/csv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/csv.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/set.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/set.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/csv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/csv.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/po.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/po.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/pv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/pv.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/sc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/sc.png -------------------------------------------------------------------------------- /olm/rabbitmq-operators/picture/csv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/rabbitmq-operators/picture/csv.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/catsrc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/catsrc.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/create.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/ip-yaml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/ip-yaml.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/local-pv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/local-pv.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/catsrc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/catsrc.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/ip-yaml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/ip-yaml.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/operator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/operator.png -------------------------------------------------------------------------------- /olm/rabbitmq-operators/picture/rabbitmq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/rabbitmq-operators/picture/rabbitmq.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/get-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/get-cluster.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/conn_mongo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/conn_mongo.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/validation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/validation.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/validation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/validation.png -------------------------------------------------------------------------------- /olm/rabbitmq-operators/picture/validation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/rabbitmq-operators/picture/validation.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/cluster-nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/cluster-nodes.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/mongo-secret.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/mongo-secret.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/storage-class.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/storage-class.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/my-postgres.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/my-postgres.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/validation2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/validation2.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/redis-operators.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/redis-operators.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/mongodb-operator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/mongodb-operator.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/get-redis-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/get-redis-cluster.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/postgres-Operator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/postgres-Operator.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/create-redis-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/create-redis-cluster.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/create-redis-cluster1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/create-redis-cluster1.png -------------------------------------------------------------------------------- /olm/redis-operators/picture/redis-operator-yaml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/redis-operators/picture/redis-operator-yaml.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/create-mongo-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/create-mongo-cluster.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/create-mongo-cluster1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/create-mongo-cluster1.png -------------------------------------------------------------------------------- /olm/mongodb-operators/picture/create-mongodb-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/mongodb-operators/picture/create-mongodb-cluster.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/create-postgres-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/create-postgres-cluster.png -------------------------------------------------------------------------------- /olm/rabbitmq-operators/picture/create-rabbitmq-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/rabbitmq-operators/picture/create-rabbitmq-cluster.png -------------------------------------------------------------------------------- /olm/postgres-Operators/picture/create-postgres-cluster1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/postgres-Operators/picture/create-postgres-cluster1.png -------------------------------------------------------------------------------- /olm/rabbitmq-operators/picture/rabbitmq-cluster-operator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/HEAD/olm/rabbitmq-operators/picture/rabbitmq-cluster-operator.png -------------------------------------------------------------------------------- /kubelet/kubelet-create.md: -------------------------------------------------------------------------------- 1 | # kubelet--创建pod篇 2 | 3 | 本篇主要介绍 `kubelet` 创建 `pod` 的流程 4 | 5 | `kubelet` 的工作核心就是围绕着不同的生产者生产出来的不同有关 `pod` 的消息来调用相应的消费者，完成不同的行为(创建和删除 `pod`等), 6 | 通过不同的时间启动这个控制循环运行(SyncLoop) 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /olm/rabbitmq-operators/yml/create-rabbitmq-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: rabbitmqcluster 5 | spec: 6 | persistence: 7 | # storageClassName 自动创建 pv 8 | storageClassName: managed-nfs-storage 9 | storage: 1Gi 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | bin 8 | testbin/* 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Kubernetes Generated files - skip generated files, except for vendored files 17 | 18 | !vendor/**/zz_generated.* 19 | 20 | # editor and IDE paraphernalia 21 | .idea 22 | .vscode 23 | *.swp 24 | *.swo 25 | 26 | # output 27 | dist 28 | 29 | vendor -------------------------------------------------------------------------------- /kubelet/README.md: -------------------------------------------------------------------------------- 1 | # Kubelet 2 | 3 | ## **概要**: 4 | 5 | kubelet 是运行在每个节点上的主要的“节点代理”，每个节点都会启动 kubelet进程，用来处理 Master 节点下发到本节点的任务， 6 | 按照 PodSpec 描述来管理Pod 和其中的容器（PodSpec 是用来描述一个 pod 的 YAML 或者 JSON 对象）。 7 | kubelet 通过各种机制（主要通过 apiserver ）获取一组 PodSpec 并保证在这些 PodSpec 中描述的容器健康运行。 8 | 9 | ## 源码: 10 | - `cmd/kubelet/kubectl.go` // cmd/kubelet/kubectl.go 为 main 函数入口，主要负责参数加载，运行框架等前期准备工作 11 | 12 | - `pkg/kubelet/kubectl.go` // pkg/kubelet/kubectl.go 主要负责功能上的逻辑代码 13 | 14 | ### 目录结构: 15 | ![img](picture/cmd-kubelet.png) 16 | ![img](picture/pkg-kubelet.png) 17 | 18 | 19 | ### kubelet--启动流程 20 | - [kubelet--启动流程](kubelet-run.md) 21 | 22 | ### kubelet--创建 pod 流程 23 | 24 | - [kubelet--创建pod流程](kubelet-create.md) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-learning 2 | 3 | ## Operator Lifecycle Manager ( OLM ) 4 | 5 | - [OLM](olm/README.md) 6 | 7 | ## Kubernetes Helm 8 | 9 | - [Helm](helm/README.md) 10 | 11 | ## kubernetes Apiserver 12 | 13 | - [Apiserver](apiserver/README.md) 14 | 15 | ## kubernetes Kubelet 16 | 17 | - [Kubelet](kubelet/README.md) 18 | 19 | ## kubernet Controller-manager 20 | 21 | - [controller-manager](controller-manager/README.md) 22 | 23 | ## kubernet Informer 24 | 25 | - [Informer](informer/README.md) 26 | 27 | ## kubernet Informer 28 | 29 | - [Operator](operator/README.md) 30 | 31 | ![image](https://github.com/chenghongxi/kubernetes-learning/blob/09e1997fbf03c949f1c2a0bb2808ca5ac0b3b689/k8s%E6%9E%B6%E6%9E%84%E5%9B%BE.jpeg) 32 | 33 | -------------------------------------------------------------------------------- /operator/README.md: -------------------------------------------------------------------------------- 1 | # Operator learning 2 | 3 | ## Rely On 4 | ```shell 5 | 1. git 6 | 2. go 1.19+ ( 需要设置 GOPROXY ) 7 | 3. kubectl 8 | ``` 9 | 10 | ## Install operator-sdk 11 | ```shell 12 | mkdir -p $GOPATH/src/github.com/operator-framework 13 | cd $GOPATH/src/github.com/operator-framework 14 | git clone https://github.com/operator-framework/operator-sdk 15 | cd operator-sdk 16 | git checkout master 17 | make install 18 | ``` 19 | 20 | ## Use Operator-SDK Create Mysql-operator 21 | ```shell 22 | mkdir -p $GOPATH/src/github.com/example-inc/ 23 | cd $GOPATH/src/github.com/example-inc/ 24 | operator-sdk init --domain example.com --repo github.com/chengxiaobai/mysql-operator 25 | operator-sdk create api --group mysql --version v1beta1 --kind mysqlcluster 26 | ``` 27 | -------------------------------------------------------------------------------- /olm/mongodb-operators/yml/create-mongodb-cluster.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | type: Opaque 5 | metadata: 6 | name: mongodb-secret 7 | data: 8 | username: YWRtaW4= 9 | password: MTIzNDU2 10 | --- 11 | apiVersion: opstreelabs.in/v1alpha1 12 | kind: MongoDBCluster 13 | metadata: 14 | name: mongodb 15 | spec: 16 | clusterSize: 3 17 | kubernetesConfig: 18 | image: 'quay.io/opstree/mongo:v5.0.6' 19 | imagePullPolicy: IfNotPresent 20 | securityContext: 21 | fsGroup: 1001 22 | storage: 23 | accessModes: 24 | - ReadWriteOnce 25 | storageSize: 1Gi 26 | storageClass: managed-nfs-storage 27 | mongoDBSecurity: 28 | mongoDBAdminUser: admin 29 | secretRef: 30 | name: mongodb-secret 31 | key: password 32 | 33 | -------------------------------------------------------------------------------- /olm/redis-operators/yml/create-redis-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: redis.redis.opstreelabs.in/v1beta1 2 | kind: RedisCluster 3 | metadata: 4 | name: redis-cluster 5 | spec: 6 | clusterSize: 3 7 | clusterVersion: v7 8 | securityContext: 9 | runAsUser: 1000 10 | fsGroup: 1000 11 | persistenceEnabled: true 12 | kubernetesConfig: 13 | image: 'quay.io/opstree/redis:v7.0.5' 14 | imagePullPolicy: IfNotPresent 15 | redisExporter: 16 | enabled: true 17 | image: 'quay.io/opstree/redis-exporter:v1.44.0' 18 | imagePullPolicy: IfNotPresent 19 | storage: 20 | # volumeClaimTemplate 自动创建 pvc 21 | volumeClaimTemplate: 22 | spec: 23 | accessModes: 24 | - ReadWriteOnce 25 | # storageClassName 自动创建 pv 26 | storageClassName: managed-nfs-storage 27 | resources: 28 | requests: 29 | storage: 1Gi 30 | 31 | -------------------------------------------------------------------------------- /olm/postgres-Operators/yml/Postgres-cluster.yml: -------------------------------------------------------------------------------- 1 | apiVersion: postgres-operator.crunchydata.com/v1beta1 2 | kind: PostgresCluster 3 | metadata: 4 | name: hippo 5 | spec: 6 | image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.5-1 7 | postgresVersion: 14 8 | instances: 9 | - name: instance1 10 | replicas: 1 11 | dataVolumeClaimSpec: 12 | accessModes: 13 | - "ReadWriteOnce" 14 | storageClassName: "local-storage" 15 | resources: 16 | requests: 17 | storage: 1Gi 18 | backups: 19 | pgbackrest: 20 | image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.40-1 21 | repos: 22 | - name: repo1 23 | volume: 24 | volumeClaimSpec: 25 | accessModes: 26 | - "ReadWriteOnce" 27 | storageClassName: "local" 28 | resources: 29 | requests: 30 | storage: 1Gi 31 | 32 | -------------------------------------------------------------------------------- /olm/postgres-Operators/yml/create-postgres-cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: postgres-operator.crunchydata.com/v1beta1 2 | kind: PostgresCluster 3 | metadata: 4 | name: hippo 5 | spec: 6 | image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.5-1 7 | postgresVersion: 14 8 | instances: 9 | - name: instance1 10 | replicas: 1 11 | dataVolumeClaimSpec: 12 | accessModes: 13 | - "ReadWriteOnce" 14 | storageClassName: "managed-nfs-storage" 15 | resources: 16 | requests: 17 | storage: 1Gi 18 | backups: 19 | pgbackrest: 20 | image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.40-1 21 | repos: 22 | - name: repo1 23 | volume: 24 | volumeClaimSpec: 25 | accessModes: 26 | - "ReadWriteOnce" 27 | storageClassName: "managed-nfs-storage" 28 | resources: 29 | requests: 30 | storage: 1Gi -------------------------------------------------------------------------------- /helm/README.md: -------------------------------------------------------------------------------- 1 | # Helm 2 | 3 | ## Helm是什么？ 4 | Helm 是一个用于 Kubernetes 应用的包管理工具，主要用来管理 Charts。 5 | 有点类似于 Ubuntu 中的 APT 或 CentOS 中的 YUM。 6 | 7 | ## 组件: 8 | - `Chart：Chart 是一个 Helm 的软件包，采用 tar 格式。包含了运行一个K8S应用程序所需的镜像、依赖关系和资源定义等，也就是包含了一个应用所需资源对象的YAML文件。` 9 | - `Repository（仓库）：是 Helm 的软件仓库，Repository 本质上是一个 Web 服务器,该服务器保存了一系列的 Chart 软件包供用户下载，并且提供了该 Repository 的 Chart 包的清单文件便于查询。` 10 | - `Config（配置数据）：部署时设置到 Chart 中的配置数据。` 11 | - `Release：使用 helm install 命令在 k8s 集群中部署的 Chart 称为 Release 。是基于 Chart 和Config 部署到 K8S 集群中运行的一个实例。一个Chart可以被部署多次，每次的 Release 都不相同。` 12 | 13 | ## Install: 14 | ```shell 15 | wget https://mirrors.huaweicloud.com/helm/v3.3.1/helm-v3.3.1-linux-amd64.tar.gz 16 | 17 | tar -zxvf helm-v3.3.1-linux-amd64.tar.gz 18 | 19 | cp linux-amd64/helm /usr/local/bin/ 20 | 21 | [root@VM-32-17-centos bin]# /usr/local/bin/helm version 22 | version.BuildInfo{Version:"v3.3.1", GitCommit:"249e5215cde0c3fa72e27eb7a30e8d55c9696144", GitTreeState:"clean", GoVersion:"go1.14.7"} 23 | ``` 24 | 25 | #### 常用命令: 26 | ```shell 27 | # 查看配置仓库 28 | helm repo list 29 | # 查看 stable 仓库中的所有软件包 30 | helm search repo stable 31 | 32 | ``` 33 | 34 | 35 | -------------------------------------------------------------------------------- /olm/rabbitmq-operators/README.md: -------------------------------------------------------------------------------- 1 | # Rabbitmq-Operators 2 | 3 | 部署和管理RabbitMQ集群 , 为 RabbitMQ 集群的生命周期（创建、升级、正常关闭）设计的自定义控制器和自定义资源定义 ( CRD ) 4 | 5 | ## Documentation 6 | https://www.rabbitmq.com/kubernetes/operator/operator-overview.html 7 | 8 | ## GitHub 9 | https://github.com/rabbitmq/cluster-operator 10 | 11 | 12 | ## Rely on 13 | - `Kubernetes 1.19.0 cluster` 14 | - 集群已安装 `OLM` 组件。安装手册参考 [OLM安装](../README.md) 15 | 16 | ## Install: 17 | 18 | 19 | ```shell 20 | 1. kubectl apply -f https://operatorhub.io/install/rabbitmq-cluster-operator.yaml 21 | ``` 22 | ![img](picture/rabbitmq-cluster-operator.png) 23 | 24 | #### `安装过程解析` 25 | - 原理同 `redis-operator` 26 | 27 | - [redis-operator](https://github.com/chenghongxi/kubernetes-learning/blob/master/olm/redis-operators/README.md#%E5%AE%89%E8%A3%85%E8%BF%87%E7%A8%8B%E8%A7%A3%E6%9E%90) 28 | 29 | 30 | 31 | 32 | 33 | ```shell 34 | 2. kubectl get csv -n operators 35 | ``` 36 | ![img](picture/csv.png) 37 | 38 | ```shell 39 | 3. kubectl create -f https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/rabbitmq-operators/yml/create-rabbitmq-cluster.yaml 40 | ``` 41 | ![img](picture/create-rabbitmq-cluster.png) 42 | 43 | [create-rabbitmq-cluster.yaml](https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/rabbitmq-operators/yml/create-rabbitmq-cluster.yaml) 44 | 45 | ## Validation 46 | ```shell 47 | 1. kubectl get po,sc,pv,pvc,secret 48 | ``` 49 | ![img](picture/validation.png) 50 | ## UnInstall 51 | - `删除步骤 3 中的资源` 52 | ```shell 53 | kubectl delete -f https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/rabbitmq-operators/yml/create-rabbitmq-cluster.yaml 54 | ``` 55 | - `删除此 Operator` 56 | ```shell 57 | 1. kubectl delete subscription -n operators 58 | 2. kubectl delete clusterserviceversion -n operators 59 | ``` -------------------------------------------------------------------------------- /olm/mongodb-operators/README.md: -------------------------------------------------------------------------------- 1 | # Mongodb-Operators 2 | 3 | 基于 golang 的操作员，用于创建和管理 MongoDB standalone or cluster 4 | 5 | MongoDB Operator 用于在 Kubernetes 和 Openshift 集群上创建、更新和管理 MongoDB standalone, replicated, and arbiter replicated。 6 | 7 | 8 | ## Documentation 9 | https://ot-mongodb-operator.netlify.app/ 10 | 11 | ## Rely on 12 | - `Kubernetes 1.17.0 cluster` 13 | - 集群已安装 `OLM` 组件。安装手册参考 [OLM安装](../README.md) 14 | 15 | ## Install: 16 | ```text 17 | 1. kubectl apply -f https://operatorhub.io/install/mongodb-operator.yaml 18 | ``` 19 | ![img](picture/mongodb-operator.png) 20 | 21 | #### `安装过程解析` 22 | - 原理同 `redis-operator` 23 | 24 | - [redis-operator](https://github.com/chenghongxi/kubernetes-learning/blob/master/olm/redis-operators/README.md#%E5%AE%89%E8%A3%85%E8%BF%87%E7%A8%8B%E8%A7%A3%E6%9E%90) 25 | 26 | 27 | ```shell 28 | 2. kubectl get csv -n operators 29 | ``` 30 | ![img](picture/csv.png) 31 | 32 | 33 | ```shell 34 | 3. kubectl apply -f https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/mongodb-operators/yml/create-mongodb-cluster.yml 35 | ``` 36 | ![img](picture/create-mongo-cluster.png) 37 | ![img](picture/create-mongo-cluster1.png) 38 | 39 | 40 | [create-mongodb-cluster](https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/mongodb-operators/yml/create-mongodb-cluster.yml) 41 | 42 | 43 | ## Validation 44 | ```shell 45 | 1. kubectl get po,sc,pv,pvc 46 | ``` 47 | ![img](picture/validation.png) 48 | ```shell 49 | 1. kubectl exec -it mongodb-cluster-0 -- /bin/bash 50 | 2. mongo 51 | 3. use pixiuDB 52 | ``` 53 | ![img](picture/conn_mongo.png) 54 | 55 | 56 | ## UnInstall 57 | - `删除步骤 3 中的资源` 58 | ```shell 59 | kubectl delete -f https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/mongodb-operators/yml/create-mongodb-cluster.yml 60 | ``` 61 | - `删除此 Operator` 62 | ```shell 63 | 1. kubectl delete subscription -n operators 64 | 2. kubectl delete clusterserviceversion -n operators 65 | ``` 66 | -------------------------------------------------------------------------------- /olm/redis-operators/README.md: -------------------------------------------------------------------------------- 1 | # Redis-operators 2 | 基于 `Golang` 的 Redos-Operator,可以在 `Kubernetes` 集群上创建 `Redis standalone/cluster` 模式。 3 | 4 | `Redis-Exporter` 提供监控功能。 5 | 6 | ## Supported redis versions: 7 | - `=> v6` 8 | 9 | ## Documentation 10 | https://ot-container-kit.github.io/redis-operator/ 11 | 12 | ## GitHub 13 | https://github.com/ot-container-kit/redis-operator 14 | 15 | ## Supported Features 16 | - `Redis 集群和独立模式` 17 | - `Redis 集群故障转移和恢复` 18 | - `内置 Redis 监控` 19 | - `Redis 密码设置` 20 | - `TLS` 21 | - `支持 Ipv4 和 Ipv6` 22 | - `支持配置领导者和追随者` 23 | - `grafana dashboard` 24 | 25 | ## Rely on 26 | - `Kubernetes 1.18.0 cluster` 27 | - 集群已安装 `OLM` 组件。安装手册参考 [OLM安装](../README.md) 28 | 29 | ## Install: 30 | 31 | 32 | ```shell 33 | 1. kubectl apply -f https://operatorhub.io/install/redis-operator.yaml 34 | ``` 35 | ![img](picture/redis-operators.png) 36 | 37 | #### `安装过程解析` 38 | - 创建 `Subscription` 对象 39 | ![img](picture/redis-operator-yaml.png) 40 | 41 | - `Subscription` 对象从 `Catalog source` 获取可用版本 42 | - `metadaata.generation` 初始值为1: 随着 `spec` 内容的改变而自增 43 | - `spec.image` 此镜像用于, 实例化一个 `registry-serve` 44 | - `spec.publisher` 发行商 45 | - `spec.sourceType` 源类型 46 | ![img](picture/catsrc.png) 47 | - `Subscription`会创建一个 `Install` 对象为 `Operator` 安装资源,根据批准策略来批准安装计划: 48 | - `如果 Subscription 的 spec.approval 字段被设置为 Automatic，则会自动批准安装计划。` 49 | - `如果 Subscription 的 spec.approval 字段被设置为 Manual，则安装计划必须由集群管理员或具有适当权限的用户手动批准。` 50 | - `批准安装计划后，OLM 会创建指定的资源，并在 Subscription 指定的命名空间中安装 Operator。` 51 | ![img](picture/ip-yaml.png) 52 | 53 | 54 | 55 | 56 | ```shell 57 | 2. kubectl get csv -n operators 58 | ``` 59 | ![img](picture/csv.png) 60 | 61 | ```shell 62 | 3. kubectl apply -f https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/redis-operators/yml/create-redis-cluster.yaml 63 | ``` 64 | ![img](picture/create-redis-cluster.png) 65 | ![img](picture/create-redis-cluster1.png) 66 | 67 | 68 | [create-redis-cluster.yaml](https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/redis-operators/yml/create-redis-cluster.yaml) 69 | 70 | 71 | ## Validation 72 | ```shell 73 | 1. kubectl get sc,pv,po 74 | ``` 75 | ![img](picture/get-cluster.png) 76 | ```text 77 | 2. kubectl exec -it redis-cluster-leader-0 -- redis-cli -c cluster nodes 78 | ``` 79 | ![img](picture/cluster-nodes.png) 80 | ![img](picture/set.png) 81 | 82 | 83 | ## UnInstall 84 | - `删除步骤 3 中的资源` 85 | ```shell 86 | kubectl delete -f https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/redis-operators/yml/create-redis-cluster.yaml 87 | ``` 88 | - `删除此 Operator` 89 | ```shell 90 | 1. kubectl delete subscription -n operators 91 | 2. kubectl delete clusterserviceversion -n operators 92 | ``` 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /olm/postgres-Operators/README.md: -------------------------------------------------------------------------------- 1 | # Postgres-Operator 2 | 3 | ## Documentation 4 | https://access.crunchydata.com/documentation/postgres-operator/v5/ 5 | 6 | ## Github 7 | https://github.com/CrunchyData/postgres-operator 8 | 9 | ## Supported Features 10 | - `集群配置: ` 11 | - `高可用性` 12 | - `灾难恢复` 13 | - `监控` 14 | - `克隆` 15 | - `TLS` 16 | - `连接池` 17 | - `亲和力和容忍度` 18 | - `完全的可定制性` 19 | 20 | ## Rely On 21 | - `Kubernetes Cluster Version > 1.19.0` 22 | - 集群已安装 `OLM` 组件。安装手册参考 [OLM安装](../README.md) 23 | 24 | ## Install: 25 | 26 | ```shell 27 | 1. kubectl apply -f https://operatorhub.io/install/postgresql.yaml 28 | ``` 29 | ![img](picture/postgres-Operator.png) 30 | 31 | #### `安装过程解析` 32 | - 创建 `Subscription` 对象 33 | 34 | ![img](picture/my-postgres.png) 35 | - `Subscription` 对象从 `Catalogsource` 获取可用版本,通过引用存储在容器 registry 中的 index image 36 | - `metadaata.generation` 初始值为1: 随着 `spec` 内容的改变而自增 37 | - `spec` 字段指明了如何构造 `pod` , 以及如何与 Operator Registry gRPC API 进行通信 38 | - `spec.image` 目录的索引镜像 39 | 40 | - `spec.sourceType` 源类型,当`spec.image` 为索引镜像时，OLM 拉取镜像并运行 pod，为兼容的 API 服务。 41 | - `spec.updateStrategy.registryPoll` 在指定的时间段内自动检查新版本以保持最新。 42 | - `status.connectionState.lastObservedState` READY: 成功建立连接。 43 | 44 | ![img](picture/catsrc.png) 45 | - `Subscription`会创建一个 `InstallPlan` 对象为 `Operator` 安装资源,根据批准策略来批准安装计划: 46 | - `如果 Subscription 的 spec.approval 字段被设置为 Automatic，则会自动批准安装计划。` 47 | - `如果 Subscription 的 spec.approval 字段被设置为 Manual，则安装计划必须由集群管理员或具有适当权限的用户手动批准。` 48 | - `批准安装计划后，OLM 会创建指定的资源，并在 Subscription 指定的命名空间中安装 Operator。` 49 | ![img](picture/ip-yaml.png) 50 | 51 | 52 | ```shell 53 | 2. kubectl get csv -n operators 54 | ``` 55 | ![img](picture/operator.png) 56 | 57 | ```shell 58 | 3. kubectl apply -f https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/postgres-Operators/yml/create-postgres-cluster.yaml 59 | ``` 60 | ![img](picture/create-postgres-cluster.png) 61 | ![img](picture/create-postgres-cluster1.png) 62 | 63 | 64 | [create-postgres-cluster.yaml](https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/postgres-Operators/yml/create-postgres-cluster.yaml) 65 | 66 | 67 | 68 | 69 | ## Validation 70 | ```shell 71 | 1. kubectl get sts,sc,pv,secret 72 | ``` 73 | `hippo-instance1-gfb5-0` `pod` 中会创建几个容器，分别为: `database`, `replication-cert-copy`, `pgbackrest`, `pgbackrest-config`,`postgres-startup (init)`, `nss-wrapper-init (init)` 74 | 75 | `hippo-repo-host-0` `pod` 中会创建几个容器，分别为:`pgbackrest`, `pgbackrest-config`, `pgbackrest-log-dir (init)`, `nss-wrapper-init (init)` 76 | 77 | ![img](picture/validation.png) 78 | ```text 79 | 1. kubectl exec -it hippo-instance1-gfb5-0 -- /bin/bash 80 | 2. psql 81 | ``` 82 | 83 | ![img](picture/validation2.png) 84 | 85 | ## UnInstall 86 | - `删除步骤 3 中的资源` 87 | ```shell 88 | kubectl delete -f https://raw.githubusercontent.com/chenghongxi/kubernetes-learning/master/olm/postgres-Operators/yml/create-postgres-cluster.yaml 89 | ``` 90 | - `删除此 Operator` 91 | ```shell 92 | 1. kubectl delete subscription -n operators 93 | 2. kubectl delete clusterserviceversion -n operators 94 | ``` 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /olm/README.md: -------------------------------------------------------------------------------- 1 | # Operator Lifecycle Manager ( OLM ) 2 | 3 | - `Website`: https://access.redhat.com/documentation/zh-cn/openshift_container_platform/4.11/html/operators/_operator-lifecycle-manager-olm 4 | - `GitHub`: https://github.com/operator-framework/operator-lifecycle-manager.git 5 | - `Operatorhub`: https://operatorhub.io/ 6 | 7 | ## Rely on 8 | - `Kubernetes v1.17+ cluster` 9 | 10 | ## Environment version 11 | 1. 机器配置：4核，8G 12 | 2. Kubernetes Version: v1.21 单节点 13 | 3. 集群安装方法: https://github.com/caoyingjunz/kubez-ansible 14 | ## OLM 介绍: 15 | ```text 16 | OLM( Operator Lifecycle Manager ) 作为 Operator Framework 的一部分，可以帮助用户进行 Operator 的自动安装， 17 | 升级及其生命周期的管理。同时 OLM 自身也是以 Operator 的形式进行安装部署，可以说它的工作方式是以 Operators 来管理 Operators， 18 | 而它面向 Operator 提供了声明式 ( declarative ) 的自动化管理能力也完全符合 Kubernetes 交互的设计理念。 19 | ``` 20 | 21 | ## OLM 组件原理: 22 | ```text 23 | OLM 由两个 Operator 构成：OLM Operator 和 Catalog Operator 24 | 25 | OLM Operator: 工作基于 ClusterServiceVersion，一旦 CSV 中声明的依赖资源在目标集群中注册成功，OLM Operator 将负责安装这些资源对应的应用实例。 26 | 27 | Catalog Operator: 主要负责解析 CSV 中声明的依赖资源定义，同时通过监听 catalog 中安装包对应 channels 的版本定义完成 CSV 对应的版本更新。 28 | ``` 29 | 30 | ## OLM 基础模型： 31 | ```text 32 | OLM Operator, Catalog Operator 其分别管理如下 CRD : 33 | ``` 34 | 35 | ![img.png](picture/1.jpg) 36 | 37 | ## 使用场景: 38 | OLM 可以帮助用户，安装，更新，和管理所有Operator ( 如: redis-operator , mongodb-operator ) 的生命周期. 39 | 40 | 41 | 42 | ### OLM Install: 43 | - `Scripted` 44 | ```shell 45 | curl -L https://github.com/operator-framework/operator-lifecycle-manager/releases/download/v0.19.1/install.sh -o install.sh 46 | chmod +x install.sh 47 | ./install.sh v0.19.1 48 | ``` 49 | 50 | ![img](picture/install.png) 51 | 52 | ##### `安装过程解析` 53 | 54 | [crds.yaml](yml/crds.yaml) 55 | 56 | [olm.yml](yml/olm.yaml) 57 | 58 | - `Operator-sdk` 59 | 60 | 请查看官方快速开始文档 61 | ```text 62 | https://olm.operatorframework.io/docs/getting-started/ 63 | ``` 64 | 65 | #### Validation: 66 | ```text 67 | kubectl get ns 68 | ``` 69 | ![img](picture/ns.png) 70 | ```text 71 | kubectl get deploy,svc,secret -n olm 72 | ``` 73 | ![img](picture/deploy.png) 74 | 75 | #### 查看`OLM` 可安装的 `Operators` 76 | ```shell 77 | kubectl get packagemanifests 78 | ``` 79 | ![img](picture/packagemanifests.png) 80 | 81 | ##### 重要提示: 82 | - `ClusterServiceVersion` 在集群的所有命名空间都可用, 如下图: 83 | ![img](picture/allcsv.png) 84 | - 这个行为由 `OperatorGroup` 控制 85 | ![img](picture/targetnamespace.png) 86 | 87 | ## OLM UnInstall 88 | ```shell 89 | 1. kubectl delete -f olm.yaml 90 | 2. kubectl delete -f crds.yaml 91 | ``` 92 | 93 | ## Operators 94 | 95 | ### Rely on 96 | 97 | - `Kubernetes v1.17+ cluster` 98 | 99 | 100 | ### Operators 是什么: 101 | ```text 102 | 简单来说，Operators 就是一组自定义控制器的集合以及由这些控制器管理着的一系列自定义资源，我们将不在关注于 pod，configmap 等基本模型元素， 103 | 而是将他们聚合为一个应用或服务，Operator 通过控制器的协调循环来使自定义应用达到我们期望的状态，我们只需要关注该应用的期望状态，通过自定义控制器协调循环逻辑， 104 | 来达到安装，更新，扩展，备份，恢复 7*24 小时不间断的应用或服务的生命周期管理。 105 | ``` 106 | 107 | ### Operator 解决了那些问题: 108 | ```text 109 | 解决数据库等复杂应用容器化问题的模式, 对复杂的有状态应用，如数据库、缓存和监控系统等，进行创建、配置和管理。 110 | ``` 111 | ### Operators 对象: 112 | ``` 113 | CRD:自定义资源的定义，Kubernetes API 服务器会为你所指定的每一个 CRD 版本生成 RESTful 的资源路径。一个 CRD 其实就是定义自己应用业务模型的地方， 114 | 可以根据业务的需求，完全定制自己所需要的资源对象，如 RedisCluster, PrometheusServer 等这些都是可以被 Kubernetes 直接操作和管理的自定义的资源对象。 115 | 116 | CR：自定义资源，即 CRD 的一个具体实例，是具体的可管理的 Kubernetes 资源对象，可以对其进行正常的生命周期管理，如创建，删除，修改，查询等， 117 | 同时 CR 对象一般还会包含运行时的状态，如当前的 CR 的真实的状态是什么，用于观察和判断，CR 对象的真正所处于的状态。 118 | 119 | Controller：其实就是控制器真正的用武之地了，它会循环处理工作队列中的动作，按照逻辑协调应用当前状态至期望状态。如观察一个 CR 对象被创建了之后， 120 | 会根据实现的逻辑来处理 CR，让 CR 对象的状态以及CR对象所负责的业务逻辑慢慢的往最终期望的状态上靠近，最终达到期望的效果，举例来说如果定义了一个 Redis 的 Operator， 121 | 那在创建 RedisCluster 的时候，就会一直协调和观察 Redis 真正的集群是不是创建好了，以及每个节点的状态和可用性是不是健康的， 122 | 一旦发现不符合期望的状态就会继续协调，就一直保持基于事件的机制，不断检查和协调，去保证期望的状态。 123 | ``` 124 | 125 | ### Application-Example 126 | 127 | [Redis-Operators](redis-operators/README.md) 128 | 129 | [MongoDB-Operators](mongodb-operators/README.md) 130 | 131 | [Rabbitmq-Operators](rabbitmq-operators/README.md) 132 | 133 | [Postgres-Operators](postgres-Operators/README.md) 134 | 135 | `...` 136 | 137 | -------------------------------------------------------------------------------- /informer/README.md: -------------------------------------------------------------------------------- 1 | # Informer 2 | 3 | ## 什么是 Informer 4 | ```text 5 | informer 是 client-go 中的核心工具包，informer 其实就是一个带有本地缓存和索引机制的，可以注册 EventHandler 的 client 本地缓存被称为 Store, 6 | 索引被称为 Index 。使用 informer 的目的是为了减轻 apiserver 数据交互的压力而抽象出来的一个 cache 层，客户端对 apiserver 数据的 读取和监听操作都通过本地 informer进行。 7 | ``` 8 | 9 | ## 为什么需要 Informer 机制 10 | ```text 11 | Kubernetes 各个组件都是通过 REST API 跟 API Server 交互通信的，而如果每次每一个组件都直接跟 API Server 交互去读取/写入到后端的etcd的话， 12 | 会对 API Server 以及etcd造成非常大的负担。 而 Informer 机制是为了保证各个组件之间通信的实时性、可靠性，并且减缓对 API Server和etcd 的负担。 13 | ``` 14 | 15 | ## 对比: 16 | 17 | ### 代码对比: 18 | - `1. 使用了 informer:` 19 | 20 | 代码链接: https://github.com/chenghongxi/go-learning/blob/main/gin/gin-informer.go 21 | ```text 22 | package main 23 | 24 | import ( 25 | "context" 26 | "log" 27 | "net/http" 28 | "path/filepath" 29 | 30 | "github.com/gin-gonic/gin" 31 | "k8s.io/apimachinery/pkg/labels" 32 | "k8s.io/apimachinery/pkg/runtime/schema" 33 | "k8s.io/client-go/informers" 34 | "k8s.io/client-go/kubernetes" 35 | "k8s.io/client-go/tools/clientcmd" 36 | "k8s.io/client-go/util/homedir" 37 | ) 38 | 39 | // 使用 informer 减轻 kubernetes apiserver 并发压力 40 | // use two clients 41 | // 一个 client 负责查资源 42 | // 一个 client 负责资源的创建，更新，删除 43 | func main() { 44 | // 从家目录读取config 45 | config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(homedir.HomeDir(), ".kube", "config")) 46 | if err != nil { 47 | panic(err) 48 | } 49 | // 使用config，返回clientset 50 | clientset, err := kubernetes.NewForConfig(config) 51 | if err != nil { 52 | panic(err) 53 | } 54 | ctx, cancel := context.WithCancel(context.Background()) 55 | defer cancel() 56 | 57 | // get all namespace sharedInformers 58 | sharedInformers := informers.NewSharedInformerFactory(clientset, 0) 59 | 60 | // get Resource 61 | gvrs := []schema.GroupVersionResource{ 62 | {Group: "apps", Version: "v1", Resource: "deployments"}, 63 | {Group: "", Version: "v1", Resource: "pods"}, 64 | } 65 | 66 | for _, gvr := range gvrs { 67 | if _, err = sharedInformers.ForResource(gvr); err != nil { 68 | panic(err) 69 | } 70 | } 71 | 72 | // Start all informers 73 | sharedInformers.Start(ctx.Done()) 74 | // Wait for all caches to sync 75 | sharedInformers.WaitForCacheSync(ctx.Done()) 76 | 77 | log.Printf("all informers has been started") 78 | 79 | // 构造 pod Lister，用于 gin 的查询 80 | podLister := sharedInformers.Core().V1().Pods().Lister() 81 | // 启动 gin router 82 | // 仅作演示， 无封装， 无异常处理 83 | // 启动之后，curl 127.0.0.1:8080/pods 84 | r := gin.Default() 85 | r.GET("/pods", func(c *gin.Context) { 86 | pods, err := podLister.List(labels.Everything()) 87 | if err != nil { 88 | panic(err) 89 | c.JSON(http.StatusBadRequest, gin.H{"message": err, "code": 400}) 90 | return 91 | } 92 | c.JSON(http.StatusOK, gin.H{"message": "pong", "code": 200, "result": pods}) 93 | }) 94 | _ = r.Run(":8080") 95 | } 96 | ``` 97 | - `2. 不使用 informer` 98 | 99 | 代码链接: https://github.com/chenghongxi/go-learning/blob/main/gin/gin-no-informer.go 100 | ```text 101 | package main 102 | 103 | import ( 104 | "context" 105 | "github.com/gin-gonic/gin" 106 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 107 | "k8s.io/client-go/kubernetes" 108 | "k8s.io/client-go/tools/clientcmd" 109 | "k8s.io/client-go/util/homedir" 110 | "net/http" 111 | "path/filepath" 112 | ) 113 | 114 | func main() { 115 | config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(homedir.HomeDir(), ".kube", "config")) 116 | if err != nil { 117 | panic(err) 118 | } 119 | clientset, err := kubernetes.NewForConfig(config) 120 | if err != nil { 121 | panic(err) 122 | } 123 | 124 | r := gin.Default() 125 | r.GET("/pods", func(c *gin.Context) { 126 | list, err := clientset.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{}) 127 | if err != nil { 128 | panic(err) 129 | c.JSON(http.StatusBadRequest, gin.H{"message": err, "code": 400}) 130 | } 131 | c.JSON(http.StatusOK, gin.H{"message": 200, "result": list}) 132 | }) 133 | _ = r.Run(":8080") 134 | } 135 | ``` 136 | 137 | ### 测试结果对比: 138 | 139 | 使用了 apipoox 进行访问测试: 140 | 141 | 测试路径: http://127.0.0.1:8080/pods 142 | 143 | 循环次数: 800 144 | 145 | 线程数: 3 146 | 147 | 148 | - `1. 使用了 informer:` 149 | 150 | ![img](picture/informer.png) 151 | - `2. 未使用 informer:` 152 | 153 | ![img](picture/no-informer.png) 154 | 155 | ### 结论: 156 | ```text 157 | 使用与不使用 informer ，平均接口请求耗时相差巨大 (这个相差量会随着数据量的变化越来越大), 我们在开发相关容器平台时需要加入 informer 机制，减轻 158 | apiserver 与 etcd 压力。 159 | ``` 160 | 161 | 162 | 163 | 164 | 165 | -------------------------------------------------------------------------------- /apiserver/README.md: -------------------------------------------------------------------------------- 1 | # apiserver源码分析(启动流程) 2 | 3 | ## 介绍: 4 | 5 | github: https://github.com/kubernetes/kubernetes 6 | 7 | Apiserver是Kubernetes的一个重要组件，在所有组件中是唯一一个对接Etcd,Kubernetes中各种资源的增删改查服务都是使用http服务的形式 8 | 9 | ## 组件: 10 | 11 | Apiserver组件中有3个server,apiserver依靠这3个server来处理不同的请求内容,具体如下: 12 | 13 | - `KubeAPIServer: 主要负责处理k8s内置资源的请求，此外还会包括通用处理，认证、鉴权等` 14 | - `APIExtensionServer: 主要负责处理CustomResourceDefination（CRD）方面的请求` 15 | - `AggregratorServer: 主要负责aggregrate方面的处理，它充当一个代理服务器，将请求转发到聚合进来的k8s service中` 16 | 17 | apiserver组件与其他组件(如:kubelet等)使用了corbra命令行框架处理启动命令,它从命令行的RunE回调函数-->Run函数，开始执行启动流程。 18 | 19 | RunE回调函数: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go#L133 20 | 21 | ```go 22 | func NewAPIServerCommand() *cobra.Command { 23 | s := options.NewServerRunOptions() 24 | cmd := &cobra.Command{ 25 | Use: "kube-apiserver", 26 | Long: `The Kubernetes API server validates and configures data 27 | for the api objects which include pods, services, replicationcontrollers, and 28 | others. The API Server services REST operations and provides the frontend to the 29 | cluster's shared state through which all other components interact.`, 30 | 31 | // stop printing usage when the command errors 32 | SilenceUsage: true, 33 | PersistentPreRunE: func (*cobra.Command, []string) error { 34 | // silence client-go warnings. 35 | // kube-apiserver loopback clients should not log self-issued warnings. 36 | rest.SetDefaultWarningHandler(rest.NoWarnings{}) 37 | return nil 38 | }, 39 | RunE: func (cmd *cobra.Command, args []string) error { 40 | verflag.PrintAndExitIfRequested() 41 | fs := cmd.Flags() 42 | 43 | // Activate logging as soon as possible, after that 44 | // show flags with the final logging configuration. 45 | if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil { 46 | return err 47 | } 48 | cliflag.PrintFlags(fs) 49 | 50 | // set default options 51 | completedOptions, err := Complete(s) 52 | if err != nil { 53 | return err 54 | } 55 | 56 | // validate options 57 | if errs := completedOptions.Validate(); len(errs) != 0 { 58 | return utilerrors.NewAggregate(errs) 59 | } 60 | // add feature enablement metrics 61 | utilfeature.DefaultMutableFeatureGate.AddMetrics() 62 | return Run(completedOptions, genericapiserver.SetupSignalHandler()) 63 | }, 64 | Args: func (cmd *cobra.Command, args []string) error { 65 | for _, arg := range args { 66 | if len(arg) > 0 { 67 | return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) 68 | } 69 | } 70 | return nil 71 | }, 72 | } 73 | 74 | fs := cmd.Flags() 75 | namedFlagSets := s.Flags() 76 | verflag.AddFlags(namedFlagSets.FlagSet("global")) 77 | globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) 78 | options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) 79 | for _, f := range namedFlagSets.FlagSets { 80 | fs.AddFlagSet(f) 81 | } 82 | 83 | cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) 84 | cliflag.SetUsageAndHelpFunc(cmd, namedFlagSets, cols) 85 | 86 | return cmd 87 | } 88 | ``` 89 | 90 | Run函数: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go#L161 91 | 92 | 93 | ```go 94 | func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) error { 95 | // To help debugging, immediately log version 96 | klog.Infof("Version: %+v", version.Get()) 97 | 98 | klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) 99 | 100 | // 注册了三个server,分别为: 101 | // 1. KubeAPIServer 102 | // 2. APIExtensionServer 103 | // 3. AggregratorServer 104 | // CreateServerChain函数为具体的注册 105 | server, err := CreateServerChain(completeOptions) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | // 为每个server注册健康检查,就绪,存活探针 111 | prepared, err := server.PrepareRun() 112 | if err != nil { 113 | return err 114 | } 115 | 116 | return prepared.Run(stopCh) 117 | } 118 | ``` 119 | 三个server的创建流程: server的创建都有相对应的config，每个config都需要依赖GenericConfig,函数结束后返回一个APIAggregator(api聚合器) 120 | - `KubeAPIServer` 121 | - `APIExtensionServer` 122 | - `AggregratorServer` 123 | 124 | CreateServerChain函数: https://github.com/kubernetes/kubernetes/blob/f87231003aeb43bf884cf9dc10b1247b8ae5cbb8/cmd/kube-apiserver/app/server.go#L181 125 | 126 | ```go 127 | func CreateServerChain(completedOptions completedServerRunOptions) (*aggregatorapiserver.APIAggregator, error) { 128 | kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions) 129 | if err != nil { 130 | return nil, err 131 | } 132 | 133 | // If additional API servers are added, they should be gated. 134 | apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount, 135 | serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIServerConfig.ExtraConfig.ProxyTransport, kubeAPIServerConfig.GenericConfig.EgressSelector, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig, kubeAPIServerConfig.GenericConfig.TracerProvider)) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | notFoundHandler := notfoundhandler.New(kubeAPIServerConfig.GenericConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey) 141 | // 创建APIExtensionsServer并注册路由 142 | apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler)) 143 | if err != nil { 144 | return nil, err 145 | } 146 | // 创建KubeAPIServer并注册路由 147 | kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer) 148 | if err != nil { 149 | return nil, err 150 | } 151 | 152 | // aggregator comes last in the chain 153 | aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, kubeAPIServerConfig.ExtraConfig.ProxyTransport, pluginInitializer) 154 | if err != nil { 155 | return nil, err 156 | } 157 | // 创建aggregatorServer并注册路由 158 | aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers) 159 | if err != nil { 160 | // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines 161 | return nil, err 162 | } 163 | 164 | return aggregatorServer, nil 165 | } 166 | ``` 167 | CreateServerChain函数解析: 168 | 169 | - `1. 通过 CreateKubeAPIServerConfig函数 返回 kubeAPIServerConfig` 170 | 171 | - `2. 每个server在创建时需要对应的config,函数结束时，返回APIAggregator` 172 | 173 | - `3. 完成server注册` 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | -------------------------------------------------------------------------------- /olm/yml/olm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: olm 6 | labels: 7 | pod-security.kubernetes.io/enforce: restricted 8 | pod-security.kubernetes.io/enforce-version: latest 9 | --- 10 | apiVersion: v1 11 | kind: Namespace 12 | metadata: 13 | name: operators 14 | labels: 15 | pod-security.kubernetes.io/enforce: baseline 16 | pod-security.kubernetes.io/enforce-version: latest 17 | --- 18 | kind: ServiceAccount 19 | apiVersion: v1 20 | metadata: 21 | name: olm-operator-serviceaccount 22 | namespace: olm 23 | --- 24 | apiVersion: rbac.authorization.k8s.io/v1 25 | kind: ClusterRole 26 | metadata: 27 | name: system:controller:operator-lifecycle-manager 28 | rules: 29 | - apiGroups: ["*"] 30 | resources: ["*"] 31 | verbs: ["*"] 32 | - nonResourceURLs: ["*"] 33 | verbs: ["*"] 34 | --- 35 | apiVersion: rbac.authorization.k8s.io/v1 36 | kind: ClusterRoleBinding 37 | metadata: 38 | name: olm-operator-binding-olm 39 | roleRef: 40 | apiGroup: rbac.authorization.k8s.io 41 | kind: ClusterRole 42 | name: system:controller:operator-lifecycle-manager 43 | subjects: 44 | - kind: ServiceAccount 45 | name: olm-operator-serviceaccount 46 | namespace: olm 47 | --- 48 | apiVersion: operators.coreos.com/v1 49 | kind: OLMConfig 50 | metadata: 51 | name: cluster 52 | --- 53 | apiVersion: apps/v1 54 | kind: Deployment 55 | metadata: 56 | name: olm-operator 57 | namespace: olm 58 | labels: 59 | app: olm-operator 60 | spec: 61 | strategy: 62 | type: RollingUpdate 63 | replicas: 1 64 | selector: 65 | matchLabels: 66 | app: olm-operator 67 | template: 68 | metadata: 69 | labels: 70 | app: olm-operator 71 | spec: 72 | securityContext: 73 | runAsNonRoot: true 74 | seccompProfile: 75 | type: RuntimeDefault 76 | serviceAccountName: olm-operator-serviceaccount 77 | containers: 78 | - name: olm-operator 79 | securityContext: 80 | allowPrivilegeEscalation: false 81 | capabilities: 82 | drop: [ "ALL" ] 83 | command: 84 | - /bin/olm 85 | args: 86 | - --namespace 87 | - $(OPERATOR_NAMESPACE) 88 | - --writeStatusName 89 | - "" 90 | image: quay.io/operator-framework/olm@sha256:2b4fee73c05069d9d2c537c7d3072241097914748abfb938b5b08c969b2f544b 91 | imagePullPolicy: IfNotPresent 92 | ports: 93 | - containerPort: 8080 94 | name: metrics 95 | livenessProbe: 96 | httpGet: 97 | path: /healthz 98 | port: 8080 99 | scheme: HTTP 100 | readinessProbe: 101 | httpGet: 102 | path: /healthz 103 | port: 8080 104 | scheme: HTTP 105 | terminationMessagePolicy: FallbackToLogsOnError 106 | env: 107 | - name: OPERATOR_NAMESPACE 108 | valueFrom: 109 | fieldRef: 110 | fieldPath: metadata.namespace 111 | - name: OPERATOR_NAME 112 | value: olm-operator 113 | resources: 114 | requests: 115 | cpu: 10m 116 | memory: 160Mi 117 | nodeSelector: 118 | kubernetes.io/os: linux 119 | --- 120 | apiVersion: apps/v1 121 | kind: Deployment 122 | metadata: 123 | name: catalog-operator 124 | namespace: olm 125 | labels: 126 | app: catalog-operator 127 | spec: 128 | strategy: 129 | type: RollingUpdate 130 | replicas: 1 131 | selector: 132 | matchLabels: 133 | app: catalog-operator 134 | template: 135 | metadata: 136 | labels: 137 | app: catalog-operator 138 | spec: 139 | securityContext: 140 | runAsNonRoot: true 141 | seccompProfile: 142 | type: RuntimeDefault 143 | serviceAccountName: olm-operator-serviceaccount 144 | containers: 145 | - name: catalog-operator 146 | securityContext: 147 | allowPrivilegeEscalation: false 148 | capabilities: 149 | drop: [ "ALL" ] 150 | command: 151 | - /bin/catalog 152 | args: 153 | - '--namespace' 154 | - olm 155 | - --configmapServerImage=quay.io/operator-framework/configmap-operator-registry:latest 156 | - --util-image 157 | - quay.io/operator-framework/olm@sha256:2b4fee73c05069d9d2c537c7d3072241097914748abfb938b5b08c969b2f544b 158 | - --set-workload-user-id=true 159 | image: quay.io/operator-framework/olm@sha256:2b4fee73c05069d9d2c537c7d3072241097914748abfb938b5b08c969b2f544b 160 | imagePullPolicy: IfNotPresent 161 | ports: 162 | - containerPort: 8080 163 | name: metrics 164 | livenessProbe: 165 | httpGet: 166 | path: /healthz 167 | port: 8080 168 | scheme: HTTP 169 | readinessProbe: 170 | httpGet: 171 | path: /healthz 172 | port: 8080 173 | scheme: HTTP 174 | terminationMessagePolicy: FallbackToLogsOnError 175 | resources: 176 | requests: 177 | cpu: 10m 178 | memory: 80Mi 179 | nodeSelector: 180 | kubernetes.io/os: linux 181 | --- 182 | kind: ClusterRole 183 | apiVersion: rbac.authorization.k8s.io/v1 184 | metadata: 185 | name: aggregate-olm-edit 186 | labels: 187 | rbac.authorization.k8s.io/aggregate-to-admin: "true" 188 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 189 | rules: 190 | - apiGroups: ["operators.coreos.com"] 191 | resources: ["subscriptions"] 192 | verbs: ["create", "update", "patch", "delete"] 193 | - apiGroups: ["operators.coreos.com"] 194 | resources: ["clusterserviceversions", "catalogsources", "installplans", "subscriptions"] 195 | verbs: ["delete"] 196 | --- 197 | kind: ClusterRole 198 | apiVersion: rbac.authorization.k8s.io/v1 199 | metadata: 200 | name: aggregate-olm-view 201 | labels: 202 | rbac.authorization.k8s.io/aggregate-to-admin: "true" 203 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 204 | rbac.authorization.k8s.io/aggregate-to-view: "true" 205 | rules: 206 | - apiGroups: ["operators.coreos.com"] 207 | resources: ["clusterserviceversions", "catalogsources", "installplans", "subscriptions", "operatorgroups"] 208 | verbs: ["get", "list", "watch"] 209 | - apiGroups: ["packages.operators.coreos.com"] 210 | resources: ["packagemanifests", "packagemanifests/icon"] 211 | verbs: ["get", "list", "watch"] 212 | --- 213 | apiVersion: operators.coreos.com/v1 214 | kind: OperatorGroup 215 | metadata: 216 | name: global-operators 217 | namespace: operators 218 | --- 219 | apiVersion: operators.coreos.com/v1 220 | kind: OperatorGroup 221 | metadata: 222 | name: olm-operators 223 | namespace: olm 224 | spec: 225 | targetNamespaces: 226 | - olm 227 | --- 228 | apiVersion: operators.coreos.com/v1alpha1 229 | kind: ClusterServiceVersion 230 | metadata: 231 | name: packageserver 232 | namespace: olm 233 | labels: 234 | olm.version: v0.22.0 235 | spec: 236 | displayName: Package Server 237 | description: Represents an Operator package that is available from a given CatalogSource which will resolve to a ClusterServiceVersion. 238 | minKubeVersion: 1.11.0 239 | keywords: ['packagemanifests', 'olm', 'packages'] 240 | maintainers: 241 | - name: Red Hat 242 | email: openshift-operators@redhat.com 243 | provider: 244 | name: Red Hat 245 | links: 246 | - name: Package Server 247 | url: https://github.com/operator-framework/operator-lifecycle-manager/tree/master/pkg/package-server 248 | installModes: 249 | - type: OwnNamespace 250 | supported: true 251 | - type: SingleNamespace 252 | supported: true 253 | - type: MultiNamespace 254 | supported: true 255 | - type: AllNamespaces 256 | supported: true 257 | install: 258 | strategy: deployment 259 | spec: 260 | clusterPermissions: 261 | - serviceAccountName: olm-operator-serviceaccount 262 | rules: 263 | - apiGroups: 264 | - authorization.k8s.io 265 | resources: 266 | - subjectaccessreviews 267 | verbs: 268 | - create 269 | - get 270 | - apiGroups: 271 | - "" 272 | resources: 273 | - configmaps 274 | verbs: 275 | - get 276 | - list 277 | - watch 278 | - apiGroups: 279 | - "operators.coreos.com" 280 | resources: 281 | - catalogsources 282 | verbs: 283 | - get 284 | - list 285 | - watch 286 | - apiGroups: 287 | - "packages.operators.coreos.com" 288 | resources: 289 | - packagemanifests 290 | verbs: 291 | - get 292 | - list 293 | deployments: 294 | - name: packageserver 295 | spec: 296 | strategy: 297 | type: RollingUpdate 298 | rollingUpdate: 299 | maxUnavailable: 1 300 | maxSurge: 1 301 | replicas: 2 302 | selector: 303 | matchLabels: 304 | app: packageserver 305 | template: 306 | metadata: 307 | labels: 308 | app: packageserver 309 | spec: 310 | securityContext: 311 | runAsNonRoot: true 312 | seccompProfile: 313 | type: RuntimeDefault 314 | serviceAccountName: olm-operator-serviceaccount 315 | nodeSelector: 316 | kubernetes.io/os: linux 317 | containers: 318 | - name: packageserver 319 | securityContext: 320 | allowPrivilegeEscalation: false 321 | capabilities: 322 | drop: [ "ALL" ] 323 | command: 324 | - /bin/package-server 325 | - -v=4 326 | - --secure-port 327 | - "5443" 328 | - --global-namespace 329 | - olm 330 | image: quay.io/operator-framework/olm@sha256:2b4fee73c05069d9d2c537c7d3072241097914748abfb938b5b08c969b2f544b 331 | imagePullPolicy: Always 332 | ports: 333 | - containerPort: 5443 334 | livenessProbe: 335 | httpGet: 336 | scheme: HTTPS 337 | path: /healthz 338 | port: 5443 339 | readinessProbe: 340 | httpGet: 341 | scheme: HTTPS 342 | path: /healthz 343 | port: 5443 344 | terminationMessagePolicy: FallbackToLogsOnError 345 | resources: 346 | requests: 347 | cpu: 10m 348 | memory: 50Mi 349 | volumeMounts: 350 | - name: tmpfs 351 | mountPath: /tmp 352 | volumes: 353 | - name: tmpfs 354 | emptyDir: {} 355 | maturity: alpha 356 | version: v0.22.0 357 | apiservicedefinitions: 358 | owned: 359 | - group: packages.operators.coreos.com 360 | version: v1 361 | kind: PackageManifest 362 | name: packagemanifests 363 | displayName: PackageManifest 364 | description: A PackageManifest is a resource generated from existing CatalogSources and their ConfigMaps 365 | deploymentName: packageserver 366 | containerPort: 5443 367 | --- 368 | apiVersion: operators.coreos.com/v1alpha1 369 | kind: CatalogSource 370 | metadata: 371 | name: operatorhubio-catalog 372 | namespace: olm 373 | spec: 374 | sourceType: grpc 375 | image: quay.io/operatorhubio/catalog:latest 376 | displayName: Community Operators 377 | publisher: OperatorHub.io 378 | updateStrategy: 379 | registryPoll: 380 | interval: 60m 381 | -------------------------------------------------------------------------------- /controller-manager/README.md: -------------------------------------------------------------------------------- 1 | # Controller-Manager 2 | 3 | ## 介绍: 4 | 5 | Github: https://github.com/kubernetes/kubernetes 6 | 7 | 在`Kubernetes`中 `Controller-Manager` 作为集群内部的管理控制中心, 负责集群内的`Node`,`pod`,`Endpoint`,`service` 8 | ,`namespace`等资源的管理,这些资源各有一个`controller`来负责。 9 | 10 | 每个`Controller`通过 `Api-server` 11 | 提供的接口监控整个集群的每个资源对象的当前状态，当发生变化时，会尝试将状态恢复到`期望状态`。 12 | 13 | Controller-Manager 由一系列的 Controller 组成: 14 | 15 | - `Certificate Controller` 16 | - `ClusterRoleAggregation Controller` 17 | - `Node Controller` 18 | - `CronJob Controller` 19 | - `Daemon Controller` 20 | - `Deployment Controller` 21 | - `StatefulSet Controller` 22 | - `Endpoint Controller` 23 | - `Endpointslice Controller` 24 | - `Garbage Collector` 25 | - `Namespace Controller` 26 | - `Job Controller` 27 | - `Pod AutoScaler` 28 | - `PodGC Controller` 29 | - `ReplicaSet Controller` 30 | - `Service Controller` 31 | - `ServiceAccount Controller` 32 | - `Volume Controller` 33 | - `Resource quota Controller` 34 | - `Disruption Controller` 35 | - ... 36 | 37 | ## 源码分析 ( 启动流程 ): 38 | Controller-Manager 组件与其他组件(如:kubelet,apiserver等)使用了corbra命令行框架处理启动命令,它从命令行的RunE回调函数-->Run函数，开始执行启动流程。 39 | RunE回调函数: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-controller-manager/app/controllermanager.go#L126 40 | ```go 41 | func NewControllerManagerCommand() *cobra.Command { 42 | s, err := options.NewKubeControllerManagerOptions() 43 | if err != nil { 44 | klog.Fatalf("unable to initialize command options: %v", err) 45 | } 46 | 47 | cmd := &cobra.Command{ 48 | Use: "kube-controller-manager", 49 | Long: `The Kubernetes controller manager is a daemon that embeds 50 | the core control loops shipped with Kubernetes. In applications of robotics and 51 | automation, a control loop is a non-terminating loop that regulates the state of 52 | the system. In Kubernetes, a controller is a control loop that watches the shared 53 | state of the cluster through the apiserver and makes changes attempting to move the 54 | current state towards the desired state. Examples of controllers that ship with 55 | Kubernetes today are the replication controller, endpoints controller, namespace 56 | controller, and serviceaccounts controller.`, 57 | PersistentPreRunE: func(*cobra.Command, []string) error { 58 | // silence client-go warnings. 59 | // kube-controller-manager generically watches APIs (including deprecated ones), 60 | // and CI ensures it works properly against matching kube-apiserver versions. 61 | restclient.SetDefaultWarningHandler(restclient.NoWarnings{}) 62 | return nil 63 | }, 64 | RunE: func(cmd *cobra.Command, args []string) error { 65 | verflag.PrintAndExitIfRequested() 66 | 67 | // Activate logging as soon as possible, after that 68 | // show flags with the final logging configuration. 69 | if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil { 70 | return err 71 | } 72 | cliflag.PrintFlags(cmd.Flags()) 73 | 74 | c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List()) 75 | if err != nil { 76 | return err 77 | } 78 | // 添加一个功能指标 79 | utilfeature.DefaultMutableFeatureGate.AddMetrics() 80 | return Run(c.Complete(), wait.NeverStop) 81 | }, 82 | Args: func(cmd *cobra.Command, args []string) error { 83 | for _, arg := range args { 84 | if len(arg) > 0 { 85 | return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) 86 | } 87 | } 88 | return nil 89 | }, 90 | } 91 | 92 | fs := cmd.Flags() 93 | namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List()) 94 | verflag.AddFlags(namedFlagSets.FlagSet("global")) 95 | globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) 96 | registerLegacyGlobalFlags(namedFlagSets) 97 | for _, f := range namedFlagSets.FlagSets { 98 | fs.AddFlagSet(f) 99 | } 100 | 101 | cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) 102 | cliflag.SetUsageAndHelpFunc(cmd, namedFlagSets, cols) 103 | 104 | return cmd 105 | } 106 | ``` 107 | Run函数: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-controller-manager/app/controllermanager.go#L180 108 | ```text 109 | func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { 110 | // To help debugging, immediately log version 111 | klog.Infof("Version: %+v", version.Get()) 112 | 113 | klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) 114 | 115 | // 开启事件处理管道 116 | c.EventBroadcaster.StartStructuredLogging(0) 117 | c.EventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: c.Client.CoreV1().Events("")}) 118 | defer c.EventBroadcaster.Shutdown() 119 | 120 | if cfgz, err := configz.New(ConfigzName); err == nil { 121 | cfgz.Set(c.ComponentConfig) 122 | } else { 123 | klog.Errorf("unable to register configz: %v", err) 124 | } 125 | 126 | // 设置 healthz 检查 127 | var checks []healthz.HealthChecker 128 | var electionChecker *leaderelection.HealthzAdaptor 129 | if c.ComponentConfig.Generic.LeaderElection.LeaderElect { 130 | electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20) 131 | checks = append(checks, electionChecker) 132 | } 133 | healthzHandler := controllerhealthz.NewMutableHealthzHandler(checks...) 134 | 135 | // Start the controller manager HTTP server 136 | // unsecuredMux is the handler for these controller *after* authn/authz filters have been applied 137 | var unsecuredMux *mux.PathRecorderMux 138 | if c.SecureServing != nil { 139 | unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, healthzHandler) 140 | if utilfeature.DefaultFeatureGate.Enabled(features.ComponentSLIs) { 141 | slis.SLIMetricsWithReset{}.Install(unsecuredMux) 142 | } 143 | handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) 144 | // TODO: handle stoppedCh and listenerStoppedCh returned by c.SecureServing.Serve 145 | if _, _, err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { 146 | return err 147 | } 148 | } 149 | 150 | clientBuilder, rootClientBuilder := createClientBuilders(c) 151 | 152 | saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController 153 | 154 | run := func(ctx context.Context, startSATokenController InitFunc, initializersFunc ControllerInitializersFunc) { 155 | controllerContext, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, ctx.Done()) 156 | if err != nil { 157 | klog.Fatalf("error building controller context: %v", err) 158 | } 159 | controllerInitializers := initializersFunc(controllerContext.LoopMode) 160 | if err := StartControllers(ctx, controllerContext, startSATokenController, controllerInitializers, unsecuredMux, healthzHandler); err != nil { 161 | klog.Fatalf("error starting controllers: %v", err) 162 | } 163 | 164 | controllerContext.InformerFactory.Start(stopCh) 165 | controllerContext.ObjectOrMetadataInformerFactory.Start(stopCh) 166 | close(controllerContext.InformersStarted) 167 | 168 | <-ctx.Done() 169 | } 170 | 171 | // 没有领导者选举，直接参选 172 | if !c.ComponentConfig.Generic.LeaderElection.LeaderElect { 173 | ctx, _ := wait.ContextForChannel(stopCh) 174 | run(ctx, saTokenControllerInitFunc, NewControllerInitializers) 175 | return nil 176 | } 177 | 178 | id, err := os.Hostname() 179 | if err != nil { 180 | return err 181 | } 182 | 183 | // add a uniquifier so that two processes on the same host don't accidentally both become active 184 | id = id + "_" + string(uuid.NewUUID()) 185 | 186 | // leaderMigrator will be non-nil if and only if Leader Migration is enabled. 187 | var leaderMigrator *leadermigration.LeaderMigrator = nil 188 | 189 | // startSATokenController will be original saTokenControllerInitFunc if leader migration is not enabled. 190 | startSATokenController := saTokenControllerInitFunc 191 | 192 | // If leader migration is enabled, create the LeaderMigrator and prepare for migration 193 | if leadermigration.Enabled(&c.ComponentConfig.Generic) { 194 | klog.Infof("starting leader migration") 195 | 196 | leaderMigrator = leadermigration.NewLeaderMigrator(&c.ComponentConfig.Generic.LeaderMigration, 197 | "kube-controller-manager") 198 | 199 | // Wrap saTokenControllerInitFunc to signal readiness for migration after starting 200 | // the controller. 201 | startSATokenController = func(ctx context.Context, controllerContext ControllerContext) (controller.Interface, bool, error) { 202 | defer close(leaderMigrator.MigrationReady) 203 | return saTokenControllerInitFunc(ctx, controllerContext) 204 | } 205 | } 206 | 207 | // 开启一个主协程锁 208 | go leaderElectAndRun(c, id, electionChecker, 209 | c.ComponentConfig.Generic.LeaderElection.ResourceLock, 210 | c.ComponentConfig.Generic.LeaderElection.ResourceName, 211 | leaderelection.LeaderCallbacks{ 212 | OnStartedLeading: func(ctx context.Context) { 213 | initializersFunc := NewControllerInitializers 214 | if leaderMigrator != nil { 215 | // If leader migration is enabled, we should start only non-migrated controllers 216 | // for the main lock. 217 | initializersFunc = createInitializersFunc(leaderMigrator.FilterFunc, leadermigration.ControllerNonMigrated) 218 | klog.Info("leader migration: starting main controllers.") 219 | } 220 | run(ctx, startSATokenController, initializersFunc) 221 | }, 222 | OnStoppedLeading: func() { 223 | klog.ErrorS(nil, "leaderelection lost") 224 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 225 | }, 226 | }) 227 | 228 | // If Leader Migration is enabled, proceed to attempt the migration lock. 229 | if leaderMigrator != nil { 230 | // Wait for Service Account Token Controller to start before acquiring the migration lock. 231 | // At this point, the main lock must have already been acquired, or the KCM process already exited. 232 | // We wait for the main lock before acquiring the migration lock to prevent the situation 233 | // where KCM instance A holds the main lock while KCM instance B holds the migration lock. 234 | <-leaderMigrator.MigrationReady 235 | 236 | // Start the migration lock. 237 | go leaderElectAndRun(c, id, electionChecker, 238 | c.ComponentConfig.Generic.LeaderMigration.ResourceLock, 239 | c.ComponentConfig.Generic.LeaderMigration.LeaderName, 240 | leaderelection.LeaderCallbacks{ 241 | OnStartedLeading: func(ctx context.Context) { 242 | klog.Info("leader migration: starting migrated controllers.") 243 | // DO NOT start saTokenController under migration lock 244 | run(ctx, nil, createInitializersFunc(leaderMigrator.FilterFunc, leadermigration.ControllerMigrated)) 245 | }, 246 | OnStoppedLeading: func() { 247 | klog.ErrorS(nil, "migration leaderelection lost") 248 | klog.FlushAndExit(klog.ExitFlushTimeout, 1) 249 | }, 250 | }) 251 | } 252 | 253 | <-stopCh 254 | return nil 255 | } 256 | ``` 257 | NewControllerInitializers函数: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-controller-manager/app/controllermanager.go#L423 258 | - `1. NewControllerInitializers 是与其 InitFunc 配对的命名控制器组的公共映射` 259 | - `2. 可以在 initFunc 中启动多个控制器组这允许结构化的下游组成和细分` 260 | - `3. 此函数完成了，下游controller的相关注册` 261 | ```text 262 | func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc { 263 | controllers := map[string]InitFunc{} 264 | 265 | // All of the controllers must have unique names, or else we will explode. 266 | register := func(name string, fn InitFunc) { 267 | if _, found := controllers[name]; found { 268 | panic(fmt.Sprintf("controller name %q was registered twice", name)) 269 | } 270 | controllers[name] = fn 271 | } 272 | 273 | register("endpoint", startEndpointController) 274 | register("endpointslice", startEndpointSliceController) 275 | register("endpointslicemirroring", startEndpointSliceMirroringController) 276 | register("replicationcontroller", startReplicationController) 277 | register("podgc", startPodGCController) 278 | register("resourcequota", startResourceQuotaController) 279 | register("namespace", startNamespaceController) 280 | register("serviceaccount", startServiceAccountController) 281 | register("garbagecollector", startGarbageCollectorController) 282 | register("daemonset", startDaemonSetController) 283 | register("job", startJobController) 284 | register("deployment", startDeploymentController) 285 | register("replicaset", startReplicaSetController) 286 | register("horizontalpodautoscaling", startHPAController) 287 | register("disruption", startDisruptionController) 288 | register("statefulset", startStatefulSetController) 289 | register("cronjob", startCronJobController) 290 | register("csrsigning", startCSRSigningController) 291 | register("csrapproving", startCSRApprovingController) 292 | register("csrcleaner", startCSRCleanerController) 293 | register("ttl", startTTLController) 294 | register("bootstrapsigner", startBootstrapSignerController) 295 | register("tokencleaner", startTokenCleanerController) 296 | register("nodeipam", startNodeIpamController) 297 | register("nodelifecycle", startNodeLifecycleController) 298 | if loopMode == IncludeCloudLoops { 299 | register("service", startServiceController) 300 | register("route", startRouteController) 301 | register("cloud-node-lifecycle", startCloudNodeLifecycleController) 302 | // TODO: volume controller into the IncludeCloudLoops only set. 303 | } 304 | register("persistentvolume-binder", startPersistentVolumeBinderController) 305 | register("attachdetach", startAttachDetachController) 306 | register("persistentvolume-expander", startVolumeExpandController) 307 | register("clusterrole-aggregation", startClusterRoleAggregrationController) 308 | register("pvc-protection", startPVCProtectionController) 309 | register("pv-protection", startPVProtectionController) 310 | register("ttl-after-finished", startTTLAfterFinishedController) 311 | register("root-ca-cert-publisher", startRootCACertPublisher) 312 | register("ephemeral-volume", startEphemeralVolumeController) 313 | if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) && 314 | utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) { 315 | register("storage-version-gc", startStorageVersionGCController) 316 | } 317 | if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DynamicResourceAllocation) { 318 | controllers["resource-claim-controller"] = startResourceClaimController 319 | } 320 | 321 | return controllers 322 | } 323 | 324 | // GetAvailableResources gets the map which contains all available resources of the apiserver 325 | // TODO: In general, any controller checking this needs to be dynamic so 326 | // users don't have to restart their controller manager if they change the apiserver. 327 | // Until we get there, the structure here needs to be exposed for the construction of a proper ControllerContext. 328 | func GetAvailableResources(clientBuilder clientbuilder.ControllerClientBuilder) (map[schema.GroupVersionResource]bool, error) { 329 | client := clientBuilder.ClientOrDie("controller-discovery") 330 | discoveryClient := client.Discovery() 331 | _, resourceMap, err := discoveryClient.ServerGroupsAndResources() 332 | if err != nil { 333 | utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err)) 334 | } 335 | if len(resourceMap) == 0 { 336 | return nil, fmt.Errorf("unable to get any supported resources from server") 337 | } 338 | 339 | allResources := map[schema.GroupVersionResource]bool{} 340 | for _, apiResourceList := range resourceMap { 341 | version, err := schema.ParseGroupVersion(apiResourceList.GroupVersion) 342 | if err != nil { 343 | return nil, err 344 | } 345 | for _, apiResource := range apiResourceList.APIResources { 346 | allResources[version.WithResource(apiResource.Name)] = true 347 | } 348 | } 349 | 350 | return allResources, nil 351 | } 352 | 353 | ``` 354 | StartControllers函数: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-controller-manager/app/controllermanager.go#L231 355 | - `完成 controller-manager注册并启动` 356 | ```text 357 | func StartControllers(ctx context.Context, controllerCtx ControllerContext, startSATokenController InitFunc, controllers map[string]InitFunc, 358 | unsecuredMux *mux.PathRecorderMux, healthzHandler *controllerhealthz.MutableHealthzHandler) error { 359 | // Always start the SA token controller first using a full-power client, since it needs to mint tokens for the rest 360 | // If this fails, just return here and fail since other controllers won't be able to get credentials. 361 | if startSATokenController != nil { 362 | if _, _, err := startSATokenController(ctx, controllerCtx); err != nil { 363 | return err 364 | } 365 | } 366 | 367 | // Initialize the cloud provider with a reference to the clientBuilder only after token controller 368 | // has started in case the cloud provider uses the client builder. 369 | if controllerCtx.Cloud != nil { 370 | controllerCtx.Cloud.Initialize(controllerCtx.ClientBuilder, ctx.Done()) 371 | } 372 | 373 | var controllerChecks []healthz.HealthChecker 374 | 375 | for controllerName, initFn := range controllers { 376 | if !controllerCtx.IsControllerEnabled(controllerName) { 377 | klog.Warningf("%q is disabled", controllerName) 378 | continue 379 | } 380 | 381 | time.Sleep(wait.Jitter(controllerCtx.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) 382 | 383 | klog.V(1).Infof("Starting %q", controllerName) 384 | ctrl, started, err := initFn(ctx, controllerCtx) 385 | if err != nil { 386 | klog.Errorf("Error starting %q", controllerName) 387 | return err 388 | } 389 | if !started { 390 | klog.Warningf("Skipping %q", controllerName) 391 | continue 392 | } 393 | check := controllerhealthz.NamedPingChecker(controllerName) 394 | if ctrl != nil { 395 | // check if the controller supports and requests a debugHandler 396 | // and it needs the unsecuredMux to mount the handler onto. 397 | if debuggable, ok := ctrl.(controller.Debuggable); ok && unsecuredMux != nil { 398 | if debugHandler := debuggable.DebuggingHandler(); debugHandler != nil { 399 | basePath := "/debug/controllers/" + controllerName 400 | unsecuredMux.UnlistedHandle(basePath, http.StripPrefix(basePath, debugHandler)) 401 | unsecuredMux.UnlistedHandlePrefix(basePath+"/", http.StripPrefix(basePath, debugHandler)) 402 | } 403 | } 404 | if healthCheckable, ok := ctrl.(controller.HealthCheckable); ok { 405 | if realCheck := healthCheckable.HealthChecker(); realCheck != nil { 406 | check = controllerhealthz.NamedHealthChecker(controllerName, realCheck) 407 | } 408 | } 409 | } 410 | controllerChecks = append(controllerChecks, check) 411 | 412 | klog.Infof("Started %q", controllerName) 413 | } 414 | 415 | healthzHandler.AddHealthChecker(controllerChecks...) 416 | 417 | return nil 418 | } 419 | ``` 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | -------------------------------------------------------------------------------- /kubelet/kubelet-run.md: -------------------------------------------------------------------------------- 1 | # Kubelet--启动篇 2 | 3 | 4 | ## cmd/kubelet/kubectl.go 源码分析: 5 | 6 | 1. kubelet 程序主入口, 使用了 corbra 命令行框架处理启动命令,它从命令行的`RunE`回调函数-->`Run`函数，开始执行启动流程。 7 | 8 | // cobra使用问题请移步: https://github.com/chenghongxi/go-learning/tree/main/cobra 9 | 10 | 11 | 代码: https://github.com/kubernetes/kubernetes/blob/master/cmd/kubelet/kubelet.go#L34 12 | ```text 13 | func main() { 14 | command := app.NewKubeletCommand() 15 | code := cli.Run(command) 16 | os.Exit(code) 17 | } 18 | ``` 19 | 20 | 2. `NewKubeletCommand` 函数主要作用，从命令行的`RunE`回调函数-->`Run`函数，开始启动流程流程,从下面的代码中可以看到,`NewKubeletCommand` 函数是做前期的工作准备， 21 | 比如：参数解析，校验，构建依赖，构建server,通过这些参数运行Kubelet,进入Run函数: 22 | 23 | 代码: https://github.com/kubernetes/kubernetes/blob/cf12a74b18b66efc577ec819c78a0c68f6d49225/cmd/kubelet/app/server.go#L124 24 | ```text 25 | func NewKubeletCommand() *cobra.Command { 26 | cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError) 27 | cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) 28 | kubeletFlags := options.NewKubeletFlags() 29 | 30 | kubeletConfig, err := options.NewKubeletConfiguration() 31 | // 错误 32 | if err != nil { 33 | klog.ErrorS(err, "Failed to create a new kubelet configuration") 34 | os.Exit(1) 35 | } 36 | 37 | cmd := &cobra.Command{ 38 | Use: componentKubelet, 39 | Long: `The kubelet is the primary "node agent" that runs on each 40 | node. It can register the node with the apiserver using one of: the hostname; a flag to 41 | override the hostname; or specific logic for a cloud provider. 42 | The kubelet works in terms of a PodSpec. A PodSpec is a YAML or JSON object 43 | that describes a pod. The kubelet takes a set of PodSpecs that are provided through 44 | various mechanisms (primarily through the apiserver) and ensures that the containers 45 | described in those PodSpecs are running and healthy. The kubelet doesn't manage 46 | containers which were not created by Kubernetes. 47 | Other than from an PodSpec from the apiserver, there are two ways that a container 48 | manifest can be provided to the Kubelet. 49 | File: Path passed as a flag on the command line. Files under this path will be monitored 50 | periodically for updates. The monitoring period is 20s by default and is configurable 51 | via a flag. 52 | HTTP endpoint: HTTP endpoint passed as a parameter on the command line. This endpoint 53 | is checked every 20 seconds (also configurable with a flag).`, 54 | DisableFlagParsing: true, 55 | SilenceUsage: true, 56 | RunE: func(cmd *cobra.Command, args []string) error { 57 | // initial flag parse, since we disable cobra's flag parsing 58 | if err := cleanFlagSet.Parse(args); err != nil { 59 | return fmt.Errorf("failed to parse kubelet flag: %w", err) 60 | } 61 | 62 | 63 | // 检查是否有命令未传入参数，若是，则返回错误和使用指引 64 | cmds := cleanFlagSet.Args() 65 | if len(cmds) > 0 { 66 | return fmt.Errorf("unknown command %+s", cmds[0]) 67 | } 68 | 69 | // short-circuit on help 70 | // 检查是否设置了帮助指引 71 | help, err := cleanFlagSet.GetBool("help") 72 | if err != nil { 73 | return errors.New(`"help" flag is non-bool, programmer error, please correct`) 74 | } 75 | if help { 76 | return cmd.Help() 77 | } 78 | 79 | 80 | // 若是查看版本信息，则返回版本信息及退出 81 | verflag.PrintAndExitIfRequested() 82 | 83 | // set feature gates from initial flags-based config 84 | if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil { 85 | return fmt.Errorf("failed to set feature gates from initial flags-based config: %w", err) 86 | } 87 | 88 | 89 | // 校验kubelet flag相关参数传入是否合法，如果合法则初始化到对应变量 90 | if err := options.ValidateKubeletFlags(kubeletFlags); err != nil { 91 | return fmt.Errorf("failed to validate kubelet flags: %w", err) 92 | } 93 | 94 | if cleanFlagSet.Changed("pod-infra-container-image") { 95 | klog.InfoS("--pod-infra-container-image will not be pruned by the image garbage collector in kubelet and should also be set in the remote runtime") 96 | } 97 | 98 | 99 | // 检测并解析kubelet配置文件 100 | if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 { 101 | kubeletConfig, err = loadConfigFile(configFile) 102 | if err != nil { 103 | return fmt.Errorf("failed to load kubelet config file, error: %w, path: %s", err, configFile) 104 | } 105 | 106 | if err := kubeletConfigFlagPrecedence(kubeletConfig, args); err != nil { 107 | return fmt.Errorf("failed to precedence kubeletConfigFlag: %w", err) 108 | } 109 | 110 | if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil { 111 | return fmt.Errorf("failed to set feature gates from initial flags-based config: %w", err) 112 | } 113 | } 114 | 115 | // 初始化日志 116 | logs.InitLogs() 117 | if err := logsapi.ValidateAndApplyAsField(&kubeletConfig.Logging, utilfeature.DefaultFeatureGate, field.NewPath("logging")); err != nil { 118 | return fmt.Errorf("initialize logging: %v", err) 119 | } 120 | //若是查看参数使用，则输出所有参数说明 121 | cliflag.PrintFlags(cleanFlagSet) 122 | 123 | // 本地命令行+配置文件验证 124 | if err := kubeletconfigvalidation.ValidateKubeletConfiguration(kubeletConfig, utilfeature.DefaultFeatureGate); err != nil { 125 | return fmt.Errorf("failed to validate kubelet configuration, error: %w, path: %s", err, kubeletConfig) 126 | } 127 | 128 | if (kubeletConfig.KubeletCgroups != "" && kubeletConfig.KubeReservedCgroup != "") && (strings.Index(kubeletConfig.KubeletCgroups, kubeletConfig.KubeReservedCgroup) != 0) { 129 | klog.InfoS("unsupported configuration:KubeletCgroups is not within KubeReservedCgroup") 130 | } 131 | 132 | // construct a KubeletServer from kubeletFlags and kubeletConfig 133 | kubeletServer := &options.KubeletServer{ 134 | KubeletFlags: *kubeletFlags, 135 | KubeletConfiguration: *kubeletConfig, 136 | } 137 | 138 | 139 | // 通过kubelet server和DefaultFeatureGate，构建kubeletDeps， 140 | // kubeletDeps主要集成了kubelet的运行依赖如：认证信息、第三方云厂商信息、事件记录、挂载、网络/卷插件、OOM管理等依赖组件调用链操作句柄 141 | kubeletDeps, err := UnsecuredDependencies(kubeletServer, utilfeature.DefaultFeatureGate) 142 | if err != nil { 143 | return fmt.Errorf("failed to construct kubelet dependencies: %w", err) 144 | } 145 | 146 | if err := checkPermissions(); err != nil { 147 | klog.ErrorS(err, "kubelet running with insufficient permissions") 148 | } 149 | 150 | config := kubeletServer.KubeletConfiguration.DeepCopy() 151 | for k := range config.StaticPodURLHeader { 152 | config.StaticPodURLHeader[k] = []string{""} 153 | } 154 | // log the kubelet's config for inspection 155 | klog.V(5).InfoS("KubeletConfiguration", "configuration", config) 156 | 157 | // 设置kubelet关闭的信号环境 158 | ctx := genericapiserver.SetupSignalContext() 159 | 160 | utilfeature.DefaultMutableFeatureGate.AddMetrics() 161 | 162 | // 启动kubelet，把ctx、kubeletServer、kubeletDeps、utilfeature.DefaultFeatureGate这些准备好的参数传入， 163 | // 执行内层核心Run函数 164 | return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) 165 | }, 166 | } 167 | 168 | kubeletFlags.AddFlags(cleanFlagSet) 169 | options.AddKubeletConfigFlags(cleanFlagSet, kubeletConfig) 170 | options.AddGlobalFlags(cleanFlagSet) 171 | cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name())) 172 | 173 | const usageFmt = "Usage:\n %s\n\nFlags:\n%s" 174 | cmd.SetUsageFunc(func(cmd *cobra.Command) error { 175 | fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2)) 176 | return nil 177 | }) 178 | cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { 179 | fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2)) 180 | }) 181 | 182 | return cmd 183 | } 184 | ``` 185 | 186 | 3. `Run` 函数主要工作: 187 | - 打印kubelet 188 | - golang设置 189 | - 初始化OS 190 | - 真正run kubelet 191 | 代码: https://github.com/kubernetes/kubernetes/blob/cf12a74b18b66efc577ec819c78a0c68f6d49225/cmd/kubelet/app/server.go#L410 192 | ```text 193 | func Run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate) error { 194 | // 为了帮助调试，记录版本 195 | klog.InfoS("Kubelet version", "kubeletVersion", version.Get()) 196 | 197 | klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) 198 | 199 | if err := initForOS(s.KubeletFlags.WindowsService, s.KubeletFlags.WindowsPriorityClass); err != nil { 200 | return fmt.Errorf("failed OS init: %w", err) 201 | } 202 | // 203 | if err := run(ctx, s, kubeDeps, featureGate); err != nil { 204 | return fmt.Errorf("failed to run Kubelet: %w", err) 205 | } 206 | return nil 207 | } 208 | ``` 209 | 4. `run` 函数主要流程： 210 | - 设置全局kubeletServer 211 | - 验证初始化的 kubeletServer 212 | - 将当前配置注册到config 213 | - 获取客户端，检测 standaloneMode 214 | - 如果处于独立模式，则将所有客户端设置为nil 215 | - 设置 event 记录器 216 | - 通知systemd,kubelet已经启动 217 | 218 | 代码: https://github.com/kubernetes/kubernetes/blob/cf12a74b18b66efc577ec819c78a0c68f6d49225/cmd/kubelet/app/server.go#L489 219 | ```text 220 | func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate) (err error) { 221 | // 设置全局kubeletServer 222 | err = utilfeature.DefaultMutableFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates) 223 | if err != nil { 224 | return err 225 | } 226 | // 开始验证初始化的 kubeletServer 227 | if err := options.ValidateKubeletServer(s); err != nil { 228 | return err 229 | } 230 | 231 | // 判断 cgroups v1是否启动了 MemoryQoS 232 | if utilfeature.DefaultFeatureGate.Enabled(features.MemoryQoS) && 233 | !isCgroup2UnifiedMode() { 234 | klog.InfoS("Warning: MemoryQoS feature only works with cgroups v2 on Linux, but enabled with cgroups v1") 235 | } 236 | // 获取 kubelet 锁文件 237 | if s.ExitOnLockContention && s.LockFilePath == "" { 238 | return errors.New("cannot exit on lock file contention: no lock file specified") 239 | } 240 | done := make(chan struct{}) 241 | if s.LockFilePath != "" { 242 | klog.InfoS("Acquiring file lock", "path", s.LockFilePath) 243 | if err := flock.Acquire(s.LockFilePath); err != nil { 244 | return fmt.Errorf("unable to acquire file lock on %q: %w", s.LockFilePath, err) 245 | } 246 | if s.ExitOnLockContention { 247 | klog.InfoS("Watching for inotify events", "path", s.LockFilePath) 248 | if err := watchForLockfileContention(s.LockFilePath, done); err != nil { 249 | return err 250 | } 251 | } 252 | } 253 | 254 | // 将当前配置注册到 /configz endpoint 255 | err = initConfigz(&s.KubeletConfiguration) 256 | if err != nil { 257 | klog.ErrorS(err, "Failed to register kubelet configuration with configz") 258 | } 259 | 260 | if len(s.ShowHiddenMetricsForVersion) > 0 { 261 | metrics.SetShowHidden() 262 | } 263 | 264 | // 获取客户端，检测 standaloneMode 265 | standaloneMode := true 266 | if len(s.KubeConfig) > 0 { 267 | standaloneMode = false 268 | } 269 | 270 | if kubeDeps == nil { 271 | kubeDeps, err = UnsecuredDependencies(s, featureGate) 272 | if err != nil { 273 | return err 274 | } 275 | } 276 | 277 | if kubeDeps.Cloud == nil { 278 | if !cloudprovider.IsExternal(s.CloudProvider) { 279 | cloudprovider.DeprecationWarningForProvider(s.CloudProvider) 280 | cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) 281 | if err != nil { 282 | return err 283 | } 284 | if cloud != nil { 285 | klog.V(2).InfoS("Successfully initialized cloud provider", "cloudProvider", s.CloudProvider, "cloudConfigFile", s.CloudConfigFile) 286 | } 287 | kubeDeps.Cloud = cloud 288 | } 289 | } 290 | 291 | hostName, err := nodeutil.GetHostname(s.HostnameOverride) 292 | if err != nil { 293 | return err 294 | } 295 | nodeName, err := getNodeName(kubeDeps.Cloud, hostName) 296 | if err != nil { 297 | return err 298 | } 299 | 300 | // 如果处于独立模式，则将所有客户端设置为nil 301 | switch { 302 | case standaloneMode: 303 | kubeDeps.KubeClient = nil 304 | kubeDeps.EventClient = nil 305 | kubeDeps.HeartbeatClient = nil 306 | klog.InfoS("Standalone mode, no API client") 307 | 308 | case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil: 309 | clientConfig, onHeartbeatFailure, err := buildKubeletClientConfig(ctx, s, kubeDeps.TracerProvider, nodeName) 310 | if err != nil { 311 | return err 312 | } 313 | if onHeartbeatFailure == nil { 314 | return errors.New("onHeartbeatFailure must be a valid function other than nil") 315 | } 316 | kubeDeps.OnHeartbeatFailure = onHeartbeatFailure 317 | 318 | kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig) 319 | if err != nil { 320 | return fmt.Errorf("failed to initialize kubelet client: %w", err) 321 | } 322 | 323 | // 为事件创建单独的客户端 324 | eventClientConfig := *clientConfig 325 | eventClientConfig.QPS = float32(s.EventRecordQPS) 326 | eventClientConfig.Burst = int(s.EventBurst) 327 | kubeDeps.EventClient, err = v1core.NewForConfig(&eventClientConfig) 328 | if err != nil { 329 | return fmt.Errorf("failed to initialize kubelet event client: %w", err) 330 | } 331 | 332 | // 禁用节流，附加超时的心跳创建一个单独的客户机 333 | heartbeatClientConfig := *clientConfig 334 | heartbeatClientConfig.Timeout = s.KubeletConfiguration.NodeStatusUpdateFrequency.Duration 335 | // The timeout is the minimum of the lease duration and status update frequency 336 | leaseTimeout := time.Duration(s.KubeletConfiguration.NodeLeaseDurationSeconds) * time.Second 337 | if heartbeatClientConfig.Timeout > leaseTimeout { 338 | heartbeatClientConfig.Timeout = leaseTimeout 339 | } 340 | 341 | heartbeatClientConfig.QPS = float32(-1) 342 | kubeDeps.HeartbeatClient, err = clientset.NewForConfig(&heartbeatClientConfig) 343 | if err != nil { 344 | return fmt.Errorf("failed to initialize kubelet heartbeat client: %w", err) 345 | } 346 | } 347 | 348 | if kubeDeps.Auth == nil { 349 | auth, runAuthenticatorCAReload, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration) 350 | if err != nil { 351 | return err 352 | } 353 | kubeDeps.Auth = auth 354 | runAuthenticatorCAReload(ctx.Done()) 355 | } 356 | 357 | var cgroupRoots []string 358 | nodeAllocatableRoot := cm.NodeAllocatableRoot(s.CgroupRoot, s.CgroupsPerQOS, s.CgroupDriver) 359 | cgroupRoots = append(cgroupRoots, nodeAllocatableRoot) 360 | kubeletCgroup, err := cm.GetKubeletContainer(s.KubeletCgroups) 361 | if err != nil { 362 | klog.InfoS("Failed to get the kubelet's cgroup. Kubelet system container metrics may be missing.", "err", err) 363 | } else if kubeletCgroup != "" { 364 | cgroupRoots = append(cgroupRoots, kubeletCgroup) 365 | } 366 | 367 | if s.RuntimeCgroups != "" { 368 | // RuntimeCgroups is optional, so ignore if it isn't specified 369 | cgroupRoots = append(cgroupRoots, s.RuntimeCgroups) 370 | } 371 | 372 | if s.SystemCgroups != "" { 373 | // SystemCgroups is optional, so ignore if it isn't specified 374 | cgroupRoots = append(cgroupRoots, s.SystemCgroups) 375 | } 376 | 377 | if kubeDeps.CAdvisorInterface == nil { 378 | imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.RemoteRuntimeEndpoint) 379 | kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.RemoteRuntimeEndpoint), s.LocalStorageCapacityIsolation) 380 | if err != nil { 381 | return err 382 | } 383 | } 384 | 385 | // 设置 event 记录器 386 | makeEventRecorder(kubeDeps, nodeName) 387 | 388 | if kubeDeps.ContainerManager == nil { 389 | if s.CgroupsPerQOS && s.CgroupRoot == "" { 390 | klog.InfoS("--cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /") 391 | s.CgroupRoot = "/" 392 | } 393 | 394 | machineInfo, err := kubeDeps.CAdvisorInterface.MachineInfo() 395 | if err != nil { 396 | return err 397 | } 398 | reservedSystemCPUs, err := getReservedCPUs(machineInfo, s.ReservedSystemCPUs) 399 | if err != nil { 400 | return err 401 | } 402 | if reservedSystemCPUs.Size() > 0 { 403 | // at cmd option validation phase it is tested either --system-reserved-cgroup or --kube-reserved-cgroup is specified, so overwrite should be ok 404 | klog.InfoS("Option --reserved-cpus is specified, it will overwrite the cpu setting in KubeReserved and SystemReserved", "kubeReservedCPUs", s.KubeReserved, "systemReservedCPUs", s.SystemReserved) 405 | if s.KubeReserved != nil { 406 | delete(s.KubeReserved, "cpu") 407 | } 408 | if s.SystemReserved == nil { 409 | s.SystemReserved = make(map[string]string) 410 | } 411 | s.SystemReserved["cpu"] = strconv.Itoa(reservedSystemCPUs.Size()) 412 | klog.InfoS("After cpu setting is overwritten", "kubeReservedCPUs", s.KubeReserved, "systemReservedCPUs", s.SystemReserved) 413 | } 414 | 415 | kubeReserved, err := parseResourceList(s.KubeReserved) 416 | if err != nil { 417 | return err 418 | } 419 | systemReserved, err := parseResourceList(s.SystemReserved) 420 | if err != nil { 421 | return err 422 | } 423 | var hardEvictionThresholds []evictionapi.Threshold 424 | // If the user requested to ignore eviction thresholds, then do not set valid values for hardEvictionThresholds here. 425 | if !s.ExperimentalNodeAllocatableIgnoreEvictionThreshold { 426 | hardEvictionThresholds, err = eviction.ParseThresholdConfig([]string{}, s.EvictionHard, nil, nil, nil) 427 | if err != nil { 428 | return err 429 | } 430 | } 431 | experimentalQOSReserved, err := cm.ParseQOSReserved(s.QOSReserved) 432 | if err != nil { 433 | return err 434 | } 435 | 436 | var cpuManagerPolicyOptions map[string]string 437 | if utilfeature.DefaultFeatureGate.Enabled(features.CPUManagerPolicyOptions) { 438 | cpuManagerPolicyOptions = s.CPUManagerPolicyOptions 439 | } else if s.CPUManagerPolicyOptions != nil { 440 | return fmt.Errorf("CPU Manager policy options %v require feature gates %q, %q enabled", 441 | s.CPUManagerPolicyOptions, features.CPUManager, features.CPUManagerPolicyOptions) 442 | } 443 | 444 | var topologyManagerPolicyOptions map[string]string 445 | if utilfeature.DefaultFeatureGate.Enabled(features.TopologyManager) { 446 | if utilfeature.DefaultFeatureGate.Enabled(features.TopologyManagerPolicyOptions) { 447 | topologyManagerPolicyOptions = s.TopologyManagerPolicyOptions 448 | } else if s.TopologyManagerPolicyOptions != nil { 449 | return fmt.Errorf("topology manager policy options %v require feature gates %q, %q enabled", 450 | s.TopologyManagerPolicyOptions, features.TopologyManager, features.TopologyManagerPolicyOptions) 451 | } 452 | } 453 | 454 | kubeDeps.ContainerManager, err = cm.NewContainerManager( 455 | kubeDeps.Mounter, 456 | kubeDeps.CAdvisorInterface, 457 | cm.NodeConfig{ 458 | RuntimeCgroupsName: s.RuntimeCgroups, 459 | SystemCgroupsName: s.SystemCgroups, 460 | KubeletCgroupsName: s.KubeletCgroups, 461 | KubeletOOMScoreAdj: s.OOMScoreAdj, 462 | CgroupsPerQOS: s.CgroupsPerQOS, 463 | CgroupRoot: s.CgroupRoot, 464 | CgroupDriver: s.CgroupDriver, 465 | KubeletRootDir: s.RootDirectory, 466 | ProtectKernelDefaults: s.ProtectKernelDefaults, 467 | NodeAllocatableConfig: cm.NodeAllocatableConfig{ 468 | KubeReservedCgroupName: s.KubeReservedCgroup, 469 | SystemReservedCgroupName: s.SystemReservedCgroup, 470 | EnforceNodeAllocatable: sets.NewString(s.EnforceNodeAllocatable...), 471 | KubeReserved: kubeReserved, 472 | SystemReserved: systemReserved, 473 | ReservedSystemCPUs: reservedSystemCPUs, 474 | HardEvictionThresholds: hardEvictionThresholds, 475 | }, 476 | QOSReserved: *experimentalQOSReserved, 477 | CPUManagerPolicy: s.CPUManagerPolicy, 478 | CPUManagerPolicyOptions: cpuManagerPolicyOptions, 479 | CPUManagerReconcilePeriod: s.CPUManagerReconcilePeriod.Duration, 480 | ExperimentalMemoryManagerPolicy: s.MemoryManagerPolicy, 481 | ExperimentalMemoryManagerReservedMemory: s.ReservedMemory, 482 | ExperimentalPodPidsLimit: s.PodPidsLimit, 483 | EnforceCPULimits: s.CPUCFSQuota, 484 | CPUCFSQuotaPeriod: s.CPUCFSQuotaPeriod.Duration, 485 | ExperimentalTopologyManagerPolicy: s.TopologyManagerPolicy, 486 | ExperimentalTopologyManagerScope: s.TopologyManagerScope, 487 | ExperimentalTopologyManagerPolicyOptions: topologyManagerPolicyOptions, 488 | }, 489 | s.FailSwapOn, 490 | kubeDeps.Recorder) 491 | 492 | if err != nil { 493 | return err 494 | } 495 | } 496 | 497 | if kubeDeps.PodStartupLatencyTracker == nil { 498 | kubeDeps.PodStartupLatencyTracker = kubeletutil.NewPodStartupLatencyTracker() 499 | } 500 | 501 | // TODO(vmarmol): Do this through container config. 502 | oomAdjuster := kubeDeps.OOMAdjuster 503 | if err := oomAdjuster.ApplyOOMScoreAdj(0, int(s.OOMScoreAdj)); err != nil { 504 | klog.InfoS("Failed to ApplyOOMScoreAdj", "err", err) 505 | } 506 | 507 | err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, kubeDeps, s.RemoteRuntimeEndpoint, s.RemoteImageEndpoint) 508 | if err != nil { 509 | return err 510 | } 511 | 512 | if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil { 513 | return err 514 | } 515 | 516 | if s.HealthzPort > 0 { 517 | mux := http.NewServeMux() 518 | healthz.InstallHandler(mux) 519 | go wait.Until(func() { 520 | err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), mux) 521 | if err != nil { 522 | klog.ErrorS(err, "Failed to start healthz server") 523 | } 524 | }, 5*time.Second, wait.NeverStop) 525 | } 526 | 527 | if s.RunOnce { 528 | return nil 529 | } 530 | 531 | // 通知systemd,kubelet已经启动 532 | go daemon.SdNotify(false, "READY=1") 533 | 534 | select { 535 | case <-done: 536 | break 537 | case <-ctx.Done(): 538 | break 539 | } 540 | 541 | return nil 542 | } 543 | ``` 544 | 545 | 546 | 547 | 548 | 549 | 550 | --------------------------------------------------------------------------------