├── .gitignore ├── calico.yaml ├── docker-compose.yaml ├── dockerfile ├── entrypoint └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | demo 2 | -------------------------------------------------------------------------------- /calico.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Source: calico/templates/calico-config.yaml 3 | # This ConfigMap is used to configure a self-hosted Calico installation. 4 | kind: ConfigMap 5 | apiVersion: v1 6 | metadata: 7 | name: calico-config 8 | namespace: kube-system 9 | data: 10 | # Typha is disabled. 11 | typha_service_name: "none" 12 | # Configure the backend to use. 13 | calico_backend: "bird" 14 | 15 | # Configure the MTU to use 16 | veth_mtu: "1440" 17 | 18 | # The CNI network configuration to install on each node. The special 19 | # values in this config will be automatically populated. 20 | cni_network_config: |- 21 | { 22 | "name": "k8s-pod-network", 23 | "cniVersion": "0.3.1", 24 | "plugins": [ 25 | { 26 | "type": "calico", 27 | "log_level": "info", 28 | "datastore_type": "kubernetes", 29 | "nodename": "__KUBERNETES_NODE_NAME__", 30 | "mtu": __CNI_MTU__, 31 | "ipam": { 32 | "type": "calico-ipam" 33 | }, 34 | "policy": { 35 | "type": "k8s" 36 | }, 37 | "kubernetes": { 38 | "kubeconfig": "__KUBECONFIG_FILEPATH__" 39 | } 40 | }, 41 | { 42 | "type": "portmap", 43 | "snat": true, 44 | "capabilities": {"portMappings": true} 45 | }, 46 | { 47 | "type": "bandwidth", 48 | "capabilities": {"bandwidth": true} 49 | } 50 | ] 51 | } 52 | 53 | --- 54 | # Source: calico/templates/kdd-crds.yaml 55 | apiVersion: apiextensions.k8s.io/v1beta1 56 | kind: CustomResourceDefinition 57 | metadata: 58 | name: felixconfigurations.crd.projectcalico.org 59 | spec: 60 | scope: Cluster 61 | group: crd.projectcalico.org 62 | version: v1 63 | names: 64 | kind: FelixConfiguration 65 | plural: felixconfigurations 66 | singular: felixconfiguration 67 | --- 68 | 69 | apiVersion: apiextensions.k8s.io/v1beta1 70 | kind: CustomResourceDefinition 71 | metadata: 72 | name: ipamblocks.crd.projectcalico.org 73 | spec: 74 | scope: Cluster 75 | group: crd.projectcalico.org 76 | version: v1 77 | names: 78 | kind: IPAMBlock 79 | plural: ipamblocks 80 | singular: ipamblock 81 | 82 | --- 83 | 84 | apiVersion: apiextensions.k8s.io/v1beta1 85 | kind: CustomResourceDefinition 86 | metadata: 87 | name: blockaffinities.crd.projectcalico.org 88 | spec: 89 | scope: Cluster 90 | group: crd.projectcalico.org 91 | version: v1 92 | names: 93 | kind: BlockAffinity 94 | plural: blockaffinities 95 | singular: blockaffinity 96 | 97 | --- 98 | 99 | apiVersion: apiextensions.k8s.io/v1beta1 100 | kind: CustomResourceDefinition 101 | metadata: 102 | name: ipamhandles.crd.projectcalico.org 103 | spec: 104 | scope: Cluster 105 | group: crd.projectcalico.org 106 | version: v1 107 | names: 108 | kind: IPAMHandle 109 | plural: ipamhandles 110 | singular: ipamhandle 111 | 112 | --- 113 | 114 | apiVersion: apiextensions.k8s.io/v1beta1 115 | kind: CustomResourceDefinition 116 | metadata: 117 | name: ipamconfigs.crd.projectcalico.org 118 | spec: 119 | scope: Cluster 120 | group: crd.projectcalico.org 121 | version: v1 122 | names: 123 | kind: IPAMConfig 124 | plural: ipamconfigs 125 | singular: ipamconfig 126 | 127 | --- 128 | 129 | apiVersion: apiextensions.k8s.io/v1beta1 130 | kind: CustomResourceDefinition 131 | metadata: 132 | name: bgppeers.crd.projectcalico.org 133 | spec: 134 | scope: Cluster 135 | group: crd.projectcalico.org 136 | version: v1 137 | names: 138 | kind: BGPPeer 139 | plural: bgppeers 140 | singular: bgppeer 141 | 142 | --- 143 | 144 | apiVersion: apiextensions.k8s.io/v1beta1 145 | kind: CustomResourceDefinition 146 | metadata: 147 | name: bgpconfigurations.crd.projectcalico.org 148 | spec: 149 | scope: Cluster 150 | group: crd.projectcalico.org 151 | version: v1 152 | names: 153 | kind: BGPConfiguration 154 | plural: bgpconfigurations 155 | singular: bgpconfiguration 156 | 157 | --- 158 | 159 | apiVersion: apiextensions.k8s.io/v1beta1 160 | kind: CustomResourceDefinition 161 | metadata: 162 | name: ippools.crd.projectcalico.org 163 | spec: 164 | scope: Cluster 165 | group: crd.projectcalico.org 166 | version: v1 167 | names: 168 | kind: IPPool 169 | plural: ippools 170 | singular: ippool 171 | 172 | --- 173 | 174 | apiVersion: apiextensions.k8s.io/v1beta1 175 | kind: CustomResourceDefinition 176 | metadata: 177 | name: hostendpoints.crd.projectcalico.org 178 | spec: 179 | scope: Cluster 180 | group: crd.projectcalico.org 181 | version: v1 182 | names: 183 | kind: HostEndpoint 184 | plural: hostendpoints 185 | singular: hostendpoint 186 | 187 | --- 188 | 189 | apiVersion: apiextensions.k8s.io/v1beta1 190 | kind: CustomResourceDefinition 191 | metadata: 192 | name: clusterinformations.crd.projectcalico.org 193 | spec: 194 | scope: Cluster 195 | group: crd.projectcalico.org 196 | version: v1 197 | names: 198 | kind: ClusterInformation 199 | plural: clusterinformations 200 | singular: clusterinformation 201 | 202 | --- 203 | 204 | apiVersion: apiextensions.k8s.io/v1beta1 205 | kind: CustomResourceDefinition 206 | metadata: 207 | name: globalnetworkpolicies.crd.projectcalico.org 208 | spec: 209 | scope: Cluster 210 | group: crd.projectcalico.org 211 | version: v1 212 | names: 213 | kind: GlobalNetworkPolicy 214 | plural: globalnetworkpolicies 215 | singular: globalnetworkpolicy 216 | 217 | --- 218 | 219 | apiVersion: apiextensions.k8s.io/v1beta1 220 | kind: CustomResourceDefinition 221 | metadata: 222 | name: globalnetworksets.crd.projectcalico.org 223 | spec: 224 | scope: Cluster 225 | group: crd.projectcalico.org 226 | version: v1 227 | names: 228 | kind: GlobalNetworkSet 229 | plural: globalnetworksets 230 | singular: globalnetworkset 231 | 232 | --- 233 | 234 | apiVersion: apiextensions.k8s.io/v1beta1 235 | kind: CustomResourceDefinition 236 | metadata: 237 | name: networkpolicies.crd.projectcalico.org 238 | spec: 239 | scope: Namespaced 240 | group: crd.projectcalico.org 241 | version: v1 242 | names: 243 | kind: NetworkPolicy 244 | plural: networkpolicies 245 | singular: networkpolicy 246 | 247 | --- 248 | 249 | apiVersion: apiextensions.k8s.io/v1beta1 250 | kind: CustomResourceDefinition 251 | metadata: 252 | name: networksets.crd.projectcalico.org 253 | spec: 254 | scope: Namespaced 255 | group: crd.projectcalico.org 256 | version: v1 257 | names: 258 | kind: NetworkSet 259 | plural: networksets 260 | singular: networkset 261 | --- 262 | # Source: calico/templates/rbac.yaml 263 | 264 | # Include a clusterrole for the kube-controllers component, 265 | # and bind it to the calico-kube-controllers serviceaccount. 266 | kind: ClusterRole 267 | apiVersion: rbac.authorization.k8s.io/v1 268 | metadata: 269 | name: calico-kube-controllers 270 | rules: 271 | # Nodes are watched to monitor for deletions. 272 | - apiGroups: [""] 273 | resources: 274 | - nodes 275 | verbs: 276 | - watch 277 | - list 278 | - get 279 | # Pods are queried to check for existence. 280 | - apiGroups: [""] 281 | resources: 282 | - pods 283 | verbs: 284 | - get 285 | # IPAM resources are manipulated when nodes are deleted. 286 | - apiGroups: ["crd.projectcalico.org"] 287 | resources: 288 | - ippools 289 | verbs: 290 | - list 291 | - apiGroups: ["crd.projectcalico.org"] 292 | resources: 293 | - blockaffinities 294 | - ipamblocks 295 | - ipamhandles 296 | verbs: 297 | - get 298 | - list 299 | - create 300 | - update 301 | - delete 302 | # Needs access to update clusterinformations. 303 | - apiGroups: ["crd.projectcalico.org"] 304 | resources: 305 | - clusterinformations 306 | verbs: 307 | - get 308 | - create 309 | - update 310 | --- 311 | kind: ClusterRoleBinding 312 | apiVersion: rbac.authorization.k8s.io/v1 313 | metadata: 314 | name: calico-kube-controllers 315 | roleRef: 316 | apiGroup: rbac.authorization.k8s.io 317 | kind: ClusterRole 318 | name: calico-kube-controllers 319 | subjects: 320 | - kind: ServiceAccount 321 | name: calico-kube-controllers 322 | namespace: kube-system 323 | --- 324 | # Include a clusterrole for the calico-node DaemonSet, 325 | # and bind it to the calico-node serviceaccount. 326 | kind: ClusterRole 327 | apiVersion: rbac.authorization.k8s.io/v1 328 | metadata: 329 | name: calico-node 330 | rules: 331 | # The CNI plugin needs to get pods, nodes, and namespaces. 332 | - apiGroups: [""] 333 | resources: 334 | - pods 335 | - nodes 336 | - namespaces 337 | verbs: 338 | - get 339 | - apiGroups: [""] 340 | resources: 341 | - endpoints 342 | - services 343 | verbs: 344 | # Used to discover service IPs for advertisement. 345 | - watch 346 | - list 347 | # Used to discover Typhas. 348 | - get 349 | # Pod CIDR auto-detection on kubeadm needs access to config maps. 350 | - apiGroups: [""] 351 | resources: 352 | - configmaps 353 | verbs: 354 | - get 355 | - apiGroups: [""] 356 | resources: 357 | - nodes/status 358 | verbs: 359 | # Needed for clearing NodeNetworkUnavailable flag. 360 | - patch 361 | # Calico stores some configuration information in node annotations. 362 | - update 363 | # Watch for changes to Kubernetes NetworkPolicies. 364 | - apiGroups: ["networking.k8s.io"] 365 | resources: 366 | - networkpolicies 367 | verbs: 368 | - watch 369 | - list 370 | # Used by Calico for policy information. 371 | - apiGroups: [""] 372 | resources: 373 | - pods 374 | - namespaces 375 | - serviceaccounts 376 | verbs: 377 | - list 378 | - watch 379 | # The CNI plugin patches pods/status. 380 | - apiGroups: [""] 381 | resources: 382 | - pods/status 383 | verbs: 384 | - patch 385 | # Calico monitors various CRDs for config. 386 | - apiGroups: ["crd.projectcalico.org"] 387 | resources: 388 | - globalfelixconfigs 389 | - felixconfigurations 390 | - bgppeers 391 | - globalbgpconfigs 392 | - bgpconfigurations 393 | - ippools 394 | - ipamblocks 395 | - globalnetworkpolicies 396 | - globalnetworksets 397 | - networkpolicies 398 | - networksets 399 | - clusterinformations 400 | - hostendpoints 401 | - blockaffinities 402 | verbs: 403 | - get 404 | - list 405 | - watch 406 | # Calico must create and update some CRDs on startup. 407 | - apiGroups: ["crd.projectcalico.org"] 408 | resources: 409 | - ippools 410 | - felixconfigurations 411 | - clusterinformations 412 | verbs: 413 | - create 414 | - update 415 | # Calico stores some configuration information on the node. 416 | - apiGroups: [""] 417 | resources: 418 | - nodes 419 | verbs: 420 | - get 421 | - list 422 | - watch 423 | # These permissions are only requried for upgrade from v2.6, and can 424 | # be removed after upgrade or on fresh installations. 425 | - apiGroups: ["crd.projectcalico.org"] 426 | resources: 427 | - bgpconfigurations 428 | - bgppeers 429 | verbs: 430 | - create 431 | - update 432 | # These permissions are required for Calico CNI to perform IPAM allocations. 433 | - apiGroups: ["crd.projectcalico.org"] 434 | resources: 435 | - blockaffinities 436 | - ipamblocks 437 | - ipamhandles 438 | verbs: 439 | - get 440 | - list 441 | - create 442 | - update 443 | - delete 444 | - apiGroups: ["crd.projectcalico.org"] 445 | resources: 446 | - ipamconfigs 447 | verbs: 448 | - get 449 | # Block affinities must also be watchable by confd for route aggregation. 450 | - apiGroups: ["crd.projectcalico.org"] 451 | resources: 452 | - blockaffinities 453 | verbs: 454 | - watch 455 | # The Calico IPAM migration needs to get daemonsets. These permissions can be 456 | # removed if not upgrading from an installation using host-local IPAM. 457 | - apiGroups: ["apps"] 458 | resources: 459 | - daemonsets 460 | verbs: 461 | - get 462 | 463 | --- 464 | apiVersion: rbac.authorization.k8s.io/v1 465 | kind: ClusterRoleBinding 466 | metadata: 467 | name: calico-node 468 | roleRef: 469 | apiGroup: rbac.authorization.k8s.io 470 | kind: ClusterRole 471 | name: calico-node 472 | subjects: 473 | - kind: ServiceAccount 474 | name: calico-node 475 | namespace: kube-system 476 | 477 | --- 478 | # Source: calico/templates/calico-node.yaml 479 | # This manifest installs the calico-node container, as well 480 | # as the CNI plugins and network config on 481 | # each master and worker node in a Kubernetes cluster. 482 | kind: DaemonSet 483 | apiVersion: apps/v1 484 | metadata: 485 | name: calico-node 486 | namespace: kube-system 487 | labels: 488 | k8s-app: calico-node 489 | spec: 490 | selector: 491 | matchLabels: 492 | k8s-app: calico-node 493 | updateStrategy: 494 | type: RollingUpdate 495 | rollingUpdate: 496 | maxUnavailable: 1 497 | template: 498 | metadata: 499 | labels: 500 | k8s-app: calico-node 501 | annotations: 502 | # This, along with the CriticalAddonsOnly toleration below, 503 | # marks the pod as a critical add-on, ensuring it gets 504 | # priority scheduling and that its resources are reserved 505 | # if it ever gets evicted. 506 | scheduler.alpha.kubernetes.io/critical-pod: '' 507 | spec: 508 | nodeSelector: 509 | kubernetes.io/os: linux 510 | hostNetwork: true 511 | tolerations: 512 | # Make sure calico-node gets scheduled on all nodes. 513 | - effect: NoSchedule 514 | operator: Exists 515 | # Mark the pod as a critical add-on for rescheduling. 516 | - key: CriticalAddonsOnly 517 | operator: Exists 518 | - effect: NoExecute 519 | operator: Exists 520 | serviceAccountName: calico-node 521 | # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force 522 | # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. 523 | terminationGracePeriodSeconds: 0 524 | priorityClassName: system-node-critical 525 | initContainers: 526 | # This container performs upgrade from host-local IPAM to calico-ipam. 527 | # It can be deleted if this is a fresh installation, or if you have already 528 | # upgraded to use calico-ipam. 529 | - name: upgrade-ipam 530 | image: calico/cni:v3.13.0 531 | command: ["/opt/cni/bin/calico-ipam", "-upgrade"] 532 | env: 533 | - name: KUBERNETES_NODE_NAME 534 | valueFrom: 535 | fieldRef: 536 | fieldPath: spec.nodeName 537 | - name: CALICO_NETWORKING_BACKEND 538 | valueFrom: 539 | configMapKeyRef: 540 | name: calico-config 541 | key: calico_backend 542 | volumeMounts: 543 | - mountPath: /var/lib/cni/networks 544 | name: host-local-net-dir 545 | - mountPath: /host/opt/cni/bin 546 | name: cni-bin-dir 547 | securityContext: 548 | privileged: true 549 | # This container installs the CNI binaries 550 | # and CNI network config file on each node. 551 | - name: install-cni 552 | image: calico/cni:v3.13.0 553 | command: ["/install-cni.sh"] 554 | env: 555 | # Name of the CNI config file to create. 556 | - name: CNI_CONF_NAME 557 | value: "10-calico.conflist" 558 | # The CNI network config to install on each node. 559 | - name: CNI_NETWORK_CONFIG 560 | valueFrom: 561 | configMapKeyRef: 562 | name: calico-config 563 | key: cni_network_config 564 | # Set the hostname based on the k8s node name. 565 | - name: KUBERNETES_NODE_NAME 566 | valueFrom: 567 | fieldRef: 568 | fieldPath: spec.nodeName 569 | # CNI MTU Config variable 570 | - name: CNI_MTU 571 | valueFrom: 572 | configMapKeyRef: 573 | name: calico-config 574 | key: veth_mtu 575 | # Prevents the container from sleeping forever. 576 | - name: SLEEP 577 | value: "false" 578 | volumeMounts: 579 | - mountPath: /host/opt/cni/bin 580 | name: cni-bin-dir 581 | - mountPath: /host/etc/cni/net.d 582 | name: cni-net-dir 583 | securityContext: 584 | privileged: true 585 | # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes 586 | # to communicate with Felix over the Policy Sync API. 587 | - name: flexvol-driver 588 | image: calico/pod2daemon-flexvol:v3.13.0 589 | volumeMounts: 590 | - name: flexvol-driver-host 591 | mountPath: /host/driver 592 | securityContext: 593 | privileged: true 594 | containers: 595 | # Runs calico-node container on each Kubernetes node. This 596 | # container programs network policy and routes on each 597 | # host. 598 | - name: calico-node 599 | image: calico/node:v3.13.0 600 | env: 601 | # Use Kubernetes API as the backing datastore. 602 | - name: DATASTORE_TYPE 603 | value: "kubernetes" 604 | # Wait for the datastore. 605 | - name: WAIT_FOR_DATASTORE 606 | value: "true" 607 | # Set based on the k8s node name. 608 | - name: NODENAME 609 | valueFrom: 610 | fieldRef: 611 | fieldPath: spec.nodeName 612 | # Choose the backend to use. 613 | - name: CALICO_NETWORKING_BACKEND 614 | valueFrom: 615 | configMapKeyRef: 616 | name: calico-config 617 | key: calico_backend 618 | # Cluster type to identify the deployment type 619 | - name: CLUSTER_TYPE 620 | value: "k8s,bgp" 621 | # Auto-detect the BGP IP address. 622 | - name: IP 623 | value: "autodetect" 624 | # Enable IPIP 625 | - name: CALICO_IPV4POOL_IPIP 626 | value: "Always" 627 | # Set MTU for tunnel device used if ipip is enabled 628 | - name: FELIX_IPINIPMTU 629 | valueFrom: 630 | configMapKeyRef: 631 | name: calico-config 632 | key: veth_mtu 633 | # The default IPv4 pool to create on startup if none exists. Pod IPs will be 634 | # chosen from this range. Changing this value after installation will have 635 | # no effect. This should fall within `--cluster-cidr`. 636 | # - name: CALICO_IPV4POOL_CIDR 637 | # value: "192.168.0.0/16" 638 | # Disable file logging so `kubectl logs` works. 639 | - name: CALICO_DISABLE_FILE_LOGGING 640 | value: "true" 641 | # Set Felix endpoint to host default action to ACCEPT. 642 | - name: FELIX_DEFAULTENDPOINTTOHOSTACTION 643 | value: "ACCEPT" 644 | # Disable IPv6 on Kubernetes. 645 | - name: FELIX_IPV6SUPPORT 646 | value: "false" 647 | # Set Felix logging to "info" 648 | - name: FELIX_LOGSEVERITYSCREEN 649 | value: "info" 650 | - name: FELIX_HEALTHENABLED 651 | value: "true" 652 | securityContext: 653 | privileged: true 654 | resources: 655 | requests: 656 | cpu: 250m 657 | livenessProbe: 658 | exec: 659 | command: 660 | - /bin/calico-node 661 | - -felix-live 662 | - -bird-live 663 | periodSeconds: 10 664 | initialDelaySeconds: 10 665 | failureThreshold: 6 666 | readinessProbe: 667 | exec: 668 | command: 669 | - /bin/calico-node 670 | - -felix-ready 671 | - -bird-ready 672 | periodSeconds: 10 673 | volumeMounts: 674 | - mountPath: /lib/modules 675 | name: lib-modules 676 | readOnly: true 677 | - mountPath: /run/xtables.lock 678 | name: xtables-lock 679 | readOnly: false 680 | - mountPath: /var/run/calico 681 | name: var-run-calico 682 | readOnly: false 683 | - mountPath: /var/lib/calico 684 | name: var-lib-calico 685 | readOnly: false 686 | - name: policysync 687 | mountPath: /var/run/nodeagent 688 | volumes: 689 | # Used by calico-node. 690 | - name: lib-modules 691 | hostPath: 692 | path: /lib/modules 693 | - name: var-run-calico 694 | hostPath: 695 | path: /var/run/calico 696 | - name: var-lib-calico 697 | hostPath: 698 | path: /var/lib/calico 699 | - name: xtables-lock 700 | hostPath: 701 | path: /run/xtables.lock 702 | type: FileOrCreate 703 | # Used to install CNI. 704 | - name: cni-bin-dir 705 | hostPath: 706 | path: /opt/cni/bin 707 | - name: cni-net-dir 708 | hostPath: 709 | path: /etc/cni/net.d 710 | # Mount in the directory for host-local IPAM allocations. This is 711 | # used when upgrading from host-local to calico-ipam, and can be removed 712 | # if not using the upgrade-ipam init container. 713 | - name: host-local-net-dir 714 | hostPath: 715 | path: /var/lib/cni/networks 716 | # Used to create per-pod Unix Domain Sockets 717 | - name: policysync 718 | hostPath: 719 | type: DirectoryOrCreate 720 | path: /var/run/nodeagent 721 | # Used to install Flex Volume Driver 722 | - name: flexvol-driver-host 723 | hostPath: 724 | type: DirectoryOrCreate 725 | path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds 726 | --- 727 | 728 | apiVersion: v1 729 | kind: ServiceAccount 730 | metadata: 731 | name: calico-node 732 | namespace: kube-system 733 | 734 | --- 735 | # Source: calico/templates/calico-kube-controllers.yaml 736 | 737 | # See https://github.com/projectcalico/kube-controllers 738 | apiVersion: apps/v1 739 | kind: Deployment 740 | metadata: 741 | name: calico-kube-controllers 742 | namespace: kube-system 743 | labels: 744 | k8s-app: calico-kube-controllers 745 | spec: 746 | # The controllers can only have a single active instance. 747 | replicas: 1 748 | selector: 749 | matchLabels: 750 | k8s-app: calico-kube-controllers 751 | strategy: 752 | type: Recreate 753 | template: 754 | metadata: 755 | name: calico-kube-controllers 756 | namespace: kube-system 757 | labels: 758 | k8s-app: calico-kube-controllers 759 | annotations: 760 | scheduler.alpha.kubernetes.io/critical-pod: '' 761 | spec: 762 | nodeSelector: 763 | kubernetes.io/os: linux 764 | tolerations: 765 | # Mark the pod as a critical add-on for rescheduling. 766 | - key: CriticalAddonsOnly 767 | operator: Exists 768 | - key: node-role.kubernetes.io/master 769 | effect: NoSchedule 770 | serviceAccountName: calico-kube-controllers 771 | priorityClassName: system-cluster-critical 772 | containers: 773 | - name: calico-kube-controllers 774 | image: calico/kube-controllers:v3.13.0 775 | env: 776 | # Choose which controllers to run. 777 | - name: ENABLED_CONTROLLERS 778 | value: node 779 | - name: DATASTORE_TYPE 780 | value: kubernetes 781 | readinessProbe: 782 | exec: 783 | command: 784 | - /usr/bin/check-status 785 | - -r 786 | 787 | --- 788 | 789 | apiVersion: v1 790 | kind: ServiceAccount 791 | metadata: 792 | name: calico-kube-controllers 793 | namespace: kube-system 794 | --- 795 | # Source: calico/templates/calico-etcd-secrets.yaml 796 | 797 | --- 798 | # Source: calico/templates/calico-typha.yaml 799 | 800 | --- 801 | # Source: calico/templates/configure-canal.yaml 802 | 803 | 804 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | node-a: 4 | dns: 5 | - 8.8.8.8 6 | build: 7 | context: . 8 | container_name: node-a 9 | image: kubernetes_node 10 | working_dir: /work 11 | stdin_open: true 12 | tty: true 13 | privileged: true 14 | volumes: 15 | - .:/work 16 | master-a: 17 | container_name: master-a 18 | build: 19 | context: . 20 | image: kubernetes_master 21 | working_dir: /work 22 | stdin_open: true 23 | tty: true 24 | privileged: true 25 | volumes: 26 | - .:/work 27 | ports: 28 | - 6443:6443 29 | 30 | 31 | -------------------------------------------------------------------------------- /dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:19.10 2 | 3 | #INSTALL COMPONENTS TO MAKE THIS CONTAINER BEHAVE AS A VM 4 | # * INSTALL SYSTEMD 5 | # * CUSTOM ENTRYPOINT THAT FIXES THINGS - from K8s SIGS Kind 6 | ################################################################# 7 | #Install known tools required for the guide 8 | #We use systemd to bootstrap the cluster components 9 | RUN apt-get update && apt-get install -y --no-install-recommends systemd bash ca-certificates curl 10 | 11 | RUN apt-get clean -y && \ 12 | rm -rf \ 13 | /var/cache/debconf/* \ 14 | /var/lib/apt/lists/* \ 15 | /var/log/* \ 16 | /tmp/* \ 17 | /var/tmp/* \ 18 | /usr/share/doc/* \ 19 | /usr/share/man/* \ 20 | /usr/share/local/* 21 | 22 | RUN find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \ 23 | && rm -f /lib/systemd/system/multi-user.target.wants/* \ 24 | && rm -f /etc/systemd/system/*.wants/* \ 25 | && rm -f /lib/systemd/system/local-fs.target.wants/* \ 26 | && rm -f /lib/systemd/system/sockets.target.wants/*udev* \ 27 | && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \ 28 | && rm -f /lib/systemd/system/basic.target.wants/* \ 29 | && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \ 30 | && ln -s "$(which systemd)" /sbin/init 31 | 32 | COPY ./entrypoint /usr/local/bin/entrypoint 33 | RUN chmod +x /usr/local/bin/entrypoint 34 | 35 | # tell systemd that it is in docker (it will check for the container env) 36 | # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ 37 | ENV container docker 38 | 39 | # systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it) 40 | # https://bugzilla.redhat.com/show_bug.cgi?id=1201657 41 | STOPSIGNAL SIGRTMIN+3 42 | 43 | ################################################################ 44 | 45 | #Install Kubernetes Packages 46 | 47 | RUN apt-get update && apt-get install -y curl wget gnupg2 nano 48 | 49 | #add GPG key + apt repo for Kubernetes 50 | RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ 51 | echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" >> /etc/apt/sources.list.d/kubernetes.list && \ 52 | apt-get update 53 | 54 | #install packages and mark as hold 55 | RUN apt-get install -y docker.io kubelet kubeadm kubectl && \ 56 | apt-mark hold docker.io kubelet kubeadm kubectl 57 | 58 | ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ] 59 | -------------------------------------------------------------------------------- /entrypoint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | fix_mount() { 22 | echo 'INFO: ensuring we can execute /bin/mount even with userns-remap' 23 | # necessary only when userns-remap is enabled on the host, but harmless 24 | # The binary /bin/mount should be owned by root and have the setuid bit 25 | chown root:root /bin/mount 26 | chmod -s /bin/mount 27 | 28 | # This is a workaround to an AUFS bug that might cause `Text file 29 | # busy` on `mount` command below. See more details in 30 | # https://github.com/moby/moby/issues/9547 31 | if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then 32 | echo 'INFO: detected aufs, calling sync' 33 | sync 34 | fi 35 | 36 | echo 'INFO: remounting /sys read-only' 37 | # systemd-in-a-container should have read only /sys 38 | # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ 39 | # however, we need other things from `docker run --privileged` ... 40 | # and this flag also happens to make /sys rw, amongst other things 41 | mount -o remount,ro /sys 42 | 43 | echo 'INFO: making mounts shared' 44 | # for mount propagation 45 | mount --make-rshared / 46 | } 47 | 48 | fix_cgroup() { 49 | echo 'INFO: fix cgroup mounts for all subsystems' 50 | # For each cgroup subsystem, Docker does a bind mount from the current 51 | # cgroup to the root of the cgroup subsystem. For instance: 52 | # /sys/fs/cgroup/memory/docker/ -> /sys/fs/cgroup/memory 53 | # 54 | # This will confuse Kubelet and cadvisor and will dump the following error 55 | # messages in kubelet log: 56 | # `summary_sys_containers.go:47] Failed to get system container stats for ".../kubelet.service"` 57 | # 58 | # This is because `/proc//cgroup` is not affected by the bind mount. 59 | # The following is a workaround to recreate the original cgroup 60 | # environment by doing another bind mount for each subsystem. 61 | local docker_cgroup_mounts 62 | docker_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep docker || true) 63 | if [[ -n "${docker_cgroup_mounts}" ]]; then 64 | local docker_cgroup cgroup_subsystems subsystem 65 | docker_cgroup=$(echo "${docker_cgroup_mounts}" | head -n 1 | cut -d' ' -f 4) 66 | cgroup_subsystems=$(echo "${docker_cgroup_mounts}" | cut -d' ' -f 5) 67 | echo "${cgroup_subsystems}" | 68 | while IFS= read -r subsystem; do 69 | mkdir -p "${subsystem}${docker_cgroup}" 70 | mount --bind "${subsystem}" "${subsystem}${docker_cgroup}" 71 | done 72 | fi 73 | } 74 | 75 | fix_machine_id() { 76 | # Deletes the machine-id embedded in the node image and generates a new one. 77 | # This is necessary because both kubelet and other components like weave net 78 | # use machine-id internally to distinguish nodes. 79 | echo 'INFO: clearing and regenerating /etc/machine-id' 80 | rm -f /etc/machine-id 81 | systemd-machine-id-setup 82 | } 83 | 84 | fix_product_name() { 85 | # this is a small fix to hide the underlying hardware and fix issue #426 86 | # https://github.com/kubernetes-sigs/kind/issues/426 87 | if [[ -f /sys/class/dmi/id/product_name ]]; then 88 | echo 'INFO: faking /sys/class/dmi/id/product_name to be "kind"' 89 | mkdir /kind && touch /kind/product_name 90 | echo 'kind' > /kind/product_name 91 | mount -o ro,bind /kind/product_name /sys/class/dmi/id/product_name 92 | fi 93 | } 94 | 95 | fix_product_uuid() { 96 | # The system UUID is usually read from DMI via sysfs, the problem is that 97 | # in the kind case this means that all (container) nodes share the same 98 | # system/product uuid, as they share the same DMI. 99 | # Note: The UUID is read from DMI, this tool is overwriting the sysfs files 100 | # which should fix the attached issue, but this workaround does not address 101 | # the issue if a tool is reading directly from DMI. 102 | # https://github.com/kubernetes-sigs/kind/issues/1027 103 | [[ ! -f /kind/product_uuid ]] && cat /proc/sys/kernel/random/uuid > /kind/product_uuid 104 | if [[ -f /sys/class/dmi/id/product_uuid ]]; then 105 | echo 'INFO: faking /sys/class/dmi/id/product_uuid to be random' 106 | mount -o ro,bind /kind/product_uuid /sys/class/dmi/id/product_uuid 107 | fi 108 | if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then 109 | echo 'INFO: faking /sys/devices/virtual/dmi/id/product_uuid as well' 110 | mount -o ro,bind /kind/product_uuid /sys/devices/virtual/dmi/id/product_uuid 111 | fi 112 | } 113 | 114 | fix_kmsg() { 115 | # In environments where /dev/kmsg is not available, the kubelet (1.15+) won't 116 | # start because it cannot open /dev/kmsg when starting the kmsgparser in the 117 | # OOM parser. 118 | # To support those environments, we link /dev/kmsg to /dev/console. 119 | # https://github.com/kubernetes-sigs/kind/issues/662 120 | if [[ ! -e /dev/kmsg ]]; then 121 | if [[ -e /dev/console ]]; then 122 | echo 'WARN: /dev/kmsg does not exist, symlinking /dev/console' >&2 123 | ln -s /dev/console /dev/kmsg 124 | else 125 | echo 'WARN: /dev/kmsg does not exist, nor does /dev/console!' >&2 126 | fi 127 | fi 128 | } 129 | 130 | configure_proxy() { 131 | # ensure all processes receive the proxy settings by default 132 | # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html 133 | mkdir -p /etc/systemd/system.conf.d/ 134 | cat </etc/systemd/system.conf.d/proxy-default-environment.conf 135 | [Manager] 136 | DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}" 137 | EOF 138 | } 139 | 140 | # run pre-init fixups 141 | fix_kmsg 142 | fix_mount 143 | fix_cgroup 144 | fix_machine_id 145 | fix_product_name 146 | fix_product_uuid 147 | configure_proxy 148 | 149 | # we want the command (expected to be systemd) to be PID1, so exec to it 150 | exec "$@" 151 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Start up our compute 3 | 4 | Since we're building a Kubernetes cluster using docker containers as compute instead of VM's 5 | we'll need to build our base dockerfile. 6 | 7 | ``` 8 | docker-compose build 9 | docker-compose up -d 10 | ``` 11 | 12 | This dockerfile would represent our "ubuntu" VM's. 13 | It has `systemd` installed with workarounds to get `systemd` working inside docker 14 | 15 | 16 | ## Access our master 17 | 18 | ``` 19 | docker exec -it master-a bash 20 | ``` 21 | 22 | 23 | ## Install Kubernetes 24 | 25 | I've already installed all Kubernetes packages in the base image to save time. 26 | See (dockerfile)[./dockerfile] 27 | 28 | When running nodes in docker we don't want to use dockers internal dns
29 | If you do, coredns will get a dns loopback error
30 | 31 | ``` 32 | bash -c 'cat </etc/resolv.conf 33 | nameserver 8.8.8.8 34 | EOF' 35 | 36 | kubeadm init --pod-network-cidr=192.168.0.0/16 --ignore-preflight-errors Swap,SystemVerification 37 | 38 | systemctl stop kubelet.service 39 | echo "failSwapOn: false" >> /var/lib/kubelet/config.yaml 40 | systemctl start kubelet.service 41 | systemctl status kubelet.service 42 | 43 | #redo init - ignore few checks since files already exists and ports are already in use 44 | 45 | kubeadm init --pod-network-cidr=192.168.0.0/16 --ignore-preflight-errors "Swap,SystemVerification,Port-6443,Port-10259,Port-10257,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Port-2379,Port-2380,DirAvailable--var-lib-etcd" 46 | 47 | systemctl stop kubelet.service 48 | echo "failSwapOn: false" >> /var/lib/kubelet/config.yaml 49 | systemctl start kubelet.service 50 | systemctl status kubelet.service 51 | 52 | #in case of failure 53 | journalctl -xeu kubelet 54 | 55 | mkdir -p $HOME/.kube 56 | cp -i /etc/kubernetes/admin.conf $HOME/.kube/config 57 | chown $(id -u):$(id -g) $HOME/.kube/config 58 | 59 | #setup the pod network 60 | kubectl apply -f calico.yaml 61 | 62 | ``` --------------------------------------------------------------------------------