├── .circleci └── config.yml ├── .gitignore ├── .travis.yml ├── AUTHORS.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── OWNERS ├── OWNERS_ALIASES ├── README.md ├── SECURITY_CONTACTS ├── architecture.md ├── build ├── build-all.sh ├── build-local.sh ├── buildconf.sh ├── funcs.sh ├── genfixed.sh ├── genvars.sh ├── portforward.sh ├── release.sh └── update-test-matrix.sh ├── code-of-conduct.md ├── config.sh ├── dind-cluster.sh ├── fixed ├── README.md ├── dind-cluster-v1.12.sh ├── dind-cluster-v1.13.sh ├── dind-cluster-v1.14.sh └── dind-cluster-v1.15.sh ├── gce-setup.sh ├── image ├── Dockerfile ├── bind9 │ ├── Dockerfile │ └── Makefile ├── dind-cluster-rbd ├── dindnet ├── dindnet.service ├── docker.service ├── hypokube.dkr ├── hypokube_base.dkr ├── kubeadm.conf.1.12.tmpl ├── kubeadm.conf.1.13.tmpl ├── kubelet.service ├── node-info ├── rundocker ├── snapshot ├── start_services └── wrapkubeadm ├── test.sh └── test ├── shared.inc.sh ├── test-multiple-clusters.sh └── test-net-connectivity.sh /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Use build/update-build-matrix.sh to update test cases & workflows 2 | # in this file (after "#cut here#" mark below) 3 | 4 | defaults: &defaults 5 | docker: 6 | - image: ubuntu:16.04 7 | working_directory: ~/kubeadm-dind-cluster 8 | 9 | prereqs: &prereqs 10 | name: Install prerequisites 11 | command: | 12 | apt-get -qq update 13 | apt-get install -qqy curl ca-certificates git bzip2 14 | 15 | prereqs_machine: &prereqs_machine 16 | name: Install prerequisites 17 | command: | 18 | sudo apt-get -qq update 19 | sudo apt-get install -qqy curl ca-certificates git 20 | 21 | env: &env 22 | DOCKER_VERSION: "17.03.0-ce" 23 | IMAGE_NAME: "mirantis/kubeadm-dind-cluster" 24 | PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.kubeadm-dind-cluster 25 | # prevent dind-cluster*.sh from trying to pull the image 26 | # during the tests 27 | DIND_SKIP_PULL: y 28 | KUBE_RSYNC_PORT: 8730 29 | STABLE_BRANCH: stable 30 | STABLE_SRC_TAG: "v1.12" 31 | PRIMARY_BRANCH: master 32 | DIND_COMMIT: current 33 | DOWNLOAD_KUBECTL: y 34 | GITHUB_RELEASE_TOOL_URL: "https://github.com/aktau/github-release/releases/download/v0.7.2/linux-amd64-github-release.tar.bz2" 35 | GITHUB_RELEASE_TOOL_SHA256: 3feae868828f57a24d1bb21a5f475de4a116df3063747e3290ad561b4292185c 36 | 37 | setup: &setup 38 | name: Set up the environment 39 | command: | 40 | apt-get -qq update 41 | apt-get install -y curl ca-certificates git liblz4-tool rsync socat tzdata 42 | curl -sSL -o "/tmp/docker-${DOCKER_VERSION}.tgz" "https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz" 43 | tar -xz -C /tmp -f "/tmp/docker-${DOCKER_VERSION}.tgz" 44 | mv /tmp/docker/* /usr/bin 45 | # Start port forwarder 46 | "${PWD}/build/portforward.sh" start 47 | 48 | display_cluster_state: &display_cluster_state 49 | command: | 50 | DIND_ROOT="$PWD" 51 | 52 | kubectl="kubectl" 53 | if [[ ${K8S_SRC:-} ]]; then 54 | cd kubernetes 55 | kubectl="cluster/kubectl.sh" 56 | fi 57 | 58 | apiserver_port="$( "${DIND_ROOT}/dind-cluster.sh" apiserver-port )" 59 | "${DIND_ROOT}/build/portforward.sh" -wait "$apiserver_port" 60 | 61 | "${kubectl}" "--server=:${apiserver_port}" version 62 | "${kubectl}" "--server=:${apiserver_port}" get all --all-namespaces -o wide 63 | "${kubectl}" "--server=:${apiserver_port}" get nodes 64 | 65 | dump_cluster: &dump_cluster 66 | command: | 67 | mkdir -p /tmp/cluster_state 68 | out="/tmp/cluster_state/dump-1.gz" 69 | if [[ -f ${out} ]]; then 70 | out="/tmp/cluster_state/dump-2.gz" 71 | fi 72 | if [[ ${K8S_SRC:-} ]]; then 73 | cd kubernetes 74 | ../dind-cluster.sh dump | gzip >"${out}" 75 | else 76 | ./dind-cluster.sh dump | gzip >"${out}" 77 | fi 78 | 79 | test: &test 80 | steps: 81 | - run: 82 | <<: *prereqs 83 | - checkout 84 | - setup_remote_docker 85 | - run: 86 | <<: *setup 87 | - attach_workspace: 88 | at: _save 89 | - run: 90 | name: Restore images 91 | command: | 92 | docker load -i _save/images.tar 93 | - run: 94 | name: Bring up the cluster 95 | command: | 96 | export DIND_PORT_FORWARDER_WAIT=1 97 | export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh" 98 | ./dind-cluster.sh up 99 | - run: 100 | name: Display cluster status (1) 101 | <<: *display_cluster_state 102 | - run: 103 | name: Dump cluster state (1) 104 | when: always 105 | <<: *dump_cluster 106 | - run: 107 | name: Bring up the cluster (again) 108 | command: | 109 | export DIND_PORT_FORWARDER_WAIT=1 110 | export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh" 111 | ./dind-cluster.sh up 112 | - run: 113 | name: Display cluster status (2) 114 | <<: *display_cluster_state 115 | - run: 116 | name: Dump cluster state (2) 117 | when: always 118 | <<: *dump_cluster 119 | - store_artifacts: 120 | path: /tmp/cluster_state 121 | - run: 122 | name: Bring down the cluster 123 | command: | 124 | ./dind-cluster.sh down 125 | - run: 126 | name: Clean the cluster 127 | command: | 128 | ./dind-cluster.sh clean 129 | 130 | test_src: &test_src 131 | steps: 132 | - run: 133 | <<: *prereqs 134 | - checkout 135 | - setup_remote_docker 136 | - run: 137 | <<: *setup 138 | - attach_workspace: 139 | at: _save 140 | - run: 141 | name: Restore images 142 | command: | 143 | docker load -i _save/images.tar 144 | - run: 145 | name: Check out Kubernetes source 146 | command: | 147 | git clone https://github.com/kubernetes/kubernetes.git 148 | cd kubernetes 149 | if [[ ${K8S_SRC_VERSION:-} ]]; then 150 | git checkout "${K8S_SRC_VERSION}" 151 | fi 152 | echo 153 | echo "*** Latest commit:" 154 | git --no-pager log -1 155 | - run: 156 | name: Bring up the cluster 157 | no_output_timeout: 35m 158 | command: | 159 | export DIND_PORT_FORWARDER_WAIT=1 160 | export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh" 161 | "${PWD}/build/portforward.sh" 8730 162 | cd kubernetes 163 | ../dind-cluster.sh up 164 | - run: 165 | name: Display cluster status (1) 166 | <<: *display_cluster_state 167 | - run: 168 | name: Dump cluster state (1) 169 | when: always 170 | <<: *dump_cluster 171 | - run: 172 | name: Bring up the cluster (again) 173 | command: | 174 | export DIND_PORT_FORWARDER_WAIT=1 175 | export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh" 176 | "${PWD}/build/portforward.sh" 8730 177 | cd kubernetes 178 | ../dind-cluster.sh up 179 | - run: 180 | name: Run some e2e tests 181 | no_output_timeout: 25m 182 | command: | 183 | export DIND_PORT_FORWARDER_WAIT=1 184 | export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh" 185 | "${PWD}/build/portforward.sh" 8730 186 | cd kubernetes 187 | ../dind-cluster.sh e2e "existing RC" 188 | - run: 189 | name: Display cluster status (2) 190 | <<: *display_cluster_state 191 | - run: 192 | name: Dump cluster state (2) 193 | when: always 194 | <<: *dump_cluster 195 | - store_artifacts: 196 | path: /tmp/cluster_state 197 | - run: 198 | name: Bring down the cluster 199 | command: | 200 | cd kubernetes 201 | ../dind-cluster.sh down 202 | - run: 203 | name: Clean the cluster 204 | command: | 205 | cd kubernetes 206 | ../dind-cluster.sh clean 207 | 208 | test_ipv6_only: &test_ipv6_only 209 | machine: true 210 | working_directory: ~/kubeadm-dind-cluster 211 | steps: 212 | - run: 213 | <<: *prereqs_machine 214 | - checkout 215 | - attach_workspace: 216 | at: _save 217 | - run: 218 | name: Restore images 219 | command: | 220 | docker load -i _save/images.tar 221 | - run: 222 | name: Run IPv6 connectivity tests 223 | no_output_timeout: 25m 224 | command: | 225 | "${PWD}/test/test-net-connectivity.sh" 226 | 227 | test_dual_stack: &test_dual_stack 228 | machine: true 229 | working_directory: ~/kubeadm-dind-cluster 230 | steps: 231 | - run: 232 | <<: *prereqs_machine 233 | - checkout 234 | - attach_workspace: 235 | at: _save 236 | - run: 237 | name: Restore images 238 | command: | 239 | docker load -i _save/images.tar 240 | - run: 241 | name: Run dual stack connectivity tests 242 | no_output_timeout: 25m 243 | command: | 244 | export IP_MODE=dual-stack 245 | "${PWD}/test/test-net-connectivity.sh" 246 | 247 | test_ipv4_only: &test_ipv4_only 248 | machine: true 249 | working_directory: ~/kubeadm-dind-cluster 250 | steps: 251 | - run: 252 | <<: *prereqs_machine 253 | - checkout 254 | - attach_workspace: 255 | at: _save 256 | - run: 257 | name: Restore images 258 | command: | 259 | docker load -i _save/images.tar 260 | - run: 261 | name: Run IPv4 connectivity tests 262 | no_output_timeout: 25m 263 | command: | 264 | export IP_MODE=ipv4 265 | "${PWD}/test/test-net-connectivity.sh" 266 | 267 | test_multiple_clusters: &test_multiple_clusters 268 | steps: 269 | - run: 270 | <<: *prereqs 271 | - checkout 272 | - setup_remote_docker 273 | - run: 274 | <<: *setup 275 | - attach_workspace: 276 | at: _save 277 | - run: 278 | name: Restore images 279 | command: | 280 | docker load -i _save/images.tar 281 | - run: 282 | name: Bring up multiple clusters in parallel 283 | no_output_timeout: 35m 284 | command: | 285 | "${PWD}/test/test-multiple-clusters.sh" 286 | 287 | version: 2 288 | jobs: 289 | build: 290 | <<: *defaults 291 | environment: 292 | <<: *env 293 | steps: 294 | - run: 295 | name: "Install prerequisites (FIXME: rm this)" 296 | command: | 297 | apt-get -qq update 298 | apt-get install -y curl ca-certificates git 299 | - run: 300 | <<: *prereqs 301 | - checkout 302 | - setup_remote_docker 303 | - run: 304 | <<: *setup 305 | - run: 306 | name: Build images 307 | command: | 308 | build/build-all.sh _save 309 | - persist_to_workspace: 310 | root: _save 311 | paths: 312 | - images.tar 313 | 314 | # test_src_release: 315 | # <<: *defaults 316 | # environment: 317 | # <<: *env 318 | # BUILD_KUBEADM: y 319 | # BUILD_HYPERKUBE: y 320 | # K8S_SRC: y 321 | # K8S_SRC_VERSION: release-1.15 322 | # <<: *test_src 323 | 324 | # test_src_master: 325 | # <<: *defaults 326 | # environment: 327 | # <<: *env 328 | # BUILD_KUBEADM: y 329 | # BUILD_HYPERKUBE: y 330 | # K8S_SRC: y 331 | # <<: *test_src 332 | 333 | release: 334 | <<: *defaults 335 | environment: 336 | <<: *env 337 | steps: 338 | - run: 339 | <<: *prereqs 340 | - checkout 341 | - setup_remote_docker 342 | - run: 343 | <<: *setup 344 | - attach_workspace: 345 | at: _save 346 | - run: 347 | name: Restore images 348 | command: | 349 | docker load -i _save/images.tar 350 | - run: 351 | name: Install github-release 352 | command: | 353 | curl -sSL -o /tmp/gh-release.tar.bz2 "${GITHUB_RELEASE_TOOL_URL}" 354 | echo "${GITHUB_RELEASE_TOOL_SHA256} /tmp/gh-release.tar.bz2" | sha256sum -c - 355 | tar --strip-components=3 -C /usr/local/bin -xvjf /tmp/gh-release.tar.bz2 356 | - run: 357 | name: Perform the release 358 | command: | 359 | if [[ ! ${CIRCLE_TAG} ]]; then 360 | echo >&2 "*** No CIRCLE_TAG" 361 | exit 1 362 | fi 363 | build/genfixed.sh 364 | build/release.sh "${CIRCLE_TAG}" 365 | 366 | push: 367 | <<: *defaults 368 | environment: 369 | <<: *env 370 | steps: 371 | - run: 372 | <<: *prereqs 373 | - checkout 374 | - setup_remote_docker 375 | - run: 376 | <<: *setup 377 | - attach_workspace: 378 | at: _save 379 | - run: 380 | name: Restore images 381 | command: | 382 | docker load -i _save/images.tar 383 | - run: 384 | name: Push images 385 | command: | 386 | if [[ ! ${DOCKER_USER:-} || ! ${DOCKER_PASS} ]]; then 387 | echo >&2 "*** WARNING: can't push images for outside forks" 388 | exit 0 389 | fi 390 | docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" 391 | docker images \ 392 | --format '{{ .Repository }}:{{ .Tag }}' \ 393 | --filter=label=mirantis.kubeadm_dind_cluster_final | 394 | while read image; do 395 | echo >&2 "*** Pushing: ${image}" 396 | docker push "${image}" 397 | done 398 | 399 | # Don't remove the "#cut here#" marker below. 400 | #cut here# 401 | 402 | test_v1.12_bridge_docker: 403 | <<: *defaults 404 | <<: *test 405 | environment: 406 | <<: *env 407 | DIND_K8S_VERSION: v1.12 408 | CNI_PLUGIN: bridge 409 | DIND_CRI: docker 410 | 411 | test_v1.12_bridge_containerd: 412 | <<: *defaults 413 | <<: *test 414 | environment: 415 | <<: *env 416 | DIND_K8S_VERSION: v1.12 417 | CNI_PLUGIN: bridge 418 | DIND_CRI: containerd 419 | 420 | test_v1.13_bridge_docker: 421 | <<: *defaults 422 | <<: *test 423 | environment: 424 | <<: *env 425 | DIND_K8S_VERSION: v1.13 426 | CNI_PLUGIN: bridge 427 | DIND_CRI: docker 428 | 429 | test_v1.13_bridge_containerd: 430 | <<: *defaults 431 | <<: *test 432 | environment: 433 | <<: *env 434 | DIND_K8S_VERSION: v1.13 435 | CNI_PLUGIN: bridge 436 | DIND_CRI: containerd 437 | 438 | test_v1.14_bridge_docker: 439 | <<: *defaults 440 | <<: *test 441 | environment: 442 | <<: *env 443 | DIND_K8S_VERSION: v1.14 444 | CNI_PLUGIN: bridge 445 | DIND_CRI: docker 446 | 447 | test_v1.14_bridge_containerd: 448 | <<: *defaults 449 | <<: *test 450 | environment: 451 | <<: *env 452 | DIND_K8S_VERSION: v1.14 453 | CNI_PLUGIN: bridge 454 | DIND_CRI: containerd 455 | 456 | test_v1.15_bridge_docker: 457 | <<: *defaults 458 | <<: *test 459 | environment: 460 | <<: *env 461 | DIND_K8S_VERSION: v1.15 462 | CNI_PLUGIN: bridge 463 | DIND_CRI: docker 464 | 465 | test_v1.15_flannel_docker: 466 | <<: *defaults 467 | <<: *test 468 | environment: 469 | <<: *env 470 | DIND_K8S_VERSION: v1.15 471 | CNI_PLUGIN: flannel 472 | DIND_CRI: docker 473 | 474 | test_v1.15_calico_docker: 475 | <<: *defaults 476 | <<: *test 477 | environment: 478 | <<: *env 479 | DIND_K8S_VERSION: v1.15 480 | CNI_PLUGIN: calico 481 | DIND_CRI: docker 482 | 483 | test_v1.15_calico-kdd_docker: 484 | <<: *defaults 485 | <<: *test 486 | environment: 487 | <<: *env 488 | DIND_K8S_VERSION: v1.15 489 | CNI_PLUGIN: calico-kdd 490 | DIND_CRI: docker 491 | 492 | test_v1.15_weave_docker: 493 | <<: *defaults 494 | <<: *test 495 | environment: 496 | <<: *env 497 | DIND_K8S_VERSION: v1.15 498 | CNI_PLUGIN: weave 499 | DIND_CRI: docker 500 | 501 | test_v1.15_bridge_containerd: 502 | <<: *defaults 503 | <<: *test 504 | environment: 505 | <<: *env 506 | DIND_K8S_VERSION: v1.15 507 | CNI_PLUGIN: bridge 508 | DIND_CRI: containerd 509 | 510 | test_v1.15_ptp_docker: 511 | <<: *defaults 512 | <<: *test 513 | environment: 514 | <<: *env 515 | DIND_K8S_VERSION: v1.15 516 | CNI_PLUGIN: ptp 517 | DIND_CRI: docker 518 | 519 | test_multiple_clusters_v1.15_bridge_docker: 520 | <<: *defaults 521 | <<: *test_multiple_clusters 522 | environment: 523 | <<: *env 524 | DIND_K8S_VERSION: v1.15 525 | CNI_PLUGIN: bridge 526 | DIND_CRI: docker 527 | 528 | test_ipv4_only_v1.15_bridge_docker: 529 | <<: *test_ipv4_only 530 | environment: 531 | <<: *env 532 | DIND_K8S_VERSION: v1.15 533 | CNI_PLUGIN: bridge 534 | DIND_CRI: docker 535 | 536 | test_ipv6_only_v1.15_bridge_docker: 537 | <<: *test_ipv6_only 538 | environment: 539 | <<: *env 540 | DIND_K8S_VERSION: v1.15 541 | CNI_PLUGIN: bridge 542 | DIND_CRI: docker 543 | 544 | test_ipv6_only_nat64_v1.15_bridge_docker: 545 | <<: *test_ipv6_only 546 | environment: 547 | <<: *env 548 | DIND_K8S_VERSION: v1.15 549 | CNI_PLUGIN: bridge 550 | DIND_CRI: docker 551 | NAT64_V4_SUBNET_PREFIX: '10.100' 552 | 553 | test_ipv6_only_ext_aaaa_v1.15_bridge_docker: 554 | <<: *test_ipv6_only 555 | environment: 556 | <<: *env 557 | DIND_K8S_VERSION: v1.15 558 | CNI_PLUGIN: bridge 559 | DIND_CRI: docker 560 | DIND_ALLOW_AAAA_USE: 'true' 561 | 562 | test_dual_stack_v4_svc_net_v1.15_bridge_docker: 563 | <<: *test_dual_stack 564 | environment: 565 | <<: *env 566 | DIND_K8S_VERSION: v1.15 567 | CNI_PLUGIN: bridge 568 | DIND_CRI: docker 569 | SERVICE_CIDR: '10.96.0.0/12' 570 | DIND_ALLOW_AAAA_USE: 'true' 571 | 572 | workflows: 573 | version: 2 574 | build-test-push: 575 | jobs: 576 | - build: 577 | filters: 578 | tags: 579 | only: 580 | - /^v[0-9].*/ 581 | - push: 582 | filters: 583 | tags: 584 | only: 585 | - /^v[0-9].*/ 586 | requires: 587 | - build 588 | - test_v1.12_bridge_docker: 589 | filters: 590 | tags: 591 | only: 592 | - /^v[0-9].*/ 593 | requires: 594 | - build 595 | - test_v1.12_bridge_containerd: 596 | filters: 597 | tags: 598 | only: 599 | - /^v[0-9].*/ 600 | requires: 601 | - build 602 | - test_v1.13_bridge_docker: 603 | filters: 604 | tags: 605 | only: 606 | - /^v[0-9].*/ 607 | requires: 608 | - build 609 | - test_v1.13_bridge_containerd: 610 | filters: 611 | tags: 612 | only: 613 | - /^v[0-9].*/ 614 | requires: 615 | - build 616 | - test_v1.14_bridge_docker: 617 | filters: 618 | tags: 619 | only: 620 | - /^v[0-9].*/ 621 | requires: 622 | - build 623 | - test_v1.14_bridge_containerd: 624 | filters: 625 | tags: 626 | only: 627 | - /^v[0-9].*/ 628 | requires: 629 | - build 630 | 631 | - test_v1.15_bridge_docker: 632 | filters: 633 | tags: 634 | only: 635 | - /^v[0-9].*/ 636 | requires: 637 | - build 638 | - test_v1.15_flannel_docker: 639 | filters: 640 | tags: 641 | only: 642 | - /^v[0-9].*/ 643 | requires: 644 | - build 645 | - test_v1.15_calico_docker: 646 | filters: 647 | tags: 648 | only: 649 | - /^v[0-9].*/ 650 | requires: 651 | - build 652 | - test_v1.15_calico-kdd_docker: 653 | filters: 654 | tags: 655 | only: 656 | - /^v[0-9].*/ 657 | requires: 658 | - build 659 | - test_v1.15_weave_docker: 660 | filters: 661 | tags: 662 | only: 663 | - /^v[0-9].*/ 664 | requires: 665 | - build 666 | - test_v1.15_bridge_containerd: 667 | filters: 668 | tags: 669 | only: 670 | - /^v[0-9].*/ 671 | requires: 672 | - build 673 | - test_v1.15_ptp_docker: 674 | filters: 675 | tags: 676 | only: 677 | - /^v[0-9].*/ 678 | requires: 679 | - build 680 | - test_multiple_clusters_v1.15_bridge_docker: 681 | filters: 682 | tags: 683 | only: 684 | - /^v[0-9].*/ 685 | requires: 686 | - build 687 | - test_ipv4_only_v1.15_bridge_docker: 688 | filters: 689 | tags: 690 | only: 691 | - /^v[0-9].*/ 692 | requires: 693 | - build 694 | - test_ipv6_only_v1.15_bridge_docker: 695 | filters: 696 | tags: 697 | only: 698 | - /^v[0-9].*/ 699 | requires: 700 | - build 701 | - test_ipv6_only_nat64_v1.15_bridge_docker: 702 | filters: 703 | tags: 704 | only: 705 | - /^v[0-9].*/ 706 | requires: 707 | - build 708 | - test_ipv6_only_ext_aaaa_v1.15_bridge_docker: 709 | filters: 710 | tags: 711 | only: 712 | - /^v[0-9].*/ 713 | requires: 714 | - build 715 | - test_dual_stack_v4_svc_net_v1.15_bridge_docker: 716 | filters: 717 | tags: 718 | only: 719 | - /^v[0-9].*/ 720 | requires: 721 | - build 722 | # - test_src_release: 723 | # filters: 724 | # tags: 725 | # only: 726 | # - /^v[0-9].*/ 727 | # requires: 728 | # - build 729 | # - test_src_master: 730 | # filters: 731 | # tags: 732 | # only: 733 | # - /^v[0-9].*/ 734 | # requires: 735 | # - build 736 | - release: 737 | filters: 738 | branches: 739 | ignore: /.*/ 740 | tags: 741 | only: 742 | - /^v[0-9].*/ 743 | requires: 744 | - push 745 | - test_v1.12_bridge_docker 746 | - test_v1.12_bridge_containerd 747 | - test_v1.13_bridge_docker 748 | - test_v1.13_bridge_containerd 749 | - test_v1.14_bridge_docker 750 | - test_v1.14_bridge_containerd 751 | - test_v1.15_bridge_docker 752 | - test_v1.15_flannel_docker 753 | - test_v1.15_calico_docker 754 | - test_v1.15_calico-kdd_docker 755 | - test_v1.15_weave_docker 756 | - test_v1.15_bridge_containerd 757 | - test_v1.15_ptp_docker 758 | - test_multiple_clusters_v1.15_bridge_docker 759 | - test_ipv4_only_v1.15_bridge_docker 760 | - test_ipv6_only_v1.15_bridge_docker 761 | - test_ipv6_only_nat64_v1.15_bridge_docker 762 | - test_ipv6_only_ext_aaaa_v1.15_bridge_docker 763 | - test_dual_stack_v4_svc_net_v1.15_bridge_docker 764 | # - test_src_release 765 | # - test_src_master 766 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /save.tar.lz4 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | services: docker 3 | language: minimal 4 | 5 | env: 6 | - TEST_CASE=1.12 7 | - TEST_CASE=1.13 8 | - TEST_CASE=1.14 9 | - TEST_CASE=1.15 10 | 11 | before_install: 12 | - sudo apt-get -qq update 13 | - sudo apt-get install -y liblz4-tool 14 | 15 | script: 16 | # building k8s may cause Travis to time out w/o output 17 | - if [ -n "${TWAIT:-}" ]; then travis_wait 35 ./test.sh; else ./test.sh; fi 18 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | Authors 2 | ------- 3 | 4 | A number of persons have contributed to predecessors of this work in one way or another: 5 | 6 | - Ivan Shvedunov 7 | - Ilya Dmitrichenko 8 | - Maru Newby 9 | - Karl Isenberg 10 | - Dr. Stefan Schimanski 11 | - the authors of wrapdocker at https://github.com/jpetazzo/dind 12 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Read the following guide if you're interested in contributing to kubeadm-dind-cluster. 4 | 5 | ## Contributor License Agreements 6 | 7 | We'd love to accept your patches! Before we can take them, we have to jump a couple of legal hurdles. 8 | 9 | Please fill out either the individual or corporate Contributor License Agreement (CLA). More information about the CLA and instructions for signing it [can be found here](https://github.com/kubernetes/community/blob/master/CLA.md). 10 | 11 | ***NOTE***: Only original source code from you and other people that have signed the CLA can be accepted into the repository. 12 | 13 | ## Finding Things That Need Help 14 | 15 | If you're new to the project and want to help, but don't know where to start, we have a semi-curated list of issues that should not need deep knowledge of the system. [Have a look and see if anything sounds interesting](https://github.com/kubernetes-sigs/kubeadm-dind-cluster/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22). Alternatively, read some of the docs on other controllers and try to write your own, file and fix any/all issues that come up, including gaps in documentation! 16 | 17 | ## Contributing a Patch 18 | 19 | 1. If you haven't already done so, sign a Contributor License Agreement (see details above). 20 | 1. Fork the desired repo, develop and test your code changes. 21 | 1. Submit a pull request. 22 | 23 | All changes must be code reviewed. Coding conventions and standards are explained in the official [developer docs](https://github.com/kubernetes/community/tree/master/contributors/devel). Expect reviewers to request that you avoid common [go style mistakes](https://github.com/golang/go/wiki/CodeReviewComments) in your PRs. 24 | 25 | ### Merge Approval 26 | 27 | kubeadm-dind-cluster maintainers may add "LGTM" (Looks Good To Me) or an equivalent comment to indicate that a PR is acceptable. Any change requires at least one LGTM. No pull requests can be merged until at least one kubeadm-dind-cluster maintainer signs off with an LGTM. 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mirantis/kubeadm-dind-cluster:bare-v4 2 | 3 | LABEL mirantis.kubeadm_dind_cluster_final=1 4 | 5 | # the following args should be set for derived images 6 | 7 | #Todo: download prebuilt hyperkube (not kubectl!) & kubeadm into /k8s 8 | ARG PREBUILT_KUBEADM_AND_HYPERKUBE 9 | ARG KUBEADM_URL 10 | ARG KUBEADM_SHA1 11 | ARG HYPERKUBE_URL 12 | ARG HYPERKUBE_SHA1 13 | ARG KUBECTL_VERSION 14 | ARG KUBECTL_LINUX_URL 15 | ARG KUBECTL_LINUX_SHA1 16 | ARG KUBECTL_DARWIN_URL 17 | ARG KUBECTL_DARWIN_SHA1 18 | 19 | RUN if [ -n "${KUBEADM_URL}" ]; then \ 20 | mkdir -p /k8s && \ 21 | curl -sSL "${KUBEADM_URL}" > /k8s/kubeadm && \ 22 | if [ -n "${KUBEADM_SHA1}" ]; then echo "${KUBEADM_SHA1} /k8s/kubeadm" | sha1sum -c; fi && \ 23 | chmod +x /k8s/kubeadm; \ 24 | fi; \ 25 | if [ -n "${HYPERKUBE_URL}" ]; then \ 26 | curl -sSL "${HYPERKUBE_URL}" > /k8s/hyperkube && \ 27 | if [ -n "${HYPERKUBE_SHA1}" ]; then echo "${HYPERKUBE_SHA1} /k8s/hyperkube" | sha1sum -c; fi && \ 28 | chmod +x /k8s/hyperkube; \ 29 | fi && \ 30 | ( echo "export KUBECTL_VERSION=${KUBECTL_VERSION}" && \ 31 | echo "export KUBECTL_LINUX_SHA1=${KUBECTL_LINUX_SHA1}" && \ 32 | echo "export KUBECTL_LINUX_URL=${KUBECTL_LINUX_URL}" && \ 33 | echo "export KUBECTL_DARWIN_SHA1=${KUBECTL_DARWIN_SHA1}" && \ 34 | echo "export KUBECTL_DARWIN_URL=${KUBECTL_DARWIN_URL}" ) >/dind-env 35 | COPY save.tar.lz4 / 36 | 37 | ENTRYPOINT ["/sbin/dind_init"] 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md 2 | 3 | approvers: 4 | - sig-cluster-lifecycle-leads 5 | - kubeadm-dind-cluster-maintainers 6 | reviewers: 7 | - sig-cluster-lifecycle-leads 8 | - kubeadm-dind-cluster-maintainers 9 | -------------------------------------------------------------------------------- /OWNERS_ALIASES: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md 2 | 3 | aliases: 4 | sig-cluster-lifecycle-leads: 5 | - lukemarsden 6 | - luxas 7 | - roberthbailey 8 | - timothysc 9 | kubeadm-dind-cluster-maintainers: 10 | - pigmej 11 | - ivan4th 12 | - lukaszo 13 | - jellonek 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubeadm-dind-cluster [![CircleCI](https://circleci.com/gh/kubernetes-sigs/kubeadm-dind-cluster/tree/master.svg?style=svg)](https://circleci.com/gh/kubernetes-sigs/kubeadm-dind-cluster/tree/master) [![Travis CI](https://travis-ci.org/kubernetes-sigs/kubeadm-dind-cluster.svg?branch=master)](https://travis-ci.org/kubernetes-sigs/kubeadm-dind-cluster) 2 | 3 | **NOTE: This project is deprecated in favor of [kind](https://kind.sigs.k8s.io/). 4 | Try [kind](https://kind.sigs.k8s.io/) today, it's great!** 5 | 6 | A Kubernetes multi-node cluster for developer _of_ Kubernetes and 7 | projects that extend Kubernetes. Based on kubeadm and DIND (Docker in 8 | Docker). 9 | 10 | Supports both local workflows and workflows utilizing powerful remote 11 | machines/cloud instances for building Kubernetes, starting test 12 | clusters and running e2e tests. 13 | 14 | If you're an application developer, you may be better off with 15 | [Minikube](https://github.com/kubernetes/minikube) because it's more 16 | mature and less dependent on the local environment, but if you're 17 | feeling adventurous you may give `kubeadm-dind-cluster` a try, too. In 18 | particular you can run `kubeadm-dind-cluster` in CI environment such 19 | as Travis without having issues with nested virtualization. 20 | 21 | ## Requirements 22 | Docker 1.12+ is recommended. If you're not using one of the 23 | preconfigured scripts (see below) and not building from source, it's 24 | better to have `kubectl` executable in your path matching the 25 | version of k8s binaries you're using (i.e. for example better don't 26 | use `kubectl` 1.13.x with `hyperkube` 1.12.x). As an alternative, 27 | you can set `DOWNLOAD_KUBECTL` to a non-empty string in your 28 | config.sh so `kubeadm-dind-cluster` will download it for you. 29 | 30 | `kubeadm-dind-cluster` supports k8s versions 1.12.x through 1.15.x. 31 | 32 | **As of now, running `kubeadm-dind-cluster` on Docker with `btrfs` 33 | storage driver is not supported.** 34 | 35 | The problems include inability to properly clean up DIND volumes due 36 | to a [docker bug](https://github.com/docker/docker/issues/9939) which 37 | is not really fixed and, more importantly, a 38 | [kubelet problem](https://github.com/kubernetes/kubernetes/issues/38337). 39 | If you want to run `kubeadm-dind-cluster` on btrfs anyway, set 40 | `RUN_ON_BTRFS_ANYWAY` environment variable to a non-empty value. 41 | 42 | By default `kubeadm-dind-cluster` uses dockerized builds, so no Go 43 | installation is necessary even if you're building Kubernetes from 44 | source. If you want you can overridde this behavior by setting 45 | `KUBEADM_DIND_LOCAL` to a non-empty value in [config.sh](config.sh). 46 | 47 | ### Mac OS X considerations 48 | 49 | When building Kubernetes from source on Mac OS X, it should be 50 | possible to build `kubectl` locally, i.e. `make WHAT=cmd/kubectl` must 51 | work. 52 | 53 | NOTE: Docker on Mac OS X, at the time of this writing, does not support 54 | IPv6 and thus clusters cannot be formed using IPv6 addresses. 55 | 56 | ## Using preconfigured scripts 57 | `kubeadm-dind-cluster` currently provides preconfigured scripts for 58 | Kubernetes versions 1.12 through 1.15 published as part of GitHub 59 | releases. Each preconfigured script is pinned to the corresponding 60 | image tag and SHA256 digest, so it will not be broken by changes 61 | in kubeadm-dind-cluster master branch. 62 | 63 | The preconfigured scripts are convenient for use with projects that 64 | extend or use Kubernetes. For example, you can start Kubernetes 1.14 65 | like this: 66 | 67 | ```shell 68 | $ wget -O dind-cluster.sh https://github.com/kubernetes-sigs/kubeadm-dind-cluster/releases/download/v0.2.0/dind-cluster-v1.14.sh 69 | $ chmod +x dind-cluster.sh 70 | 71 | $ # start the cluster 72 | $ ./dind-cluster.sh up 73 | 74 | $ # add kubectl directory to PATH 75 | $ export PATH="$HOME/.kubeadm-dind-cluster:$PATH" 76 | 77 | $ kubectl get nodes 78 | NAME STATUS ROLES AGE VERSION 79 | kube-master Ready master 4m v1.14.0 80 | kube-node-1 Ready 2m v1.14.0 81 | kube-node-2 Ready 2m v1.14.0 82 | 83 | $ # k8s dashboard available at http://localhost:/api/v1/namespaces/kube-system/services/kubernetes-dashboard:/proxy. See your console for the URL. 84 | 85 | $ # restart the cluster, this should happen much quicker than initial startup 86 | $ ./dind-cluster.sh up 87 | 88 | $ # stop the cluster 89 | $ ./dind-cluster.sh down 90 | 91 | $ # remove DIND containers and volumes 92 | $ ./dind-cluster.sh clean 93 | ``` 94 | 95 | Replace 1.14 with 1.13 or 1.12 to use older Kubernetes versions. 96 | **Important note:** you need to do `./dind-cluster.sh clean` when 97 | you switch between Kubernetes versions (but no need to do this between 98 | rebuilds if you use `BUILD_HYPERKUBE=y` like described below). 99 | 100 | ## Using with Kubernetes source 101 | ```shell 102 | $ git clone https://github.com/kubernetes-sigs/kubeadm-dind-cluster.git ~/dind 103 | 104 | $ cd ~/work/kubernetes/src/k8s.io/kubernetes 105 | 106 | $ export BUILD_KUBEADM=y 107 | $ export BUILD_HYPERKUBE=y 108 | 109 | $ # build binaries+images and start the cluster 110 | $ ~/dind/dind-cluster.sh up 111 | 112 | $ kubectl get nodes 113 | NAME STATUS AGE 114 | kube-master Ready,master 1m 115 | kube-node-1 Ready 34s 116 | kube-node-2 Ready 34s 117 | 118 | $ # k8s dashboard available at http://localhost:8080/ui 119 | 120 | $ # run conformance tests 121 | $ ~/dind/dind-cluster.sh e2e 122 | 123 | $ # restart the cluster rebuilding 124 | $ ~/dind/dind-cluster.sh up 125 | 126 | $ # run particular e2e test based on substring 127 | $ ~/dind/dind-cluster.sh e2e "existing RC" 128 | 129 | $ # shut down the cluster 130 | $ ~/dind/dind-cluster.sh down 131 | ``` 132 | 133 | The first `dind/dind-cluster.sh up` invocation can be slow because it 134 | needs to build the base image and Kubernetes binaries. Subsequent 135 | invocations are much faster. 136 | 137 | ## Controlling network usage 138 | Kubeadm-dind-cluster uses several networks for operation, and allows 139 | the user to customize the networks used. Check your network assignments 140 | for your setup, and adjust things, if there are conflicts. This section 141 | will describe how to adjust settings. 142 | 143 | NOTE: Docker will define networks for bridges, which kubeadm-dind-cluster 144 | tries to avoid by default, but based on your setup, you may need to choose 145 | different subnets. Typically, docker uses 172.17.0.0/16, 172.18.0.0/16,... 146 | 147 | ### Management network 148 | For the management network, the user can set MGMT_CIDRS to a string 149 | representing the CIDR to use for the network. This is used in conjunction 150 | with the CLUSTER_ID, when creating multple clusters. If single cluster, 151 | the cluster ID will be zero. 152 | 153 | For IPv4, this must be a /24 and the third octet is reserved for the 154 | multi-cluster number (0 when in single-cluster mode). For example, 155 | use 10.192.0.0/24, for an IPv4 cluster that will have nodes 10.192.0.2, 156 | 10.192.0.3, etc. A cluster with ID "5" would have nodes 10.192.5.2, 157 | 10.192.5.3, etc. 158 | 159 | For IPv6, the CIDR must have room for a hextet to be reserved for the 160 | multi-cluster number. For example, fd00:10:20::/64 would be for an IPv6 161 | cluster with ID "10" (considered in hex) with nodes fd00:10:20:10::2, 162 | fd00:10:20:10::3, etc. If the cluster ID was "0" (single cluster mode), 163 | the nodes would be fd00:10:20:0::2, fd00:10:20:0::3, etc. 164 | 165 | The defaults are 10.192.0.0/24 for IPv4, and fd00:20::/64 for IPv6. 166 | 167 | For dual-stack mode, a comma separated list with IPv4 and IPv6 can be 168 | specified. Any omitted CIDR will use the default value above, based on 169 | the IP mode. 170 | 171 | ### Service network 172 | The service network CIDR, can be specified by SERVICE_CIDR. For IPv4, the 173 | default is 10.96.0.0/12. For IPv6, the default is fd00:30::/110. 174 | 175 | ### Pod network 176 | For the pod network the POD_NETWORK_CIDR environment variable can be set 177 | to specify the pod sub-networks. One subnet will be created for each node 178 | in the cluster. 179 | 180 | For IPv4, the value must be a /16, of which this will be split into multiple 181 | /24 subnets. The master node will set the third octet to 2, and the minion 182 | nodes will set the third octet to 3+. For example, with 10.244.0.0/16, pods 183 | on the master node will be 10.244.2.X, on minion kube-node-1 will be 10.244.3.Y, 184 | on minion kube-node-2 will be 10.244.4.Z, etc. 185 | 186 | For IPv6, the CIDR will again be split into subnets, eigth bits smaller. For 187 | example, with fd00:10:20:30::/72, the master node would have a CIDR of 188 | fd00:10:20:30:2::/80 with pods fd00:10:20:30:2::X. If the POD_NETWORK_CIDR, 189 | instead was fd00:10:20:30::/64, the master node woudl have a CIDR of 190 | fd00:10:20:30:0200::/72, and pods would be fd00:10:20:30:0200::X. 191 | 192 | The defaults are 10.244.0.0/16 for IPv4, and fd00:40::/72 for IPv6. 193 | 194 | For dual-stack mode, a comma separated list with IPv4 and IPv6 CIDR can be 195 | specified. Any omitted CIDR will use the default value above, based on the 196 | IP mode. 197 | 198 | ## Kube-router 199 | Instead of using kube-proxy and static routes (with bridge CNI plugin), 200 | kube-router can be used. Kube-router uses the bridge plugin, but uses 201 | IPVS kernel module, instead of iptables. This results in better performance 202 | and scalability. Kube-router also uses iBGP, so that static routes are not 203 | required for pods to communicate across nodes. 204 | 205 | To use kube-router, set the CNI_PLUGIN environment variable to "kube-router". 206 | 207 | NOTE: Currently pinning kube-router to v0.2.0, because of issue seen with 208 | cleanup, when using newer (latest) kube-router. 209 | 210 | NOTE: This has only been tested with Kubernetes 1.11, and currently fails 211 | when using Kuberentes 1.12+ 212 | 213 | ## IPv6 Mode 214 | To run Kubernetes in IPv6 only mode, set the environment variable IP_MODE 215 | to "ipv6". There are additional customizations that you can make for IPv6, 216 | to set the prefix used for DNS64, subnet prefix to use for DinD, and 217 | the service subnet CIDR (among other settings - see dind-cluster.sh): 218 | 219 | ```shell 220 | export EMBBEDDED_CONFIG=y 221 | export DNS64_PREFIX=fd00:77:64:ff9b:: 222 | export DIND_SUBNET=fd00:77:: 223 | export SERVICE_CIDR=fd00:77:30::/110 224 | export NAT64_V4_SUBNET_PREFIX=172.20 225 | ``` 226 | 227 | NOTE: The DNS64 and NAT64 containers that are created on the host, persist 228 | beyond the `down` operation. This is to reduce startup time, if doing multiple 229 | down/up cycles. When `clean` is done, these containers are removed. 230 | 231 | NOTE: In multi-cluster, there will be DNS and NAT64 containers for each cluster, 232 | with thier names including the cluster suffix (e.g. bind9-cluster-50). 233 | 234 | NOTE: At this time, there is no isolation between clusters. Nodes on one cluster 235 | can ping nodes on another cluster (appears to be isolation iptables rules, instead 236 | of ip6tables rules). 237 | 238 | NOTE: The IPv4 mapping subnet used by NAT64, can be overridden from the default of 239 | 172.18.0.0/16, by specifying the first two octets in NAT64_V4_SUBNET_PREFIX (you 240 | cannot change the size). This prefix must be within the 10.0.0.0/8 or 172.16.0.0/12 241 | private network ranges. Be aware, that, in a multi-cluster setup, the cluster ID, 242 | which defaults to zero, will be added to the second octet of the prefix. You must 243 | ensure that the resulting prefix is still within the private network's range. For 244 | example, if CLUSTER_ID="10", the default NAT64_V4_SUBNET_PREFIX will be 245 | "172.28", forming a subnet 172.28.0.0/16. 246 | 247 | NOTE: If you use `kube-router` for networking, IPv6 is not supported, as of 248 | July 2018. 249 | 250 | ## Configuration 251 | You may edit `config.sh` to override default settings. See comments in 252 | [the file](config.sh) for more info. In particular, you can specify 253 | CNI plugin to use via `CNI_PLUGIN` variable (`bridge`, `ptp`, 254 | `flannel`, `calico`, `calico-kdd`, `weave`, `kube-router`). 255 | 256 | You can also edit the version appropriate kubeadm.conf.#.##.tmpl file 257 | in the image/ directory, to customize how KubeAdm works. This will require 258 | that you build a new image using build/build-local.sh and then setting this 259 | environment variable: 260 | 261 | ``` 262 | export DIND_IMAGE=mirantis/kubeadm-dind-cluster:local 263 | ``` 264 | 265 | Note: the DIND_IMAGE environment variable will work only with `./dind-cluster.sh` script. 266 | It will not work with preconfigured scripts. 267 | 268 | Just keep in mind, there are some parameters in double curly-brackets that 269 | are used to substitue settings, based on other dind-cluster.sh config settings. 270 | 271 | ## Remote Docker / GCE 272 | It's possible to build Kubernetes on a remote machine running Docker. 273 | kubeadm-dind-cluster can consume binaries directly from the build 274 | data container without copying them back to developer's machine. 275 | An example utilizing GCE instance is provided in [gce-setup.sh](gce-setup.sh). 276 | You may try running it using `source` (`.`) so that docker-machine 277 | shell environment is preserved, e.g. 278 | ```shell 279 | . gce-setup.sh 280 | ``` 281 | The example is based on sample commands from 282 | [build/README.md](https://github.com/kubernetes/kubernetes/blob/master/build/README.md#really-remote-docker-engine) 283 | in Kubernetes source. 284 | 285 | When using a remote machine, you need to use ssh port forwarding 286 | to forward `KUBE_RSYNC_PORT` and `APISERVER_PORT`. 287 | 288 | If you do not explicitly set `APISERVER_PORT`, that port will be randomized. To 289 | help with that `./dind-cluster.sh` will call a user-defined executable as soon 290 | as the port is allocated and the kubectl context is set up. For that to happen 291 | you need to set `DIND_PORT_FORWARDER` to a path to an executable, which will be 292 | called with the allocated port as a first argument. If you keep 293 | `DIND_PORT_FORWARDER` empty, that mechanism will not kick in. 294 | 295 | ## Dumping cluster state 296 | 297 | In case of CI environment such as Travis CI or Circle CI, it's often 298 | desirable to get detailed cluster state for a failed job. Moreover, in case 299 | of e.g. Travis CI there's no way to store the artefacts without using 300 | an external service such as Amazon S3. Because of this, 301 | kubeadm-dind-cluster supports dumping cluster state as a text block 302 | that can be later split into individual files. For cases where there 303 | are limits on the log size (e.g. 4 Mb log limit in Travis CI) it's also 304 | possible to dump the lzma-compressed text block using base64 encoding. 305 | 306 | The following commands can be used to work with cluster state dumps: 307 | * `./dind-cluster.sh dump` dumps the cluster state as a text block 308 | * `./dind-cluster.sh dump64` dumps the cluster state as a base64 blob 309 | * `./dind-cluster.sh split-dump` splits the text block into individual 310 | files using `@@@ filename @@@` markers which are generated by 311 | `dump`. The output is stored in `cluster-dump/` subdirectory of the 312 | current directory. 313 | * `./dind-cluster.sh split-dump64` splits the base64 blob into 314 | separate files. The blob has start and end markers so it can be 315 | extracted automatically from a build job log. The output is stored 316 | in `cluster-dump/` subdirectory of the current directory. 317 | 318 | All of the above commands work with 'fixed' scripts, too. 319 | kubeadm-dind-cluster's own Travis CI jobs dump base64 blobs in case of 320 | failure. Such blocks can be then extracted directly from the output of 321 | `travis` command line utility, e.g. 322 | 323 | ```shell 324 | travis logs NNN.N | ./dind-cluster.sh split-dump64 325 | ``` 326 | 327 | The following information is currently stored in the dump: 328 | * status and logs for the following systemd units on each DIND node, if the exist: 329 | `kubelet.service`, `dindnet.service`, `criproxy.service` and 330 | `dockershim.service` (the latter two are used by [CRI Proxy](https://github.com/Mirantis/criproxy)) 331 | * `ps auxww`, `docker ps -a`, `ip a` and `ip r` output for each DIND node 332 | * the logs of all the containers of each pod in the cluster 333 | * the output of `kubectl get all --all-namespaces -o wide`, 334 | `kubectl describe all --all-namespaces` and `kubectl get nodes -o wide` 335 | 336 | ## Running multiple clusters in parallel 337 | 338 | `dind-cluster.sh` can be used to create and manage multiple dind clusters. 339 | 340 | Normally, default names will be used for docker resources and the kubectl context. 341 | For example, `kube-master` (container name), `kubeadm-dind-kube-master` (volume name), 342 | `dind` (context name), etc. Likewise, the management, pod, and service IPs will use 343 | the defaults or user specified values (via environment variables). This would occur 344 | when CLUSTER_ID is not set, or set to "0". 345 | 346 | For each additional cluster, the user can set a unique CLUSTER_ID to a string that 347 | represents a number from 1..254. The number will be used on all management network IP 348 | addresses. 349 | 350 | For IPv4, the cluster ID will be used as the third octet of the management address 351 | (whether default or user specified). For example, with cluster ID "10", the default 352 | management network CIDR will be 10.192.10.0/24. For IPv6, the cluster ID will be 353 | placed as the hextet before the double colon, for the management CIDR. For example, 354 | a management ntwork CIDR of fd00:20::/64 will become fd00:20:2::/64, for a cluster 355 | ID of '2'. 356 | 357 | NOTE: The cluster ID can be limited in some cases. For IPv6 mode, the cluster ID is 358 | also used in the NAT64 prefix, and that prifix must be within one of the RFC-1918 359 | private network ranges. If the 172.16.0.0/12 private network is used, the cluster ID 360 | cannot be more than 15 (and less, if a higher base prefix is specified by the 361 | NAT64_V4_SUBNET_PREFIX, like the default 172.18, which would allow cluster IDs up to 362 | 13). 363 | 364 | Note: If the MGMT_CIDR (or legacy DIND_SUBNET/DIND_SUBNET_SIZE) environment variables 365 | are set for the management network, they must be able to accommodate the cluster ID 366 | injection. 367 | 368 | In addition to the management network, the resource names will have the suffix 369 | "-cluster-#", where # is the CLUSTER_ID. The context for kubectl will be "dind-cluster-#". 370 | 371 | For legacy support (or if a user wants a custom cluster name), setting the DIND_LABEL 372 | will create a resource suffix "-{DIND_LABEL}-#", where # is the cluster ID. If no 373 | cluster ID is specified, as would be for backwards-compatibility, or it is zero, the 374 | resource names will just use the DIND_LABEL, and a pseudo-random number from 1..13 will 375 | be used for the cluster ID to be applied to the management network, and in case of IPv6, 376 | the NAT64 V4 mapping subnet prefix (hence the limitation). 377 | 378 | Example usage: 379 | 380 | ```shell 381 | $ # creates a 'default' cluster 382 | $ ./dind-cluster up 383 | $ # creates a cluster with an ID of 10 384 | $ CLUSTER_ID="10" ./dind-cluster.sh up 385 | $ # creates an additional cluster with the label 'foo' and random cluster ID assigned 386 | $ DIND_LABEL="foo" ./dind-cluster.sh up 387 | ``` 388 | 389 | Example containers: 390 | 391 | ```shell 392 | $ docker ps --format '{{ .ID }} - {{ .Names }} -- {{ .Labels }}' 393 | 394 | 8178227e567c - kube-node-2 -- mirantis.kubeadm_dind_cluster=1,mirantis.kubeadm_dind_cluster_runtime= 395 | 6ea1822303bf - kube-node-1 -- mirantis.kubeadm_dind_cluster=1,mirantis.kubeadm_dind_cluster_runtime= 396 | 7bc6b28be0b4 - kube-master -- mirantis.kubeadm_dind_cluster=1,mirantis.kubeadm_dind_cluster_runtime= 397 | 398 | ce3fa6eaecfe - kube-node-2-cluster-10 -- cluster-10=,mirantis.kubeadm_dind_cluster=1 399 | 12c18cf3edb7 - kube-node-1-cluster-10 -- cluster-10=,mirantis.kubeadm_dind_cluster=1 400 | 963a6e7c1e40 - kube-master-cluster-10 -- cluster-10=,mirantis.kubeadm_dind_cluster=1 401 | 402 | b05926f06642 - kube-node-2-foo -- mirantis.kubeadm_dind_cluster=1,foo= 403 | ddb961f1cc95 - kube-node-1-foo -- mirantis.kubeadm_dind_cluster=1,foo= 404 | 2efc46f9dafd - kube-master-foo -- foo=,mirantis.kubeadm_dind_cluster=1 405 | ``` 406 | 407 | Example `kubectl` access: 408 | 409 | ```shell 410 | $ # to access the 'default' cluster 411 | $ kubectl --context dind get all 412 | $ # to access the additional clusters 413 | $ kubectl --context dind-cluster-10 get all 414 | $ kubectl --context dind-foo get all 415 | ``` 416 | 417 | ## Dual-stack Operation 418 | By setting the `IP_MODE` environment variable to `dual-stack`, the cluster 419 | created will be in dual-stack mode. This means there will be an IPv4 and 420 | IPv6 address for pods (pod net) and nodes (mgmt net). The service network 421 | will still be single mode, based on the CIDR used (default is IPv6 mode 422 | with fd00:30::/110). 423 | 424 | The MGMT_CIDRS and POD_NETWORK_CIDR environment variables can be used to 425 | customize the management and pod networks, respectively. 426 | 427 | For this mode, static routes will be created on each node, for both IPv4 428 | and IPv6, to allow pods to communicate across nodes. 429 | 430 | ### Limitations 431 | Dual-stack mode for k-d-c is only available when using the bridge or PTP 432 | CNI plugins. 433 | 434 | The initial version will not be using DNS64/NAT64, meaning that the cluster 435 | must have access to the outside via IPv6 (or use an external DNS64/NAT64). 436 | This implies that it will not work, out of the box, with GCE, which provides 437 | only IPv4 access to the outside world. 438 | 439 | The functionality of the cluster in dual-stack mode, depends on the 440 | implementation of the dual-stack KEP. As of this commit, implementation 441 | of the KEP is only beginning, so some things will not work yet. Consider 442 | this commit as support for a WIP. 443 | 444 | One known current limitation is that the service network must use the IPv4 445 | family, as currently IPv4 is preferred, when both are available and logic 446 | doesn't force famliy to IPv6. As a result, endpoints are still IPv4, when 447 | service network is IPv6 (and doesn't work correctly). 448 | 449 | ## Motivation 450 | `hack/local-up-cluster.sh` is widely used for k8s development. It has 451 | a couple of serious issues though. First of all, it only supports 452 | single node clusters, which means that it's hard to use it to work on 453 | e.g. scheduler-related issues and e2e tests that require several nodes 454 | can't be run. Another problem is that it has little resemblance to 455 | real clusters. 456 | 457 | There's also k8s vagrant provider, but it's quite slow. Besides, 458 | `cluster/` directory in k8s source is now considered deprecated. 459 | 460 | Another widely suggested solution for development clusters is 461 | [minikube](https://github.com/kubernetes/minikube), but currently it's 462 | not very well suited for development of Kubernetes itself. Besides, 463 | it's currently only supports single node, too, unless used with 464 | additional DIND layer like [nkube](https://github.com/marun/nkube). 465 | 466 | [kubernetes-dind-cluster](https://github.com/sttts/kubernetes-dind-cluster) 467 | is very nice & useful but uses a custom method of cluster setup 468 | (same as 2nd problem with local-up-cluster). 469 | 470 | There's also sometimes a need to use a powerful remote machine or a 471 | cloud instance to build and test Kubernetes. Having Docker as the only 472 | requirement for such machine would be nice. Builds and unit tests are 473 | already covered by 474 | [jbeda's work](https://github.com/kubernetes/kubernetes/pull/30787) on 475 | dockerized builds, but being able to quickly start remote test 476 | clusters and run e2e tests is also important. 477 | 478 | kubeadm-dind-cluster uses kubeadm to create a cluster consisting of 479 | docker containers instead of VMs. That's somewhat of a compromise but 480 | allows one to (re)start clusters quickly which is quite important when 481 | making changes to k8s source. 482 | 483 | Moreover, some projects that extend Kubernetes such as 484 | [Virtlet](https://github.com/Mirantis/virtlet) need a way to start 485 | kubernetes cluster quickly in CI environment without involving nested 486 | virtulization. Current kubeadm-dind-cluster version provides means to 487 | do this without the need to build Kubernetes locally. 488 | 489 | ## Additional notes 490 | At the moment, all non-serial `[Conformance]` e2e tests pass for 491 | clusters created by kubeadm-dind-cluster. `[Serial]...[Conformance]` tests 492 | currently have some issues. You may still try running them though: 493 | ``` 494 | $ dind/dind-cluster.sh e2e-serial 495 | ``` 496 | 497 | When restoring a cluster (either using `restore` or by doing `down` and 498 | then `up`), be sure to use the same IP mode. The DinD network that is 499 | created as part of the `up` operation, will persist after a `down` 500 | command and will not have the correct configuration, if the IP mode has 501 | changed. 502 | 503 | ## Contributing to & Testing `kubeadm-dind-cluster` 504 | 505 | ### Test setup 506 | 507 | There are currently two CI systems in place which automatically test PRs to 508 | kubeadm-dind-cluster: 509 | - [CircleCI](https://circleci.com/gh/kubernetes-sigs/kubeadm-dind-cluster) 510 | - [TravisCI](https://travis-ci.org/kubernetes-sigs/kubeadm-dind-cluster) 511 | 512 | #### CircleCI, `./.circleci/config.yml` 513 | 514 | All new tests should run on CircleCI, thus need to be configured in 515 | `./.circleci/config.yml`. 516 | 517 | There are some tests completely implemented in `./.circleci/config.yml`. There 518 | are also other tests which are implemented in a script in `./test/` and then 519 | CircleCI just calls that script. This makes it easier to run a CircleCI test 520 | case also locally, by just calling the script: 521 | 522 | ```shell 523 | $ DIND_ALLOW_AAAA_USE='true' TEST_K8S_VER='v1.10' ./test/test-ipv6-only.sh 524 | ``` 525 | 526 | CircleCI config can be re-generated using `build/update-test-matrix.sh` 527 | 528 | #### TravisCI, `./test.sh` 529 | 530 | There are some tests in `./test.sh`, those will run on TravisCI. 531 | New tests should be added to CircleCI and not to `./test.sh` / the TravisCI 532 | setup. 533 | 534 | To run a specific test from `./test.sh` use the following mechanism to discover 535 | and run a specific test: 536 | 537 | ```shell 538 | # See all test cases: 539 | $ grep 'function test-case-' ./test.sh 540 | # run a specific test: 541 | $ TEST_CASE= ./test.sh 542 | ``` 543 | 544 | ### IPv6 tests 545 | 546 | All of the IPv6 related tests currently run on 547 | [CircleCI](https://circleci.com/gh/kubernetes-sigs/kubeadm-dind-cluster). 548 | Those tests run with the `machine executor` (and not as docker containers), so 549 | that we have IPv6 available for the test cases. Note, that while internal IPv6 550 | is configured, external IPv6 is not available. 551 | 552 | There are two slightly different kind of tests which run for all version 553 | starting from `v1.12`: 554 | 555 | #### `TEST_K8S_VER='1.x' ./test/test-ipv6-only.sh` 556 | 557 | The cluster is setup with IPv6 support. The tests check if the IP resolution on 558 | nodes and pods works as expected. DNS64 is always used, and external IPv6 559 | traffic goes throught NAT64. Both NAT64 and DNS64 are automatically deployed 560 | as docker containers, alongside the `kube-master` and `kube-node-X` containers 561 | running in the outer docker daemon. 562 | 563 | These IPv6 tests do not depend on the host machine of the 564 | outer docker daemon actually having external IPv6 connectivity. 565 | 566 | The tests cover, on pods, nodes and host: 567 | * IP address lookups 568 | * internal `ping6`s (pod to pod on different nodes) 569 | * external `ping6`s (to IPv4-only and IPv6-enabled targets) 570 | 571 | #### `TEST_K8S_VER='1.x' DIND_ALLOW_AAAA_USE=true ./test/test-ipv6-only.sh` 572 | 573 | Those tests use the public AAAA records when available. Specifically for hosts 574 | which have a AAAA record, the IP address is used, traffic to those hosts does 575 | not get routed through NAT64. In that case the host running the outer docker 576 | daemon would need to have external IPv6 available to actually communicate with 577 | external IPv6 hosts. Therefore (because none of our CI systems can provide 578 | external IPv6) we skip the external ping tests and instead print a warning 579 | about external IPv6 not being available. If a host does not have a public AAAA 580 | record, the IPv4 address is used, embedded into a synthesized IPv6 address, and 581 | routed through NAT64. 582 | 583 | In summary: 584 | The same test suites as above run, except for external ping tests which are 585 | intentionally disabled. Internal ping tests still run. 586 | 587 | ## Related work 588 | 589 | * kubeadm-dind-cluster was initially derived from 590 | [kubernetes-dind-cluster](https://github.com/sttts/kubernetes-dind-cluster), 591 | although as of now the code was completely rewritten. 592 | kubernetes-dind-cluster is somewhat faster but uses less standard 593 | way of k8s deployment. It also doesn't include support for consuming 594 | binaries from remote dockerized builds. 595 | * [kubeadm-ci-dind](https://github.com/errordeveloper/kubeadm-ci-dind), 596 | [kubeadm-ci-packager](https://github.com/errordeveloper/kubeadm-ci-packager) and 597 | [kubeadm-ci-tester](https://github.com/errordeveloper/kubeadm-ci-tester). 598 | These projects are similar to kubeadm-dind-cluster but are intended primarily for CI. 599 | They include packaging step which is too slow for the purpose of having 600 | convenient k8s "playground". kubeadm-dind-cluster uses Docker images 601 | from `kubeadm-ci-dind`. 602 | * [nkube](https://github.com/marun/nkube) starts 603 | Kubernetes-in-Kubernetes clusters. 604 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Product Security Team to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | pigmej 14 | ivan4th 15 | lukaszo 16 | -------------------------------------------------------------------------------- /architecture.md: -------------------------------------------------------------------------------- 1 | # Architecture of kubeadm-dind-cluster 2 | 3 | Base image: 4 | 5 | * contains systemd, docker and helper scripts 6 | * contains saved+archived pre-pulled images as .tar 7 | * contains hypokube image in the same .tar as pre-pulled images 8 | * used as the base for prebuilt images 9 | * when used directly expects prebuilt binaries 10 | * specified either by url 11 | * or as [build] to pull them from build containers 12 | 13 | Important points: 14 | * there's separate volume mounted at /k8s holding hyperkube and 15 | kubeadm binaries which is created during initial cluster startup and 16 | is updated when k8s is rebuilt from source (we can't just use k8s 17 | build data container in the latter case because it may be removed at 18 | any moment). 19 | * kubectl is a symlink to hyperkube binary in that volume 20 | * loading pre-pulled images is skipped if earlier /dind exists 21 | * when kubeadm is started, cached docker directories may be reused 22 | but filesystem differences are not applied 23 | * kubeadm is executed 24 | * upon the first cluster startup on this particular docker 25 | * after ./dind-cluster.sh clean 26 | * when k8s binaries are rebuilt 27 | 28 | How binary injection works in `wrapkubeadm`: 29 | 30 | 1. Start docker daemon 31 | 1. Purge any remaining containers 32 | 1. Remove saved filesystem diffs for the current node 33 | 1. Get a hash of hyperkube binary 34 | 1. Check if hypokube image with tag = hyperkube hash exists 35 | 1. If it doesn't exist, remove any non-base hypokube images and build new one with binaries 36 | 1. Enable kubelet service and start kubeadm 37 | 1. Patch kube-proxy daemonset so it 38 | * disables conntrack 39 | * mounts /hyperkube from hostPath /k8s/hyperkube 40 | 1. Patch apiserver static pod so it 41 | * has necessary feature gates 42 | * mounts /hyperkube from hostPath /k8s/hyperkube 43 | 1. Patch controller-manager and scheduler static pods so they mount 44 | /hyperkube from hostPath /k8s/hyperkube 45 | -------------------------------------------------------------------------------- /build/build-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | 21 | save_to_dir="${1:-}" 22 | 23 | DIND_ROOT=$(dirname "${BASH_SOURCE}")/.. 24 | source "$DIND_ROOT/build/funcs.sh" 25 | 26 | dind::build-base 27 | images_to_save=(${BARE_IMAGE_NAME}) 28 | for v in ${K8S_VERSIONS}; do 29 | version="${v//./_}" 30 | eval "HYPERKUBE_URL=\${HYPERKUBE_URL_${version}}" 31 | eval "HYPERKUBE_SHA1=\${HYPERKUBE_SHA1_${version}}" 32 | eval "KUBEADM_URL=\${KUBEADM_URL_${version}}" 33 | eval "KUBEADM_SHA1=\${KUBEADM_SHA1_${version}}" 34 | eval "KUBECTL_LINUX_URL=\${KUBECTL_LINUX_URL_${version}}" 35 | eval "KUBECTL_LINUX_SHA1=\${KUBECTL_LINUX_SHA1_${version}}" 36 | eval "KUBECTL_DARWIN_URL=\${KUBECTL_DARWIN_URL_${version}}" 37 | eval "KUBECTL_DARWIN_SHA1=\${KUBECTL_DARWIN_SHA1_${version}}" 38 | commit="$(git rev-parse HEAD)" 39 | tag="${commit}-v${v}" 40 | cur_image="${IMAGE_NAME}:${tag}" 41 | dind::build-image "${cur_image}" 42 | images_to_save+=("${cur_image}") 43 | done 44 | if [[ ${save_to_dir} ]]; then 45 | mkdir -p _save 46 | docker save "${images_to_save[@]}" >_save/images.tar 47 | else 48 | echo "${images_to_save[*]}" 49 | fi 50 | -------------------------------------------------------------------------------- /build/build-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | 21 | DIND_ROOT=$(dirname "${BASH_SOURCE}")/.. 22 | source "$DIND_ROOT/build/funcs.sh" 23 | 24 | dind::build-base 25 | dind::build-image "${image_name}:local" 26 | -------------------------------------------------------------------------------- /build/buildconf.sh: -------------------------------------------------------------------------------- 1 | # Version 1.12.10 2 | KUBEADM_URL_1_12='https://storage.googleapis.com/kubernetes-release/release/v1.12.10/bin/linux/amd64/kubeadm' 3 | KUBEADM_SHA1_1_12=5b12635420b19c267663e2d78c4a544fb5f63feb 4 | HYPERKUBE_URL_1_12='https://storage.googleapis.com/kubernetes-release/release/v1.12.10/bin/linux/amd64/hyperkube' 5 | HYPERKUBE_SHA1_1_12=cdec1f1f6f8660e37f3f109ecc151e1d58b52501 6 | KUBECTL_LINUX_URL_1_12='https://storage.googleapis.com/kubernetes-release/release/v1.12.10/bin/linux/amd64/kubectl' 7 | KUBECTL_LINUX_SHA1_1_12=cc3606e6bcc65c685772f5491f5bd1bb53a55416 8 | KUBECTL_DARWIN_URL_1_12='https://storage.googleapis.com/kubernetes-release/release/v1.12.10/bin/darwin/amd64/kubectl' 9 | KUBECTL_DARWIN_SHA1_1_12=89d1db0bdb061d083329c52f833609970bc3ecf5 10 | 11 | # Version 1.13.8 12 | KUBEADM_URL_1_13='https://storage.googleapis.com/kubernetes-release/release/v1.13.8/bin/linux/amd64/kubeadm' 13 | KUBEADM_SHA1_1_13=79b2076ab759c72c7f3b3ee1b5049cde9c24ed52 14 | HYPERKUBE_URL_1_13='https://storage.googleapis.com/kubernetes-release/release/v1.13.8/bin/linux/amd64/hyperkube' 15 | HYPERKUBE_SHA1_1_13=6a420462afd661f91f6d55a16abc55bb5d0f6628 16 | KUBECTL_LINUX_URL_1_13='https://storage.googleapis.com/kubernetes-release/release/v1.13.8/bin/linux/amd64/kubectl' 17 | KUBECTL_LINUX_SHA1_1_13=21e2f9c768c991a5897582b8fae5c234a32cc088 18 | KUBECTL_DARWIN_URL_1_13='https://storage.googleapis.com/kubernetes-release/release/v1.13.8/bin/darwin/amd64/kubectl' 19 | KUBECTL_DARWIN_SHA1_1_13=dc17274dd617ddebe041f5b90863273eb7be3fc5 20 | 21 | # Version 1.14.4 22 | KUBEADM_URL_1_14='https://storage.googleapis.com/kubernetes-release/release/v1.14.4/bin/linux/amd64/kubeadm' 23 | KUBEADM_SHA1_1_14=9b0103936d79b5a5ed42b1e7c7a62c580fa34a64 24 | HYPERKUBE_URL_1_14='https://storage.googleapis.com/kubernetes-release/release/v1.14.4/bin/linux/amd64/hyperkube' 25 | HYPERKUBE_SHA1_1_14=112a3cd6f84bfbc3bd7389b5f9b6ab6c435a2863 26 | KUBECTL_LINUX_URL_1_14='https://storage.googleapis.com/kubernetes-release/release/v1.14.4/bin/linux/amd64/kubectl' 27 | KUBECTL_LINUX_SHA1_1_14=c055ce78bebd3af14050d3e3f6ea4306daa37b5f 28 | KUBECTL_DARWIN_URL_1_14='https://storage.googleapis.com/kubernetes-release/release/v1.14.4/bin/darwin/amd64/kubectl' 29 | KUBECTL_DARWIN_SHA1_1_14=2ddaa5874a854eff191722622f977f6d8da29e2f 30 | 31 | # Version 1.15.0 32 | KUBEADM_URL_1_15='https://storage.googleapis.com/kubernetes-release/release/v1.15.0/bin/linux/amd64/kubeadm' 33 | KUBEADM_SHA1_1_15=f806e926a71eb034063ef12e1585a463082d126b 34 | HYPERKUBE_URL_1_15='https://storage.googleapis.com/kubernetes-release/release/v1.15.0/bin/linux/amd64/hyperkube' 35 | HYPERKUBE_SHA1_1_15=3c7f951c7f3e38997a836580ea5f7b2e014a64f4 36 | KUBECTL_LINUX_URL_1_15='https://storage.googleapis.com/kubernetes-release/release/v1.15.0/bin/linux/amd64/kubectl' 37 | KUBECTL_LINUX_SHA1_1_15=d604417c2efba1413e2441f16de3be84d3d9b1ae 38 | KUBECTL_DARWIN_URL_1_15='https://storage.googleapis.com/kubernetes-release/release/v1.15.0/bin/darwin/amd64/kubectl' 39 | KUBECTL_DARWIN_SHA1_1_15=57f15cab86e15e9351edcb0e0a2a7d9eee7acff7 40 | 41 | # url and sha1sum of hyperkube binary -- only used for prebuilt hyperkube 42 | KUBEADM_URL=${KUBEADM_URL:-${KUBEADM_URL_1_15}} 43 | KUBEADM_SHA1=${KUBEADM_SHA1:-${KUBEADM_SHA1_1_15}} 44 | HYPERKUBE_URL=${HYPERKUBE_URL:-${HYPERKUBE_URL_1_15}} 45 | HYPERKUBE_SHA1=${HYPERKUBE_SHA1:-${HYPERKUBE_SHA1_1_15}} 46 | KUBECTL_LINUX_URL=${KUBECTL_LINUX_URL:-${KUBECTL_LINUX_URL_1_15}} 47 | KUBECTL_LINUX_SHA1=${KUBECTL_LINUX_SHA1:-${KUBECTL_LINUX_SHA1_1_15}} 48 | KUBECTL_DARWIN_URL=${KUBECTL_DARWIN_URL:-${KUBECTL_DARWIN_URL_1_15}} 49 | KUBECTL_DARWIN_SHA1=${KUBECTL_DARWIN_SHA1:-${KUBECTL_DARWIN_SHA1_1_15}} 50 | 51 | K8S_VERSIONS='1.12 1.13 1.14 1.15' 52 | -------------------------------------------------------------------------------- /build/funcs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | source "${DIND_ROOT}/build/buildconf.sh" 17 | 18 | IMAGE_CACHE_DIR="${IMAGE_CACHE_DIR:-}" 19 | IMAGE_NAME="${IMAGE_NAME:-mirantis/kubeadm-dind-cluster}" 20 | 21 | KUBEADM_URL="${KUBEADM_URL:-}" 22 | HYPERKUBE_URL="${HYPERKUBE_URL:-}" 23 | KUBEADM_SHA1=${KUBEADM_SHA1:-} 24 | HYPERKUBE_SHA1=${HYPERKUBE_SHA1:-} 25 | GH_RELEASE_USER="kubernetes-sigs" 26 | GH_REPO="kubeadm-dind-cluster" 27 | GH_RELEASE_TEST_USER="ivan4th" 28 | 29 | # FIXME: use k8s version specific list of images to pre-pull 30 | # prepull_images=(gcr.io/google_containers/etcd-amd64:3.0.17 31 | # gcr.io/google_containers/kube-discovery-amd64:1.0 32 | # gcr.io/google_containers/kubedns-amd64:1.7 33 | # gcr.io/google_containers/exechealthz-amd64:1.1 34 | # gcr.io/google_containers/kube-dnsmasq-amd64:1.3 35 | # gcr.io/google_containers/pause-amd64:3.0 36 | # gcr.io/google_containers/etcd-amd64:2.2.5 37 | # gcr.io/google_containers/etcd:2.2.1) 38 | 39 | hypokube_base_image=mirantis/hypokube:base 40 | image_version_suffix=v4 41 | image_name="mirantis/kubeadm-dind-cluster" 42 | BARE_IMAGE_NAME="${image_name}:bare-${image_version_suffix}" 43 | 44 | function dind::step { 45 | local OPTS="" 46 | if [ "$1" = "-n" ]; then 47 | shift 48 | OPTS+="-n" 49 | fi 50 | GREEN="$1" 51 | shift 52 | if [ -t 1 ] ; then 53 | echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2 54 | else 55 | echo ${OPTS} "* ${GREEN} $*" 1>&2 56 | fi 57 | } 58 | 59 | function dind::build-hypokube { 60 | dind::step "Building hypokube image" 61 | docker build -t ${hypokube_base_image} -f image/hypokube_base.dkr image 62 | } 63 | 64 | function dind::image-archive-name { 65 | local images=("$@") 66 | image_hash="$(echo -n "${images[@]}" | md5sum | head -c32)" 67 | # echo "images-${image_hash}.tar.gz" 68 | echo "images-${image_hash}.tar" 69 | } 70 | 71 | function dind::save-images { 72 | local archive="$1" 73 | shift 74 | local images=("$@") 75 | dind::step "Pulling images: ${images[*]}" 76 | for image in "${images[@]}"; do 77 | if [[ ${image} != ${hypokube_base_image} ]]; then 78 | docker pull "${image}" 79 | fi 80 | done 81 | dind::step "Saving images to:" "${archive}" 82 | docker save "${images[@]}" | lz4 -9 > "${archive}" 83 | } 84 | 85 | # Rationale behind storing compressed prepulled images inside the image: 86 | # this way they're pulled just once, otherwise it would be necessary to 87 | # re-download them after each 'clean' 88 | function dind::copy-saved-images { 89 | local dest="$1" 90 | shift 91 | local images=("$@") 92 | if [[ ! ${IMAGE_CACHE_DIR} ]]; then 93 | dind::save-images "${dest}" "${images[@]}" 94 | else 95 | local cached_archive="${IMAGE_CACHE_DIR}/$(dind::image-archive-name ${images[@]})" 96 | if [[ ! -f ${cached_archive} ]]; then 97 | mkdir -p "${IMAGE_CACHE_DIR}" 98 | dind::save-images "${cached_archive}" "${images[@]}" 99 | fi 100 | cp "${cached_archive}" "${dest}" 101 | fi 102 | } 103 | 104 | function dind::build-base { 105 | docker build -t "${BARE_IMAGE_NAME}" image/ 106 | } 107 | 108 | function dind::build-image { 109 | local name="$1" 110 | # local -a images=("${prepull_images[@]}") 111 | local -a images=() 112 | 113 | dind::build-hypokube 114 | images+=("${hypokube_base_image}") 115 | 116 | if [[ ${KUBECTL_LINUX_URL} =~ /(v[0-9.]*)/ ]]; then 117 | kubectl_version="${BASH_REMATCH[1]}" 118 | else 119 | echo >&2 "can't get kubectl version from url: ${KUBECTL_LINUX_URL}" 120 | exit 1 121 | fi 122 | 123 | dind::copy-saved-images save.tar.lz4 "${images[@]}" 124 | docker build -t "${name}" \ 125 | --build-arg KUBEADM_URL="${KUBEADM_URL}" \ 126 | --build-arg KUBEADM_SHA1="${KUBEADM_SHA1}" \ 127 | --build-arg HYPERKUBE_URL="${HYPERKUBE_URL}" \ 128 | --build-arg HYPERKUBE_SHA1="${HYPERKUBE_SHA1}" \ 129 | --build-arg KUBECTL_VERSION=${kubectl_version} \ 130 | --build-arg KUBECTL_LINUX_SHA1="${KUBECTL_LINUX_SHA1}" \ 131 | --build-arg KUBECTL_LINUX_URL="${KUBECTL_LINUX_URL}" \ 132 | --build-arg KUBECTL_DARWIN_SHA1="${KUBECTL_DARWIN_SHA1}" \ 133 | --build-arg KUBECTL_DARWIN_URL="${KUBECTL_DARWIN_URL}" \ 134 | . 135 | } 136 | 137 | function release_description { 138 | local -a tag="${1}" 139 | shift 140 | git tag -l --format='%(contents:body)' "${tag}" 141 | echo 142 | echo "SHA256 sums for the files:" 143 | echo '```' 144 | (cd fixed && sha256sum "$@") 145 | echo '```' 146 | } 147 | 148 | function release { 149 | local tag="${1}" 150 | shift 151 | local gh_user="${GH_RELEASE_USER}" 152 | if [[ ${tag} =~ test ]]; then 153 | gh_user="${GH_RELEASE_TEST_USER}" 154 | fi 155 | local -a opts=(--user "${gh_user}" --repo "${GH_REPO}" --tag "${tag}") 156 | local -a files=($(cd fixed && ls dind-cluster-v*.sh)) 157 | local description="$(release_description "${tag}" "${files[@]}")" 158 | local pre_release= 159 | if [[ ${tag} =~ -(test|pre).*$ ]]; then 160 | pre_release="--pre-release" 161 | fi 162 | if github-release --quiet delete "${opts[@]}"; then 163 | echo >&2 "Replacing the old release" 164 | fi 165 | github-release release "${opts[@]}" \ 166 | --name "$(git tag -l --format='%(contents:subject)' "${tag}")" \ 167 | --description "${description}" \ 168 | ${pre_release} 169 | for filename in "${files[@]}"; do 170 | echo >&2 "Uploading: ${filename}" 171 | github-release upload "${opts[@]}" \ 172 | --name "${filename}" \ 173 | --replace \ 174 | --file "fixed/${filename}" 175 | done 176 | } 177 | -------------------------------------------------------------------------------- /build/genfixed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | 21 | DIND_ROOT=$(dirname "${BASH_SOURCE}")/.. 22 | source "$DIND_ROOT/build/funcs.sh" 23 | 24 | image_tag_prefix= 25 | if [[ ${1:-} ]]; then 26 | image_tag_prefix="${1}-" 27 | fi 28 | 29 | fixed_dir="${DIND_ROOT}/fixed" 30 | mkdir -p "${fixed_dir}" 31 | 32 | for tag in v1.12 v1.13 v1.14 v1.15; do 33 | dest="${fixed_dir}/dind-cluster-${tag}.sh" 34 | commit="$(cd "${DIND_ROOT}"; git rev-parse HEAD)" 35 | image="mirantis/kubeadm-dind-cluster:${commit}-${tag}" 36 | # invoke docker pull to get the digest 37 | docker pull "${image}" 38 | digest="$(docker inspect --format='{{index .RepoDigests 0}}' "${image}" | sed 's/.*@//')" 39 | vars=(EMBEDDED_CONFIG=y 40 | DOWNLOAD_KUBECTL=y 41 | DIND_K8S_VERSION="${tag}" 42 | DIND_IMAGE_DIGEST="${digest}" 43 | DIND_COMMIT="$(cd "${DIND_ROOT}" && git rev-parse HEAD)") 44 | var_str=$(IFS=';'; echo "${vars[*]}") 45 | sed "s@#%CONFIG%@${var_str}@" \ 46 | "${DIND_ROOT}/dind-cluster.sh" >"${dest}" 47 | chmod +x "${dest}" 48 | done 49 | -------------------------------------------------------------------------------- /build/genvars.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | VERSIONS=(1.12.10 1.13.8 1.14.4 1.15.0) 21 | 22 | first=1 23 | for version in ${VERSIONS[@]}; do 24 | base_url="https://storage.googleapis.com/kubernetes-release/release/v${version}/bin" 25 | linux_url="${base_url}/linux/amd64" 26 | darwin_url="${base_url}/darwin/amd64" 27 | if [[ ${first} ]]; then 28 | first= 29 | else 30 | echo 31 | fi 32 | echo "# Version ${version}" 33 | suffix="$(tr .- _ <<<"${version}" | sed 's/^\([0-9]*_[0-9]*\).*/\1/')" 34 | echo "KUBEADM_URL_${suffix}='${linux_url}/kubeadm'" 35 | echo "KUBEADM_SHA1_${suffix}=$(curl -sSL "${linux_url}/kubeadm.sha1")" 36 | echo "HYPERKUBE_URL_${suffix}='${linux_url}/hyperkube'" 37 | echo "HYPERKUBE_SHA1_${suffix}=$(curl -sSL "${linux_url}/hyperkube.sha1")" 38 | echo "KUBECTL_LINUX_URL_${suffix}='${linux_url}/kubectl'" 39 | echo "KUBECTL_LINUX_SHA1_${suffix}=$(curl -sSL "${linux_url}/kubectl.sha1")" 40 | echo "KUBECTL_DARWIN_URL_${suffix}='${darwin_url}/kubectl'" 41 | echo "KUBECTL_DARWIN_SHA1_${suffix}=$(curl -sSL "${darwin_url}/kubectl.sha1")" 42 | done 43 | -------------------------------------------------------------------------------- /build/portforward.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Portforward hack for CircleCI remote docker 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | set -o errtrace 7 | 8 | wait= 9 | if [[ ${1:-} = -wait ]]; then 10 | wait=1 11 | shift 12 | fi 13 | if [ "${DIND_PORT_FORWARDER_WAIT:-}" = "1" ]; then 14 | wait=1 15 | fi 16 | 17 | if [[ ${1:-} = start ]]; then 18 | docker run -d -it \ 19 | --name portforward --net=host \ 20 | --entrypoint /bin/sh \ 21 | bobrik/socat -c "while true; do sleep 1000; done" 22 | elif [[ ${1} ]]; then 23 | port="${1}" 24 | mode="" 25 | localhost="localhost" 26 | if [[ "${IP_MODE:-ipv4}" = "ipv6" ]]; then 27 | mode="6" 28 | localhost="[::1]" 29 | fi 30 | socat "TCP-LISTEN:${port},reuseaddr,fork" \ 31 | EXEC:"'docker exec -i portforward socat STDIO TCP${mode}:${localhost}:${port}'" & 32 | if [[ ${wait} ]]; then 33 | for ((n = 0; n < 20; n++)); do 34 | if socat - "TCP${mode}:localhost:${port}" &2 42 | exit 1 43 | fi 44 | -------------------------------------------------------------------------------- /build/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -o errexit 16 | set -o nounset 17 | set -o pipefail 18 | set -o errtrace 19 | 20 | DIND_ROOT=$(dirname "${BASH_SOURCE}")/.. 21 | source "$DIND_ROOT/build/funcs.sh" 22 | 23 | release "$@" 24 | -------------------------------------------------------------------------------- /build/update-test-matrix.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # ^ use newer bash on Mac 3 | # 4 | # Copyright 2018 Mirantis 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # FIXME: should test kube-router with 1.13 instead 19 | 20 | set -o errexit 21 | set -o nounset 22 | set -o pipefail 23 | set -o errtrace 24 | 25 | # TODO: test dual stack with IPv6 service network, when implemented 26 | # TODO: test dual stack with NAT64/DNS64, if/when implemented 27 | 28 | DIND_ROOT=$(dirname "${BASH_SOURCE}")/.. 29 | 30 | test_matrix=( 31 | # "No defaults" should be set for the tests with machine executor 32 | # Test name k8s CNI CRI No defaults 33 | "test v1.12 " 34 | "test v1.12 bridge containerd " 35 | "test v1.13 " 36 | "test v1.13 bridge containerd " 37 | "test v1.14 " 38 | "test v1.14 bridge containerd " 39 | "test v1.15 " 40 | "test v1.15 flannel " 41 | "test v1.15 calico " 42 | "test v1.15 calico-kdd " 43 | "test v1.15 weave " 44 | "test v1.15 bridge containerd " 45 | "test v1.15 ptp " 46 | "test_multiple_clusters v1.15 " 47 | "test_ipv4_only v1.15 bridge docker y" 48 | "test_ipv6_only v1.15 bridge docker y" 49 | "test_ipv6_only_nat64 v1.15 bridge docker y" 50 | "test_ipv6_only_ext_aaaa v1.15 bridge docker y" 51 | "test_dual_stack_v4_svc_net v1.15 bridge docker y" 52 | "test_src_release " 53 | "test_src_master " 54 | ) 55 | 56 | declare -A subst vars 57 | subst[test_ipv6_only_nat64]=test_ipv6_only 58 | vars[test_ipv6_only_nat64]="NAT64_V4_SUBNET_PREFIX: '10.100'" 59 | 60 | subst[test_ipv6_only_ext_aaaa]=test_ipv6_only 61 | vars[test_ipv6_only_ext_aaaa]="DIND_ALLOW_AAAA_USE: 'true'" 62 | 63 | subst[test_dual_stack_v4_svc_net]=test_dual_stack 64 | vars[test_dual_stack_v4_svc_net]="SERVICE_CIDR: '10.96.0.0/12';DIND_ALLOW_AAAA_USE: 'true'" 65 | 66 | function test_name { 67 | local test_name k8s_ver cni cri nodef 68 | read test_name k8s_ver cni cri nodef <<<"$1" 69 | if [[ ${test_name} =~ ^test_src.* ]]; then 70 | echo "${test_name}" 71 | else 72 | echo "${test_name}_${k8s_ver}_${cni:-bridge}_${cri:-docker}" 73 | fi 74 | } 75 | 76 | function generate_test_case_circleci { 77 | local test_name k8s_ver cni cri nodef 78 | read test_name k8s_ver cni cri nodef <<<"$1" 79 | if [[ ${test_name} =~ ^test_src.* ]]; then 80 | # src tests are already defined above the marker 81 | return 82 | fi 83 | local cur_subst="${subst[${test_name}]:-}" 84 | cat <>"${DIND_ROOT}/.circleci/config.yml" 163 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /config.sh: -------------------------------------------------------------------------------- 1 | if [[ ${IP_MODE} = "ipv4" ]]; then 2 | # DIND_SUBNET="10.192.0.0" 3 | # DIND_SUBNET_SIZE=16 4 | : 5 | else 6 | # DinD subnet (expected to be /64) 7 | DIND_SUBNET="${DIND_SUBNET:-fd00:10::}" 8 | fi 9 | 10 | # Apiserver port 11 | # APISERVER_PORT=${APISERVER_PORT:-8080} 12 | 13 | # Number of nodes. 0 nodes means just one master node. 14 | # In case of NUM_NODES=0 'node-role.kubernetes.io/master' taint is removed 15 | # from the master node. 16 | NUM_NODES=${NUM_NODES:-2} 17 | 18 | # Use non-dockerized build 19 | # KUBEADM_DIND_LOCAL= 20 | 21 | # Image name base for k-d-c 22 | DIND_IMAGE_BASE="${DIND_IMAGE_BASE:-mirantis/kubeadm-dind-cluster}" 23 | 24 | # Specify DIND image to use. mirantis/kubeadm-dind-cluster:local 25 | # is the one that is built locally using build/build-local.sh 26 | DIND_IMAGE="${DIND_IMAGE:-${DIND_IMAGE_BASE}:local}" 27 | 28 | # Set DOWNLOAD_KUBECTL to non-empty string to download 29 | # kubectl. Should not be used with BUILD_KUBEADM / BUILD_HYPERKUBE 30 | # DOWNLOAD_KUBECTL=y 31 | 32 | # Set to non-empty string to enable building kubeadm 33 | # BUILD_KUBEADM=y 34 | 35 | # Set to non-empty string to enable building hyperkube 36 | # BUILD_HYPERKUBE=y 37 | 38 | # Use pre-built Kubernetes binaries (hyperkube and kubeadm) on the 39 | # host, located in the specified directory. When this environment 40 | # variable is set, BUILD_KUBEADM and BUILD_HYPERKUBE will be ignored. 41 | # This will not work with a remote docker engine (e.g. started via 42 | # docker-machine on GCE) unless the file is placed on the target machine. 43 | DIND_K8S_BIN_DIR="${DIND_K8S_BIN_DIR:-}" 44 | 45 | # Set custom URL for Dashboard yaml file 46 | # DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml}" 47 | # or for versions >= 1.15, 48 | # DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml}" 49 | 50 | # CNI plugin to use (bridge, flannel, calico, calico-kdd, weave). Defaults to 'bridge' 51 | # In case of 'bridge' plugin, additional hacks are employed to bridge 52 | # DIND containers together. 53 | CNI_PLUGIN="${CNI_PLUGIN:-bridge}" 54 | 55 | # When using Calico with Kubernetes as the datastore (calico-kdd) your 56 | # controller manager needs to be started with `--cluster-cidr=192.168.0.0/16`. 57 | # More information here: http://docs.projectcalico.org/v2.3/getting-started/kubernetes/installation/hosted/kubernetes-datastore/ 58 | # POD_NETWORK_CIDR="192.168.0.0/16" 59 | 60 | # Set SKIP_SNAPSHOT to non-empty string to skip making the snapshot. 61 | # This may be useful for CI environment where the cluster is never 62 | # restarted after it's created. 63 | # SKIP_SNAPSHOT=y 64 | 65 | # Disable parallel running of e2e tests. Use this if you use a resource 66 | # constrained machine for e2e tests and get some flakes. 67 | # DIND_NO_PARALLEL_E2E=y 68 | 69 | # Any options to be passed to the docker run both on init and reup. 70 | # By default it's empty 71 | # MASTER_EXTRA_OPTS=" " 72 | 73 | # Define which DNS service to run 74 | # possible values are coredns (default) and kube-dns 75 | DNS_SERVICE="${DNS_SERVICE:-coredns}" 76 | 77 | # Feature Gates 78 | # This value will be passed to kube-apiserver, kube-controller-manager and kube-scheduler 79 | # you can set special value 'none' not to set any feature gates on them. 80 | # FEATURE_GATES="" 81 | 82 | # Kubelet Feature Gates 83 | # you can set special value 'none' not to set any feature gates on kubelet. 84 | # KUBELET_FEATURE_GATES="" 85 | 86 | # You can configure extra component args for kube-apiservers 87 | # APISERVER_underscored_option_name will be converted --hyphenated-option-name 88 | # e.g. APISERVER_admission_control=xxx,yyy -> --admission-control=xxx,yyy 89 | # APISERVER_xxx_yyy=zzz 90 | 91 | # Extra component args for kube-controller-manager 92 | # CONTROLLER_MANAGER_underscored_option_name will be converted --hyphenated-option-name 93 | # CONTROLLER_MANAGER_xxx=yyy 94 | 95 | # Extra component args for kube-scheduler 96 | # SCHEDULER_underscored_option_name will be converted --hyphenated-option-name 97 | # SCHEDULER_xxx=yyy 98 | 99 | # Enable Ceph support. DANGER: you must take care of unmapping all 100 | # the RBDs (e.g. by removing all the pods that use RBDs) before 101 | # stopping / restarting the cluster, or they'll get stuck possibly 102 | # blocking even system reboot. 103 | # ENABLE_CEPH=y 104 | -------------------------------------------------------------------------------- /fixed/README.md: -------------------------------------------------------------------------------- 1 | This directory is deprecated. Please use scripts from the GitHub 2 | releases of kubeadm-dind-cluster. These scripts are actually copies of 3 | the released scripts, we'll keep updating them for a while but at some 4 | point this directory will be removed. 5 | -------------------------------------------------------------------------------- /gce-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | if [ $(uname) = Darwin ]; then 17 | readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} 18 | else 19 | readlinkf(){ readlink -f "$1"; } 20 | fi 21 | DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" 22 | KUBE_DIND_GCE_PROJECT="${KUBE_DIND_GCE_PROJECT:-$(gcloud config list --format 'value(core.project)' 2>/dev/null)}" 23 | KUBE_DIND_GCE_ZONE="${KUBE_DIND_GCE_ZONE:-$(gcloud config list --format 'value(compute.zone)' 2>/dev/null)}" 24 | # Based on instructions from k8s build/README.md 25 | if [ -z "${KUBE_DIND_GCE_PROJECT:-}" ]; then 26 | echo >&2 "Please set KUBE_DIND_GCE_PROJECT or use 'gcloud config set project NAME'" 27 | return 1 28 | fi 29 | 30 | set -x 31 | KUBE_DIND_VM="${KUBE_DIND_VM:-k8s-dind}" 32 | export GCE_HOSTED=true 33 | export KUBE_RSYNC_PORT=8730 34 | export APISERVER_PORT=8899 35 | IP_MODE="${IP_MODE:-ipv4}" 36 | opts=("-L ${KUBE_RSYNC_PORT}:localhost:${KUBE_RSYNC_PORT}") 37 | 38 | if [[ "${IP_MODE}" = "ipv4" ]]; then 39 | opts+=("-L ${APISERVER_PORT}:localhost:${APISERVER_PORT}") 40 | else 41 | opts+=("-L ${APISERVER_PORT}:[::1]:${APISERVER_PORT}") 42 | fi 43 | docker-machine create \ 44 | --driver=google \ 45 | --google-project=${KUBE_DIND_GCE_PROJECT} \ 46 | --google-machine-image=ubuntu-os-cloud/global/images/ubuntu-1604-xenial-v20170307 \ 47 | --google-zone=${KUBE_DIND_GCE_ZONE} \ 48 | --google-machine-type=n1-standard-8 \ 49 | --google-disk-size=50 \ 50 | --google-disk-type=pd-ssd \ 51 | --engine-storage-driver=overlay2 \ 52 | ${KUBE_DIND_VM} 53 | eval $(docker-machine env ${KUBE_DIND_VM}) 54 | docker-machine ssh ${KUBE_DIND_VM} ${opts[*]} -N& 55 | if [ ! -z "${DIND_IMAGE:-}" ]; then 56 | place=`pwd` 57 | cd "${DIND_ROOT}" 58 | build/build-local.sh 59 | cd "$place" 60 | fi 61 | time "${DIND_ROOT}"/dind-cluster.sh up 62 | set +x 63 | -------------------------------------------------------------------------------- /image/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Mirantis 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Some parts are taken from here: 16 | # https://github.com/kubernetes/test-infra/blob/master/dind/base/Dockerfile 17 | 18 | # Copyright 2017 The Kubernetes Authors. 19 | # 20 | # Licensed under the Apache License, Version 2.0 (the "License"); 21 | # you may not use this file except in compliance with the License. 22 | # You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | 32 | # ci-xenial-systemd image source: https://github.com/errordeveloper/kubeadm-ci-dind 33 | # The tag includes commit id 34 | FROM k8s.gcr.io/debian-base-amd64:0.4.1 35 | 36 | STOPSIGNAL SIGRTMIN+3 37 | 38 | LABEL mirantis.kubeadm_dind_cluster=1 39 | 40 | ENV ARCH amd64 41 | ENV CRICTL_VERSION=v1.12.0 42 | ENV CRICTL_SHA256=e7d913bcce40bf54e37ab1d4b75013c823d0551e6bc088b217bc1893207b4844 43 | ENV CNI_VERSION=v0.7.1 44 | ENV CNI_ARCHIVE=cni-plugins-"${ARCH}"-"${CNI_VERSION}".tgz 45 | ENV CNI_SHA1=fb29e20401d3e9598a1d8e8d7992970a36de5e05 46 | 47 | # Specify CONTAINERD_VERSION if containerd which ships with Docker needs to be 48 | # replaced. Note that containerd version must be specified w/o "v" prefix, 49 | # e.g. v1.2.1 50 | ENV CONTAINERD_VERSION=1.2.1 51 | ENV CONTAINERD_SHA256=9818e3af4f9aac8d55fc3f66114346db1d1acd48d45f88b2cefd3d3bafb380e0 52 | 53 | # make systemd behave correctly in Docker container 54 | # (e.g. accept systemd.setenv args, etc.) 55 | ENV container docker 56 | 57 | ARG DEBIAN_FRONTEND=noninteractive 58 | RUN clean-install \ 59 | apt-transport-https \ 60 | bash \ 61 | bridge-utils \ 62 | ca-certificates \ 63 | curl \ 64 | e2fsprogs \ 65 | ebtables \ 66 | ethtool \ 67 | gnupg2 \ 68 | ipcalc \ 69 | iptables \ 70 | iproute2 \ 71 | iputils-ping \ 72 | ipset \ 73 | ipvsadm \ 74 | jq \ 75 | kmod \ 76 | lsb-core \ 77 | less \ 78 | lzma \ 79 | liblz4-tool \ 80 | mount \ 81 | net-tools \ 82 | procps \ 83 | socat \ 84 | software-properties-common \ 85 | util-linux \ 86 | vim \ 87 | systemd \ 88 | systemd-sysv \ 89 | sysvinit-utils 90 | 91 | # Install docker. 92 | RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - && apt-key fingerprint 0EBFCD88 && add-apt-repository \ 93 | "deb [arch=amd64] https://download.docker.com/linux/debian \ 94 | $(lsb_release -cs) \ 95 | stable" 96 | RUN clean-install docker-ce=5:18.09.0~3-0~debian-stretch && \ 97 | sed -i '/^disabled_plugins/d' /etc/containerd/config.toml 98 | 99 | RUN curl -sSL --retry 5 https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz >/tmp/crictl.tar.gz && \ 100 | echo "${CRICTL_SHA256} /tmp/crictl.tar.gz" | sha256sum -c && \ 101 | tar -C /usr/local/bin -xvzf /tmp/crictl.tar.gz && \ 102 | rm -f /tmp/crictl.tar.gz && \ 103 | if [ ${CONTAINERD_VERSION} ]; then \ 104 | curl -sSL --retry 5 https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz >/tmp/containerd.tar.gz && \ 105 | echo "${CONTAINERD_SHA256} /tmp/containerd.tar.gz" | sha256sum -c && \ 106 | tar -C /usr -xvzf /tmp/containerd.tar.gz && \ 107 | rm -f /tmp/containerd.tar.gz; \ 108 | fi 109 | 110 | RUN mkdir -p /hypokube /etc/systemd/system/docker.service.d /var/lib/kubelet 111 | 112 | COPY hypokube.dkr /hypokube/ 113 | COPY kubelet.service /lib/systemd/system/ 114 | COPY dindnet.service /lib/systemd/system/ 115 | COPY docker.service /lib/systemd/system/docker.service 116 | COPY wrapkubeadm /usr/local/bin/ 117 | COPY start_services /usr/local/bin/ 118 | COPY rundocker /usr/local/bin 119 | COPY dindnet /usr/local/bin 120 | COPY snapshot /usr/local/bin 121 | COPY kubeadm.conf.1.12.tmpl /etc/kubeadm.conf.1.12.tmpl 122 | COPY kubeadm.conf.1.13.tmpl /etc/kubeadm.conf.1.13.tmpl 123 | COPY dind-cluster-rbd /usr/bin/rbd 124 | COPY node-info / 125 | 126 | # Remove unwanted systemd services. 127 | # TODO: use 'systemctl mask' to disable units 128 | # See here for example: https://github.com/docker/docker/issues/27202#issuecomment-253579916 129 | RUN for i in /lib/systemd/system/sysinit.target.wants/*; do [ "${i##*/}" = "systemd-tmpfiles-setup.service" ] || rm -f "$i"; done; \ 130 | rm -f /lib/systemd/system/multi-user.target.wants/*;\ 131 | rm -f /etc/systemd/system/*.wants/*;\ 132 | rm -f /lib/systemd/system/local-fs.target.wants/*; \ 133 | rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ 134 | rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ 135 | rm -f /lib/systemd/system/basic.target.wants/*;\ 136 | rm -f /lib/systemd/system/anaconda.target.wants/*; 137 | 138 | RUN chmod +x /usr/local/bin/rundocker /usr/local/bin/dindnet /usr/local/bin/snapshot && \ 139 | mkdir -p /opt/cni/bin && \ 140 | curl -sSL --retry 5 https://github.com/containernetworking/plugins/releases/download/"${CNI_VERSION}"/"${CNI_ARCHIVE}" >"/tmp/${CNI_ARCHIVE}" && \ 141 | echo "${CNI_SHA1} /tmp/${CNI_ARCHIVE}" | sha1sum -c && \ 142 | tar -C /opt/cni/bin -xzf "/tmp/${CNI_ARCHIVE}" && \ 143 | rm -f "/tmp/${CNI_ARCHIVE}" && \ 144 | mkdir -p /etc/systemd/system/docker.service.wants && \ 145 | ln -s /lib/systemd/system/dindnet.service /etc/systemd/system/docker.service.wants/ && \ 146 | ln -s /dind/containerd /var/lib/containerd && \ 147 | mkdir -p /dind/containerd && \ 148 | ln -s /k8s/hyperkube /usr/bin/kubectl && \ 149 | ln -s /k8s/hyperkube /usr/bin/kubelet && \ 150 | ln -s /k8s/kubeadm /usr/bin/kubeadm 151 | 152 | # TODO: move getty target removal to the base image 153 | 154 | EXPOSE 8080 155 | 156 | RUN ln -fs /sbin/init /sbin/dind_init 157 | ENTRYPOINT ["/sbin/dind_init"] 158 | -------------------------------------------------------------------------------- /image/bind9/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM alpine:latest 16 | 17 | LABEL maintainer "leblancd@cisco.com" 18 | 19 | RUN apk --update add bind 20 | 21 | # Move the 'named' binary to a different location. This is done in order to 22 | # support running the bind9 container in privileged mode in a Docker-in-Docker 23 | # (DinD) environment on a host that happens to have active AppArmor profile 24 | # for 'named'. Without this, AppArmor defaults to using the host's AppArmor 25 | # profile for 'named' based on the '/usr/sbin/named' path, and permissions 26 | # errors are generated when 'named' in the container tries to access 27 | # libraries upon which it depends. 28 | RUN mv /usr/sbin/named /usr/bin/named 29 | 30 | EXPOSE 53 31 | 32 | CMD ["named", "-c", "/etc/bind/named.conf", "-g", "-u", "named"] 33 | -------------------------------------------------------------------------------- /image/bind9/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Usage: 16 | # To build and push an image to DockerHub, run: 17 | # make IMG= 18 | # To just do a local build without a push to DockerHub: 19 | # make image IMG= 20 | 21 | IMG = "diverdane/bind9:latest" 22 | 23 | all: image 24 | 25 | image: 26 | docker build --no-cache -t "$(IMG)" . 27 | 28 | push: image 29 | gcloud docker -- push "$(IMG)" 30 | 31 | .PHONY: all image push 32 | -------------------------------------------------------------------------------- /image/dind-cluster-rbd: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # https://github.com/rook/rook/blob/cd2b69915958e7453b3fc5031f59179058163dcd/tests/scripts/dind-cluster-rbd 3 | echo "RUN: $*" >>/tmp/rbd.log 4 | DOCKER_HOST=unix:///opt/outer-docker.sock /usr/bin/docker run --rm -v /sys:/sys --net=host --privileged=true ceph/base rbd "$@" 5 | -------------------------------------------------------------------------------- /image/dindnet: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | set -x 21 | 22 | function get_interface_info_for { 23 | local from_intf=$1 24 | IPV4_CIDR=$(ip addr show $from_intf | grep -w inet | awk '{ print $2; }') 25 | IPV4="$(echo ${IPV4_CIDR} | sed 's,/.*,,')" 26 | if [[ "${IP_MODE}" = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then 27 | IPV6_CIDR="$(ip addr show $from_intf | grep -w inet6 | grep -i global | head -1 | awk '{ print $2; }')" 28 | IPV6="$(echo ${IPV6_CIDR} | sed 's,/.*,,')" 29 | fi 30 | } 31 | 32 | function dind::setup-bridge { 33 | if [[ ${CNI_PLUGIN} = "ptp" ]]; then 34 | return 35 | fi 36 | 37 | if [[ ! -z "$(ip addr show dind0 2>/dev/null)" ]]; then 38 | return # Bridge is already set up 39 | fi 40 | 41 | ip link add dind0 type bridge 42 | local i=0 43 | if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then 44 | ip addr add "${pod_prefixes[$i]}.1/${pod_sizes[$i]}" dev dind0 45 | i=$(( i+1 )) 46 | fi 47 | if [[ "${IP_MODE}" = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then 48 | ip -6 addr add "${pod_prefixes[$i]}::1/${pod_sizes[$i]}" dev dind0 49 | fi 50 | # To prevent MAC on dind0 from dynamically changing, create a dummy I/F enslaved in 51 | # dind0 and set the MAC of dind0 to match. 52 | ip link add v1 type veth peer name v2 53 | ip link set dev v1 master dind0 54 | DIND0_MAC="$(ip a show v1 | grep link/ether | awk '{ print $2; }')" 55 | ip link set dev dind0 address $DIND0_MAC 56 | 57 | ip link set dind0 up 58 | } 59 | 60 | function ptp-cni-contents-start { 61 | cat <${CONFIG_FILE} 154 | echo "Config file created: ${CONFIG_FILE}" 155 | } 156 | 157 | function dind::make-kubelet-extra-dns-args { 158 | if [[ "${SERVICE_NET_MODE}" = "ipv6" ]]; then 159 | # Create drop-in file here, where we know the DNS IP. 160 | mkdir -p /etc/systemd/system/kubelet.service.d 161 | cat >/etc/systemd/system/kubelet.service.d/20-extra-dns-args.conf < /etc/hosts.updated 194 | cp /etc/hosts /etc/hosts.orig 195 | cat /etc/hosts.updated >/etc/hosts 196 | fi 197 | 198 | if [[ "${IP_MODE}" = "ipv6" ]]; then 199 | if ! grep -q '#fixed#' /etc/resolv.conf; then 200 | # Removed embedded docker DNS, as we'll only be using IPv6, which is already in resolv.conf 201 | sed "s/^nameserver.*127\.0\.0\.11/# Removed 127.0.0.11/" /etc/resolv.conf >/etc/resolv.conf.updated 202 | cp /etc/resolv.conf /etc/resolv.conf.orig 203 | cat /etc/resolv.conf.updated >/etc/resolv.conf 204 | (echo; echo '#fixed#') >>/etc/resolv.conf 205 | echo "Host and DNS info updated" 206 | fi 207 | echo "Setup completed for IPv6 mode" 208 | set +x 209 | while true; do 210 | sleep 1 # Keep service running, so actions not attempted multiple times 211 | done 212 | else # IPv4 or dual-stack 213 | if ! grep -q '#fixed#' /etc/resolv.conf; then 214 | # make docker's kube-dns friendly 215 | old_ns="$(awk '/^nameserver/ {print $2; exit}' /etc/resolv.conf)" 216 | if [[ -z ${old_ns} ]]; then 217 | echo "WARNING: couldn't get nameserver" >&2 218 | exit 1 219 | fi 220 | # sed -i doesn't work here because of docker's handling of /etc/resolv.conf 221 | sed "s/^nameserver.*/nameserver ${IPV4}/" /etc/resolv.conf >/etc/resolv.conf.updated 222 | cp /etc/resolv.conf /etc/resolv.conf.orig 223 | cat /etc/resolv.conf.updated >/etc/resolv.conf 224 | (echo; echo '#fixed#') >>/etc/resolv.conf 225 | echo "Setup completed for ${IP_MODE} mode" 226 | else 227 | # Already switched from built-in DNS server, so use original value for socat 228 | old_ns="127.0.0.11" 229 | fi 230 | while true; do 231 | socat udp4-recvfrom:53,reuseaddr,fork,bind=${IPV4} UDP:${old_ns}:53 || true 232 | echo "WARNING: restarting socat" >&2 233 | done 234 | fi 235 | -------------------------------------------------------------------------------- /image/dindnet.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=DIND network setup 3 | 4 | [Service] 5 | ExecStart=/usr/local/bin/dindnet 6 | 7 | [Install] 8 | WantedBy=docker.service 9 | -------------------------------------------------------------------------------- /image/docker.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Docker Application Container Engine 3 | Documentation=https://docs.docker.com 4 | BindsTo=containerd.service 5 | After=network-online.target firewalld.service 6 | Wants=network-online.target 7 | 8 | [Service] 9 | Type=notify 10 | # the default is not to use systemd for cgroups because the delegate issues still 11 | # exists and systemd currently does not support the cgroup feature set required 12 | # for containers run by docker 13 | ExecStart=/usr/local/bin/rundocker 14 | ExecReload=/bin/kill -s HUP $MAINPID 15 | TimeoutSec=0 16 | RestartSec=2 17 | Restart=always 18 | 19 | # Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. 20 | # Both the old, and new location are accepted by systemd 229 and up, so using the old location 21 | # to make them work for either version of systemd. 22 | StartLimitBurst=3 23 | 24 | # Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. 25 | # Both the old, and new name are accepted by systemd 230 and up, so using the old name to make 26 | # this option work for either version of systemd. 27 | StartLimitInterval=60s 28 | 29 | # Having non-zero Limit*s causes performance problems due to accounting overhead 30 | # in the kernel. We recommend using cgroups to do container-local accounting. 31 | LimitNOFILE=infinity 32 | LimitNPROC=infinity 33 | LimitCORE=infinity 34 | 35 | # Comment TasksMax if your systemd version does not supports it. 36 | # Only systemd 226 and above support this option. 37 | TasksMax=infinity 38 | 39 | # set delegate yes so that systemd does not reset the cgroups of docker containers 40 | Delegate=yes 41 | 42 | # kill only the docker process, not all processes in the cgroup 43 | KillMode=process 44 | 45 | [Install] 46 | WantedBy=multi-user.target 47 | -------------------------------------------------------------------------------- /image/hypokube.dkr: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Mirantis 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM mirantis/hypokube:base 16 | 17 | COPY hyperkube /hyperkube 18 | -------------------------------------------------------------------------------- /image/hypokube_base.dkr: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Mirantis 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # based on cluster/images/hyperkube/Dockerfile from k8s 16 | FROM k8s.gcr.io/debian-hyperkube-base-amd64:0.10 17 | 18 | RUN DEBIAN_FRONTEND=noninteractive apt-get update -y \ 19 | && DEBIAN_FRONTEND=noninteractive apt-get -yy -q install ipvsadm \ 20 | && DEBIAN_FRONTEND=noninteractive apt-get autoremove -y \ 21 | && DEBIAN_FRONTEND=noninteractive apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # CACHEBUST 22 | 23 | RUN ln -s /hyperkube /apiserver \ 24 | && ln -s /hyperkube /controller-manager \ 25 | && ln -s /hyperkube /kubectl \ 26 | && ln -s /hyperkube /kubelet \ 27 | && ln -s /hyperkube /proxy \ 28 | && ln -s /hyperkube /scheduler \ 29 | && ln -s /hyperkube /aggregator \ 30 | && ln -s /hyperkube /usr/local/bin/kube-apiserver \ 31 | && ln -s /hyperkube /usr/local/bin/kube-controller-manager \ 32 | && ln -s /hyperkube /usr/local/bin/kubectl \ 33 | && ln -s /hyperkube /usr/local/bin/kubelet \ 34 | && ln -s /hyperkube /usr/local/bin/kube-proxy \ 35 | && ln -s /hyperkube /usr/local/bin/kube-scheduler 36 | -------------------------------------------------------------------------------- /image/kubeadm.conf.1.12.tmpl: -------------------------------------------------------------------------------- 1 | # Customize this, as needed, for your situation, but keep 2 | # the variables defined herein: 3 | # ADV_ADDR, POD_SUBNET_DISABLE, POD_NETWORK_CIDR, KUBE_MASTER_NAME 4 | # SVC_SUBNET, BIND_ADDR, BIND_PORT, KUBEADM_VERSION 5 | apiEndpoint: 6 | advertiseAddress: "{{ADV_ADDR}}" 7 | bindPort: 6443 8 | apiVersion: kubeadm.k8s.io/v1alpha3 9 | kind: InitConfiguration 10 | nodeRegistration: 11 | criSocket: "{{CRI_SOCKET}}" 12 | name: {{KUBE_MASTER_NAME}} 13 | taints: 14 | - effect: NoSchedule 15 | key: node-role.kubernetes.io/master 16 | --- 17 | # The extra indent is important here because of templating issues. 18 | # 1.13 has extra nesting level here. 19 | apiServerExtraArgs: 20 | insecure-bind-address: "{{BIND_ADDR}}" 21 | insecure-port: "{{BIND_PORT}}" 22 | {{COMPONENT_FEATURE_GATES}} 23 | {{APISERVER_EXTRA_ARGS}} 24 | apiVersion: kubeadm.k8s.io/v1alpha3 25 | auditPolicy: 26 | logDir: /var/log/kubernetes/audit 27 | logMaxAge: 2 28 | path: "" 29 | certificatesDir: /etc/kubernetes/pki 30 | controlPlaneEndpoint: "" 31 | # The extra indent is important here because of templating issues. 32 | # 1.13 has extra nesting level here. 33 | controllerManagerExtraArgs: 34 | {{COMPONENT_FEATURE_GATES}} 35 | {{CONTROLLER_MANAGER_EXTRA_ARGS}} 36 | etcd: 37 | local: 38 | dataDir: /var/lib/etcd 39 | image: "" 40 | featureGates: {{FEATURE_GATES}} 41 | imageRepository: k8s.gcr.io 42 | kind: ClusterConfiguration 43 | kubernetesVersion: "{{KUBEADM_VERSION}}" 44 | networking: 45 | {{POD_SUBNET_DISABLE}}podSubnet: "{{POD_NETWORK_CIDR}}" 46 | serviceSubnet: "{{SVC_SUBNET}}" 47 | # The extra indent is important here because of templating issues. 48 | # 1.13 has extra nesting level here. 49 | schedulerExtraArgs: 50 | {{COMPONENT_FEATURE_GATES}} 51 | {{SCHEDULER_EXTRA_ARGS}} 52 | unifiedControlPlaneImage: mirantis/hypokube:final 53 | apiServerExtraVolumes: 54 | - name: hyperkube 55 | hostPath: /k8s/hyperkube 56 | mountPath: /hyperkube 57 | controllerManagerExtraVolumes: 58 | - name: hyperkube 59 | hostPath: /k8s/hyperkube 60 | mountPath: /hyperkube 61 | schedulerExtraVolumes: 62 | - name: hyperkube 63 | hostPath: /k8s/hyperkube 64 | mountPath: /hyperkube 65 | -------------------------------------------------------------------------------- /image/kubeadm.conf.1.13.tmpl: -------------------------------------------------------------------------------- 1 | # Customize this, as needed, for your situation, but keep 2 | # the variables defined herein: 3 | # ADV_ADDR, POD_SUBNET_DISABLE, POD_NETWORK_CIDR, KUBE_MASTER_NAME 4 | # SVC_SUBNET, BIND_ADDR, BIND_PORT, KUBEADM_VERSION 5 | apiVersion: kubeadm.k8s.io/v1beta1 6 | kind: InitConfiguration 7 | localAPIEndpoint: 8 | advertiseAddress: "{{ADV_ADDR}}" 9 | bindPort: 6443 10 | bootstrapTokens: 11 | - groups: 12 | - system:bootstrappers:kubeadm:default-node-token 13 | # token: ... 14 | ttl: 24h0m0s 15 | usages: 16 | - signing 17 | - authentication 18 | nodeRegistration: 19 | criSocket: "{{CRI_SOCKET}}" 20 | name: {{KUBE_MASTER_NAME}} 21 | taints: 22 | - effect: NoSchedule 23 | key: node-role.kubernetes.io/master 24 | --- 25 | apiVersion: kubeadm.k8s.io/v1beta1 26 | kind: ClusterConfiguration 27 | apiServer: 28 | extraVolumes: 29 | - name: hyperkube 30 | hostPath: /k8s/hyperkube 31 | mountPath: /hyperkube 32 | extraArgs: 33 | insecure-bind-address: "{{BIND_ADDR}}" 34 | insecure-port: "{{BIND_PORT}}" 35 | {{COMPONENT_FEATURE_GATES}} 36 | {{APISERVER_EXTRA_ARGS}} 37 | controllerManager: 38 | extraVolumes: 39 | - name: hyperkube 40 | hostPath: /k8s/hyperkube 41 | mountPath: /hyperkube 42 | extraArgs: 43 | {{COMPONENT_FEATURE_GATES}} 44 | {{CONTROLLER_MANAGER_EXTRA_ARGS}} 45 | scheduler: 46 | extraVolumes: 47 | - name: hyperkube 48 | hostPath: /k8s/hyperkube 49 | mountPath: /hyperkube 50 | extraArgs: 51 | {{COMPONENT_FEATURE_GATES}} 52 | {{SCHEDULER_EXTRA_ARGS}} 53 | certificatesDir: /etc/kubernetes/pki 54 | # clusterName: kubernetes 55 | controlPlaneEndpoint: "" 56 | etcd: 57 | local: 58 | dataDir: /var/lib/etcd 59 | featureGates: {{FEATURE_GATES}} 60 | useHyperKubeImage: true 61 | kubernetesVersion: "{{KUBEADM_VERSION}}" 62 | networking: 63 | dnsDomain: cluster.local 64 | {{POD_SUBNET_DISABLE}}podSubnet: "{{POD_NETWORK_CIDR}}" 65 | serviceSubnet: "{{SVC_SUBNET}}" 66 | --- 67 | apiVersion: kubeproxy.config.k8s.io/v1alpha1 68 | bindAddress: "{{BIND_ADDR}}" 69 | clientConnection: 70 | acceptContentTypes: "" 71 | burst: 10 72 | contentType: application/vnd.kubernetes.protobuf 73 | kubeconfig: /var/lib/kube-proxy/kubeconfig.conf 74 | qps: 5 75 | # clusterCIDR: "" 76 | configSyncPeriod: 15m0s 77 | # conntrack: 78 | # max: null 79 | # maxPerCore: 32768 80 | # min: 131072 81 | # tcpCloseWaitTimeout: 1h0m0s 82 | # tcpEstablishedTimeout: 24h0m0s 83 | enableProfiling: false 84 | healthzBindAddress: 0.0.0.0:10256 85 | hostnameOverride: "" 86 | iptables: 87 | masqueradeAll: false 88 | masqueradeBit: 14 89 | minSyncPeriod: 0s 90 | syncPeriod: 30s 91 | ipvs: 92 | excludeCIDRs: null 93 | minSyncPeriod: 0s 94 | scheduler: "" 95 | syncPeriod: 30s 96 | kind: KubeProxyConfiguration 97 | metricsBindAddress: 127.0.0.1:10249 98 | mode: "" 99 | nodePortAddresses: null 100 | oomScoreAdj: -999 101 | portRange: "" 102 | resourceContainer: /kube-proxy 103 | udpIdleTimeout: 250ms 104 | --- 105 | address: 0.0.0.0 106 | apiVersion: kubelet.config.k8s.io/v1beta1 107 | authentication: 108 | anonymous: 109 | enabled: false 110 | webhook: 111 | cacheTTL: 2m0s 112 | enabled: true 113 | x509: 114 | clientCAFile: /etc/kubernetes/pki/ca.crt 115 | authorization: 116 | mode: Webhook 117 | webhook: 118 | cacheAuthorizedTTL: 5m0s 119 | cacheUnauthorizedTTL: 30s 120 | cgroupDriver: cgroupfs 121 | cgroupsPerQOS: true 122 | clusterDNS: 123 | - "{{DNS_SVC_IP}}" 124 | clusterDomain: cluster.local 125 | configMapAndSecretChangeDetectionStrategy: Watch 126 | containerLogMaxFiles: 5 127 | containerLogMaxSize: 10Mi 128 | contentType: application/vnd.kubernetes.protobuf 129 | cpuCFSQuota: true 130 | cpuCFSQuotaPeriod: 100ms 131 | cpuManagerPolicy: none 132 | cpuManagerReconcilePeriod: 10s 133 | enableControllerAttachDetach: true 134 | enableDebuggingHandlers: true 135 | enforceNodeAllocatable: 136 | - pods 137 | eventBurst: 10 138 | eventRecordQPS: 5 139 | evictionHard: 140 | imagefs.available: 15% 141 | memory.available: 100Mi 142 | nodefs.available: 10% 143 | nodefs.inodesFree: 5% 144 | evictionPressureTransitionPeriod: 5m0s 145 | failSwapOn: true 146 | fileCheckFrequency: 20s 147 | hairpinMode: promiscuous-bridge 148 | healthzBindAddress: 127.0.0.1 149 | healthzPort: 10248 150 | httpCheckFrequency: 20s 151 | imageGCHighThresholdPercent: 85 152 | imageGCLowThresholdPercent: 80 153 | imageMinimumGCAge: 2m0s 154 | iptablesDropBit: 15 155 | iptablesMasqueradeBit: 14 156 | kind: KubeletConfiguration 157 | kubeAPIBurst: 10 158 | kubeAPIQPS: 5 159 | makeIPTablesUtilChains: true 160 | maxOpenFiles: 1000000 161 | maxPods: 110 162 | nodeLeaseDurationSeconds: 40 163 | nodeStatusUpdateFrequency: 10s 164 | oomScoreAdj: -999 165 | podPidsLimit: -1 166 | port: 10250 167 | registryBurst: 10 168 | registryPullQPS: 5 169 | resolvConf: /etc/resolv.conf 170 | rotateCertificates: true 171 | runtimeRequestTimeout: 2m0s 172 | serializeImagePulls: true 173 | staticPodPath: /etc/kubernetes/manifests 174 | streamingConnectionIdleTimeout: 4h0m0s 175 | syncFrequency: 1m0s 176 | volumeStatsAggPeriod: 1m0s 177 | -------------------------------------------------------------------------------- /image/kubelet.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=kubelet: The Kubernetes Node Agent 3 | Documentation=http://kubernetes.io/docs/ 4 | 5 | [Service] 6 | Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf" 7 | Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests" 8 | Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" 9 | Environment="KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local" 10 | Environment="KUBELET_EVICTION_ARGS=--eviction-hard='memory.available<100Mi,nodefs.available<100Mi,nodefs.inodesFree<1000'" 11 | Environment="KUBELET_DIND_ARGS=" 12 | Environment="KUBELET_LOG_ARGS=--v=4" 13 | Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" 14 | Environment="KUBELET_VERSION_SPECIFIC_FLAGS=" 15 | EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env 16 | EnvironmentFile=-/etc/default/kubelet 17 | ExecStart= 18 | ExecStart=/k8s/hyperkube kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_FEATURE_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_NETWORK_ARGS $KUBELET_DNS_ARGS $KUBELET_EVICTION_ARGS $KUBELET_DIND_ARGS $KUBELET_LOG_ARGS $KUBELET_VERSION_SPECIFIC_FLAGS 19 | Restart=always 20 | StartLimitInterval=0 21 | RestartSec=10 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | -------------------------------------------------------------------------------- /image/node-info: -------------------------------------------------------------------------------- 1 | dual-stack-support 2 | -------------------------------------------------------------------------------- /image/rundocker: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Some parts are based on https://github.com/docker/docker/blob/master/hack/dind 17 | # Copyright 2012-2016 Docker, Inc. 18 | # Original version by Jerome Petazzoni 19 | # 20 | # Licensed under the Apache License, Version 2.0 (the "License"); 21 | # you may not use this file except in compliance with the License. 22 | # You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | 32 | set -o errexit 33 | set -o nounset 34 | set -o pipefail 35 | set -o errtrace 36 | 37 | DIND_STORAGE_DRIVER="${DIND_STORAGE_DRIVER:-overlay2}" 38 | 39 | # Ensure shared mount propagation to ensure volume mounting works for Kubernetes. 40 | # Took it from here: 41 | # https://github.com/marun/dkr-systemd-dind/blob/25702cf7a4a73bcc355a492f4e5ca96cc562a5a5/dind-setup.sh#L46 42 | mount --make-shared / 43 | 44 | # securityfs is mounted by systemd itself if it isn't being run in a container. 45 | # In case of containerized systemd, it doesn't want to have systemd mounted 46 | # on startup because it wants to avoid dealing with IMA in this case. 47 | # See https://github.com/systemd/systemd/blob/master/src/shared/ima-util.c 48 | # IMA stands for Integrity Measurement Architecture 49 | # https://sourceforge.net/p/linux-ima/wiki/Home/#integrity-measurement-architecture-ima 50 | # Still, we need to have securityfs in DIND for AppArmor detection and 51 | # support for --privileged mode, so we're mounting it there. 52 | if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security; then 53 | mount -t securityfs none /sys/kernel/security || { 54 | echo >&2 'Could not mount /sys/kernel/security.' 55 | echo >&2 'AppArmor detection and --privileged mode might break.' 56 | } 57 | fi 58 | 59 | # Mount /tmp (conditionally) 60 | if ! mountpoint -q /tmp; then 61 | mount -t tmpfs none /tmp 62 | fi 63 | 64 | echo "Trying to load overlay module (this may fail)" >&2 65 | modprobe overlay || true 66 | 67 | extra_opts=() 68 | if [[ ${DIND_STORAGE_DRIVER} == overlay2 ]] && grep -q '[[:blank:]]overlay[[:blank:]]*' /proc/filesystems; then 69 | # --storage-opt overlay2.override_kernel_check=true is needed for CentOS systems 70 | extra_opts+=(--storage-driver="${DIND_STORAGE_DRIVER}" 71 | --storage-opt overlay2.override_kernel_check=true) 72 | else 73 | extra_opts+=(--storage-driver="${DIND_STORAGE_DRIVER}") 74 | fi 75 | 76 | # FIXME: this fix really concerns kubelet, so perhaps it should go 77 | # into another place 78 | # Kubelet will try to contact GCE metadata server on startup if 79 | # it detects a GCE VM and this breaks kdc on Travis 80 | if [[ $(cat /sys/class/dmi/id/product_name) =~ Google ]]; then 81 | echo "KDC" >/dmi_product_name 82 | mount --bind /dmi_product_name /sys/class/dmi/id/product_name 83 | fi 84 | 85 | # Directories bind-mounted from /dind are not managed by the Docker's snapshotting mechanism, e.g. overlayfs. 86 | # Some apps expect /var/lib/kubelet/pods/{podid}/volumes/kubernetes.io~empty-dir to be non-overlayfs. 87 | mkdir -p /dind/pods /var/lib/kubelet/pods 88 | grep "/var/lib/kubelet/pods[[:space:]]" /proc/mounts || mount --bind /dind/pods /var/lib/kubelet/pods 89 | 90 | # `kubectl logs -f` expects /var/log/pods to be non-overlayfs. 91 | mkdir -p /dind/logs /var/log/pods 92 | grep "/var/log/pods[[:space:]]" /proc/mounts || mount --bind /dind/logs /var/log/pods 93 | 94 | if [[ ${DIND_CRI} = containerd ]]; then 95 | extra_opts+=("--cri-containerd") 96 | fi 97 | 98 | exec /usr/bin/dockerd -H unix:// "${extra_opts[@]}" --data-root /dind/docker "$@" 99 | -------------------------------------------------------------------------------- /image/snapshot: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | 21 | function snapshot::retry { 22 | # based on retry function in hack/jenkins/ scripts in k8s source 23 | for i in {1..10}; do 24 | "$@" && return 0 || sleep ${i} 25 | done 26 | "$@" 27 | } 28 | 29 | function snapshot::clean-containerd { 30 | if [[ ${DIND_CRI} != containerd ]]; then 31 | return 32 | fi 33 | crictl="crictl -r unix:///var/run/containerd/containerd.sock" 34 | # first, we need to stop the containers 35 | ${crictl} ps -q | xargs -r ${crictl} stop -t 0 36 | # then we remove the containers 37 | ${crictl} ps -qa | xargs -r ${crictl} rm 38 | # then we stop the pod sandboxes 39 | ${crictl} pods -q | xargs -r ${crictl} stopp 40 | # and then we remove the pod sandboxes 41 | ${crictl} pods -q | xargs -r ${crictl} rmp 42 | systemctl stop containerd 43 | } 44 | 45 | function snapshot::text_join { 46 | local IFS="$1"; shift; echo "$*" 47 | } 48 | 49 | function snapshot::is-service-running { 50 | kubectl get pods -n kube-system -o name | egrep -q "^pods*/($(snapshot::text_join '|' ${@/%/-}))" 51 | } 52 | 53 | function snapshot::scale-down-service { 54 | local service 55 | for service in "$@"; do 56 | if snapshot::is-service-running ${service} ; then 57 | snapshot::retry kubectl scale deployment --replicas=0 -n kube-system ${service} 58 | else 59 | echo "${service} not found, skip it" 60 | fi 61 | done 62 | 63 | local n=80 64 | while snapshot::is-service-running $@; do 65 | if ((--n == 0)); then 66 | echo "WARNING: controller manager glitch: $(snapshot::text_join '&' $@) won't stop; pods may 'blink' for some time after restore" >&2 67 | systemctl stop kubelet docker 68 | snapshot::clean-containerd 69 | return 70 | fi 71 | sleep 0.3 72 | done 73 | } 74 | 75 | function snapshot::prepare { 76 | if [[ -f /etc/kubernetes/manifests/kube-controller-manager.json || -f /etc/kubernetes/manifests/kube-controller-manager.yaml ]]; then 77 | # remove kube-dns and kube-dashboard pods to avoid 'blinking' 78 | # FIXME: when using k8s 1.6 kube-dns pods stay in 'Terminating' state 79 | # Possibly related: https://github.com/kubernetes/kubernetes/issues/42685 80 | 81 | DNS_SERVICE="$(kubectl get deployment -n kube-system -l k8s-app=kube-dns -o name | cut -d / -f 2)" 82 | snapshot::scale-down-service ${DNS_SERVICE} kubernetes-dashboard 83 | 84 | mkdir /manifests.bak 85 | # stop controller manager so it doesn't launch new proxy daemon pods 86 | mv /etc/kubernetes/manifests/kube-controller-manager.* /manifests.bak/ 87 | n=100 88 | while kubectl get pod kube-controller-manager-kube-master -n kube-system -o name 2>/dev/null | grep -q pods*/; do 89 | if ((--n == 0)); then 90 | echo "WARNING: kubelet glitch: kube-controller-manager won't stop; pods may 'blink' for some time after restore" >&2 91 | break 92 | fi 93 | sleep 0.3 94 | done 95 | 96 | # delete proxy pods so they don't "blink" after cluster restart 97 | snapshot::retry kubectl delete pod --now -l k8s-app=kube-proxy -n kube-system 2>/dev/null 98 | n=40 99 | while kubectl get pod -n kube-system -o name -l k8s-app=kube-proxy | grep -q '^pods*/'; do 100 | if ((--n == 0)); then 101 | echo "WARNING: cluster glitch: proxy pods aren't removed; pods may 'blink' for some time after restore" >&2 102 | break 103 | fi 104 | sleep 0.3 105 | done 106 | snapshot::retry kubectl get pods -n kube-system 107 | systemctl stop kubelet docker 108 | snapshot::clean-containerd 109 | # now it's safe to put back the manifest 110 | mv /manifests.bak/* /etc/kubernetes/manifests/ 111 | rmdir /manifests.bak 112 | else 113 | systemctl stop kubelet docker 114 | snapshot::clean-containerd 115 | fi 116 | } 117 | 118 | function snapshot::save { 119 | # the scripts expects output the output of 'docker diff' as input 120 | grep -v '^D ' | 121 | sed 's@^. /@@' | 122 | egrep -v '^(tmp|var/log|etc/mtab|run/|var/lib/containerd)' | 123 | while read path; do 124 | # Here we only add dirs if they're empty, as non-empty dirs will 125 | # be added automatically by tar 126 | if [[ ! -d ${path} || -h ${path} || ! $(ls -A "${path}") ]]; then 127 | echo "${path}" 128 | fi 129 | done | 130 | tar -C / -cf /dind/snapshot.tar -T - 131 | if [[ -f /var/run/kubernetes/kubelet-client.key ]]; then 132 | tar -C / -cf /dind/var-run-kubernetes.tar var/run/kubernetes 133 | fi 134 | 135 | systemctl start kubelet docker 136 | } 137 | 138 | function snapshot::restore { 139 | tar -C / -xf /dind/snapshot.tar 140 | if [[ -f /dind/var-run-kubernetes.tar ]]; then 141 | tar -C / -xf /dind/var-run-kubernetes.tar 142 | fi 143 | if [[ ${1:-} = "-u" ]]; then 144 | wrapkubeadm ensure-binaries 145 | fi 146 | start_services docker kubelet 147 | } 148 | 149 | case "${1:-}" in 150 | prepare) 151 | snapshot::prepare 152 | ;; 153 | save) 154 | snapshot::save 155 | ;; 156 | restore) 157 | snapshot::restore 158 | ;; 159 | update_and_restore) 160 | snapshot::restore -u 161 | ;; 162 | *) 163 | echo "usage:" >&2 164 | echo " $0 save" >&2 165 | echo " $0 restore" >&2 166 | exit 1 167 | ;; 168 | esac 169 | -------------------------------------------------------------------------------- /image/start_services: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | 21 | # wait for systemd to start in the container 22 | (( timeout = 60 + SECONDS )) 23 | until systemctl status >/dev/null 2>&1; do 24 | if (( SECONDS >= timeout )); then 25 | echo 'Timed out waiting for systemd.' >&2 26 | exit 1 27 | fi 28 | sleep 1 29 | done 30 | 31 | systemctl daemon-reload 32 | 33 | if [ "${1:-}" = "docker" ]; then 34 | systemctl stop docker 35 | fi 36 | 37 | for service in "$@"; do 38 | if ! systemctl start "${service}"; then 39 | echo >&2 "${service} failed to start. Diagnostics below:" 40 | systemctl status "${service}" 41 | exit 1 42 | fi 43 | done 44 | -------------------------------------------------------------------------------- /image/wrapkubeadm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | 21 | HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE:-keep://}" 22 | KUBEADM_SOURCE="${KUBEADM_SOURCE:-keep://}" 23 | 24 | hypokube_base_image="mirantis/hypokube:base" 25 | hypokube_final_image="mirantis/hypokube:final" 26 | build_src_dir="/go/src/k8s.io/kubernetes" 27 | binary_dir="/k8s" 28 | # FIXME: don't hardcode 29 | platform="linux/amd64" 30 | 31 | apiserver_static_pod="/etc/kubernetes/manifests/kube-apiserver.yaml" 32 | scheduler_static_pod="/etc/kubernetes/manifests/kube-scheduler.yaml" 33 | controller_manager_static_pod="/etc/kubernetes/manifests/kube-controller-manager.yaml" 34 | 35 | # jq filters follow. 36 | # This filter updates static pod definitions to mount /hyperkube from hosts: 37 | mount_binaries='.spec.volumes|=.+[{"name":"hyperkube",hostPath:{"path":"/k8s/hyperkube"}}]|.spec.containers[0].volumeMounts|=.+[{"name":"hyperkube", "mountPath":"/hyperkube"}]' 38 | proxy_mount_binaries='.items[0].spec.template.spec.volumes|=.+[{"name":"hyperkube",hostPath:{"path":"/k8s/hyperkube"}}]|.items[0].spec.template.spec.containers[0].volumeMounts|=.+[{"name":"hyperkube", "mountPath":"/hyperkube"}]' 39 | # Make apiserver listen insecurely on 0.0.0.0, it's ~ok because it's done in a container 40 | # TODO: think about more secure possibilities 41 | apiserver_insecure_bind_address='.spec.containers[0].command|=map(select(startswith("--insecure-bind-address=")|not))+["--insecure-bind-address=0.0.0.0"]' 42 | # Make apiserver accept insecure connections on port 8080 43 | # TODO: don't use insecure port 44 | apiserver_insecure_bind_port='.spec.containers[0].command|=map(select(startswith("--insecure-port=")|not))+["--insecure-port=8080"]' 45 | 46 | function dind::cluster-cidr { 47 | if [[ ! -e "/v6-mode" ]]; then 48 | ip addr show docker0 | grep -w inet | awk '{ print $2; }' 49 | fi 50 | } 51 | 52 | # Update kube-proxy CIDR, enable --masquerade-all and disable conntrack (see dind::frob-proxy below) 53 | function dind::proxy-cidr-and-no-conntrack { 54 | local cluster_cidr="$(dind::cluster-cidr)" 55 | 56 | echo ".items[0].spec.template.spec.containers[0].command |= .+ [\"--cluster-cidr=${cluster_cidr}\", \"--masquerade-all\", \"--conntrack-max=0\", \"--conntrack-max-per-core=0\"]" 57 | } 58 | 59 | is_master= 60 | if [[ "$(hostname)" == kube-master* ]]; then 61 | is_master=y 62 | fi 63 | 64 | # unpack saved /boot & /lib/modules if there are any 65 | if [[ -f /dind-sys/sys.tar ]]; then 66 | tar -C / -xf /dind-sys/sys.tar 67 | fi 68 | 69 | function dind::retry { 70 | # based on retry function in hack/jenkins/ scripts in k8s source 71 | for i in {1..10}; do 72 | "$@" && return 0 || sleep ${i} 73 | done 74 | "$@" 75 | } 76 | 77 | function dind::get-binary-from-build-data-container { 78 | local filename="$1" 79 | local src_path="${build_src_dir}/_output/dockerized/bin/${platform}/${filename}" 80 | local dest_path="${binary_dir}/${filename}" 81 | if [[ -f "${src_path}" ]]; then 82 | cp "${src_path}" "${binary_dir}/${filename}" 83 | else 84 | echo "Cannot locate the file in build data container: ${src_path}" >&2 85 | return 1 86 | fi 87 | } 88 | 89 | function dind::verify-binary { 90 | local filename="$1" 91 | local dest_path="${binary_dir}/${filename}" 92 | if [[ ! -f "${dest_path}" ]]; then 93 | echo "Binary not found in the container: ${dest_path}" >&2 94 | exit 1 95 | fi 96 | } 97 | 98 | function dind::get-binary-from-url { 99 | local filename="$1" 100 | local url="$2" 101 | local dest_path="${binary_dir}/${filename}" 102 | if [[ -f "${dest_path}" ]]; then 103 | return 0 104 | fi 105 | wget -O "${dest_path}" "${url}" 106 | chmod +x "${dest_path}" 107 | } 108 | 109 | function dind::get-binary { 110 | local filename="$1" 111 | local src="$2" 112 | 113 | if [[ ${src} = "build://" ]]; then 114 | dind::get-binary-from-build-data-container "${filename}" 115 | elif [[ ${src} = "keep://" ]]; then 116 | dind::verify-binary "${filename}" 117 | elif [[ ${src} = gs://* ]]; then 118 | if [ ! -d "google-cloud-sdk" ] 119 | then 120 | dind::install-google-cloud-sdk 121 | fi 122 | ./google-cloud-sdk/bin/gsutil cp ${src} ${binary_dir} 123 | chmod 755 ${binary_dir}/${filename} 124 | else 125 | dind::get-binary-from-url "${filename}" "${src}" 126 | fi 127 | } 128 | 129 | function dind::get-binaries { 130 | dind::get-binary hyperkube "${HYPERKUBE_SOURCE}" 131 | dind::get-binary kubeadm "${KUBEADM_SOURCE}" 132 | } 133 | 134 | function dind::image-exists { 135 | local name="$1" 136 | if ! docker inspect "${name}" >&/dev/null; then 137 | return 1 138 | fi 139 | } 140 | 141 | function dind::install-google-cloud-sdk { 142 | cd 143 | apt-get -y install wget python 144 | wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-186.0.0-linux-x86_64.tar.gz 145 | tar -xvf ./google-cloud-sdk-186.0.0-linux-x86_64.tar.gz 146 | ./google-cloud-sdk/install.sh -q 147 | rm ./google-cloud-sdk-186.0.0-linux-x86_64.tar.gz 148 | } 149 | 150 | 151 | function dind::ensure-binaries-and-hypokube { 152 | # avoid race - if binaries need to be fetched, 153 | # do it in just one container 154 | if [[ ${is_master} ]] || [[ ${HYPERKUBE_SOURCE} = gs://* ]]; then 155 | dind::get-binaries 156 | fi 157 | local hyperkube_bin="${binary_dir}/hyperkube" 158 | # in prebuilt (complete) DIND images, final hypokube image is 159 | # already present 160 | if ! dind::image-exists "${hypokube_final_image}" || [[ ${HYPERKUBE_SOURCE} = build:// ]] || [[ ${HYPERKUBE_SOURCE} = gs://* ]] ; then 161 | # We use current hyperkube binary to build final hypokube image. 162 | # It will be used for initial cluster deployment. 163 | # After we "frob" the cluster, the hyperkube image is taken from /k8s directory. 164 | cp "${hyperkube_bin}" /hypokube/ 165 | docker build -t "${hypokube_final_image}" -f /hypokube/hypokube.dkr /hypokube 166 | hypokube_extra_tags=() 167 | # In k8s 1.13+, we can't use unifiedControlPlaneImage anymore, 168 | # so we have to resort to a hack by pretending we have already 169 | # pulled the hyperkube image 170 | # https://github.com/kubernetes/kubernetes/pull/70793 171 | if kubeadm version -o short >&/dev/null; then 172 | image_version="$(kubeadm version -o short|sed 's/-.*//')" 173 | hypokube_extra_tags=("k8s.gcr.io/hyperkube:${image_version}") 174 | fi 175 | for tag in "${hypokube_extra_tags[@]}"; do 176 | docker tag "${hypokube_final_image}" "${tag}" 177 | done 178 | fi 179 | if [[ ${DIND_CRI} = containerd ]]; then 180 | docker image save mirantis/hypokube:final "${hypokube_extra_tags[@]}" | 181 | ctr -n k8s.io images import - 182 | fi 183 | } 184 | 185 | function dind::replace { 186 | local path="$1" 187 | # Replace the file after reading all of the standard input 188 | # via: https://github.com/stedolan/jq/issues/105#issuecomment-272712293 189 | # Could use 'sponge' from moreutils, but don't want to make the image larger 190 | awk 'BEGIN{RS="";getline<"-";print>ARGV[1]}' "${path}" 191 | } 192 | 193 | function dind::join-filters { 194 | local IFS="|" 195 | echo "$*" 196 | } 197 | 198 | function dind::frob-proxy-configmap { 199 | # This function modifies config.conf inside kube-proxy configmap 200 | # to apply the following settings (clusterCIDR may be different): 201 | # 202 | # clusterCIDR: "172.17.0.1/16" 203 | # conntrack: 204 | # max: 0 205 | # maxPerCore: 0 206 | # masqueradeAll: true 207 | local cluster_cidr="$(dind::cluster-cidr)" 208 | local confdir="$(mktemp -d kube-proxy-conf-XXXXXX)" 209 | local conffile="${confdir}/config.conf" 210 | kubectl get configmap -n kube-system kube-proxy -o jsonpath='{.data.config\.conf}' >"${conffile}" 211 | # sadly we can't easily convery yaml to json in this case 212 | # (kubectl convert will not work for kube-proxy config), 213 | # so we have to use sed on the yaml file 214 | sed -i "s@\bclusterCIDR:.*@clusterCIDR: \"${cluster_cidr}\"@" "${conffile}" 215 | # FIXME: here we assume that max: and maxPerCore: 216 | # only occur within conntrack: section 217 | sed -i "s/\bmax:.*/max: 0/;s/\bmaxPerCore:.*/maxPerCore: 0/" "${conffile}" 218 | sed -i "s/\bmasqueradeAll:.*/masqueradeAll: true/" "${conffile}" 219 | # Use a simple hack described in comments for 220 | # https://github.com/kubernetes/kubernetes/issues/30558 221 | # to replace just one key in the configmap 222 | kubectl create configmap \ 223 | -n kube-system --dry-run -o yaml --from-file=config.conf="${conffile}" kube-proxy | 224 | kubectl apply -f - 225 | 226 | # only remove the temp dir (inside the container) if the 227 | # commands above have succeeded 228 | rm -rf "${confdir}" 229 | } 230 | 231 | function dind::frob-proxy { 232 | # Trying to change conntrack settings fails even in priveleged containers, 233 | # so we need to avoid it. Here's sample error message from kube-proxy: 234 | # I1010 21:53:00.525940 1 conntrack.go:57] Setting conntrack hashsize to 49152 235 | # Error: write /sys/module/nf_conntrack/parameters/hashsize: operation not supported 236 | # write /sys/module/nf_conntrack/parameters/hashsize: operation not supported 237 | # 238 | # Recipe by @errordeveloper: 239 | # https://github.com/kubernetes/kubernetes/pull/34522#issuecomment-253248985 240 | 241 | # sometimes this fails with 'the server doesn't have a resource type "daemonsets"' 242 | # if done too early 243 | dind::retry kubectl -n kube-system get ds kube-proxy >/dev/null 244 | 245 | local filters 246 | # in case if the configmap is used, command line args will be overridden by it 247 | if kubectl get configmap -n kube-system -l app=kube-proxy -o jsonpath='{.items[*].data.config\.conf}' | grep -q apiVersion; then 248 | dind::frob-proxy-configmap 249 | filters="${proxy_mount_binaries}" 250 | else 251 | filters="$(dind::join-filters "${proxy_mount_binaries}" "$(dind::proxy-cidr-and-no-conntrack)")" 252 | fi 253 | 254 | kubectl -n kube-system get ds -l k8s-app=kube-proxy -o json | 255 | jq "${filters}" | 256 | kubectl apply --force -f - 257 | kubectl -n kube-system delete pods --now -l "k8s-app=kube-proxy" || true 258 | } 259 | 260 | kubelet_dind_args=(--fail-swap-on=false 261 | --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf ) 262 | 263 | KUBELET_FEATURE_GATES=${KUBELET_FEATURE_GATES:-none} 264 | if [ "${KUBELET_FEATURE_GATES}" != 'none' ]; then 265 | kubelet_dind_args+=(--feature-gates=${KUBELET_FEATURE_GATES}) 266 | fi 267 | 268 | function dind::set-kubelet-dind-args { 269 | if [[ ${#kubelet_dind_args[@]} -gt 0 ]]; then 270 | sed -i "s@KUBELET_DIND_ARGS=@KUBELET_DIND_ARGS=${kubelet_dind_args[*]}@" /lib/systemd/system/kubelet.service 271 | fi 272 | } 273 | 274 | function dind::prepare-for-kubeadm { 275 | start_services docker 276 | systemctl enable docker 277 | 278 | if ! docker images gcr.io/google_containers/pause-amd64|grep -q 'gcr\.io/'; then 279 | time lz4 -dc /save.tar.lz4 | docker load 280 | fi 281 | dind::ensure-binaries-and-hypokube 282 | 283 | # Ensure that DNS drop-in is created 284 | systemctl start dindnet 285 | 286 | dind::set-kubelet-dind-args 287 | 288 | systemctl daemon-reload 289 | 290 | # enable kubelet right before starting kubeadm to avoid 291 | # longer delays between container restarts 292 | start_services kubelet 293 | systemctl enable kubelet 294 | } 295 | 296 | function dind::kubeadm-reset { 297 | if kubeadm reset --help 2>/dev/null|grep -q -- --force; then 298 | kubeadm reset --force || true 299 | else 300 | kubeadm reset || true 301 | fi 302 | } 303 | 304 | function dind::do-kubeadm { 305 | if ! time kubeadm "$@"; then 306 | echo "*** 'kubeadm $*' failed, doing kubeadm reset ***" >&2 307 | cp -av /etc/cni /etc/cni.bak 308 | dind::kubeadm-reset 309 | rm -rf /etc/cni 310 | mv /etc/cni.bak /etc/cni 311 | systemctl restart kubelet 312 | return 1 313 | fi 314 | } 315 | 316 | 317 | if [[ ${1:-} = ensure-binaries ]]; then 318 | dind::ensure-binaries-and-hypokube 319 | else 320 | # Weave depends on /etc/machine-id being unique 321 | rm -f /etc/machine-id 322 | systemd-machine-id-setup 323 | dind::prepare-for-kubeadm 324 | 325 | dind::retry dind::do-kubeadm "$@" 326 | 327 | if [[ ${is_master} ]]; then 328 | dind::frob-proxy 329 | fi 330 | fi 331 | 332 | # TODO: only mount /k8s/hyperkube into containers in case if build:// source is used 333 | # TODO: after 'frobbing' the components, wait for them to restart (restart count must increase) 334 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # Copyright 2017 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o errtrace 20 | 21 | if [ $(uname) = Darwin ]; then 22 | readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} 23 | else 24 | readlinkf(){ readlink -f "$1"; } 25 | fi 26 | DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" 27 | 28 | . "${DIND_ROOT}"/build/buildconf.sh 29 | 30 | NOBUILD="${NOBUILD:-}" 31 | TEST_CASE="${TEST_CASE:-}" 32 | # pin 'master' tests to a specific PR number 33 | # (e.g. K8S_PR=44143) 34 | K8S_PR="${K8S_PR:-}" 35 | 36 | tempdir="$(mktemp -d)" 37 | export KUBECTL_DIR="${tempdir}" 38 | export KUBECONFIG="${KUBECTL_DIR}/kube.conf" 39 | export DOWNLOAD_KUBECTL=y 40 | 41 | function cleanup { 42 | if [[ ${TRAVIS:-} && $? -ne 0 ]]; then 43 | # Use temp file to avoid mixing error messages from the script 44 | # with base64 content 45 | export PATH="${KUBECTL_DIR}:${PATH}" 46 | "${DIND_ROOT}"/dind-cluster.sh dump64 >"${tempdir}/dump" 47 | cat "${tempdir}/dump" 48 | fi 49 | rm -rf '${tempdir}' 50 | } 51 | trap cleanup EXIT 52 | 53 | # FIXME: 192.168.0.0/16 causes problems with Travis(?) 54 | export POD_NETWORK_CIDR="10.244.0.0/16" 55 | 56 | if [[ ${NOBUILD} ]]; then 57 | bash -x "${DIND_ROOT}"/dind-cluster.sh clean 58 | else 59 | export DIND_IMAGE=mirantis/kubeadm-dind-cluster:local 60 | fi 61 | 62 | function test-cluster { 63 | local kubectl="${KUBECTL_DIR}/kubectl" 64 | local defaultContext='dind' 65 | 66 | if [[ ${BUILD_HYPERKUBE:-} ]]; then 67 | kubectl="${PWD}/cluster/kubectl.sh" 68 | fi 69 | if [[ ! ${NOBUILD} ]]; then 70 | ( 71 | cd "${DIND_ROOT}" 72 | ./build/build-local.sh 73 | ) 74 | fi 75 | bash -x "${DIND_ROOT}"/dind-cluster.sh clean 76 | time bash -x "${DIND_ROOT}"/dind-cluster.sh up 77 | "${kubectl}" --context="$defaultContext" get pods --all-namespaces | egrep 'coredns|kube-dns' 78 | time bash -x "${DIND_ROOT}"/dind-cluster.sh up 79 | "${kubectl}" --context="$defaultContext" get pods --all-namespaces | egrep 'coredns|kube-dns' 80 | bash -x "${DIND_ROOT}"/dind-cluster.sh down 81 | bash -x "${DIND_ROOT}"/dind-cluster.sh clean 82 | } 83 | 84 | function test-cluster-src { 85 | ( 86 | local version="${1:-}" 87 | if [[ ! -d "kubernetes" ]]; then 88 | git clone https://github.com/kubernetes/kubernetes.git 89 | fi 90 | cd kubernetes 91 | if [[ ${version} ]]; then 92 | git checkout "${version}" 93 | elif [[ ${K8S_PR} ]]; then 94 | git fetch origin "pull/${K8S_PR}/head:testbranch" 95 | git checkout testbranch 96 | fi 97 | export BUILD_KUBEADM=y 98 | export BUILD_HYPERKUBE=y 99 | test-cluster 100 | ) 101 | } 102 | 103 | function test-case-1.12 { 104 | ( 105 | export KUBEADM_URL="${KUBEADM_URL_1_12}" 106 | export KUBEADM_SHA1="${KUBEADM_SHA1_1_12}" 107 | export HYPERKUBE_URL="${HYPERKUBE_URL_1_12}" 108 | export HYPERKUBE_SHA1="${HYPERKUBE_SHA1_1_12}" 109 | export KUBECTL_LINUX_URL=${KUBECTL_LINUX_URL_1_12}" 110 | export KUBECTL_LINUX_SHA1=${KUBECTL_LINUX_SHA1_1_12}" 111 | export KUBECTL_DARWIN_URL=${KUBECTL_DARWIN_URL_1_12}" 112 | export KUBECTL_DARWIN_SHA1=${KUBECTL_DARWIN_SHA1_1_12}" 113 | if [[ ${NOBUILD} ]]; then 114 | export DIND_K8S_VERSION=v1.12 115 | docker pull "${DIND_IMAGE}" 116 | fi 117 | test-cluster 118 | ) 119 | } 120 | 121 | function test-case-1.13 { 122 | ( 123 | export KUBEADM_URL="${KUBEADM_URL_1_13}" 124 | export KUBEADM_SHA1="${KUBEADM_SHA1_1_13}" 125 | export HYPERKUBE_URL="${HYPERKUBE_URL_1_13}" 126 | export HYPERKUBE_SHA1="${HYPERKUBE_SHA1_1_13}" 127 | export KUBECTL_LINUX_URL=${KUBECTL_LINUX_URL_1_13}" 128 | export KUBECTL_LINUX_SHA1=${KUBECTL_LINUX_SHA1_1_13}" 129 | export KUBECTL_DARWIN_URL=${KUBECTL_DARWIN_URL_1_13}" 130 | export KUBECTL_DARWIN_SHA1=${KUBECTL_DARWIN_SHA1_1_13}" 131 | if [[ ${NOBUILD} ]]; then 132 | export DIND_K8S_VERSION=v1.13 133 | docker pull "${DIND_IMAGE}" 134 | fi 135 | test-cluster 136 | ) 137 | } 138 | 139 | function test-case-1.14 { 140 | ( 141 | export KUBEADM_URL="${KUBEADM_URL_1_14}" 142 | export KUBEADM_SHA1="${KUBEADM_SHA1_1_14}" 143 | export HYPERKUBE_URL="${HYPERKUBE_URL_1_14}" 144 | export HYPERKUBE_SHA1="${HYPERKUBE_SHA1_1_14}" 145 | export KUBECTL_LINUX_URL=${KUBECTL_LINUX_URL_1_14}" 146 | export KUBECTL_LINUX_SHA1=${KUBECTL_LINUX_SHA1_1_14}" 147 | export KUBECTL_DARWIN_URL=${KUBECTL_DARWIN_URL_1_14}" 148 | export KUBECTL_DARWIN_SHA1=${KUBECTL_DARWIN_SHA1_1_14}" 149 | if [[ ${NOBUILD} ]]; then 150 | export DIND_K8S_VERSION=v1.14 151 | docker pull "${DIND_IMAGE}" 152 | fi 153 | test-cluster 154 | ) 155 | } 156 | 157 | function test-case-1.15 { 158 | ( 159 | export KUBEADM_URL="${KUBEADM_URL_1_15}" 160 | export KUBEADM_SHA1="${KUBEADM_SHA1_1_15}" 161 | export HYPERKUBE_URL="${HYPERKUBE_URL_1_15}" 162 | export HYPERKUBE_SHA1="${HYPERKUBE_SHA1_1_15}" 163 | export KUBECTL_LINUX_URL=${KUBECTL_LINUX_URL_1_15}" 164 | export KUBECTL_LINUX_SHA1=${KUBECTL_LINUX_SHA1_1_15}" 165 | export KUBECTL_DARWIN_URL=${KUBECTL_DARWIN_URL_1_15}" 166 | export KUBECTL_DARWIN_SHA1=${KUBECTL_DARWIN_SHA1_1_15}" 167 | if [[ ${NOBUILD} ]]; then 168 | export DIND_K8S_VERSION=v1.15 169 | docker pull "${DIND_IMAGE}" 170 | fi 171 | test-cluster 172 | ) 173 | } 174 | 175 | function test-case-src-master { 176 | test-cluster-src master 177 | } 178 | 179 | function test-case-dump-succeeds() { 180 | local d="${DIND_ROOT}/dind-cluster.sh" 181 | 182 | "$d" up 183 | "$d" dump >/dev/null || { 184 | fail "Expected '$d dump' to succeed" 185 | } 186 | } 187 | 188 | function test-case-restore() { 189 | local d="${DIND_ROOT}/dind-cluster.sh" 190 | 191 | "$d" up 192 | APISERVER_PORT=8083 "$d" up || { 193 | fail 'Expected to be able to restore the cluster with the APIServer listening on 8083' 194 | } 195 | 196 | "$d" clean 197 | } 198 | 199 | function fail() { 200 | local msg="$1" 201 | echo -e "\033[1;31m${msg}\033[0m" >&2 202 | return 1 203 | } 204 | 205 | if [[ ! ${TEST_CASE} ]]; then 206 | test-case-1.12 207 | test-case-1.13 208 | test-case-1.14 209 | test-case-1.15 210 | # test-case-src-master 211 | test-case-dump-succeeds 212 | test-case-restore 213 | else 214 | "test-case-${TEST_CASE}" 215 | fi 216 | 217 | echo "*** OK ***" 218 | 219 | # TODO: build k8s master daily using Travis cron feature 220 | -------------------------------------------------------------------------------- /test/shared.inc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | if [ $(uname) = Darwin ]; then 17 | readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} 18 | else 19 | readlinkf(){ readlink -f "$1"; } 20 | fi 21 | DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")") && cd .. ; pwd)" 22 | 23 | tempdir="$(mktemp -d)" 24 | trap 'rm -rf -- "$tempdir"' EXIT 25 | export KUBECTL_DIR="${tempdir}" 26 | export KUBECONFIG="${KUBECTL_DIR}/kube.conf" 27 | 28 | function fail() { 29 | local msg="$1" 30 | echo -e "\033[1;31m${msg}\033[0m" >&2 31 | return 1 32 | } 33 | 34 | function get-ip-for-pod() { 35 | local pod_name="$1" 36 | local mode="$2" 37 | local ip="" 38 | 39 | kubectl="${kubectl:-kubectl}" 40 | 41 | if [[ ${mode} = "ipv6" ]]; then 42 | ip="$( $kubectl exec $pod_name ip addr list eth0 | grep -i global | grep -w inet6 | head -1 | awk '$1 == "inet6" {print $2}' | cut -f 1 -d"/" )" 43 | else 44 | ip="$( $kubectl exec $pod_name ip addr list eth0 | grep -i global | grep -w inet | head -1 | awk '$1 == "inet" {print $2}' | cut -f 1 -d"/" )" 45 | fi 46 | 47 | if [[ -n "$ip" ]]; then 48 | echo "$ip" 49 | else 50 | echo "UNKNOWN" 51 | fi 52 | } 53 | 54 | function get-node-selector-override() { 55 | local node="${1}" 56 | echo '{ "apiVersion": "v1", "spec": { "nodeSelector": { "kubernetes.io/hostname": "'"$node"'" } } }' 57 | } 58 | 59 | function get-hostname-override() { 60 | local name="${1}" 61 | echo '{ "apiVersion": "v1", "spec": { "hostname": "'"$name"'" } }' 62 | } 63 | 64 | function starts-with() { 65 | local needle="$1" 66 | local haystack="$2" 67 | [[ "$haystack" = "$needle"* ]] 68 | } 69 | -------------------------------------------------------------------------------- /test/test-multiple-clusters.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o xtrace 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o errtrace 21 | 22 | . "$( dirname "$0" )/shared.inc.sh" 23 | 24 | set -eu 25 | set -o pipefail 26 | 27 | function test-case-multiple-instances { 28 | local defaultLabel='mirantis.kubeadm_dind_cluster_runtime' 29 | local secondClusterID="20" 30 | local thirdClusterLabel="third" 31 | local d="./dind-cluster.sh" 32 | 33 | export PATH="${KUBECTL_DIR}:${PATH}" 34 | export DIND_PORT_FORWARDER_WAIT=1 35 | export DIND_PORT_FORWARDER="${PWD}/build/portforward.sh" 36 | 37 | "${d}" up 38 | CLUSTER_ID="$secondClusterID" "${d}" up 39 | DIND_LABEL="$thirdClusterLabel" "${d}" up 40 | 41 | # masters 42 | test "$(countContainersWithExactName "kube-master")" -eq 1 || { 43 | fail 'Expected exactly one container with name "kube-master" to exist - cluster created with default label' 44 | } 45 | test "$(countContainersWithExactName "kube-master-cluster-${secondClusterID}")" -eq 1 || { 46 | fail "Expected exactly one container with name 'kube-master-cluster-${secondClusterID}' to exist - cluster created with custom ID" 47 | } 48 | test "$(countContainersWithExactName "kube-master-${thirdClusterLabel}")" -eq 1 || { 49 | fail "Expected exactly one container with name 'kube-master-${thirdClusterLabel}' to exist - cluster created with custom label and assigned ID" 50 | } 51 | 52 | # nodes 53 | test "$(countContainersWithExactName "kube-node-\\d{1}")" -ge 1 || { 54 | fail 'Expected at least one container with name "kube-node-" to exist - cluster created with default label' 55 | } 56 | test "$(countContainersWithExactName "kube-node-\\d{1}-cluster-${secondClusterID}")" -ge 1 || { 57 | fail "Expected at least one container with name 'kube-node--cluster-${secondClusterID}' to exist - cluster created with custom ID" 58 | } 59 | test "$(countContainersWithExactName "kube-node-\\d{1}-${thirdClusterLabel}")" -ge 1 || { 60 | fail "Expected at least one container with name 'kube-node--${thirdClusterLabel}' to exist - cluster created with custom label and assigned ID" 61 | } 62 | 63 | # volumes 64 | test "$(countVolumesWithFilter "name=kubeadm-dind-kube-master$")" -eq 1 || { 65 | fail 'Expected one volume for the kube master to exist - cluster created with default label' 66 | } 67 | test "$(countVolumesWithFilter "name=kubeadm-dind-kube-node-\\d+$")" -ge 1 || { 68 | fail 'Expected one volume for the kube nodes to exist - cluster created with default label' 69 | } 70 | 71 | test "$(countVolumesWithFilter "name=kubeadm-dind-kube-master-cluster-${secondClusterID}$")" -eq 1 || { 72 | fail 'Expected one volume for the kube master to exist - cluster created with custom ID' 73 | } 74 | test "$(countVolumesWithFilter "name=kubeadm-dind-kube-node-\\d+-cluster-${secondClusterID}$")" -ge 1 || { 75 | fail 'Expected one volume for the kube nodes to exist - cluster created with custom ID' 76 | } 77 | 78 | test "$(countVolumesWithFilter "name=kubeadm-dind-kube-master-${thirdClusterLabel}$")" -eq 1 || { 79 | fail 'Expected one volume for the kube master to exist - cluster created with custom label' 80 | } 81 | test "$(countVolumesWithFilter "name=kubeadm-dind-kube-node-\\d+-${thirdClusterLabel}$")" -ge 1 || { 82 | fail 'Expected one volume for the kube nodes to exist - cluster created with custom label' 83 | } 84 | 85 | if usesLinuxKit 86 | then 87 | test "$(countVolumesWithFilter "name=kubeadm-dind-sys$")" -eq 1 || { 88 | fail 'Expected one volume for the sys to exist - cluster created with default label' 89 | } 90 | test "$(countVolumesWithFilter "name=kubeadm-dind-sys-cluster-${secondClusterID}$")" -eq 1 || { 91 | fail 'Expected one volume for the sys to exist - cluster created with custom ID' 92 | } 93 | test "$(countVolumesWithFilter "name=kubeadm-dind-sys-${thirdClusterLabel}$")" -eq 1 || { 94 | fail 'Expected one volume for the sys to exist - cluster created with custom label' 95 | } 96 | fi 97 | 98 | # networks 99 | test "$(countNetworksWithFilter "name=kubeadm-dind-net$")" -eq 1 || { 100 | fail 'Expected one network to exist - cluster created with default label' 101 | } 102 | test "$(countNetworksWithFilter "name=kubeadm-dind-net-cluster-${secondClusterID}$")" -eq 1 || { 103 | fail 'Expected one network to exist - cluster created with custom ID' 104 | } 105 | test "$(countNetworksWithFilter "name=kubeadm-dind-net-${thirdClusterLabel}$")" -eq 1 || { 106 | fail 'Expected one network to exist - cluster created with custom label' 107 | } 108 | 109 | # contexts 110 | hasKubeContext "dind" || { 111 | fail 'Expected to have context - cluster created with default label' 112 | } 113 | hasKubeContext "dind-cluster-${secondClusterID}" || { 114 | fail 'Expected to have context - cluster created with custom ID' 115 | } 116 | hasKubeContext "dind-${thirdClusterLabel}" || { 117 | fail 'Expected to have context - cluster created with custom label' 118 | } 119 | 120 | "${d}" clean 121 | CLUSTER_ID="$secondClusterID" "${d}" clean 122 | DIND_LABEL="$thirdClusterLabel" "${d}" clean 123 | } 124 | 125 | function hasKubeContext() { 126 | kubectl config get-contexts --no-headers \ 127 | | sed 's/^\s*\**\s*//g' \ 128 | | grep -q "^${1}\\s" 129 | } 130 | 131 | function fail() { 132 | local msg="$1" 133 | echo -e "\033[1;31m${msg}\033[0m" >&2 134 | return 1 135 | } 136 | 137 | function countNetworksWithFilter() { 138 | docker network ls -q --filter="$1" | wc -l | xargs 139 | } 140 | 141 | function countVolumesWithFilter() { 142 | docker volume ls -q --filter="$1" | wc -l | xargs 143 | } 144 | 145 | function countContainersWithExactName() { 146 | countContainersWithFilter "name=${1}$" 147 | } 148 | 149 | function countContainersWithLabel() { 150 | countContainersWithFilter "label=${1}" 151 | } 152 | 153 | function countContainersWithFilter() { 154 | docker ps -q --filter="${1}" | wc -l | xargs 155 | } 156 | 157 | function usesLinuxKit() { 158 | local docker_info_output 159 | docker_info_output="$(docker info)" || return $? 160 | if ! echo "$docker_info_output"|grep -s '^ *Operating System: .*Docker for Windows' > /dev/null 2>&1 ; then 161 | if echo "$docker_info_output"|grep -s '^ *Kernel Version: .*-moby$' >/dev/null 2>&1 || 162 | echo "$docker_info_output"|grep -s '^ *Kernel Version: .*-linuxkit' > /dev/null 2>&1 ; then 163 | return 0 164 | fi 165 | fi 166 | return 1 167 | } 168 | 169 | test-case-multiple-instances 170 | -------------------------------------------------------------------------------- /test/test-net-connectivity.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 Mirantis 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o xtrace 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o errtrace 21 | 22 | . "$( dirname "$0" )/shared.inc.sh" 23 | 24 | function setup-test() { 25 | export IP_MODE="${IP_MODE:-ipv6}" 26 | 27 | # set TEST_K8S_VER to something specific, by default testing v1.15 for now 28 | local v="${TEST_K8S_VER:-v1.15}" 29 | export LOCAL_KUBECTL_VERSION="$v" 30 | export DIND_K8S_VERSION="${v}" 31 | if [[ -z ${DIND_SKIP_PULL+x} ]]; then 32 | docker pull "${DIND_IMAGE}" 33 | fi 34 | 35 | # Add the directory we will download kubectl into to the PATH 36 | # Make sure the kubectl path has precedence so we use the donwloaded binary 37 | # instead of a potential locally available one. 38 | export PATH="${KUBECTL_DIR}:${PATH}" 39 | 40 | # We set it here explicitely, even though dind-cluster.sh would default it, to 41 | # later in the test be able to check if a resolved IP is prefixed with the 42 | # specified $DNS64_PREFIX 43 | export DNS64_PREFIX='fd00:10:64:ff9b::' 44 | 45 | # some shared config for the ipv6 tests 46 | export ipv6_enabled_host="ds.test-ipv6.noroutetohost.net" 47 | export ipv4_only_host="ipv4.test-ipv6.noroutetohost.net" 48 | export ipv6_enabled_host_ns='8.8.8.8' 49 | 50 | # the image to use for containers in k8s 51 | export k8s_img='tutum/dnsutils' 52 | 53 | # runs ./dind-cluster.sh with `xtrace`/`-x` by detault. To turn `xtrace` off 54 | # set TEST_DIND_RUN_FLAGS='', to change the flags use 55 | # TEST_DIND_RUN_FLAGS='-x -e -u -o pipefail' 56 | export dind_run_flags="${TEST_DIND_RUN_FLAGS--x}" 57 | } 58 | 59 | function dns-lookup-ip-by-type() { 60 | local name="$1" 61 | local type="${2:-A}" 62 | local ns="${3:-}" 63 | 64 | test -n "$ns" && ns="@${ns}" 65 | 66 | dig +short ${name} ${type} ${ns} | sed 's/\r//g' 67 | } 68 | 69 | function dns-lookup-ip-by-type-on-pod() { 70 | local pod="$1" 71 | 72 | local name="$2" 73 | local type="${3:-A}" 74 | local ns="${4:-}" 75 | 76 | test -n "$ns" && ns="@${ns}" 77 | 78 | kubectl exec -ti "$pod" -- dig +short "${name}" "${type}" ${ns} | sed 's/\r//g' 79 | } 80 | 81 | function dns-lookup-ip-by-type-on-node() { 82 | local node="$1" 83 | local name="$2" 84 | local type='' 85 | 86 | case "${3:-}" in 87 | AAAA) type='ahostsv6' ;; 88 | A) type='ahostsv4' ;; 89 | *) echo "unknown type '${3:-}' for dns-lookup-ip-by-type-on-node" >&2 ; return 1 ;; 90 | esac 91 | 92 | docker exec -ti "$node" getent "$type" "$name" | awk '/ STREAM /{ print $1 }' 93 | } 94 | 95 | function create-persistent-pod() { 96 | kubectl run pod-on-node1 --image=${k8s_img} --restart=Never \ 97 | --overrides="$( get-node-selector-override kube-node-1 )" \ 98 | --command -- /bin/sh -c 'hostname ; /bin/sleep 3600' \ 99 | >/dev/null 2>&1 100 | 101 | # Wait for the pod to be up... 102 | local tries=100 103 | local is_up="" 104 | while (( tries-- )) 105 | do 106 | is_up="$( kubectl get pod pod-on-node1 -o 'go-template={{ .status.phase }}' )" 107 | if [[ "${is_up}" = 'Running' ]]; then 108 | break 109 | fi 110 | sleep 1 111 | done 112 | echo "${is_up}" 113 | } 114 | 115 | function ping-tests-external() { 116 | if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then 117 | kubectl exec -ti pod-on-node1 -- ping -n -c1 $ipv4_only_host || { 118 | fail "Expected ping to $ipv4_only_host to succeed" 119 | } 120 | fi 121 | if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then 122 | if [ -z "${DIND_ALLOW_AAAA_USE:-}" ]; then 123 | kubectl exec -ti pod-on-node1 -- ping6 -n -c1 $ipv6_enabled_host || { 124 | fail "Expected ping6 to $ipv6_enabled_host (using synthesized address) to succeed" 125 | } 126 | else 127 | echo 'NOTE: Skipping direct IPv6 external ping - as not supported by CI' >&2 128 | fi 129 | fi 130 | if [[ ${IP_MODE} = "ipv6" ]]; then # NOTE: IPv5 only does not have direct access to IPv4 hosts 131 | kubectl exec -ti pod-on-node1 -- ping6 -n -c1 $ipv4_only_host || { 132 | fail "Expected ping6 to $ipv4_only_host (using synthesized address) to succeed" 133 | } 134 | fi 135 | } 136 | 137 | function ping-tests-internal() { 138 | if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then 139 | pod_on_node1_ip="$( get-ip-for-pod pod-on-node1 ipv4 )" 140 | kubectl run pod-on-node2 --attach --image=${k8s_img} --restart=Never --rm \ 141 | --overrides="$( get-node-selector-override kube-node-2 )" \ 142 | --command -- /bin/ping -n -c 1 "$pod_on_node1_ip" || { 143 | fail 'Expected to be able to ping a pod by IP on a different node' 144 | } 145 | fi 146 | if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then 147 | pod_on_node1_ip="$( get-ip-for-pod pod-on-node1 ipv6 )" 148 | kubectl run pod-on-node2 --attach --image=${k8s_img} --restart=Never --rm \ 149 | --overrides="$( get-node-selector-override kube-node-2 )" \ 150 | --command -- /bin/ping6 -n -c 1 "$pod_on_node1_ip" || { 151 | fail 'Expected to be able to ping6 a pod by IP on a different node' 152 | } 153 | fi 154 | } 155 | 156 | function test-namespace-lookup() { 157 | if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then 158 | verify-lookup-using-ipv4-for-ipv4-hosts "${ipv4_only_host}" 159 | fi 160 | if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then 161 | if [[ -z "${DIND_ALLOW_AAAA_USE:-}" ]]; then 162 | verify-lookup-always-using-dns64-prefix "${ipv6_enabled_host}" 163 | if [[ ${IP_MODE} = "ipv6" ]]; then 164 | verify-lookup-using-dns64-prefix-for-ipv4-hosts "${ipv4_only_host}" 165 | fi 166 | else 167 | verify-lookup-using-ipv6-address-for-ipv6-hosts "${ipv6_enabled_host}" 168 | fi 169 | fi 170 | } 171 | 172 | function resolve-host() { 173 | local target="$1" 174 | 175 | aaaa_from_node="$( dns-lookup-ip-by-type-on-node 'kube-node-1' "${target}" AAAA )" || { 176 | fail "Expected to successfully resolve ${target}'s IPv6 address on kube-node-1" 177 | } 178 | 179 | aaaa_from_pod="$( dns-lookup-ip-by-type-on-pod 'pod-on-node1' "${target}" AAAA )" || { 180 | fail "Expected to successfully resolve ${target}'s IPv6 address on a pod" 181 | } 182 | 183 | aaaa_from_host="$( dns-lookup-ip-by-type "${target}" AAAA )" || { 184 | fail "Expected to successfully resolve ${target}'s IPv6 from the host" 185 | } 186 | 187 | test "${aaaa_from_node}" = "${aaaa_from_pod}" || { 188 | fail "Expected ${target} to resolve to the same IPv6 address on a node and on a pod" 189 | } 190 | 191 | # We verified, that AAAA is the same from Pod & Node, so we can just return 192 | # one of those 193 | local aaaa_from_k8s="${aaaa_from_pod}" 194 | 195 | echo "${aaaa_from_host},${aaaa_from_k8s}" 196 | } 197 | 198 | function verify-lookup-using-ipv4-for-ipv4-hosts() { 199 | local target="$1" 200 | 201 | a_from_node="$( dns-lookup-ip-by-type-on-node 'kube-node-1' "${target}" A )" || { 202 | fail "Expected to successfully resolve ${target}'s IPv4 address on kube-node-1" 203 | } 204 | 205 | a_from_pod="$( dns-lookup-ip-by-type-on-pod 'pod-on-node1' "${target}" A )" || { 206 | fail "Expected to successfully resolve ${target}'s IPv4 address on a pod" 207 | } 208 | 209 | test "${a_from_node}" = "${a_from_pod}" || { 210 | fail "Expected ${target} to resolve to the same IPv4 address on a node and on a pod" 211 | } 212 | } 213 | 214 | function verify-lookup-always-using-dns64-prefix() { 215 | local target="$1" 216 | 217 | IFS=, read aaaa_from_host aaaa_from_k8s <<<"$( resolve-host "$target" )" 218 | 219 | if ! starts-with "$DNS64_PREFIX" "$aaaa_from_k8s" 220 | then 221 | fail "Expected ${target} to resolve to something starting with the DNS64 prefix on pods/nodes, but got ${aaaa_from_k8s}" 222 | fi 223 | 224 | if starts-with "$DNS64_PREFIX" "$aaaa_from_host" 225 | then 226 | fail "Expected ${target} not to resolve to something starting with the DNS64 prefix on the host" 227 | fi 228 | } 229 | 230 | function verify-lookup-using-ipv6-address-for-ipv6-hosts() { 231 | local target="$1" 232 | 233 | IFS=, read aaaa_from_host aaaa_from_k8s <<<"$( resolve-host "$target" )" 234 | 235 | if starts-with "$DNS64_PREFIX" "$aaaa_from_k8s" 236 | then 237 | fail "Expected ${target} not to resolve to something starting with the DNS64 prefix on pods/nodes, but got ${aaaa_from_k8s}" 238 | fi 239 | 240 | if starts-with "$DNS64_PREFIX" "$aaaa_from_host" 241 | then 242 | fail "Expected ${target} not to resolve to something starting with the DNS64 prefix on the host" 243 | fi 244 | 245 | if [ "$( dns-lookup-ip-by-type "$target" 'AAAA' "$ipv6_enabled_host_ns" )" != "$aaaa_from_k8s" ] 246 | then 247 | fail "Expected ${target} to resolve to the same as on ${ipv6_enabled_host_ns}" 248 | fi 249 | } 250 | 251 | function verify-lookup-using-dns64-prefix-for-ipv4-hosts() { 252 | local target="$1" 253 | 254 | IFS=, read aaaa_from_host aaaa_from_k8s <<<"$( resolve-host "$target" )" 255 | 256 | # Hosts which don't have a AAAA record still get one from our DNS64 range 257 | if ! starts-with "$DNS64_PREFIX" "$aaaa_from_k8s" 258 | then 259 | fail "Expected ${target} to resolve to something starting with the DNS64 prefix on pods/nodes, but got ${aaaa_from_k8s}" 260 | fi 261 | 262 | if [ "$aaaa_from_host" != "" ] 263 | then 264 | fail "Expected ${target} not to resolve on the host" 265 | fi 266 | } 267 | 268 | # Testing based on IP_MODE: 269 | # 270 | # MODE PING ACROSS NODES EXTERNAL PING NS LOOKUPS 271 | # ipv4 using v4 v4 using v4 v4 using v4 272 | # ipv6 using v6 v4/v6s/v6 using v6*/v6+/v6- v4/v6s/v6 using v6*/v6+/v6 273 | # dual-stack using v4, v6 v4/v6s/v6 using v4/v6%/v6- v4/v6s/v6 using v4/v6%/v6 274 | # 275 | # v4 = IPv4, v6 = IPv6, v6s = IPv6 synthesized address (with IPv4 embedded) 276 | # 277 | # (*) NOTE: using synthesized IPv6 address with embedded IPv4 address. 278 | # (+) NOTE: uses A record for IPv4 address and embeds in synthesized IPv6 address. 279 | # (-) NOTE: pinging direct IPv6 address is not supported by CI. 280 | # (%) NOTE: using synthesized IPv6 address is not supported currently. 281 | # 282 | function test-case-net-connectivity() { 283 | setup-test 284 | 285 | bash ${dind_run_flags} ./dind-cluster.sh up 286 | 287 | if [[ "$( create-persistent-pod )" = "Running" ]]; then 288 | ping-tests-internal 289 | ping-tests-external 290 | test-namespace-lookup 291 | fi 292 | bash ${dind_run_flags} ./dind-cluster.sh clean 293 | } 294 | 295 | function main() { 296 | test-case-net-connectivity 297 | echo "*** OK: $0 ***" 298 | } 299 | 300 | main "$@" 301 | --------------------------------------------------------------------------------