├── .gitignore ├── versions.tf ├── manifests ├── oidc-cluster-admins.yaml ├── bgppeer.yaml.tpl └── deployment.yaml ├── .envrc ├── bin └── apply ├── ignition ├── bootstrap.yaml ├── bootstrap.ipxe ├── bootstrap.json ├── worker.yaml ├── base.yaml └── controlplane.yaml ├── oidc.conf ├── flake.lock ├── default.nix ├── flake.nix ├── nix ├── sources.json └── sources.nix ├── .terraform.lock.hcl ├── README.md ├── provision.tf └── LICENSE.txt /.gitignore: -------------------------------------------------------------------------------- 1 | terraform.tfstate* 2 | .terraform 3 | .vscode 4 | .direnv 5 | admin.conf 6 | .envrc.local 7 | cloud-sa.json 8 | ca.key 9 | .terraform.tfstate.lock.info 10 | -------------------------------------------------------------------------------- /versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13" 3 | required_providers { 4 | metal = { 5 | source = "equinix/metal" 6 | # version = "1.0.0" 7 | } 8 | ct = { 9 | source = "poseidon/ct" 10 | version = "~> 0.6.1" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /manifests/oidc-cluster-admins.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: oidc-cluster-admin 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: cluster-admin 9 | subjects: 10 | - kind: User 11 | name: https://oidc.arianvp.me#sGk9k-YC-fT853b3cZPlQjqgMLQ 12 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | if ! has nix_direnv_version || ! nix_direnv_version 1.4.0; then 2 | source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/1.4.0/direnvrc" "sha256-4XfVDjv75eHMWN4G725VW7BoOV4Vl3vAabK4YXIfPyE=" 3 | fi 4 | use flake 5 | PATH_add ./bin 6 | export DATASTORE_TYPE=kubernetes 7 | export KUBECONFIG=$PWD/admin.conf:$PWD/oidc.conf 8 | [[ -f .envrc.local ]] && source_env .envrc.local 9 | -------------------------------------------------------------------------------- /bin/apply: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | echo "Terraforming..." 4 | terraform apply 5 | 6 | echo "Fetching kubeconfig from master..." 7 | scp core@[$(terraform output -raw controlplane_ipv6)]:.kube/config ./admin.conf || (echo "Master node didn't come up yet. Try again in a bit" && exit 1) 8 | 9 | echo "Applying bgp peers..." 10 | terraform output -raw calico_bgp_peers | kubectl apply -f - 11 | 12 | -------------------------------------------------------------------------------- /manifests/bgppeer.yaml.tpl: -------------------------------------------------------------------------------- 1 | %{ for device in workers ~} 2 | %{ for network in device.network ~} 3 | %{ if network.family == 6 ~} 4 | --- 5 | apiVersion: projectcalico.org/v3 6 | kind: BGPPeer 7 | metadata: 8 | name: "${device.hostname}" 9 | spec: 10 | peerIP: "${network.gateway}" 11 | asNumber: ${metal_asn} 12 | node: "${device.hostname}" 13 | %{ endif ~} 14 | %{ endfor ~} 15 | %{ endfor ~} 16 | -------------------------------------------------------------------------------- /ignition/bootstrap.yaml: -------------------------------------------------------------------------------- 1 | systemd: 2 | units: 3 | - name: flatcar-install.service 4 | enabled: true 5 | contents: | 6 | [Unit] 7 | After=network-online.target 8 | Wants=network-online.target 9 | [Service] 10 | Type=oneshot 11 | ExecStart=/usr/bin/flatcar-install -d /dev/sda -o packet 12 | ExecStart=/usr/bin/systemctl reboot 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /ignition/bootstrap.ipxe: -------------------------------------------------------------------------------- 1 | #!ipxe 2 | 3 | set base-url http://beta.release.flatcar-linux.net/amd64-usr/current 4 | 5 | kernel ${base-url}/flatcar_production_pxe.vmlinuz initrd=flatcar_production_pxe_image.cpio.gz flatcar.first_boot=1 flatcar.oem.id=packet console=ttyS1,115200n8 flatcar.autologin ignition.config.url=https://raw.githubusercontent.com/arianvp/packet-ipv6-kubernetes/metal/ignition/bootstrap.json 6 | initrd ${base-url}/flatcar_production_pxe_image.cpio.gz 7 | boot 8 | -------------------------------------------------------------------------------- /manifests/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nginx 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: nginx 9 | replicas: 5 10 | template: 11 | metadata: 12 | labels: 13 | app: nginx 14 | spec: 15 | containers: 16 | - name: nginx 17 | image: nginx 18 | resources: 19 | limits: 20 | memory: "128Mi" 21 | cpu: "500m" 22 | ports: 23 | - containerPort: 80 24 | -------------------------------------------------------------------------------- /oidc.conf: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | clusters: null 3 | contexts: null 4 | current-context: "" 5 | kind: Config 6 | preferences: {} 7 | users: 8 | - name: oidc 9 | user: 10 | exec: 11 | apiVersion: client.authentication.k8s.io/v1beta1 12 | args: 13 | - oidc-login 14 | - get-token 15 | - --oidc-issuer-url=https://oidc.arianvp.me 16 | - --oidc-client-id=ASF4Os1wJysH6uWvJV9PvyNiph4y4O84tGCHj1FZEE8 17 | command: kubectl 18 | env: null 19 | provideClusterInfo: false 20 | -------------------------------------------------------------------------------- /ignition/bootstrap.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignition": { 3 | "config": {}, 4 | "security": { 5 | "tls": {} 6 | }, 7 | "timeouts": {}, 8 | "version": "2.3.0" 9 | }, 10 | "networkd": {}, 11 | "passwd": {}, 12 | "storage": {}, 13 | "systemd": { 14 | "units": [ 15 | { 16 | "contents": "[Unit]\nAfter=network-online.target\nWants=network-online.target\n[Service]\nType=oneshot\nExecStart=/usr/bin/flatcar-install -d /dev/sda -o packet\nExecStart=/usr/bin/systemctl reboot\n[Install]\nWantedBy=multi-user.target\n", 17 | "enabled": true, 18 | "name": "flatcar-install.service" 19 | } 20 | ] 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixpkgs": { 4 | "locked": { 5 | "lastModified": 1633621759, 6 | "narHash": "sha256-Mw29zuYjOozICTWsc9RvjuR7hW5D0H83onQh/WoVrMs=", 7 | "owner": "NixOS", 8 | "repo": "nixpkgs", 9 | "rev": "6755a884b24038a73dd4c8022dbb05375feef0a7", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "id": "nixpkgs", 14 | "type": "indirect" 15 | } 16 | }, 17 | "root": { 18 | "inputs": { 19 | "nixpkgs": "nixpkgs", 20 | "utils": "utils" 21 | } 22 | }, 23 | "utils": { 24 | "locked": { 25 | "lastModified": 1631561581, 26 | "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", 27 | "owner": "numtide", 28 | "repo": "flake-utils", 29 | "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", 30 | "type": "github" 31 | }, 32 | "original": { 33 | "owner": "numtide", 34 | "repo": "flake-utils", 35 | "type": "github" 36 | } 37 | } 38 | }, 39 | "root": "root", 40 | "version": 7 41 | } 42 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import (import ./nix/sources.nix).nixpkgs {}}: 2 | let 3 | kubectl = pkgs.stdenv.mkDerivation { 4 | name = "kubectl"; 5 | src = pkgs.fetchurl { 6 | url = "https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl"; 7 | sha256 = "03x77fz6xvqyxclicp523c50mis3a03rqqwsk4rzazs80lphvfvr"; 8 | }; 9 | dontUnpack = true; 10 | dontBuild = true; 11 | installPhase = '' 12 | cp $src kubectl 13 | install -Dm0755 kubectl -t $out/bin 14 | ''; 15 | }; 16 | kubeadm = pkgs.stdenv.mkDerivation { 17 | name = "kubeadm"; 18 | src = pkgs.fetchurl { 19 | url = "https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubeadm"; 20 | sha256 = "17v5v08f3j3vzcqhgmrkiag82ak4zbjbkakrwvv4g21d632pvkl8"; 21 | }; 22 | dontUnpack = true; 23 | dontBuild = true; 24 | installPhase = '' 25 | cp $src kubeadm 26 | install -Dm0755 kubeadm -t $out/bin 27 | ''; 28 | }; 29 | in 30 | pkgs.mkShell { 31 | name = "packet-kubernetes"; 32 | buildInputs = [ pkgs.terraform_0_13 kubectl kubeadm ]; 33 | } 34 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "EKS Experiment"; 3 | 4 | inputs.utils.url = "github:numtide/flake-utils"; 5 | 6 | outputs = { self, utils, nixpkgs }: 7 | utils.lib.eachDefaultSystem (system: 8 | let pkgs = nixpkgs.legacyPackages.${system}; in 9 | { 10 | packages.calicoctl = pkgs.stdenv.mkDerivation { 11 | name = "calicoctl"; 12 | src = pkgs.fetchurl { 13 | url = "https://github.com/projectcalico/calicoctl/releases/download/v3.16.0/calicoctl"; 14 | sha256 = "14qv4wv8ndz9g7v2bq8w1fgzw14dfhn5byzp5jqp6q26z9vp0qx7"; 15 | }; 16 | dontUnpack = true; 17 | dontBuild = true; 18 | installPhase = '' 19 | cp $src calicoctl 20 | install -Dm0755 calicoctl -t $out/bin 21 | ''; 22 | }; 23 | devShell = with pkgs; mkShell { 24 | nativeBuildInputs = [ 25 | self.packages.${system}.calicoctl 26 | bashInteractive 27 | mkcert 28 | kubectl 29 | kubelogin-oidc 30 | terraform 31 | ]; 32 | }; 33 | }); 34 | } 35 | 36 | -------------------------------------------------------------------------------- /nix/sources.json: -------------------------------------------------------------------------------- 1 | { 2 | "niv": { 3 | "branch": "master", 4 | "description": "Easy dependency management for Nix projects", 5 | "homepage": "https://github.com/nmattia/niv", 6 | "owner": "nmattia", 7 | "repo": "niv", 8 | "rev": "e0ca65c81a2d7a4d82a189f1e23a48d59ad42070", 9 | "sha256": "1pq9nh1d8nn3xvbdny8fafzw87mj7gsmp6pxkdl65w2g18rmcmzx", 10 | "type": "tarball", 11 | "url": "https://github.com/nmattia/niv/archive/e0ca65c81a2d7a4d82a189f1e23a48d59ad42070.tar.gz", 12 | "url_template": "https://github.com///archive/.tar.gz" 13 | }, 14 | "nixpkgs": { 15 | "branch": "nixpkgs-unstable", 16 | "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", 17 | "homepage": "https://github.com/NixOS/nixpkgs", 18 | "owner": "NixOS", 19 | "repo": "nixpkgs-channels", 20 | "rev": "502845c3e31ef3de0e424f3fcb09217df2ce6df6", 21 | "sha256": "0fcqpsy6y7dgn0y0wgpa56gsg0b0p8avlpjrd79fp4mp9bl18nda", 22 | "type": "tarball", 23 | "url": "https://github.com/NixOS/nixpkgs-channels/archive/502845c3e31ef3de0e424f3fcb09217df2ce6df6.tar.gz", 24 | "url_template": "https://github.com///archive/.tar.gz" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /ignition/worker.yaml: -------------------------------------------------------------------------------- 1 | systemd: 2 | units: 3 | - name: setup-lvm.service 4 | enabled: true 5 | contents: | 6 | [Unit] 7 | ConditionFirstBoot=yes 8 | [Service] 9 | ExecStart=/usr/sbin/vgcreate lvmvg /dev/sda 10 | [Install] 11 | WantedBy=multi-user.target 12 | - name: kubeadm.service 13 | enabled: true 14 | contents: | 15 | [Unit] 16 | ConditionPathExists=!/etc/kubernetes/kubelet.conf 17 | After=coreos-metadata.service 18 | Requires=coreos-metadata.service 19 | StartLimitInterval=1200s 20 | StartLimitBurst=5 21 | 22 | [Service] 23 | EnvironmentFile=/run/metadata/coreos 24 | Environment=PATH=/usr/bin:/usr/sbin:/opt/bin 25 | Type=oneshot 26 | ExecStartPre=/bin/sh -c 'cat /etc/kubeadm/config.yaml.tpl | envsubst > /etc/kubeadm/config.yaml' 27 | ExecStart=/opt/bin/kubeadm join --config /etc/kubeadm/config.yaml 28 | Restart=on-failure 29 | RestartSec=120s 30 | [Install] 31 | WantedBy=multi-user.target 32 | storage: 33 | files: 34 | - path: "/etc/kubeadm/config.yaml.tpl" 35 | contents: 36 | inline: | 37 | --- 38 | apiVersion: kubeadm.k8s.io/v1beta2 39 | kind: JoinConfiguration 40 | nodeRegistration: 41 | criSocket: /run/containerd/containerd.sock 42 | kubeletExtraArgs: 43 | volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" 44 | # needed; otherwise kube-proxy will not come up 45 | # Kubernetes just takes the first IP it sees; which is the wrong one :) 46 | node-ip: "$${COREOS_PACKET_IPV6_PUBLIC_0}" 47 | discovery: 48 | bootstrapToken: 49 | token: "${token}" 50 | apiServerEndpoint: "${control_plane_endpoint}" 51 | unsafeSkipCAVerification: true 52 | %{ if certificate_key != null} 53 | controlPlane: 54 | certificateKey: ${certificate_key} 55 | %{ endif ~} 56 | -------------------------------------------------------------------------------- /ignition/base.yaml: -------------------------------------------------------------------------------- 1 | networkd: 2 | units: 3 | - name: 05-bond0.network 4 | dropins: 5 | - name: 00-forwarding.conf 6 | contents: | 7 | [Network] 8 | IPForward=ipv6 9 | # TODO: I expected this to _override_ DNS but currently it does not. 10 | # See https://github.com/systemd/systemd/issues/16959 11 | - name: 01-dns.conf 12 | contents: | 13 | [Network] 14 | DNS= 15 | DNS=2001:4860:4860::8888 16 | DNS=2001:4860:4860::8844 17 | systemd: 18 | units: 19 | # We'll install https://github.com/kinvolk/flatcar-linux-update-operator for updates instead 20 | - name: locksmithd.service 21 | mask: true 22 | - name: update-engine.service 23 | enabled: true 24 | 25 | # It gets lazily started by docker but that's too late for kubeadm 26 | - name: containerd.service 27 | enabled: true 28 | 29 | - name: kubelet.service 30 | enabled: true 31 | contents: | 32 | [Unit] 33 | Description=kubelet: The Kubernetes Node Agent 34 | Documentation=https://kubernetes.io/docs/home/ 35 | Wants=network-online.target 36 | After=network-online.target 37 | 38 | [Service] 39 | Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" 40 | Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" 41 | # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically 42 | EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env 43 | ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS 44 | Restart=always 45 | StartLimitInterval=0 46 | RestartSec=10 47 | 48 | [Install] 49 | WantedBy=multi-user.target 50 | storage: 51 | directories: 52 | - path: /opt/bin 53 | - path: /opt/cni/bin 54 | # Fetches ssh keys from equinix api 55 | links: 56 | - path: /etc/systemd/system/multi-user.target.wants/coreos-metadata-sshkeys@core.service 57 | target: /usr/lib/systemd/system/coreos-metadata-sshkeys@.service 58 | files: 59 | - path: "/etc/modules-load.d/00-k8s.conf" 60 | contents: 61 | inline: | 62 | br_netfilter 63 | - path: "/etc/sysctl.d/00-k8s.conf" 64 | contents: 65 | inline: | 66 | net.bridge.bridge-nf-call-ip6tables = 1 67 | net.bridge.bridge-nf-call-iptables = 1 68 | - path: "/opt/bin/kubeadm" 69 | mode: 755 70 | contents: 71 | remote: 72 | url: "${KUBEADM_URL}" 73 | - path: "/opt/bin/kubelet" 74 | mode: 755 75 | contents: 76 | remote: 77 | url: "${KUBELET_URL}" 78 | - path: "/opt/bin/kubectl" 79 | mode: 755 80 | contents: 81 | remote: 82 | url: "${KUBECTL_URL}" 83 | - path: "/opt/bin/calicoctl" 84 | mode: 755 85 | contents: 86 | remote: 87 | url: "${CALICOCTL_URL}" 88 | -------------------------------------------------------------------------------- /.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/equinix/metal" { 5 | version = "3.2.0" 6 | hashes = [ 7 | "h1:SzC4gFRhDkT/+KY9SaqXPEU0VJKtiMh572eMTCcfuXY=", 8 | "zh:4a53b56d7cab5a75dca16c4a285438f13fe07a37c45967668dae8e4a70b78a85", 9 | "zh:4fa88d43bd2370e420de6238f02f8ad668a0cd9e7f010a24a8bc0a76982ff3bc", 10 | "zh:525b0f6d977437388b7ee7428bccd351e46af39a0883ce08f624b4dd173de8c2", 11 | "zh:5c906ee75fbd1daa63dc40221015bfdcea7cec24ed7aa3687fcc18265556e379", 12 | "zh:6153771a66d1f5058e977a686e706f3ced64cbcb7cd623a9adf7e8ed3f0d08d8", 13 | "zh:79dd26afcc0846864ede59e79ae07fdf6cc084315cd9e50f4e09c700d9d8dfb4", 14 | "zh:992e410067995537565f38ad9f93f4d66fabdc25a8303a1be418aefaf172daae", 15 | "zh:9e52d2d008846cb6db8d0cd0aac7e1c6c635fc65c192b0217b2e673d165def0f", 16 | "zh:c2ca5c36358c03c996d97b8baca6809675f54010c021d5704124bd3815660c86", 17 | "zh:e2d4bfae4489142cfac31fb7939047226335c0fddf59c3544faeb1f3029a36d8", 18 | "zh:e5c6727b6404beb3aa54bbc7eb3145bac29bab8135771f21754e093c178ee23f", 19 | "zh:ecf34c7c8b3eb8cd61b381d8f26ec61b3d8b5d57a87fff97cb24331230286fef", 20 | "zh:fcae7646861ccf3393828ac53d05487ba8547cf9f8a9da13553dcedf8dd2646e", 21 | ] 22 | } 23 | 24 | provider "registry.terraform.io/hashicorp/null" { 25 | version = "3.1.0" 26 | hashes = [ 27 | "h1:vpC6bgUQoJ0znqIKVFevOdq+YQw42bRq0u+H3nto8nA=", 28 | "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", 29 | "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", 30 | "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", 31 | "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", 32 | "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", 33 | "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", 34 | "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", 35 | "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", 36 | "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", 37 | "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", 38 | "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", 39 | ] 40 | } 41 | 42 | provider "registry.terraform.io/hashicorp/random" { 43 | version = "3.1.0" 44 | hashes = [ 45 | "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", 46 | "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", 47 | "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", 48 | "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", 49 | "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", 50 | "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", 51 | "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", 52 | "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", 53 | "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", 54 | "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", 55 | "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", 56 | "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", 57 | ] 58 | } 59 | 60 | provider "registry.terraform.io/poseidon/ct" { 61 | version = "0.6.1" 62 | constraints = "~> 0.6.1" 63 | hashes = [ 64 | "h1:rgEmsUB26uTY7u0/Fsqi9xf4/BCp6Y6ddOef9NeThWs=", 65 | "zh:139e9f8e3e324fa430e14aaf97bf10dc6fcb7e940ab0969af95a860a9c387c41", 66 | "zh:3adb71a854af48593caa6d38ab275327942cfe4576e562d602f1116f0224bc6a", 67 | "zh:8010e6722713f9f2b4e3767cfb6abaa759c809972c759ebb12f3374dce881e9e", 68 | "zh:ee4a110eb3876ae86158dfd364a13ff5c51e31ee3be28b80649a2e69c1621c8d", 69 | ] 70 | } 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IPV6 cluster on Packet 2 | 3 | Sets up IPV6-native cluster using packet. 4 | 5 | This shows how a modern container fabric should look like. Without any NAT or 6 | overlay nonsense; but a pure IP/Layer-3 networking fabric. 7 | 8 | Instead of using an overlay network, each pod is directly assigned an IPV6 9 | address. Routes to pods are announced directly to the packet router BGP. 10 | 11 | Furthermore ClusterIPs are also allocated from a public range. This means we can 12 | publically announce kubernetes services directly; without the need of dedicated cloud load-balancers! 13 | Because Packet's routers support ECMP; traffic will be load-balanced between nodes that announce the service automatically. 14 | 15 | The IPV6-native world is a bit different than you're used to. Normally in k8s 16 | we kind of rely on NAT and overlaynetworks for a false sense of security, and 17 | hence most people don't even set up NetworkPolicy's for their services and pods 18 | properly. However, with IPV6 there are no overlays and NAT; so we are forced to 19 | take security seriously and explicitly define NetworkPolicies to make sure only 20 | the things that _should_ talk to eachother _can_ talk to eachother. 21 | 22 | # Usage 23 | 24 | Create an `.envrc.local` with your secrets: 25 | ``` 26 | export TF_VAR_kubeadm_certificate_key= 27 | export TF_VAR_kubeadm_token= 28 | export PACKET_AUTH_TOKEN= 29 | ``` 30 | 31 | ``` 32 | $ direnv allow 33 | $ apply 34 | ``` 35 | 36 | `apply` might fail to get the `kubeconfig` on first try; as the cluster might still be bootstrapping. 37 | Type `apply` again to get it. The script is idempotent. It is important to make run it until it succeeds 38 | as this also configures the `BGPPeer`s with the packet routers. 39 | 40 | 41 | # Stream notes 42 | 43 | https://www.youtube.com/watch?v=ohwHzr4O6Fo 44 | ## Gotchas 45 | * Enable ipv6 forwarding 46 | * Set up ipv6 dns servers on host as coredns will use the host's dns servers 47 | * Docuemtnation for both kubeadm and calico incomplete or outdated: 48 | * need to set kubelet node-ip 49 | * need to disable ipv4 autodetection on calico 50 | 51 | 52 | ## Packet, pods, ipv6 53 | 54 | ![packet](https://docs.projectcalico.org/images/anatomy-of-a-packet.svg) 55 | 56 | * Pod network is flat. all pods should reach eachother without NAT 57 | * pod cidr != node cidr != service cidr 58 | * Overlay networks (usually; but not neccessarily; E.g. GCP sets up routes directly for pods) 59 | * Pod network usually a private range. 60 | ![vxlan](https://docs.projectcalico.org/images/anatomy-of-an-overlay-packet.svg) 61 | * What if each pod just got a public ip address? => No need for overlays 62 | * Very possible with IPv6 (Packet gives you a /56) 63 | * Packet allows us to announce routes using BGP. 64 | * Calico can announce pod ip addresses to BGP automatically 65 | * https://docs.projectcalico.org/reference/architecture/design/l3-interconnect-fabric 66 | * _downward default_ network 67 | ![downward](https://docs.projectcalico.org/images/l3-fabric-downward-default.png) 68 | 69 | 70 | ## Advertising ClusterIPs 71 | * We can directly advertise kubernetes services using BGP too! 72 | * Will use ECMP routing to loadbalance between nodes. So highly available LB 73 | * Just make sure that the ip range is also a public ipv6 range 74 | * No need for external load-balancer; or metallb, or whatever! All services are 75 | reachable through public IP Address 76 | 77 | ## externalTrafficPolicy: Local vs Cluster 78 | * Show what it means for routes 79 | 80 | ## bbbbut What about security?! (Out of scope probably due to time constraints!) 81 | * Or maybe David can teach me NetworkPolicy?? 82 | 83 | * Overlay networks and NAT give a false sense of security. This is what IPv6 advocates yell all day 84 | * Instead, we want fine-grained routing policies that tell which pods and reach what services, and what services are publically reachable 85 | * We can define Kubernetes NetworkPolicies for this! https://kubernetes.io/docs/concepts/services-networking/network-policies/ 86 | * _Zero Trust networks_ https://docs.projectcalico.org/security/adopt-zero-trust 87 | * Service Mesh like Istio 88 | 89 | ## Whats next - Security 90 | 91 | -------------------------------------------------------------------------------- /ignition/controlplane.yaml: -------------------------------------------------------------------------------- 1 | systemd: 2 | units: 3 | - name: coreos-metadata.service 4 | dropins: 5 | # coreos-metdata.service by default doesn't expose the default gateway; 6 | # so we extend it to do so We need this to set up BGP Peering for the 7 | # controlplane node 8 | - name: 00-gateway.conf 9 | contents: | 10 | [Service] 11 | ExecStart=/bin/sh -c 'echo "COREOS_PACKET_IPV6_GATEWAY=$(curl https://metadata.packet.net/metadata | jq -r \'.network.addresses[] | select(.address_family == 6) | .gateway\')" >> /run/metadata/flatcar' 12 | ExecStart=/bin/sh -c 'echo "COREOS_PACKET_IPV6_PARENT_BLOCK=$(curl https://metadata.packet.net/metadata | jq -r \'.network.addresses[] | select(.address_family == 6) | .parent_block\')" >> /run/metadata/flatcar' 13 | - name: kubeadm.service 14 | enabled: true # We manually start it in terraform for bootstrap reasons 15 | contents: | 16 | [Unit] 17 | ConditionPathExists=!/etc/kubernetes/kubelet.conf 18 | After=coreos-metadata.service 19 | Requires=coreos-metadata.service 20 | [Service] 21 | EnvironmentFile=/run/metadata/coreos 22 | Environment=PATH=/usr/bin:/usr/sbin:/opt/bin 23 | Environment=KUBECONFIG=/etc/kubernetes/admin.conf 24 | Environment=DATASTORE_TYPE=kubernetes 25 | Type=oneshot 26 | ExecStart=/bin/sh -c 'cat /etc/kubeadm/config.yaml.tpl | envsubst > /etc/kubeadm/config.yaml' 27 | ExecStart=/opt/bin/kubeadm init --config /etc/kubeadm/config.yaml --upload-certs 28 | 29 | ExecStart=/usr/bin/sudo -u core mkdir -p /home/core/.kube 30 | ExecStart=/usr/bin/cp /etc/kubernetes/admin.conf /home/core/.kube/config 31 | ExecStart=/usr/bin/chown core:core /home/core/.kube/config 32 | 33 | ExecStart=/opt/bin/kubectl create -f https://docs.projectcalico.org/manifests/tigera-operator.yaml 34 | ExecStart=/opt/bin/kubectl create -f /etc/resources/calico-install.yaml 35 | ExecStart=/bin/sh -c 'cat /etc/resources/calico.yaml.tpl | envsubst > /etc/resources/calico.yaml' 36 | ExecStart=/opt/bin/calicoctl create -f /etc/resources/calico.yaml 37 | [Install] 38 | WantedBy=multi-user.target 39 | storage: 40 | files: 41 | - path: "/etc/kubeadm/config.yaml.tpl" 42 | contents: 43 | inline: | 44 | --- 45 | apiVersion: kubeadm.k8s.io/v1beta2 46 | kind: InitConfiguration 47 | bootstrapTokens: 48 | - token: "${token}" 49 | certificateKey: "${certificate_key}" 50 | localAPIEndpoint: 51 | advertiseAddress: "::" 52 | bindPort: 6443 53 | nodeRegistration: 54 | # taints: [] # control-plane should allow jobs 55 | criSocket: /run/containerd/containerd.sock 56 | kubeletExtraArgs: 57 | volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" 58 | # needed; otherwise kube-proxy will not come up 59 | # Kubernetes just takes the first IP it sees; which is the wrong one :) 60 | node-ip: "$${COREOS_PACKET_IPV6_PUBLIC_0}" 61 | --- 62 | apiVersion: kubeadm.k8s.io/v1beta2 63 | kind: ClusterConfiguration 64 | controlPlaneEndpoint: "$${COREOS_PACKET_IPV6_PUBLIC_0}" 65 | apiServer: 66 | extraArgs: 67 | oidc-issuer-url: https://oidc.arianvp.me 68 | oidc-client-id: ASF4Os1wJysH6uWvJV9PvyNiph4y4O84tGCHj1FZEE8 69 | controllerManager: 70 | extraArgs: 71 | flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" 72 | networking: 73 | serviceSubnet: "${service_cidr_range}" 74 | podSubnet: "${pod_cidr_range}" 75 | - path: "/etc/resources/calico-install.yaml" 76 | contents: 77 | inline: | 78 | --- 79 | apiVersion: operator.tigera.io/v1 80 | kind: Installation 81 | metadata: 82 | name: default 83 | spec: 84 | calicoNetwork: 85 | nodeAddressAutodetectionV4: {} 86 | nodeAddressAutodetectionV6: 87 | interface: bond0 88 | flexVolumePath: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" 89 | --- 90 | apiVersion: operator.tigera.io/v1 91 | kind: APIServer 92 | metadata: 93 | name: default 94 | spec: {} 95 | - path: "/etc/resources/calico.yaml.tpl" 96 | contents: 97 | inline: | 98 | --- 99 | apiVersion: projectcalico.org/v3 100 | kind: BGPConfiguration 101 | metadata: 102 | name: default 103 | spec: 104 | logSeverityScreen: Info 105 | nodeToNodeMeshEnabled: false 106 | asNumber: 65000 107 | serviceClusterIPs: 108 | - cidr: "${service_cidr_range}" 109 | --- 110 | apiVersion: projectcalico.org/v3 111 | kind: BGPPeer 112 | metadata: 113 | name: "controlplane" 114 | spec: 115 | peerIP: "$${COREOS_PACKET_IPV6_GATEWAY}" 116 | asNumber: 65530 117 | node: "controlplane" 118 | -------------------------------------------------------------------------------- /nix/sources.nix: -------------------------------------------------------------------------------- 1 | # This file has been generated by Niv. 2 | 3 | let 4 | 5 | # 6 | # The fetchers. fetch_ fetches specs of type . 7 | # 8 | 9 | fetch_file = pkgs: spec: 10 | if spec.builtin or true then 11 | builtins_fetchurl { inherit (spec) url sha256; } 12 | else 13 | pkgs.fetchurl { inherit (spec) url sha256; }; 14 | 15 | fetch_tarball = pkgs: name: spec: 16 | let 17 | ok = str: ! builtins.isNull (builtins.match "[a-zA-Z0-9+-._?=]" str); 18 | # sanitize the name, though nix will still fail if name starts with period 19 | name' = stringAsChars (x: if ! ok x then "-" else x) "${name}-src"; 20 | in 21 | if spec.builtin or true then 22 | builtins_fetchTarball { name = name'; inherit (spec) url sha256; } 23 | else 24 | pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; 25 | 26 | fetch_git = spec: 27 | builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; }; 28 | 29 | fetch_local = spec: spec.path; 30 | 31 | fetch_builtin-tarball = name: throw 32 | ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. 33 | $ niv modify ${name} -a type=tarball -a builtin=true''; 34 | 35 | fetch_builtin-url = name: throw 36 | ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. 37 | $ niv modify ${name} -a type=file -a builtin=true''; 38 | 39 | # 40 | # Various helpers 41 | # 42 | 43 | # The set of packages used when specs are fetched using non-builtins. 44 | mkPkgs = sources: 45 | let 46 | sourcesNixpkgs = 47 | import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {}; 48 | hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; 49 | hasThisAsNixpkgsPath = == ./.; 50 | in 51 | if builtins.hasAttr "nixpkgs" sources 52 | then sourcesNixpkgs 53 | else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then 54 | import {} 55 | else 56 | abort 57 | '' 58 | Please specify either (through -I or NIX_PATH=nixpkgs=...) or 59 | add a package called "nixpkgs" to your sources.json. 60 | ''; 61 | 62 | # The actual fetching function. 63 | fetch = pkgs: name: spec: 64 | 65 | if ! builtins.hasAttr "type" spec then 66 | abort "ERROR: niv spec ${name} does not have a 'type' attribute" 67 | else if spec.type == "file" then fetch_file pkgs spec 68 | else if spec.type == "tarball" then fetch_tarball pkgs name spec 69 | else if spec.type == "git" then fetch_git spec 70 | else if spec.type == "local" then fetch_local spec 71 | else if spec.type == "builtin-tarball" then fetch_builtin-tarball name 72 | else if spec.type == "builtin-url" then fetch_builtin-url name 73 | else 74 | abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; 75 | 76 | # Ports of functions for older nix versions 77 | 78 | # a Nix version of mapAttrs if the built-in doesn't exist 79 | mapAttrs = builtins.mapAttrs or ( 80 | f: set: with builtins; 81 | listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) 82 | ); 83 | 84 | # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 85 | range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); 86 | 87 | # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 88 | stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); 89 | 90 | # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 91 | stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); 92 | concatStrings = builtins.concatStringsSep ""; 93 | 94 | # fetchTarball version that is compatible between all the versions of Nix 95 | builtins_fetchTarball = { url, name, sha256 }@attrs: 96 | let 97 | inherit (builtins) lessThan nixVersion fetchTarball; 98 | in 99 | if lessThan nixVersion "1.12" then 100 | fetchTarball { inherit name url; } 101 | else 102 | fetchTarball attrs; 103 | 104 | # fetchurl version that is compatible between all the versions of Nix 105 | builtins_fetchurl = { url, sha256 }@attrs: 106 | let 107 | inherit (builtins) lessThan nixVersion fetchurl; 108 | in 109 | if lessThan nixVersion "1.12" then 110 | fetchurl { inherit url; } 111 | else 112 | fetchurl attrs; 113 | 114 | # Create the final "sources" from the config 115 | mkSources = config: 116 | mapAttrs ( 117 | name: spec: 118 | if builtins.hasAttr "outPath" spec 119 | then abort 120 | "The values in sources.json should not have an 'outPath' attribute" 121 | else 122 | spec // { outPath = fetch config.pkgs name spec; } 123 | ) config.sources; 124 | 125 | # The "config" used by the fetchers 126 | mkConfig = 127 | { sourcesFile ? ./sources.json 128 | , sources ? builtins.fromJSON (builtins.readFile sourcesFile) 129 | , pkgs ? mkPkgs sources 130 | }: rec { 131 | # The sources, i.e. the attribute set of spec name to spec 132 | inherit sources; 133 | 134 | # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers 135 | inherit pkgs; 136 | }; 137 | in 138 | mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } 139 | -------------------------------------------------------------------------------- /provision.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "kubeadm_token" { 3 | type = string 4 | description = "The token use for bootstrapping the kubernetes cluster.\nGenerate with: \n$ kubeadm token generate" 5 | } 6 | 7 | variable "kubeadm_certificate_key" { 8 | type = string 9 | description = "The key used to encrypt the control-plane certificates.\nGenerate with: \n$ kubeadm alpha certs certificate-key\n" 10 | } 11 | 12 | locals { 13 | worker_count = 2 14 | pod_cidr_range = cidrsubnet(data.metal_precreated_ip_block.addresses.cidr_notation, 8, 1) 15 | service_cidr_range_ = cidrsubnet(data.metal_precreated_ip_block.addresses.cidr_notation, 8, 2) 16 | # NOTE: subnet size for services in kubernetes can only be 20 bits in size; 17 | # hence allocate a smaller block in the larger /64 block 18 | service_cidr_range = cidrsubnet(local.service_cidr_range_, 44, 0) 19 | external_cidr_range = cidrsubnet(data.metal_precreated_ip_block.addresses.cidr_notation, 8, 3) 20 | 21 | # NOTE: If we run kube-proxy and calico-node on the control-plane, we can use this as the control_plane_endpoint!!! 22 | kubernetes_service = cidrhost(local.service_cidr_range, 1) 23 | 24 | metal_asn = 65530 # NOTE: this wasn't actually documented anywhere? I found it "somewhere" 25 | 26 | K8S_VERSION = "v1.22.3" 27 | KUBEADM_URL = "https://storage.googleapis.com/kubernetes-release/release/${local.K8S_VERSION}/bin/linux/amd64/kubeadm" 28 | 29 | KUBELET_URL = "https://storage.googleapis.com/kubernetes-release/release/${local.K8S_VERSION}/bin/linux/amd64/kubelet" 30 | 31 | KUBECTL_URL = "https://storage.googleapis.com/kubernetes-release/release/${local.K8S_VERSION}/bin/linux/amd64/kubectl" 32 | 33 | CALICOCTL_URL = "https://github.com/projectcalico/calicoctl/releases/download/v3.20.2/calicoctl" 34 | 35 | # TODO: Change to HTTPS once ISRG X1 is used. Sent a request for that to flatcar folks. Other option is to reverse proxy flatcar 36 | # re: https://community.letsencrypt.org/t/production-chain-changes/150739/1 37 | # re: https://github.com/ipxe/ipxe/pull/116 re: 38 | # https://github.com/ipxe/ipxe/pull/112 re: 39 | # https://lists.ipxe.org/pipermail/ipxe-devel/2020-May/007042.html 40 | IPXE_URL = "https://raw.githubusercontent.com/arianvp/packet-ipv6-kubernetes/metal/ignition/bootstrap.ipxe" 41 | 42 | 43 | ignition_base = templatefile("./ignition/base.yaml", { 44 | KUBEADM_URL = local.KUBEADM_URL 45 | KUBELET_URL = local.KUBELET_URL 46 | KUBECTL_URL = local.KUBECTL_URL 47 | CALICOCTL_URL = local.CALICOCTL_URL 48 | }) 49 | 50 | } 51 | 52 | # This is a pre-existing project, where I create and delete a device, such that 53 | # the 54 | data "metal_project" "kubernetes" { 55 | name = "kubernetes" 56 | } 57 | 58 | # This only works if the project we refer to had a device created at some point 59 | # so create and delete a dummy device in the UI before this data source actually works 60 | data "metal_precreated_ip_block" "addresses" { 61 | facility = "ams1" 62 | project_id = data.metal_project.kubernetes.id 63 | address_family = 6 64 | public = true 65 | } 66 | 67 | resource "metal_device" "controlplane" { 68 | hostname = "controlplane" 69 | plan = "t1.small.x86" 70 | facilities = ["ams1"] 71 | operating_system = "custom_ipxe" 72 | ipxe_script_url = local.IPXE_URL 73 | billing_cycle = "hourly" 74 | project_id = data.metal_project.kubernetes.id 75 | user_data = data.ct_config.controlplane.rendered 76 | 77 | } 78 | 79 | data "ct_config" "controlplane" { 80 | content = local.ignition_base 81 | snippets = [ 82 | templatefile("./ignition/controlplane.yaml", { 83 | # TODO: There is a cycle. instead use coreos-metadata? 84 | # node_ip = metal_device.controlplane.access_public_ipv6 85 | pod_cidr_range = local.pod_cidr_range 86 | service_cidr_range = local.service_cidr_range 87 | external_cidr_range = local.external_cidr_range 88 | certificate_key = var.kubeadm_certificate_key 89 | metal_asn = local.metal_asn 90 | token = var.kubeadm_token 91 | }) 92 | ] 93 | } 94 | 95 | resource "metal_bgp_session" "controlplane" { 96 | device_id = metal_device.controlplane.id 97 | address_family = "ipv6" 98 | } 99 | 100 | resource "random_pet" "worker" { 101 | count = local.worker_count 102 | prefix = "worker" 103 | separator = "-" 104 | keepers = { 105 | # NOTE: Done because user_data might contain sensitive data 106 | user_data = sha256(data.ct_config.worker.rendered) 107 | } 108 | } 109 | 110 | resource "metal_device" "worker" { 111 | count = local.worker_count 112 | hostname = random_pet.worker[count.index].id 113 | plan = "t1.small.x86" 114 | facilities = ["ams1"] 115 | operating_system = "custom_ipxe" 116 | ipxe_script_url = local.IPXE_URL 117 | billing_cycle = "hourly" 118 | project_id = data.metal_project.kubernetes.id 119 | user_data = data.ct_config.worker.rendered 120 | } 121 | 122 | data "ct_config" "worker" { 123 | content = local.ignition_base 124 | snippets = [ 125 | templatefile("./ignition/worker.yaml", { 126 | token = var.kubeadm_token 127 | control_plane_endpoint = "[${local.kubernetes_service}]:443" 128 | certificate_key = var.kubeadm_certificate_key 129 | }) 130 | ] 131 | } 132 | 133 | resource "metal_bgp_session" "worker" { 134 | count = local.worker_count 135 | device_id = metal_device.worker[count.index].id 136 | address_family = "ipv6" 137 | } 138 | 139 | output "controlplane_ipv4" { 140 | value = metal_device.controlplane.access_public_ipv4 141 | } 142 | 143 | output "controlplane_ipv6" { 144 | value = metal_device.controlplane.access_public_ipv6 145 | } 146 | 147 | output "calico_bgp_peers" { 148 | description = "A calico manifest describing the topology of the cluster. You should apply this to thhe cluster to set up all the needed routes." 149 | sensitive = true 150 | value = templatefile("manifests/bgppeer.yaml.tpl", { 151 | workers = metal_device.worker 152 | controlplane = metal_device.controlplane 153 | metal_asn = local.metal_asn 154 | service_cidr_range = local.service_cidr_range 155 | external_cidr_range = local.external_cidr_range 156 | }) 157 | } 158 | 159 | resource "null_resource" "write_kubeconfig" { 160 | provisioner "local-exec" { 161 | command = < oidc.conf 163 | kubectl config set-credentials oidc \ 164 | --kubeconfig=oidc.conf \ 165 | --exec-api-version=client.authentication.k8s.io/v1beta1 \ 166 | --exec-command=kubectl \ 167 | --exec-arg=oidc-login \ 168 | --exec-arg=get-token \ 169 | --exec-arg=--oidc-issuer-url=https://oidc.arianvp.me \ 170 | --exec-arg=--oidc-client-id=ASF4Os1wJysH6uWvJV9PvyNiph4y4O84tGCHj1FZEE8 171 | EOF 172 | } 173 | } 174 | 175 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | --------------------------------------------------------------------------------