├── docs ├── 10-configure-oc-command.md ├── 8-rhcos-files.md ├── 5-openshift-files.md ├── 13-upgrade.md ├── 3-web-server.md ├── 12-post-installation.md ├── 6-cluster-files.md ├── 0-prerequisites.md ├── 1-variables.md ├── 11-installation.md ├── 9-modify-isos.md ├── 99-tips-and-tricks.md ├── 7-ignition-files.md ├── 2-load-balancer.md ├── 4-dns.md └── 14-verification.md ├── .gitignore ├── README.md └── LICENSE /docs/10-configure-oc-command.md: -------------------------------------------------------------------------------- 1 | # oc command 2 | 3 | Configure `oc` to be system:admin as: 4 | 5 | ```bash 6 | mkdir -p ~/.kube/ 7 | cp ~/ocp-clusters/${CLUSTER_NAME}/auth/kubeconfig ~/.kube/config 8 | ``` 9 | 10 | [<< Previous: Modify ISOs](9-modify-isos.md) | [README](../README.md) | [Next: Installation >>](11-installation.md) 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/vim 3 | # Edit at https://www.gitignore.io/?templates=vim 4 | 5 | ### Vim ### 6 | # Swap 7 | [._]*.s[a-v][a-z] 8 | [._]*.sw[a-p] 9 | [._]s[a-rt-v][a-z] 10 | [._]ss[a-gi-z] 11 | [._]sw[a-p] 12 | 13 | # Session 14 | Session.vim 15 | Sessionx.vim 16 | 17 | # Temporary 18 | .netrwhist 19 | *~ 20 | # Auto-generated tag files 21 | tags 22 | # Persistent undo 23 | [._]*.un~ 24 | 25 | # End of https://www.gitignore.io/api/vim 26 | -------------------------------------------------------------------------------- /docs/8-rhcos-files.md: -------------------------------------------------------------------------------- 1 | # RHCOS files 2 | 3 | Download the RHCOS ISO, BIOS and UEFI image files: 4 | 5 | ```bash 6 | for asset in 'installer.iso' 'metal-bios.raw.gz' 'metal-uefi.raw.gz'; do 7 | curl -J -L https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.1/${OCPVERSION}/rhcos-${RHCOSVERSION}-x86_64-${asset} \ 8 | -o ${NGINX_DIRECTORY}/rhcos-${RHCOSVERSION}-x86_64-${asset} 9 | done 10 | ``` 11 | 12 | [<< Previous: Ignition files](7-ignition-files.md) | [README](../README.md) | [Next: Modify ISOs >>](9-modify-isos.md) 13 | -------------------------------------------------------------------------------- /docs/5-openshift-files.md: -------------------------------------------------------------------------------- 1 | # OpenShift files 2 | 3 | Download and extract the `openshift-install` and `oc/kubectl` binaries into the 4 | helper node 5 | 6 | ```bash 7 | curl -sL https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${OCPVERSION}/openshift-client-linux-${OCPVERSION}.tar.gz | \ 8 | sudo tar -C /usr/local/bin -xzf - oc kubectl 9 | 10 | # Use https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/linux/oc.tar.gz for latest oc 11 | # It doesn't include kubectl, though 12 | 13 | curl -sL https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${OCPVERSION}/openshift-install-linux-${OCPVERSION}.tar.gz | \ 14 | sudo tar -C /usr/local/bin -xzf - openshift-install 15 | 16 | sudo chmod 755 /usr/local/bin/{oc,kubectl,openshift-install} 17 | ``` 18 | 19 | [<< Previous: DNS](4-dns.md) | [README](../README.md) | [Next: Cluster files >>](6-cluster-files.md) 20 | -------------------------------------------------------------------------------- /docs/13-upgrade.md: -------------------------------------------------------------------------------- 1 | # Upgrade 2 | 3 | Upgrade to the latest bits: 4 | 5 | ```bash 6 | oc adm upgrade --to-latest 7 | ``` 8 | 9 | To switch to any other channel (such as prerelease-4.1): 10 | 11 | ```bash 12 | oc patch \ 13 | --patch='{"spec": {"channel": "prerelease-4.1"}}' \ 14 | --type=merge \ 15 | clusterversion/version 16 | ``` 17 | 18 | In order to force the update to a specific version/hash, first, get the hash of 19 | the image version of the release to upgrade to: 20 | 21 | ```bash 22 | curl -sH 'Accept: application/json' 'https://api.openshift.com/api/upgrades_info/v1/graph?channel=prerelease-4.1' | jq . 23 | ``` 24 | 25 | Then force apply the update: 26 | 27 | ```bash 28 | # RC.9 sha256 = 49c4b6bf70061e522e3525aed534d087c9abfba7c39cbcbdd1bd770ab096bf9e 29 | oc adm upgrade --force=true \ 30 | --to-image=quay.io/openshift-release-dev/ocp-release@sha256:49c4b6bf70061e522e3525aed534d087c9abfba7c39cbcbdd1bd770ab096bf9e 31 | ``` 32 | 33 | [<< Previous: Post Installation](12-post-installation.md) | [README](../README.md) | [Next: Verification >>](14-verification.md) 34 | -------------------------------------------------------------------------------- /docs/3-web-server.md: -------------------------------------------------------------------------------- 1 | # Web server 2 | 3 | To quickly spin up a web server, we will use the official NGINX container 4 | image: 5 | 6 | ```bash 7 | mkdir -p ${NGINX_DIRECTORY} 8 | 9 | podman run -d \ 10 | --expose=8001 \ 11 | -p 8001:80 \ 12 | -v ${NGINX_DIRECTORY}:/usr/share/nginx/html:z \ 13 | --name nginx \ 14 | nginx:latest 15 | ``` 16 | 17 | Open the 8001 port to the outside world 18 | 19 | ```bash 20 | sudo firewall-cmd --zone="$(firewall-cmd --get-default-zone)" \ 21 | --add-port=8001/tcp --permanent 22 | ``` 23 | 24 | Reload the firewall: 25 | 26 | ```bash 27 | sudo firewall-cmd --reload 28 | ``` 29 | 30 | ## Systemd user unit 31 | 32 | A systemd user unit can be created to automatically start/stop the podman 33 | container as a user: 34 | 35 | ```bash 36 | # This shouldn't be needed if has been previously done for the HAProxy pod 37 | # Enable start user unit without being logged first 38 | # sudo loginctl enable-linger ocp 39 | 40 | # Create the folder and unit file 41 | mkdir -p ~/.config/systemd/user/ 42 | cat > ~/.config/systemd/user/nginx.service << EOF 43 | [Unit] 44 | Description=nginx 45 | 46 | [Service] 47 | Restart=always 48 | ExecStart=/usr/bin/podman start -a nginx 49 | ExecStop=/usr/bin/podman stop -t 10 nginx 50 | KillMode=process 51 | 52 | [Install] 53 | WantedBy=multi-user.target 54 | EOF 55 | 56 | # Reload the service and enable/start the service 57 | systemctl --user daemon-reload 58 | systemctl --user enable nginx.service --now 59 | ``` 60 | 61 | [<< Previous: Load Balancer](2-load-balancer.md) | [README](../README.md) | [Next: DNS >>](4-dns.md) 62 | -------------------------------------------------------------------------------- /docs/12-post-installation.md: -------------------------------------------------------------------------------- 1 | # Post-installation 2 | 3 | * Remove bootstrap from the load balancer: 4 | 5 | ```bash 6 | cp ${HAPROXY_DIRECTORY}/haproxy.cfg{,.orig} 7 | sed -i -e '/server bootstrap/d' ${HAPROXY_DIRECTORY}/haproxy.cfg 8 | systemctl --user restart haproxy 9 | ``` 10 | 11 | * As the environment has a single worker, scale down the router replica to 1 pod: 12 | 13 | ```bash 14 | oc patch \ 15 | --namespace=openshift-ingress-operator \ 16 | --patch='{"spec": {"replicas": 1}}' \ 17 | --type=merge \ 18 | ingresscontroller/default 19 | ``` 20 | 21 | * Configure authentication backend (htpasswd) The following script will create 22 | an "admin" user with password "admin" with cluster-admin role: 23 | 24 | ```bash 25 | user=admin 26 | password=admin 27 | htpasswd=$(printf "$user:$(openssl passwd -apr1 $password)\n") 28 | htpasswd=$(echo $htpasswd | base64) 29 | 30 | oc apply -f - <>](13-upgrade.md) 62 | -------------------------------------------------------------------------------- /docs/6-cluster-files.md: -------------------------------------------------------------------------------- 1 | # Cluster files 2 | 3 | `openshift-install` requires some files (pull secret and install-config) and 4 | creates some other assets (manifests, ignition files, logs, etc.). In order to 5 | have a proper directory structure, they will be stored in 6 | `~/ocp-clusters/`: 7 | 8 | ```bash 9 | mkdir -p ~/ocp-clusters/${CLUSTER_NAME}/ 10 | ``` 11 | 12 | ## Pull secret 13 | 14 | Visit [cloud.openshift.com](cloud.openshift.com), download your pull secret and 15 | copy it into `~/ocp-clusters/pull_secret.json` (to be used by all clusters) as: 16 | 17 | ```bash 18 | cat ~/ocp-clusters/pull_secret.json 19 | # pull_secret content... 20 | export PULL_SECRET=$(cat ~/ocp-clusters/pull_secret.json) 21 | ``` 22 | 23 | ## install-config.yaml 24 | 25 | Instead creating the file directly (it will be removed by openshift-install), 26 | it is created with the cluster prefix, then copied to the proper location: 27 | 28 | ```bash 29 | cat > ~/ocp-clusters/${CLUSTER_NAME}-install-config.yaml << EOF 30 | apiVersion: v1 31 | baseDomain: ${DOMAIN_NAME} 32 | compute: 33 | - name: worker 34 | replicas: 0 35 | controlPlane: 36 | name: master 37 | replicas: 3 38 | metadata: 39 | name: ${CLUSTER_NAME} 40 | networking: 41 | clusterNetworks: 42 | - cidr: 10.128.0.0/14 43 | hostPrefix: 23 44 | networkType: OpenShiftSDN 45 | serviceNetwork: 46 | - 172.30.0.0/16 47 | platform: 48 | none: {} 49 | pullSecret: | 50 | ${PULL_SECRET} 51 | sshKey: | 52 | ${SSH_KEY} 53 | EOF 54 | 55 | cp ~/ocp-clusters/${CLUSTER_NAME}-install-config.yaml \ 56 | ~/ocp-clusters/${CLUSTER_NAME}/install-config.yaml 57 | ``` 58 | 59 | [<< Previous: OpenShift files](5-openshift-files.md) | [README](../README.md) | [Next: Ignition files >>](7-ignition-files.md) 60 | -------------------------------------------------------------------------------- /docs/0-prerequisites.md: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | 3 | The helper node used in this document is a Fedora 29 x86_64 VM and it will use 4 | a regular user instead root user (with some exceptions) 5 | 6 | If a regular user is not created: 7 | 8 | ```bash 9 | useradd -m ocp 10 | echo ocp:password | chpasswd 11 | echo "ocp ALL=(root) NOPASSWD:ALL" | tee -a /etc/sudoers.d/ocp_nopassword 12 | visudo -cf /etc/sudoers.d/ocp_nopassword 13 | ``` 14 | 15 | Allow rootless containers for the `ocp` user: 16 | 17 | ```bash 18 | sudo usermod --add-subuids 10000-75535 ocp 19 | sudo usermod --add-subgids 10000-75535 ocp 20 | ``` 21 | 22 | Install podman and other required utils 23 | 24 | ```bash 25 | dnf clean all 26 | dnf install -y podman jq libguestfs-tools-c 27 | dnf update -y 28 | ``` 29 | 30 | Disable all not needed interfaces in the host to avoid messing networking stuff 31 | with containers as well as IPv6 if not needed: 32 | 33 | ```bash 34 | # As root 35 | for interface in 'eno2' 'eth2' 'eth3'; do 36 | cat > /etc/sysconfig/network-scripts/ifcfg-${interface} << EOF 37 | DEVICE=${interface} 38 | BOOTPROTO=none 39 | ONBOOT=no 40 | NETWORKING_IPV6=no 41 | IPV6_AUTOCONF=no 42 | EOF 43 | done 44 | 45 | # Disable IPV6 46 | cat > /etc/sysctl.d/ipv6.conf << EOF 47 | net.ipv6.conf.all.disable_ipv6 = 1 48 | net.ipv6.conf.default.disable_ipv6 = 1 49 | net.ipv6.conf.lo.disable_ipv6 = 1 50 | EOF 51 | 52 | sed -i \ 53 | -e 's/IPV6_AUTOCONF.*/IPV6_AUTOCONF=no/g' \ 54 | -e 's/IPV6INIT.*/IPV6INIT=no/g' \ 55 | /etc/sysconfig/network-scripts/* 56 | 57 | sysctl -p /etc/sysctl.d/ipv6.conf 58 | # Disable virbr0 59 | rm -f /etc/libvirt/qemu/networks/autostart/default.xml 60 | systemctl stop libvirtd 61 | 62 | nmcli connection reload 63 | systemctl restart NetworkManager 64 | # Or even better 65 | # reboot 66 | ``` 67 | 68 | Switch to the `ocp` user created before: 69 | 70 | ```bash 71 | su - ocp 72 | ``` 73 | 74 | Create an ssh key to be injected in the OCP hosts: 75 | 76 | ```bash 77 | ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa 78 | ``` 79 | 80 | [<< Back to README](../README.md) | [Next: Variables >>](1-variables.md) 81 | -------------------------------------------------------------------------------- /docs/1-variables.md: -------------------------------------------------------------------------------- 1 | # Variables 2 | 3 | Those variables will allow other environments/versions and modifications to 4 | where the files are hosted: 5 | 6 | ```bash 7 | cat > ~/vars << EOF 8 | # RHCOS and OCP4 versions 9 | # From https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.1/ 10 | export RHCOSVERSION="4.1.0" 11 | # From https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/ 12 | export OCPVERSION="4.1.0" 13 | 14 | # Where to store the required files 15 | export NGINX_DIRECTORY="/home/ocp/containers/nginx" 16 | export HAPROXY_DIRECTORY="/home/ocp/containers/haproxy" 17 | export COREDNS_DIRECTORY="/home/ocp/containers/coredns" 18 | 19 | # Network details 20 | export DOMAIN_NAME="minwi.lan" 21 | export CLUSTER_NAME="ocp4" 22 | export GATEWAY="192.168.32.1" 23 | export NETMASK="255.255.255.0" 24 | export DNSFORWARDER="8.8.8.8" 25 | 26 | # Hosts 27 | export BOOTSTRAP_IP="192.168.32.99" 28 | export MASTER0_IP="192.168.32.100" 29 | export MASTER1_IP="192.168.32.101" 30 | export MASTER2_IP="192.168.32.102" 31 | export WORKER0_IP="192.168.32.200" 32 | 33 | # We will use a single interface for the OCP4 cluster network traffic (same one in all hosts) 34 | export NET_INTERFACE="eno2" 35 | 36 | # iDRAC details 37 | # Select bios/uefi depending on the hardware 38 | # https://downloads.dell.com/solutions/servers-solution-resources/BootModeWhitepaper.pdf 39 | # 'racadm get bios.BiosBootSettings.BootMode' 40 | export BIOSMODE="bios" 41 | export BOOTSTRAP_IDRAC_IP="192.168.31.99" 42 | export MASTER0_IDRAC_IP="192.168.31.100" 43 | export MASTER1_IDRAC_IP="192.168.31.101" 44 | export MASTER2_IDRAC_IP="192.168.31.102" 45 | export WORKER0_IDRAC_IP="192.168.31.200" 46 | export IDRACUSER="root" 47 | # sshpass requires this particular variable 48 | export SSHPASS="calvin" 49 | 50 | # We will use this host as DNS, static assets server and haproxy 51 | export MY_IP="192.168.32.2" 52 | export DNS="\${MY_IP}" 53 | export URL="http://\${MY_IP}:8001" 54 | export LB_IP="\${MY_IP}" 55 | 56 | # Required to extract the ISO content with guestfish without any virtualization stuff installed 57 | export LIBGUESTFS_BACKEND=direct 58 | 59 | export SSH_KEY=\$(cat ~/.ssh/id_rsa.pub) 60 | # This may not work until the pull_secret is created 61 | export PULL_SECRET=\$(cat ~/ocp-clusters/pull_secret.json) 62 | EOF 63 | ``` 64 | 65 | Then, use the vars file 66 | 67 | ```bash 68 | source ~/vars 69 | ``` 70 | 71 | Also, because `XDG_RUNTIME_DIR` [is not set under 72 | `sudo`](https://unix.stackexchange.com/questions/346841/why-does-sudo-i-not-set-xdg-runtime-dir-for-the-target-user) 73 | and in order to make the systemd unit files: 74 | 75 | ```bash 76 | export XDG_RUNTIME_DIR=/run/user/$UID 77 | ``` 78 | 79 | [<< Previous: Prerequisites](0-prerequisites.md) | [README](../README.md) | [Next: Load balancer >>](2-load-balancer.md) 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Intro 2 | 3 | The objective of this document is to provide instructions (automated ish) to install OCP4 on baremetal: 4 | 5 | * without PXE (pretty common scenario in big companies) 6 | * avoid installing stuff and use containers instead (instead yum/dnf install httpd, haproxy,... use containers) 7 | * use rootless containers if possible 8 | * use Fedora29/RHEL8 stuff (nmcli, firewalld, etc.) 9 | 10 | > **DISCLAIMER**: This is unofficial and unsupported procedure. Use the official [OpenShift](https://docs.openshift.com/) documentation for the supported scenarios. 11 | 12 | # Current status 13 | 14 | OCP4.1 GA installed 15 | 16 | # Environment 17 | 18 | In order to have a frictionless environment, a helper node will be used to achieve the OpenShift prerrequisites, including 19 | DNS entries, a load balancer and a httpd server to host some files required for the installation. 20 | If your enviorment has already a DNS server, load balancer and/or httpd server, you can use those instead. 21 | 22 | | Usage | Hostname | IP | NOTES | 23 | |-----------|--------------------------|----------------|-----------------------------------------------| 24 | | Helper | ocp4-helper.minwi.lan | 192.168.32.2 | DNS, httpd & load balancer | 25 | | Bootstrap | ocp4-bootstrap.minwi.lan | 192.168.32.99 | To be removed from the cluster once installed | 26 | | Master-0 | ocp4-master-0.minwi.lan | 192.168.32.100 | | 27 | | Master-1 | ocp4-master-1.minwi.lan | 192.168.32.101 | | 28 | | Master-2 | ocp4-master-2.minwi.lan | 192.168.32.102 | | 29 | | Worker-0 | ocp4-worker-0.minwi.lan | 192.168.32.200 | | 30 | 31 | NOTE: Those baremetal servers are Dell based, so `racadm` will be used in order to manage the iDRAC to map a virtual cd, power off/on, etc. 32 | 33 | # References 34 | 35 | * https://github.com/christianh814/openshift-toolbox/tree/master/ocp4_upi 36 | * https://docs.openshift.com/container-platform/4.1/installing/installing_bare_metal/installing-bare-metal.html 37 | 38 | # Steps 39 | 40 | * [0 - Prerequisites](docs/0-prerequisites.md) 41 | * [1 - Variables](docs/1-variables.md) 42 | * [2 - Load balancer](docs/2-load-balancer.md) 43 | * [3 - Web server](docs/3-web-server.md) 44 | * [4 - DNS](docs/4-dns.md) 45 | * [5 - OpenShift files](docs/5-openshift-files.md) 46 | * [6 - Cluster files](docs/6-cluster-files.md) 47 | * [7 - Ignition files](docs/7-ignition-files.md) 48 | * [8 - RHCOS files](docs/8-rhcos-files.md) 49 | * [9 - Modify ISOs](docs/9-modify-isos.md) 50 | * [10 - Configure `oc` command](docs/10-configure-oc-command.md) 51 | * [11 - Installation](docs/11-installation.md) 52 | * [12 - Post installation](docs/12-post-installation.md) 53 | * [13 - Upgrade](docs/13-upgrade.md) 54 | * [14 - Verification](docs/14-verification.md) 55 | * [99 - Tips and tricks](docs/99-tips-and-tricks.md) 56 | -------------------------------------------------------------------------------- /docs/11-installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | With everything prepared, it is time to install the baremetal servers. If the 4 | previous steps have been doing properly, the ISOs are modified to avoid 5 | requiring manual input and the ignition files shall configure the networking to 6 | use the DNS we have set. 7 | 8 | NOTE: In my environment, it is required to have an iDRAC version >= 2.60.60.60 9 | to be able to map virtual media iso from http. 10 | 11 | We will be using a racadm directly from the iDRAC connecting via ssh to avoid 12 | installing it locally via [sshpass](https://sourceforge.net/projects/sshpass/). 13 | 14 | ## Install bootstrap 15 | 16 | ```bash 17 | # Note: racadm commands will complain about certificates 18 | sudo yum install -y sshpass 19 | 20 | # Bootstrap 21 | export IDRACIP=${BOOTSTRAP_IDRAC_IP} 22 | racadm="sshpass -e ssh -oStrictHostKeyChecking=no ${IDRACUSER}@${IDRACIP} racadm" 23 | $racadm remoteimage -d 24 | $racadm remoteimage -c -u "foo" -p "bar" -l ${URL}/bootstrap.iso 25 | $racadm jobqueue delete -i JID_CLEARALL 26 | $racadm set BIOS.OneTimeBoot.OneTimeBootMode OneTimeBootSeq 27 | $racadm set BIOS.OneTimeBoot.OneTimeBootSeqDev Optical.iDRACVirtual.1-1 28 | $racadm jobqueue create BIOS.Setup.1-1 -r pwrcycle 29 | ``` 30 | 31 | ## Install hosts 32 | 33 | ```bash 34 | install_host(){ 35 | racadm="sshpass -e ssh -oStrictHostKeyChecking=no ${IDRACUSER}@${IDRACIP} racadm" 36 | $racadm remoteimage -d 37 | $racadm remoteimage -c -u "foo" -p "bar" -l ${URL}/${NODE}.iso 38 | $racadm jobqueue delete -i JID_CLEARALL 39 | $racadm set BIOS.OneTimeBoot.OneTimeBootMode OneTimeBootSeq 40 | $racadm set BIOS.OneTimeBoot.OneTimeBootSeqDev Optical.iDRACVirtual.1-1 41 | $racadm jobqueue create BIOS.Setup.1-1 -r pwrcycle 42 | } 43 | 44 | # Masters 45 | export IDRACIP=${MASTER0_IDRAC_IP} 46 | export NODE='master-0' 47 | install_host 48 | 49 | export IDRACIP=${MASTER1_IDRAC_IP} 50 | export NODE='master-1' 51 | install_host 52 | 53 | export IDRACIP=${MASTER2_IDRAC_IP} 54 | export NODE='master-2' 55 | install_host 56 | 57 | # Workers 58 | export IDRACIP=${WORKER0_IDRAC_IP} 59 | export NODE='worker-0' 60 | install_host 61 | ``` 62 | 63 | After a while, the hosts will be running RHCOS, then wait until the bootstrap process ends: 64 | 65 | ```bash 66 | openshift-install --dir=$(readlink -f ~/ocp-clusters/${CLUSTER_NAME}) --log-level debug \ 67 | wait-for bootstrap-complete 68 | ``` 69 | 70 | Then, wait until the install is complete: 71 | 72 | ```bash 73 | openshift-install --dir=$(readlink -f ~/ocp-clusters/${CLUSTER_NAME}) --log-level debug \ 74 | wait-for install-complete 75 | ``` 76 | 77 | WARNING: The installation won't finish until the image registry has been 78 | deployed. As there is no storage available, emptydir will be used (not 79 | recommended for production!!!): 80 | 81 | ```bash 82 | oc patch configs.imageregistry.operator.openshift.io cluster --type merge --patch '{"spec":{"storage":{"emptyDir":{}}}}' 83 | ``` 84 | 85 | [<< Previous: Configure `oc` command](10-configure-oc-command.md) | [README](../README.md) | [Next: Post installation >>](12-post-installation.md) 86 | -------------------------------------------------------------------------------- /docs/9-modify-isos.md: -------------------------------------------------------------------------------- 1 | # Modify ISOs 2 | 3 | It is required to add some parameters to the kernel line for the RHCOS installation: 4 | 5 | * `coreos.inst=yes` 6 | * `coreos.inst.install_dev=sda` 7 | 8 | To set the image and ignition file location: 9 | 10 | * `coreos.inst.image_url=` 11 | * `coreos.inst.ignition_url=http://example.com/config.ign` 12 | 13 | To provide the hosts fixed IPs it is required to specify the IP address as: 14 | 15 | * `ip=::::::none` 16 | 17 | To specify the DNS nameserver IP: 18 | 19 | * `nameserver=` 20 | 21 | Instead of doing it manually, different ISOs will be created (for bootstrap, 22 | masters and nodes) as: 23 | 24 | ```bash 25 | export VOLID=$(isoinfo -d -i ${NGINX_DIRECTORY}/rhcos-${RHCOSVERSION}-x86_64-installer.iso | \ 26 | awk '/Volume id/ { print $3 }') 27 | TEMPDIR=$(mktemp -d) 28 | 29 | cd ${TEMPDIR} 30 | # Extract the ISO content using guestfish (to avoid sudo mount) 31 | guestfish -a ${NGINX_DIRECTORY}/rhcos-${RHCOSVERSION}-x86_64-installer.iso \ 32 | -m /dev/sda tar-out / - | tar xvf - 33 | 34 | # Helper function to modify the config files 35 | modify_cfg(){ 36 | for file in "EFI/fedora/grub.cfg" "isolinux/isolinux.cfg"; do 37 | # Append the proper image and ignition urls 38 | sed -e '/coreos.inst=yes/s|$| coreos.inst.install_dev=sda coreos.inst.image_url='"${URL}"'\/rhcos-'"${RHCOSVERSION}"'-x86_64-metal-'"${BIOSMODE}"'.raw.gz coreos.inst.ignition_url='"${URL}"'\/'"${NODE}"'.ign ip='"${IP}"'::'"${GATEWAY}"':'"${NETMASK}"':'"${FQDN}"':'"${NET_INTERFACE}"':none nameserver='"${DNS}"'|' ${file} > $(pwd)/${NODE}_${file##*/} 39 | # Boot directly in the installation 40 | sed -i -e 's/default vesamenu.c32/default linux/g' -e 's/timeout 600/timeout 10/g' $(pwd)/${NODE}_${file##*/} 41 | done 42 | } 43 | 44 | # BOOTSTRAP 45 | TYPE="bootstrap" 46 | NODE="bootstrap" 47 | IP=${BOOTSTRAP_IP} 48 | FQDN="${CLUSTER_NAME}-bootstrap.${DOMAIN_NAME}" 49 | modify_cfg 50 | 51 | # MASTERS 52 | TYPE="master" 53 | # MASTER-0 54 | NODE="master-0" 55 | IP=${MASTER0_IP} 56 | FQDN="${CLUSTER_NAME}-${NODE}.${DOMAIN_NAME}" 57 | modify_cfg 58 | 59 | # MASTER-1 60 | NODE="master-1" 61 | IP=${MASTER1_IP} 62 | FQDN="${CLUSTER_NAME}-${NODE}.${DOMAIN_NAME}" 63 | modify_cfg 64 | 65 | # MASTER-2 66 | NODE="master-2" 67 | IP=${MASTER2_IP} 68 | FQDN="${CLUSTER_NAME}-${NODE}.${DOMAIN_NAME}" 69 | modify_cfg 70 | 71 | # WORKERS 72 | TYPE="worker" 73 | # WORKER-0 74 | NODE="worker-0" 75 | IP=${WORKER0_IP} 76 | FQDN="${CLUSTER_NAME}-${NODE}.${DOMAIN_NAME}" 77 | modify_cfg 78 | 79 | # Generate the images, one per node as the IP configuration is different... 80 | # https://github.com/coreos/coreos-assembler/blob/master/src/cmd-buildextend-installer#L97-L103 81 | for node in master-0 master-1 master-2 worker-0 bootstrap; do 82 | # Overwrite the grub.cfg and isolinux.cfg files for each node type 83 | for file in "EFI/fedora/grub.cfg" "isolinux/isolinux.cfg"; do 84 | cp $(pwd)/${node}_${file##*/} ${file} 85 | done 86 | # As regular user! 87 | genisoimage -verbose -rock -J -joliet-long -volset ${VOLID} \ 88 | -eltorito-boot isolinux/isolinux.bin -eltorito-catalog isolinux/boot.cat \ 89 | -no-emul-boot -boot-load-size 4 -boot-info-table \ 90 | -eltorito-alt-boot -efi-boot images/efiboot.img -no-emul-boot \ 91 | -o ${NGINX_DIRECTORY}/${node}.iso . 92 | done 93 | 94 | # Optionally, clean up 95 | # cd 96 | # rm -Rf ${TEMPDIR} 97 | ``` 98 | 99 | [<< Previous: RHCOS files](8-rhcos-files.md) | [README](../README.md) | [Next: Configure `oc` command >>](10-configure-oc-command.md) 100 | -------------------------------------------------------------------------------- /docs/99-tips-and-tricks.md: -------------------------------------------------------------------------------- 1 | # Tips and tricks 2 | 3 | This section explains some tips and tricks to familiarize with OpenShift 4. 4 | 5 | NOTE: [I wrote 6 | this](https://github.com/openshift/training/commit/eab9062e2b5c6a31f510dc2a0e813ee5221e4452) 7 | in the 8 | [openshift/training](https://github.com/openshift/training/blob/master/docs/97-tips-and-tricks.md) 9 | repository back in the day and it still applies. 10 | 11 | ## NTP configuration 12 | 13 | RHCOS uses chronyd to synchronize the system time. The default configuration 14 | uses the `*.rhel.pool.ntp.org` servers: 15 | 16 | ```bash 17 | $ grep -v -E '^#|^$' /etc/chrony.conf 18 | server 0.rhel.pool.ntp.org iburst 19 | server 1.rhel.pool.ntp.org iburst 20 | server 2.rhel.pool.ntp.org iburst 21 | server 3.rhel.pool.ntp.org iburst 22 | driftfile /var/lib/chrony/drift 23 | makestep 1.0 3 24 | rtcsync 25 | logdir /var/log/chrony 26 | ``` 27 | 28 | As the hosts configuration shouldn't be managed manually, in order to configure 29 | chronyd to use custom servers or a custom setting, it is required to use the 30 | `machine-config-operator` to modify the files used by the masters and workers 31 | by the following procedure: 32 | 33 | * Create the proper file with your custom tweaks and encode it as base64: 34 | 35 | ```bash 36 | cat << EOF | base64 37 | server clock.redhat.com iburst 38 | driftfile /var/lib/chrony/drift 39 | makestep 1.0 3 40 | rtcsync 41 | logdir /var/log/chrony 42 | EOF 43 | ``` 44 | 45 | * Create the MachineConfig file with the base64 string from the previous command 46 | as: 47 | 48 | ```bash 49 | cat << EOF > ./masters-chrony-configuration.yaml 50 | apiVersion: machineconfiguration.openshift.io/v1 51 | kind: MachineConfig 52 | metadata: 53 | labels: 54 | machineconfiguration.openshift.io/role: master 55 | name: masters-chrony-configuration 56 | spec: 57 | config: 58 | ignition: 59 | config: {} 60 | security: 61 | tls: {} 62 | timeouts: {} 63 | version: 2.2.0 64 | networkd: {} 65 | passwd: {} 66 | storage: 67 | files: 68 | - contents: 69 | source: data:text/plain;charset=utf-8;base64,c2VydmVyIGNsb2NrLnJlZGhhdC5jb20gaWJ1cnN0CmRyaWZ0ZmlsZSAvdmFyL2xpYi9jaHJvbnkvZHJpZnQKbWFrZXN0ZXAgMS4wIDMKcnRjc3luYwpsb2dkaXIgL3Zhci9sb2cvY2hyb255Cg== 70 | verification: {} 71 | filesystem: root 72 | mode: 420 73 | path: /etc/chrony.conf 74 | osImageURL: "" 75 | EOF 76 | ``` 77 | 78 | Substitute the base64 string with your own. 79 | 80 | * Apply it 81 | 82 | ```bash 83 | oc apply -f ./masters-chrony-configuration.yaml 84 | ``` 85 | 86 | ## OCP Master configuration 87 | The master configuration is now stored in a `configMap`. During the installation 88 | process, a few `configMaps` are created, so in order to get the latest: 89 | 90 | ```bash 91 | oc get cm -n openshift-kube-apiserver | grep config 92 | ``` 93 | 94 | Observe the latest id and then: 95 | 96 | ```bash 97 | oc get cm -n openshift-kube-apiserver config-ID 98 | ``` 99 | 100 | To get the output in a human-readable form, use: 101 | 102 | ```bash 103 | oc get cm -n openshift-kube-apiserver config-ID \ 104 | -o jsonpath='{.data.config\.yaml}' | jq 105 | ``` 106 | 107 | For the OpenShift API configuration: 108 | 109 | ```bash 110 | oc get cm -n openshift-apiserver config -o jsonpath='{.data.config\.yaml}' | jq 111 | ``` 112 | 113 | ## Delete 'Completed' pods 114 | 115 | During the installation process, a few temporary pods are created. Keeping those 116 | pods as 'Completed' doesn't harm nor waste resources but if you want to delete 117 | them to have only 'running' pods in your environment you can use the following 118 | command: 119 | 120 | ```bash 121 | oc get pods --all-namespaces | \ 122 | awk '{if ($4 == "Completed") system ("oc delete pod " $2 " -n " $1 )}' 123 | ``` 124 | 125 | ## Get pods not running nor completed 126 | 127 | A handy one liner to see the pods having issues (such as CrashLoopBackOff): 128 | 129 | ```bash 130 | oc get pods --all-namespaces | grep -v -E 'Completed|Running' 131 | ``` 132 | 133 | [<< Previous: Verification](14-verification.md) | [Back to README](../README.md) 134 | -------------------------------------------------------------------------------- /docs/7-ignition-files.md: -------------------------------------------------------------------------------- 1 | # Ignition files 2 | 3 | Create the ignition files once the `install-config.yaml` file has been created: 4 | 5 | ```bash 6 | openshift-install create ignition-configs \ 7 | --dir=$(readlink -f ~/ocp-clusters/${CLUSTER_NAME}) 8 | 9 | # They are going to be modified and be served by the NGINX container 10 | cp ~/ocp-clusters/${CLUSTER_NAME}/*.ign ${NGINX_DIRECTORY} 11 | ``` 12 | 13 | NOTE: The ignition files include certificates that are only valid for 24h 14 | 15 | ## Ignition configs modifications 16 | 17 | In order to be able to set static IP addresses for the hosts, it is required to 18 | inject the proper configuration 19 | (`/etc/sysconfig/network-scripts/ifcfg-` & `/etc/hostname`) via 20 | ignition: 21 | 22 | ```bash 23 | create_ifcfg(){ 24 | cat > ${NGINX_DIRECTORY}/${HOST}-eno2 << EOF 25 | DEVICE=eno2 26 | BOOTPROTO=none 27 | ONBOOT=yes 28 | NETMASK=${NETMASK} 29 | IPADDR=${IP} 30 | GATEWAY=${GATEWAY} 31 | PEERDNS=no 32 | DNS1=${DNS} 33 | IPV6INIT=no 34 | EOF 35 | 36 | ENO2=$(cat ${NGINX_DIRECTORY}/${HOST}-eno2 | base64 -w0) 37 | rm ${NGINX_DIRECTORY}/${HOST}-eno2 38 | 39 | cat > ${NGINX_DIRECTORY}/${HOST}-ifcfg-eno2.json << EOF 40 | { 41 | "append" : false, 42 | "mode" : 420, 43 | "filesystem" : "root", 44 | "path" : "/etc/sysconfig/network-scripts/ifcfg-eno2", 45 | "contents" : { 46 | "source" : "data:text/plain;charset=utf-8;base64,${ENO2}", 47 | "verification" : {} 48 | }, 49 | "user" : { 50 | "name" : "root" 51 | }, 52 | "group": { 53 | "name": "root" 54 | } 55 | } 56 | EOF 57 | 58 | cat > ${NGINX_DIRECTORY}/${HOST}-eno1 << EOF 59 | DEVICE=eno1 60 | BOOTPROTO=none 61 | ONBOOT=no 62 | EOF 63 | ENO1=$(cat ${NGINX_DIRECTORY}/${HOST}-eno1 | base64 -w0) 64 | rm ${NGINX_DIRECTORY}/${HOST}-eno1 65 | cat > ${NGINX_DIRECTORY}/${HOST}-ifcfg-eno1.json << EOF 66 | { 67 | "append" : false, 68 | "mode" : 420, 69 | "filesystem" : "root", 70 | "path" : "/etc/sysconfig/network-scripts/ifcfg-eno1", 71 | "contents" : { 72 | "source" : "data:text/plain;charset=utf-8;base64,${ENO1}", 73 | "verification" : {} 74 | }, 75 | "user" : { 76 | "name" : "root" 77 | }, 78 | "group": { 79 | "name": "root" 80 | } 81 | } 82 | EOF 83 | 84 | cat > ${NGINX_DIRECTORY}/${HOST}-hostname << EOF 85 | ${CLUSTER_NAME}-${HOST}.${DOMAIN_NAME} 86 | EOF 87 | HN=$(cat ${NGINX_DIRECTORY}/${HOST}-hostname | base64 -w0) 88 | rm ${NGINX_DIRECTORY}/${HOST}-hostname 89 | cat > ${NGINX_DIRECTORY}/${HOST}-hostname.json << EOF 90 | { 91 | "append" : false, 92 | "mode" : 420, 93 | "filesystem" : "root", 94 | "path" : "/etc/hostname", 95 | "contents" : { 96 | "source" : "data:text/plain;charset=utf-8;base64,${HN}", 97 | "verification" : {} 98 | }, 99 | "user" : { 100 | "name" : "root" 101 | }, 102 | "group": { 103 | "name": "root" 104 | } 105 | } 106 | EOF 107 | } 108 | 109 | # Disable set hostname via reverse lookup 110 | # Common to all hosts 111 | cat > ${NGINX_DIRECTORY}/hostname-mode << EOF 112 | [main] 113 | hostname-mode=none 114 | EOF 115 | HM=$(cat ${NGINX_DIRECTORY}/hostname-mode | base64 -w0) 116 | rm ${NGINX_DIRECTORY}/hostname-mode 117 | cat > ${NGINX_DIRECTORY}/hostname-mode.json << EOF 118 | { 119 | "append" : false, 120 | "mode" : 420, 121 | "filesystem" : "root", 122 | "path" : "/etc/NetworkManager/conf.d/hostname-mode.conf", 123 | "contents" : { 124 | "source" : "data:text/plain;charset=utf-8;base64,${HM}", 125 | "verification" : {} 126 | }, 127 | "user" : { 128 | "name" : "root" 129 | }, 130 | "group": { 131 | "name": "root" 132 | } 133 | } 134 | EOF 135 | 136 | modify_ignition(){ 137 | cp -u ${NGINX_DIRECTORY}/${TYPE}.ign ${NGINX_DIRECTORY}/${HOST}.ign.orig 138 | jq '.storage.files += [inputs]' ${NGINX_DIRECTORY}/${HOST}.ign.orig ${NGINX_DIRECTORY}/${HOST}-hostname.json ${HOST}-ifcfg-eno1.json ${HOST}-ifcfg-eno2.json ${NGINX_DIRECTORY}/hostname-mode.json > ${NGINX_DIRECTORY}/${HOST}.ign 139 | rm -f ${NGINX_DIRECTORY}/${HOST}-hostname.json ${HOST}-ifcfg-eno1.json ${HOST}-ifcfg-eno2.json 140 | } 141 | 142 | HOST="bootstrap" 143 | TYPE="bootstrap" 144 | IP=${BOOTSTRAP_IP} 145 | create_ifcfg 146 | modify_ignition 147 | 148 | TYPE="master" 149 | HOST=master-0 150 | IP=${MASTER0_IP} 151 | create_ifcfg 152 | modify_ignition 153 | 154 | HOST=master-1 155 | IP=${MASTER1_IP} 156 | create_ifcfg 157 | modify_ignition 158 | 159 | HOST=master-2 160 | IP=${MASTER2_IP} 161 | create_ifcfg 162 | modify_ignition 163 | 164 | TYPE="worker" 165 | HOST=worker-0 166 | IP=${WORKER0_IP} 167 | create_ifcfg 168 | modify_ignition 169 | ``` 170 | 171 | NOTE: I'm 100% sure this whole code block can be improved… any suggestions appreciated :) 172 | 173 | [<< Previous: Cluster files](6-cluster-files.md) | [README](../README.md) | [Next: RHCOS files >>](8-rhcos-files.md) 174 | -------------------------------------------------------------------------------- /docs/2-load-balancer.md: -------------------------------------------------------------------------------- 1 | # Haproxy 2 | 3 | To quickly spin up an HAProxy server, we will use the HAProxy official 4 | container image: 5 | 6 | ```bash 7 | mkdir -p ${HAPROXY_DIRECTORY} 8 | 9 | cat > ${HAPROXY_DIRECTORY}/haproxy.cfg << EOF 10 | defaults 11 | mode http 12 | log global 13 | option httplog 14 | option dontlognull 15 | option forwardfor except 127.0.0.0/8 16 | option redispatch 17 | retries 3 18 | timeout http-request 10s 19 | timeout queue 1m 20 | timeout connect 10s 21 | timeout client 300s 22 | timeout server 300s 23 | timeout http-keep-alive 10s 24 | timeout check 10s 25 | maxconn 20000 26 | 27 | # Useful for debugging, dangerous for production 28 | listen stats 29 | bind :9000 30 | mode http 31 | stats enable 32 | stats uri / 33 | 34 | frontend openshift-api-server 35 | bind *:6443 36 | default_backend openshift-api-server 37 | mode tcp 38 | option tcplog 39 | 40 | backend openshift-api-server 41 | balance source 42 | mode tcp 43 | server bootstrap ${BOOTSTRAP_IP}:6443 check 44 | server master-0 ${MASTER0_IP}:6443 check 45 | server master-1 ${MASTER1_IP}:6443 check 46 | server master-2 ${MASTER2_IP}:6443 check 47 | 48 | frontend machine-config-server 49 | bind *:22623 50 | default_backend machine-config-server 51 | mode tcp 52 | option tcplog 53 | 54 | backend machine-config-server 55 | balance source 56 | mode tcp 57 | server bootstrap ${BOOTSTRAP_IP}:22623 check 58 | server master-0 ${MASTER0_IP}:22623 check 59 | server master-1 ${MASTER1_IP}:22623 check 60 | server master-2 ${MASTER2_IP}:22623 check 61 | 62 | # As we are using rootless containers, we will bind to the 8080/tcp port in the host 63 | # so we can map 1:1 (and then firewalld will perform the redirection) 64 | frontend ingress-http 65 | bind *:8080 66 | default_backend ingress-http 67 | mode tcp 68 | option tcplog 69 | 70 | backend ingress-http 71 | balance source 72 | mode tcp 73 | server worker-0 ${WORKER0_IP}:80 check 74 | 75 | # As we are using rootless containers, we will bind to the 8443/tcp port in the host 76 | # so we can map 1:1 (and then firewalld will perform the redirection) 77 | frontend ingress-https 78 | bind *:8443 79 | default_backend ingress-https 80 | mode tcp 81 | option tcplog 82 | 83 | backend ingress-https 84 | balance source 85 | mode tcp 86 | server worker-0 ${WORKER0_IP}:443 check 87 | EOF 88 | ``` 89 | 90 | Then, run the container: 91 | 92 | ```bash 93 | podman run -d \ 94 | --expose=9000 --expose=22623 --expose=6443 --expose=8080 --expose=8443 \ 95 | -p 9000:9000 -p 22623:22623 -p 6443:6443 -p 8080:8080 -p 8443:8443 \ 96 | -v ${HAPROXY_DIRECTORY}/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:Z \ 97 | --name haproxy \ 98 | haproxy:alpine 99 | ``` 100 | 101 | Configure firewall to forward 8443 to 443 and 8080 to 80 102 | 103 | ```bash 104 | sudo firewall-cmd --zone="$(firewall-cmd --get-default-zone)" \ 105 | --add-forward-port=port=443:proto=tcp:toport=8443 --permanent 106 | sudo firewall-cmd --zone="$(firewall-cmd --get-default-zone)" \ 107 | --add-forward-port=port=80:proto=tcp:toport=8080 --permanent 108 | ``` 109 | 110 | Open those ports to the outside world 111 | 112 | ```bash 113 | for service in http https 114 | do 115 | sudo firewall-cmd --zone="$(firewall-cmd --get-default-zone)" \ 116 | --add-service=${service} --permanent 117 | done 118 | 119 | for port in 9000 22623 6443 120 | do 121 | sudo firewall-cmd --zone="$(firewall-cmd --get-default-zone)" \ 122 | --add-port=${port}/tcp --permanent 123 | done 124 | ``` 125 | 126 | Reload the firewall 127 | 128 | ```bash 129 | sudo firewall-cmd --reload 130 | ``` 131 | 132 | ## Systemd user unit 133 | 134 | A systemd user unit can be created to automatically start/stop the podman 135 | container as a user: 136 | 137 | ```bash 138 | # Enable start user unit without being logged first 139 | sudo loginctl enable-linger ocp 140 | 141 | # Create the folder and unit file 142 | mkdir -p ~/.config/systemd/user/ 143 | cat > ~/.config/systemd/user/haproxy.service << EOF 144 | [Unit] 145 | Description=HAProxy 146 | 147 | [Service] 148 | Restart=always 149 | ExecStart=/usr/bin/podman start -a haproxy 150 | ExecStop=/usr/bin/podman stop -t 10 haproxy 151 | KillMode=process 152 | 153 | [Install] 154 | WantedBy=multi-user.target 155 | EOF 156 | 157 | # Reload the service and enable/start the service 158 | systemctl --user daemon-reload 159 | systemctl --user enable haproxy.service --now 160 | ``` 161 | 162 | [<< Previous: Variables](1-variables.md) | [README](../README.md) | [Next: Web server >>](3-web-server.md) 163 | -------------------------------------------------------------------------------- /docs/4-dns.md: -------------------------------------------------------------------------------- 1 | # DNS 2 | 3 | To quickly spin up a DNS server, we will use the official CoreDNS container 4 | image: 5 | 6 | ```bash 7 | mkdir -p ${COREDNS_DIRECTORY} 8 | 9 | # CoreDNS configuration file 10 | cat > ${COREDNS_DIRECTORY}/Corefile << EOF 11 | .:53 { 12 | log 13 | errors 14 | forward . ${DNSFORWARDER} 15 | } 16 | 17 | ${DOMAIN_NAME}:53 { 18 | log 19 | errors 20 | file /etc/coredns/db.${DOMAIN_NAME} 21 | } 22 | EOF 23 | 24 | ``` 25 | 26 | Then, the proper zone file, including the SRV records, CNAMES, etc. To avoid 27 | escaping dollar symbols, etc. we create the template file first then replace 28 | the variables using `sed`: 29 | 30 | ```bash 31 | cat > ${COREDNS_DIRECTORY}/db.${DOMAIN_NAME} << 'EOF' 32 | $ORIGIN DOMAIN_NAME. 33 | $TTL 10800 ; 3 hours 34 | @ 3600 IN SOA sns.dns.icann.org. noc.dns.icann.org. ( 35 | 2019010101 ; serial 36 | 7200 ; refresh (2 hours) 37 | 3600 ; retry (1 hour) 38 | 1209600 ; expire (2 weeks) 39 | 3600 ; minimum (1 hour) 40 | ) 41 | 42 | _etcd-server-ssl._tcp.CLUSTER_NAME.DOMAIN_NAME. 8640 IN SRV 0 10 2380 etcd-0.CLUSTER_NAME.DOMAIN_NAME. 43 | _etcd-server-ssl._tcp.CLUSTER_NAME.DOMAIN_NAME. 8640 IN SRV 0 10 2380 etcd-1.CLUSTER_NAME.DOMAIN_NAME. 44 | _etcd-server-ssl._tcp.CLUSTER_NAME.DOMAIN_NAME. 8640 IN SRV 0 10 2380 etcd-2.CLUSTER_NAME.DOMAIN_NAME. 45 | 46 | api.CLUSTER_NAME.DOMAIN_NAME. A LB_IP 47 | api-int.CLUSTER_NAME.DOMAIN_NAME. A LB_IP 48 | CLUSTER_NAME-master-0.DOMAIN_NAME. A MASTER0_IP 49 | CLUSTER_NAME-master-1.DOMAIN_NAME. A MASTER1_IP 50 | CLUSTER_NAME-master-2.DOMAIN_NAME. A MASTER2_IP 51 | CLUSTER_NAME-worker-0.DOMAIN_NAME. A WORKER0_IP 52 | CLUSTER_NAME-bootstrap.DOMAIN_NAME. A BOOTSTRAP_IP 53 | etcd-0.CLUSTER_NAME.DOMAIN_NAME. IN CNAME CLUSTER_NAME-master-0.DOMAIN_NAME. 54 | etcd-1.CLUSTER_NAME.DOMAIN_NAME. IN CNAME CLUSTER_NAME-master-1.DOMAIN_NAME. 55 | etcd-2.CLUSTER_NAME.DOMAIN_NAME. IN CNAME CLUSTER_NAME-master-2.DOMAIN_NAME. 56 | 57 | $ORIGIN apps.CLUSTER_NAME.DOMAIN_NAME. 58 | * A LB_IP 59 | EOF 60 | 61 | sed -i -e "s/MASTER0_IP/${MASTER0_IP}/g" \ 62 | -e "s/MASTER1_IP/${MASTER1_IP}/g" \ 63 | -e "s/MASTER2_IP/${MASTER2_IP}/g" \ 64 | -e "s/WORKER0_IP/${WORKER0_IP}/g" \ 65 | -e "s/CLUSTER_NAME/${CLUSTER_NAME}/g" \ 66 | -e "s/DOMAIN_NAME/${DOMAIN_NAME}/g" \ 67 | -e "s/BOOTSTRAP_IP/${BOOTSTRAP_IP}/g" \ 68 | -e "s/LB_IP/${LB_IP}/g" \ 69 | ${COREDNS_DIRECTORY}/db.${DOMAIN_NAME} 70 | ``` 71 | 72 | I've not been able to query from localhost or `$(hostname -I)` from the host 73 | running the coredns container when running rootless. `/etc/resolv.conf` 74 | doesn't allow specific ports and redirecting localhost/ip is really messy, 75 | hence, this container runs with `sudo ` (and binds to :53): 76 | 77 | ```bash 78 | sudo podman run -d \ 79 | --expose=53 --expose=53/udp \ 80 | -p ${DNS}:53:53 -p ${DNS}:53:53/udp \ 81 | -v ${COREDNS_DIRECTORY}:/etc/coredns:z \ 82 | --name coredns \ 83 | coredns/coredns:latest -conf /etc/coredns/Corefile 84 | 85 | sudo firewall-cmd --zone="$(firewall-cmd --get-default-zone)" \ 86 | --add-service=dns --permanent 87 | sudo firewall-cmd --reload 88 | ``` 89 | 90 | ## Systemd unit 91 | 92 | A systemd unit can be created to automatically start/stop the podman container. 93 | In this case, as we require root, we will create a regular systemd unit: 94 | 95 | ```bash 96 | sudo bash -c 'cat > /etc/systemd/system/coredns.service << EOF 97 | [Unit] 98 | Description=CoreDNS 99 | 100 | [Service] 101 | Restart=always 102 | ExecStart=/usr/bin/podman start -a coredns 103 | ExecStop=/usr/bin/podman stop -t 10 coredns 104 | KillMode=process 105 | 106 | [Install] 107 | WantedBy=multi-user.target 108 | EOF' 109 | 110 | # Reload the service and enable the service 111 | systemctl daemon-reload 112 | systemctl enable coredns.service 113 | ``` 114 | 115 | To see if it works: 116 | 117 | ```bash 118 | $ sudo podman ps 119 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 120 | abf77a3da374 docker.io/coredns/coredns:latest /coredns -conf /e... 9 minutes ago Up 2 seconds ago 10.19.138.7:53->53/tcp, 10.19.138.7:53->53/udp coredns 121 | 122 | $ sudo systemctl stop coredns.service 123 | $ sudo podman ps 124 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 125 | ``` 126 | 127 | Then, modify `/etc/resolv.conf` to use the CoreDNS container: 128 | 129 | ```bash 130 | sudo nmcli con mod 'System eno1' ipv4.ignore-auto-dns yes 131 | sudo nmcli con mod 'System eno1' ipv4.dns "${DNS}" 132 | sudo systemctl restart NetworkManager 133 | ``` 134 | 135 | [<< Previous: Web server](3-web-server.md) | [README](../README.md) | [Next: OpenShift files >>](5-openshift-files.md) 136 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /docs/14-verification.md: -------------------------------------------------------------------------------- 1 | # Verification 2 | 3 | * `oc get nodes` 4 | 5 | ```bash 6 | NAME STATUS ROLES AGE VERSION 7 | ocp4-master-0.minwi.lan Ready master 22m v1.13.4+cb455d664 8 | ocp4-master-1.minwi.lan Ready master 22m v1.13.4+cb455d664 9 | ocp4-master-2.minwi.lan Ready master 22m v1.13.4+cb455d664 10 | ocp4-worker-0.minwi.lan Ready worker 23m v1.13.4+cb455d664 11 | ``` 12 | 13 | * `oc get clusterversion` 14 | 15 | ```bash 16 | NAME VERSION AVAILABLE PROGRESSING SINCE STATUS 17 | version 4.1.0 True False 11s Cluster version is 4.1.0 18 | ``` 19 | 20 | * `oc get clusteroperators` 21 | 22 | ```bash 23 | NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE 24 | authentication 4.1.0 True False False 49s 25 | cloud-credential 4.1.0 True False False 21m 26 | cluster-autoscaler 4.1.0 True False False 21m 27 | console 4.1.0 True False False 5m16s 28 | dns 4.1.0 True False False 21m 29 | image-registry 4.1.0 True False False 15m 30 | ingress 4.1.0 True False False 16m 31 | kube-apiserver 4.1.0 True False False 20m 32 | kube-controller-manager 4.1.0 True False False 19m 33 | kube-scheduler 4.1.0 True False False 19m 34 | machine-api 4.1.0 True False False 21m 35 | machine-config 4.1.0 True False False 20m 36 | marketplace 4.1.0 True False False 15m 37 | monitoring 4.1.0 True False False 14m 38 | network 4.1.0 True False False 21m 39 | node-tuning 4.1.0 True False False 19m 40 | openshift-apiserver 4.1.0 True False False 18m 41 | openshift-controller-manager 4.1.0 True False False 21m 42 | openshift-samples 4.1.0 True False False 10m 43 | operator-lifecycle-manager 4.1.0 True False False 21m 44 | operator-lifecycle-manager-catalog 4.1.0 True False False 21m 45 | service-ca 4.1.0 True False False 21m 46 | service-catalog-apiserver 4.1.0 True False False 19m 47 | service-catalog-controller-manager 4.1.0 True False False 19m 48 | storage 4.1.0 True False False 16m 49 | ``` 50 | 51 | * `oc get pods --all-namespaces | grep -v -E 'Running|Completed'` 52 | 53 | ```bash 54 | NAMESPACE NAME READY STATUS RESTARTS AGE 55 | ``` 56 | 57 | * `oc get pods --all-namespaces -o name | wc -l` 58 | 59 | ```bash 60 | 162 61 | ``` 62 | 63 | * `oc get pods --all-namespaces -o wide` 64 | 65 | ```bash 66 | NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES 67 | openshift-apiserver-operator openshift-apiserver-operator-85cb746d55-kvjgn 1/1 Running 1 21h 10.129.0.4 ocp4-master-2.minwi.lan 68 | openshift-apiserver apiserver-6ks2j 1/1 Running 0 21h 10.131.0.27 ocp4-master-1.minwi.lan 69 | openshift-apiserver apiserver-hcrvg 1/1 Running 0 21h 10.130.0.28 ocp4-master-0.minwi.lan 70 | openshift-apiserver apiserver-sqbk8 1/1 Running 0 21h 10.129.0.45 ocp4-master-2.minwi.lan 71 | openshift-authentication-operator authentication-operator-69d5d8bf84-lcbrp 1/1 Running 0 21h 10.129.0.51 ocp4-master-2.minwi.lan 72 | openshift-authentication oauth-openshift-7bb77f879-8sxvf 1/1 Running 0 21h 10.130.0.35 ocp4-master-0.minwi.lan 73 | openshift-authentication oauth-openshift-7bb77f879-nslpl 1/1 Running 0 21h 10.129.0.53 ocp4-master-2.minwi.lan 74 | openshift-cloud-credential-operator cloud-credential-operator-74b9b4bff6-b6cjj 1/1 Running 0 21h 10.129.0.11 ocp4-master-2.minwi.lan 75 | openshift-cluster-machine-approver machine-approver-7cd7f97455-8wk8p 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 76 | openshift-cluster-node-tuning-operator cluster-node-tuning-operator-8598b6c957-89pww 1/1 Running 0 21h 10.129.0.32 ocp4-master-2.minwi.lan 77 | openshift-cluster-node-tuning-operator tuned-2vcqf 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 78 | openshift-cluster-node-tuning-operator tuned-mf8mf 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 79 | openshift-cluster-node-tuning-operator tuned-vsqvq 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 80 | openshift-cluster-node-tuning-operator tuned-xlwzr 1/1 Running 0 21h 192.168.32.200 ocp4-worker-0.minwi.lan 81 | openshift-cluster-samples-operator cluster-samples-operator-6b48ccf677-zj79s 1/1 Running 0 21h 10.131.0.21 ocp4-master-1.minwi.lan 82 | openshift-cluster-storage-operator cluster-storage-operator-868dbc4698-qpjdg 1/1 Running 0 21h 10.131.0.20 ocp4-master-1.minwi.lan 83 | openshift-cluster-version cluster-version-operator-6f8fc78789-k4ln8 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 84 | openshift-console-operator console-operator-f6d5f6d4f-2r66h 1/1 Running 0 21h 10.131.0.35 ocp4-master-1.minwi.lan 85 | openshift-console console-7996445b88-pwx8f 1/1 Running 0 21h 10.129.0.50 ocp4-master-2.minwi.lan 86 | openshift-console console-7996445b88-tks4n 1/1 Running 0 21h 10.130.0.34 ocp4-master-0.minwi.lan 87 | openshift-console downloads-65877c7d-65rfw 1/1 Running 0 21h 10.131.0.17 ocp4-master-1.minwi.lan 88 | openshift-console downloads-65877c7d-6vx2p 1/1 Running 0 21h 10.130.0.15 ocp4-master-0.minwi.lan 89 | openshift-controller-manager-operator openshift-controller-manager-operator-7d7b899bdf-xlgcp 1/1 Running 1 21h 10.129.0.12 ocp4-master-2.minwi.lan 90 | openshift-controller-manager controller-manager-bwzj8 1/1 Running 0 3h23m 10.130.0.41 ocp4-master-0.minwi.lan 91 | openshift-controller-manager controller-manager-gnl6m 1/1 Running 0 3h22m 10.131.0.45 ocp4-master-1.minwi.lan 92 | openshift-controller-manager controller-manager-scgqr 1/1 Running 0 3h24m 10.129.0.59 ocp4-master-2.minwi.lan 93 | openshift-dns-operator dns-operator-7f54c7fd95-pzr2g 1/1 Running 0 21h 10.131.0.12 ocp4-master-1.minwi.lan 94 | openshift-dns dns-default-f8wlt 2/2 Running 0 21h 10.131.0.2 ocp4-master-1.minwi.lan 95 | openshift-dns dns-default-k9nhs 2/2 Running 0 21h 10.128.0.2 ocp4-worker-0.minwi.lan 96 | openshift-dns dns-default-rglzx 2/2 Running 0 21h 10.129.0.13 ocp4-master-2.minwi.lan 97 | openshift-dns dns-default-twbfk 2/2 Running 0 21h 10.130.0.2 ocp4-master-0.minwi.lan 98 | openshift-etcd etcd-member-ocp4-master-0.minwi.lan 2/2 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 99 | openshift-etcd etcd-member-ocp4-master-1.minwi.lan 2/2 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 100 | openshift-etcd etcd-member-ocp4-master-2.minwi.lan 2/2 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 101 | openshift-image-registry cluster-image-registry-operator-5fc86678cf-sw567 1/1 Running 0 21h 10.129.0.37 ocp4-master-2.minwi.lan 102 | openshift-image-registry image-registry-684768fc9c-6ppcg 1/1 Running 0 21h 10.128.0.10 ocp4-worker-0.minwi.lan 103 | openshift-image-registry node-ca-7cr2d 1/1 Running 0 21h 10.129.0.43 ocp4-master-2.minwi.lan 104 | openshift-image-registry node-ca-bjb26 1/1 Running 0 21h 10.131.0.24 ocp4-master-1.minwi.lan 105 | openshift-image-registry node-ca-pfkxk 1/1 Running 0 21h 10.130.0.22 ocp4-master-0.minwi.lan 106 | openshift-image-registry node-ca-qdb92 1/1 Running 0 21h 10.128.0.11 ocp4-worker-0.minwi.lan 107 | openshift-ingress-operator ingress-operator-7694cfbdb7-xh2wj 1/1 Running 0 21h 10.131.0.19 ocp4-master-1.minwi.lan 108 | openshift-ingress router-default-6c5cf4dccc-lnxtd 1/1 Running 0 21h 192.168.32.200 ocp4-worker-0.minwi.lan 109 | openshift-kube-apiserver-operator kube-apiserver-operator-7d8b4bd84-pw2xf 1/1 Running 1 21h 10.129.0.6 ocp4-master-2.minwi.lan 110 | openshift-kube-apiserver installer-2-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.8 ocp4-master-0.minwi.lan 111 | openshift-kube-apiserver installer-2-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.5 ocp4-master-1.minwi.lan 112 | openshift-kube-apiserver installer-2-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.29 ocp4-master-2.minwi.lan 113 | openshift-kube-apiserver installer-3-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.11 ocp4-master-0.minwi.lan 114 | openshift-kube-apiserver installer-3-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.18 ocp4-master-1.minwi.lan 115 | openshift-kube-apiserver installer-6-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.29 ocp4-master-0.minwi.lan 116 | openshift-kube-apiserver installer-6-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.26 ocp4-master-1.minwi.lan 117 | openshift-kube-apiserver installer-6-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.46 ocp4-master-2.minwi.lan 118 | openshift-kube-apiserver installer-7-ocp4-master-0.minwi.lan 0/1 Completed 0 3h35m 10.130.0.37 ocp4-master-0.minwi.lan 119 | openshift-kube-apiserver installer-7-ocp4-master-1.minwi.lan 0/1 Completed 0 3h39m 10.131.0.37 ocp4-master-1.minwi.lan 120 | openshift-kube-apiserver installer-7-ocp4-master-2.minwi.lan 0/1 Completed 0 3h37m 10.129.0.55 ocp4-master-2.minwi.lan 121 | openshift-kube-apiserver installer-8-ocp4-master-0.minwi.lan 0/1 Completed 0 3h21m 10.130.0.42 ocp4-master-0.minwi.lan 122 | openshift-kube-apiserver installer-8-ocp4-master-1.minwi.lan 0/1 Completed 0 3h25m 10.131.0.40 ocp4-master-1.minwi.lan 123 | openshift-kube-apiserver installer-8-ocp4-master-2.minwi.lan 0/1 Completed 0 3h23m 10.129.0.61 ocp4-master-2.minwi.lan 124 | openshift-kube-apiserver kube-apiserver-ocp4-master-0.minwi.lan 2/2 Running 0 3h21m 192.168.32.100 ocp4-master-0.minwi.lan 125 | openshift-kube-apiserver kube-apiserver-ocp4-master-1.minwi.lan 2/2 Running 0 3h25m 192.168.32.101 ocp4-master-1.minwi.lan 126 | openshift-kube-apiserver kube-apiserver-ocp4-master-2.minwi.lan 2/2 Running 0 3h23m 192.168.32.102 ocp4-master-2.minwi.lan 127 | openshift-kube-apiserver revision-pruner-2-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.12 ocp4-master-0.minwi.lan 128 | openshift-kube-apiserver revision-pruner-2-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.13 ocp4-master-1.minwi.lan 129 | openshift-kube-apiserver revision-pruner-2-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.36 ocp4-master-2.minwi.lan 130 | openshift-kube-apiserver revision-pruner-3-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.17 ocp4-master-0.minwi.lan 131 | openshift-kube-apiserver revision-pruner-3-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.25 ocp4-master-1.minwi.lan 132 | openshift-kube-apiserver revision-pruner-6-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.32 ocp4-master-0.minwi.lan 133 | openshift-kube-apiserver revision-pruner-6-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.30 ocp4-master-1.minwi.lan 134 | openshift-kube-apiserver revision-pruner-6-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.48 ocp4-master-2.minwi.lan 135 | openshift-kube-apiserver revision-pruner-7-ocp4-master-0.minwi.lan 0/1 Completed 0 3h33m 10.130.0.38 ocp4-master-0.minwi.lan 136 | openshift-kube-apiserver revision-pruner-7-ocp4-master-1.minwi.lan 0/1 Completed 0 3h37m 10.131.0.38 ocp4-master-1.minwi.lan 137 | openshift-kube-apiserver revision-pruner-7-ocp4-master-2.minwi.lan 0/1 Completed 0 3h35m 10.129.0.56 ocp4-master-2.minwi.lan 138 | openshift-kube-apiserver revision-pruner-8-ocp4-master-0.minwi.lan 0/1 Completed 0 3h19m 10.130.0.43 ocp4-master-0.minwi.lan 139 | openshift-kube-apiserver revision-pruner-8-ocp4-master-1.minwi.lan 0/1 Completed 0 3h23m 10.131.0.43 ocp4-master-1.minwi.lan 140 | openshift-kube-apiserver revision-pruner-8-ocp4-master-2.minwi.lan 0/1 Completed 0 3h21m 10.129.0.62 ocp4-master-2.minwi.lan 141 | openshift-kube-controller-manager-operator kube-controller-manager-operator-7f585f879c-87ctc 1/1 Running 1 21h 10.129.0.7 ocp4-master-2.minwi.lan 142 | openshift-kube-controller-manager installer-2-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.21 ocp4-master-2.minwi.lan 143 | openshift-kube-controller-manager installer-3-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.7 ocp4-master-0.minwi.lan 144 | openshift-kube-controller-manager installer-3-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.14 ocp4-master-1.minwi.lan 145 | openshift-kube-controller-manager installer-3-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.26 ocp4-master-2.minwi.lan 146 | openshift-kube-controller-manager installer-4-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.24 ocp4-master-0.minwi.lan 147 | openshift-kube-controller-manager installer-4-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.29 ocp4-master-1.minwi.lan 148 | openshift-kube-controller-manager installer-4-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.39 ocp4-master-2.minwi.lan 149 | openshift-kube-controller-manager installer-5-ocp4-master-0.minwi.lan 0/1 Completed 0 3h24m 10.130.0.39 ocp4-master-0.minwi.lan 150 | openshift-kube-controller-manager installer-5-ocp4-master-1.minwi.lan 0/1 Completed 0 3h24m 10.131.0.41 ocp4-master-1.minwi.lan 151 | openshift-kube-controller-manager installer-5-ocp4-master-2.minwi.lan 0/1 Completed 0 3h25m 10.129.0.57 ocp4-master-2.minwi.lan 152 | openshift-kube-controller-manager kube-controller-manager-ocp4-master-0.minwi.lan 2/2 Running 0 3h24m 192.168.32.100 ocp4-master-0.minwi.lan 153 | openshift-kube-controller-manager kube-controller-manager-ocp4-master-1.minwi.lan 2/2 Running 0 3h24m 192.168.32.101 ocp4-master-1.minwi.lan 154 | openshift-kube-controller-manager kube-controller-manager-ocp4-master-2.minwi.lan 2/2 Running 0 3h25m 192.168.32.102 ocp4-master-2.minwi.lan 155 | openshift-kube-controller-manager revision-pruner-2-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.27 ocp4-master-2.minwi.lan 156 | openshift-kube-controller-manager revision-pruner-3-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.13 ocp4-master-0.minwi.lan 157 | openshift-kube-controller-manager revision-pruner-3-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.15 ocp4-master-1.minwi.lan 158 | openshift-kube-controller-manager revision-pruner-3-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.34 ocp4-master-2.minwi.lan 159 | openshift-kube-controller-manager revision-pruner-4-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.26 ocp4-master-0.minwi.lan 160 | openshift-kube-controller-manager revision-pruner-4-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.32 ocp4-master-1.minwi.lan 161 | openshift-kube-controller-manager revision-pruner-4-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.42 ocp4-master-2.minwi.lan 162 | openshift-kube-controller-manager revision-pruner-5-ocp4-master-0.minwi.lan 0/1 Completed 0 3h24m 10.130.0.40 ocp4-master-0.minwi.lan 163 | openshift-kube-controller-manager revision-pruner-5-ocp4-master-1.minwi.lan 0/1 Completed 0 3h23m 10.131.0.44 ocp4-master-1.minwi.lan 164 | openshift-kube-controller-manager revision-pruner-5-ocp4-master-2.minwi.lan 0/1 Completed 0 3h24m 10.129.0.58 ocp4-master-2.minwi.lan 165 | openshift-kube-scheduler-operator openshift-kube-scheduler-operator-6dc6d5c469-48tzl 1/1 Running 1 21h 10.129.0.8 ocp4-master-2.minwi.lan 166 | openshift-kube-scheduler installer-2-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.17 ocp4-master-2.minwi.lan 167 | openshift-kube-scheduler installer-3-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.22 ocp4-master-2.minwi.lan 168 | openshift-kube-scheduler installer-4-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.9 ocp4-master-0.minwi.lan 169 | openshift-kube-scheduler installer-4-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.16 ocp4-master-1.minwi.lan 170 | openshift-kube-scheduler installer-4-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.25 ocp4-master-2.minwi.lan 171 | openshift-kube-scheduler installer-5-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.25 ocp4-master-0.minwi.lan 172 | openshift-kube-scheduler installer-5-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.31 ocp4-master-1.minwi.lan 173 | openshift-kube-scheduler installer-5-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.41 ocp4-master-2.minwi.lan 174 | openshift-kube-scheduler openshift-kube-scheduler-ocp4-master-0.minwi.lan 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 175 | openshift-kube-scheduler openshift-kube-scheduler-ocp4-master-1.minwi.lan 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 176 | openshift-kube-scheduler openshift-kube-scheduler-ocp4-master-2.minwi.lan 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 177 | openshift-kube-scheduler revision-pruner-2-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.20 ocp4-master-2.minwi.lan 178 | openshift-kube-scheduler revision-pruner-3-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.24 ocp4-master-2.minwi.lan 179 | openshift-kube-scheduler revision-pruner-4-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.14 ocp4-master-0.minwi.lan 180 | openshift-kube-scheduler revision-pruner-4-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.22 ocp4-master-1.minwi.lan 181 | openshift-kube-scheduler revision-pruner-4-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.35 ocp4-master-2.minwi.lan 182 | openshift-kube-scheduler revision-pruner-5-ocp4-master-0.minwi.lan 0/1 Completed 0 21h 10.130.0.27 ocp4-master-0.minwi.lan 183 | openshift-kube-scheduler revision-pruner-5-ocp4-master-1.minwi.lan 0/1 Completed 0 21h 10.131.0.33 ocp4-master-1.minwi.lan 184 | openshift-kube-scheduler revision-pruner-5-ocp4-master-2.minwi.lan 0/1 Completed 0 21h 10.129.0.44 ocp4-master-2.minwi.lan 185 | openshift-machine-api cluster-autoscaler-operator-65cbb65f5c-99g8k 1/1 Running 0 21h 10.129.0.3 ocp4-master-2.minwi.lan 186 | openshift-machine-api machine-api-operator-54bf977cc8-679r9 1/1 Running 0 21h 10.129.0.9 ocp4-master-2.minwi.lan 187 | openshift-machine-config-operator etcd-quorum-guard-66b78568d6-8vqcw 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 188 | openshift-machine-config-operator etcd-quorum-guard-66b78568d6-jm5sv 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 189 | openshift-machine-config-operator etcd-quorum-guard-66b78568d6-vgrtt 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 190 | openshift-machine-config-operator machine-config-controller-69876f9fc5-w2pms 1/1 Running 0 21h 10.129.0.14 ocp4-master-2.minwi.lan 191 | openshift-machine-config-operator machine-config-daemon-g85cl 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 192 | openshift-machine-config-operator machine-config-daemon-m9pjv 1/1 Running 0 21h 192.168.32.200 ocp4-worker-0.minwi.lan 193 | openshift-machine-config-operator machine-config-daemon-pl2vv 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 194 | openshift-machine-config-operator machine-config-daemon-tk6rm 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 195 | openshift-machine-config-operator machine-config-operator-84f487cc5d-z24lq 1/1 Running 0 21h 10.129.0.5 ocp4-master-2.minwi.lan 196 | openshift-machine-config-operator machine-config-server-97l6k 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 197 | openshift-machine-config-operator machine-config-server-9b4m6 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 198 | openshift-machine-config-operator machine-config-server-jjg7q 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 199 | openshift-marketplace certified-operators-679bc8bccd-85kff 1/1 Running 0 21h 10.128.0.3 ocp4-worker-0.minwi.lan 200 | openshift-marketplace community-operators-7969fd8f5f-978g2 1/1 Running 0 16h 10.128.0.21 ocp4-worker-0.minwi.lan 201 | openshift-marketplace marketplace-operator-fc68ffc58-cxzh8 1/1 Running 0 21h 10.130.0.19 ocp4-master-0.minwi.lan 202 | openshift-marketplace redhat-operators-78b7b5b467-vk26p 1/1 Running 0 21h 10.128.0.5 ocp4-worker-0.minwi.lan 203 | openshift-monitoring alertmanager-main-0 3/3 Running 0 21h 10.128.0.13 ocp4-worker-0.minwi.lan 204 | openshift-monitoring alertmanager-main-1 3/3 Running 0 21h 10.128.0.18 ocp4-worker-0.minwi.lan 205 | openshift-monitoring alertmanager-main-2 3/3 Running 0 21h 10.128.0.19 ocp4-worker-0.minwi.lan 206 | openshift-monitoring cluster-monitoring-operator-6b875c9f45-pzn4t 1/1 Running 0 21h 10.130.0.18 ocp4-master-0.minwi.lan 207 | openshift-monitoring grafana-7cbddfd4f6-kg7dt 2/2 Running 0 21h 10.128.0.9 ocp4-worker-0.minwi.lan 208 | openshift-monitoring kube-state-metrics-76dbd866ff-6g29v 3/3 Running 0 21h 10.128.0.6 ocp4-worker-0.minwi.lan 209 | openshift-monitoring node-exporter-bp2fx 2/2 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 210 | openshift-monitoring node-exporter-fcwtr 2/2 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 211 | openshift-monitoring node-exporter-hml9t 2/2 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 212 | openshift-monitoring node-exporter-tlc9c 2/2 Running 0 21h 192.168.32.200 ocp4-worker-0.minwi.lan 213 | openshift-monitoring prometheus-adapter-6669f77fcc-49xx8 1/1 Running 0 3h23m 10.128.0.25 ocp4-worker-0.minwi.lan 214 | openshift-monitoring prometheus-adapter-6669f77fcc-5tl9t 1/1 Running 0 3h23m 10.128.0.24 ocp4-worker-0.minwi.lan 215 | openshift-monitoring prometheus-k8s-0 6/6 Running 1 21h 10.128.0.16 ocp4-worker-0.minwi.lan 216 | openshift-monitoring prometheus-k8s-1 6/6 Running 1 21h 10.128.0.17 ocp4-worker-0.minwi.lan 217 | openshift-monitoring prometheus-operator-7bfd67bf6c-ws5cr 1/1 Running 0 21h 10.128.0.12 ocp4-worker-0.minwi.lan 218 | openshift-monitoring telemeter-client-d5d6757bb-9f7ps 3/3 Running 0 21h 10.128.0.8 ocp4-worker-0.minwi.lan 219 | openshift-multus multus-8mlgf 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 220 | openshift-multus multus-9r2c2 1/1 Running 0 21h 192.168.32.200 ocp4-worker-0.minwi.lan 221 | openshift-multus multus-h9rpm 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 222 | openshift-multus multus-pk8bx 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 223 | openshift-network-operator network-operator-5f8b568759-b64gg 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 224 | openshift-operator-lifecycle-manager catalog-operator-7ff7b858bb-2kqfr 1/1 Running 0 21h 10.129.0.18 ocp4-master-2.minwi.lan 225 | openshift-operator-lifecycle-manager olm-operator-6b6bc6fc8f-n55xv 1/1 Running 0 21h 10.129.0.16 ocp4-master-2.minwi.lan 226 | openshift-operator-lifecycle-manager olm-operators-t7t2w 1/1 Running 0 21h 10.129.0.23 ocp4-master-2.minwi.lan 227 | openshift-operator-lifecycle-manager packageserver-76b7856754-cz768 1/1 Running 0 3h24m 10.129.0.60 ocp4-master-2.minwi.lan 228 | openshift-operator-lifecycle-manager packageserver-76b7856754-qm8jr 1/1 Running 0 3h23m 10.131.0.42 ocp4-master-1.minwi.lan 229 | openshift-sdn ovs-4cbk7 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 230 | openshift-sdn ovs-9vwmh 1/1 Running 0 21h 192.168.32.200 ocp4-worker-0.minwi.lan 231 | openshift-sdn ovs-d9fd4 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 232 | openshift-sdn ovs-wgl97 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 233 | openshift-sdn sdn-2j7fk 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 234 | openshift-sdn sdn-controller-g5lkj 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 235 | openshift-sdn sdn-controller-mmkqj 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 236 | openshift-sdn sdn-controller-xsnhm 1/1 Running 0 21h 192.168.32.102 ocp4-master-2.minwi.lan 237 | openshift-sdn sdn-dpw5w 1/1 Running 0 21h 192.168.32.200 ocp4-worker-0.minwi.lan 238 | openshift-sdn sdn-rp777 1/1 Running 0 21h 192.168.32.100 ocp4-master-0.minwi.lan 239 | openshift-sdn sdn-v6zwf 1/1 Running 0 21h 192.168.32.101 ocp4-master-1.minwi.lan 240 | openshift-service-ca-operator service-ca-operator-9654d9559-xwbf2 1/1 Running 0 21h 10.129.0.10 ocp4-master-2.minwi.lan 241 | openshift-service-ca apiservice-cabundle-injector-5bf59477d7-ndr8q 1/1 Running 0 21h 10.130.0.3 ocp4-master-0.minwi.lan 242 | openshift-service-ca configmap-cabundle-injector-6cfcd498d7-6788j 1/1 Running 0 21h 10.129.0.15 ocp4-master-2.minwi.lan 243 | openshift-service-ca service-serving-cert-signer-7789d64745-xdmrt 1/1 Running 0 21h 10.131.0.3 ocp4-master-1.minwi.lan 244 | openshift-service-catalog-apiserver-operator openshift-service-catalog-apiserver-operator-54dcb96555-ms9k4 1/1 Running 0 21h 10.131.0.10 ocp4-master-1.minwi.lan 245 | openshift-service-catalog-controller-manager-operator openshift-service-catalog-controller-manager-operator-f8cfjc2wj 1/1 Running 0 21h 10.131.0.9 ocp4-master-1.minwi.lan 246 | ``` 247 | 248 | * `oc get nodes -o wide --show-labels` 249 | 250 | ```bash 251 | NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME LABELS 252 | ocp4-master-0.minwi.lan Ready master 22h v1.13.4+cb455d664 192.168.32.100 Red Hat Enterprise Linux CoreOS 410.8.20190520.0 (Ootpa) 4.18.0-80.1.2.el8_0.x86_64 cri-o://1.13.9-1.rhaos4.1.gitd70609a.el8 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=ocp4-master-0.minwi.lan,node-role.kubernetes.io/master=,node.openshift.io/os_id=rhcos,node.openshift.io/os_version=4.1 253 | ocp4-master-1.minwi.lan Ready master 22h v1.13.4+cb455d664 192.168.32.101 Red Hat Enterprise Linux CoreOS 410.8.20190520.0 (Ootpa) 4.18.0-80.1.2.el8_0.x86_64 cri-o://1.13.9-1.rhaos4.1.gitd70609a.el8 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=ocp4-master-1.minwi.lan,node-role.kubernetes.io/master=,node.openshift.io/os_id=rhcos,node.openshift.io/os_version=4.1 254 | ocp4-master-2.minwi.lan Ready master 22h v1.13.4+cb455d664 192.168.32.102 Red Hat Enterprise Linux CoreOS 410.8.20190520.0 (Ootpa) 4.18.0-80.1.2.el8_0.x86_64 cri-o://1.13.9-1.rhaos4.1.gitd70609a.el8 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=ocp4-master-2.minwi.lan,node-role.kubernetes.io/master=,node.openshift.io/os_id=rhcos,node.openshift.io/os_version=4.1 255 | ocp4-worker-0.minwi.lan Ready worker 22h v1.13.4+cb455d664 192.168.32.200 Red Hat Enterprise Linux CoreOS 410.8.20190520.0 (Ootpa) 4.18.0-80.1.2.el8_0.x86_64 cri-o://1.13.9-1.rhaos4.1.gitd70609a.el8 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=ocp4-worker-0.minwi.lan,node-role.kubernetes.io/worker=,node.openshift.io/os_id=rhcos,node.openshift.io/os_version=4.1 256 | ``` 257 | 258 | [<< Previous: Upgrade](13-upgrade.md) | [README](../README.md) | [Next: Tips and Tricks >>](99-tips-and-tricks.md) 259 | --------------------------------------------------------------------------------