├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── LICENSE ├── README.md ├── backup_restore ├── README.md ├── master-backup.yaml ├── ocp-etcd3-pods-backup.sh ├── ocp-master-cert-backup.sh └── ocp-project-backup.sh ├── branding ├── ansible-playbook-openshift-custom-login-page │ ├── .gitignore │ ├── README.md │ ├── playbook.yaml │ ├── requirements.yaml │ └── result.png └── ansible-playbook-openshift-custom-webconsole-logo │ ├── .gitignore │ ├── README.md │ ├── playbook.yaml │ ├── requirements.yaml │ └── result.png ├── capacity-dashboard ├── README.md ├── capacity-planning-3.11.json ├── capacity-planning.json ├── capacity_management_rules.yaml ├── example │ ├── README.md │ └── cluster-quota.yaml └── media │ ├── assumptions_setup.png │ └── dashboard-screenshot.png ├── cluster-management ├── README.md └── garbage-collection │ └── docker-gc-setup.yaml ├── custom-autoscaler ├── README.md └── scale.py ├── custom-dashboards ├── .applier │ ├── group_vars │ │ └── all.yml │ ├── host_vars │ │ └── localhost.yml │ └── hosts ├── .openshift │ ├── dashboards │ │ ├── capacity │ │ │ ├── capacity-volume-patch.yml │ │ │ ├── dashboard-capacity-3.11.yml │ │ │ └── dashboard-capacity.yml │ │ └── cluster-status │ │ │ ├── manifests │ │ │ ├── dashboard-master-api.yml │ │ │ ├── dashboard-pods.yml │ │ │ ├── dashboard-router.yml │ │ │ └── dashboard-traffic.yml │ │ │ ├── master-api-volume-patch.yml │ │ │ ├── pods-volume-patch.yml │ │ │ ├── router-volume-patch.yml │ │ │ └── traffic-volume-patch.yml │ ├── manifest_templates │ │ └── grafana-clusterrolebinding.yml.j2 │ ├── manifests │ │ ├── grafana-custom-config-secret.yml │ │ ├── grafana-dashboards-cm.yml │ │ ├── grafana-deployment.yml │ │ ├── grafana-route.yml │ │ ├── grafana-sa.yml │ │ ├── grafana-service.yml │ │ └── openshift-state-metrics │ │ │ ├── openshift-state-metrics-cluster-role-binding.yaml │ │ │ ├── openshift-state-metrics-cluster-role.yaml │ │ │ ├── openshift-state-metrics-deployment.yaml │ │ │ ├── openshift-state-metrics-role-binding.yaml │ │ │ ├── openshift-state-metrics-role.yaml │ │ │ ├── openshift-state-metrics-service-account.yaml │ │ │ ├── openshift-state-metrics-service-monitor.yaml │ │ │ └── openshift-state-metrics-service.yaml │ ├── namespace │ │ └── namespace.yml.j2 │ ├── patches │ │ └── openshift-state-metrics-service-monitor.yaml │ └── secrets │ │ ├── grafana_config.yml.j2 │ │ ├── grafana_datasources.yml.j2 │ │ └── grafana_proxy.yml.j2 ├── README.md ├── mdt-secret-discovery-requirements.yml ├── mdt-secret-discovery │ ├── meta │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── monitoring.yml └── requirements.yml ├── disconnected_registry ├── README.md ├── docker-registry-sync.py └── docker_tags.json ├── etcd-procedures ├── README.adoc ├── docs │ ├── README.adoc │ ├── add_etcd_node.adoc │ └── replace_etcd_nodes.adoc ├── playbooks │ ├── README.adoc │ ├── etcd_scaleup.yml │ └── templates │ │ └── etcd_container.service.j2 └── src │ ├── etcd_container.service.sh │ └── etcdconf.vars.sh ├── fix-project-terminating ├── README.md └── fix-terminating.sh ├── health_check ├── README.md ├── elasticsearch-health-check-ocp33.sh ├── elasticsearch-health-check-ocp34.sh └── metrics-health-check.sh ├── image-management ├── README.md ├── applier │ ├── image-builder-inventory │ │ ├── group_vars │ │ │ └── seed-hosts.yml │ │ └── hosts │ ├── manifests │ │ ├── jboss-eap70-openshift.yml │ │ └── openjdk18-openshift.yml │ ├── operator-inventory │ │ ├── group_vars │ │ │ └── seed-hosts.yml │ │ └── hosts │ ├── params │ │ ├── builder-rolebinding │ │ ├── image-builds │ │ │ ├── myorg-eap-oraclejdk │ │ │ └── myorg-openjdk18 │ │ └── image-management-project │ ├── patches │ │ ├── jboss-eap70-openshift.yml │ │ └── openjdk18-openshift.yml │ └── templates │ │ ├── docker-build-git.yml │ │ ├── project.yml │ │ └── rolebinding.yml ├── img │ └── workflow.png ├── myorg-eap-oraclejdk │ └── Dockerfile ├── myorg-openjdk18 │ ├── Dockerfile │ └── certs │ │ └── myorg-rootCA.pem └── requirements.yml ├── image-scanning ├── README.adoc ├── inventory │ ├── group_vars │ │ └── seed-hosts.yml │ └── hosts ├── params │ ├── image-scan-base │ ├── pipeline │ └── scanning-scc-jenkins ├── projects │ └── project.yml ├── registry-viewer.yml ├── requirements.yml └── templates │ ├── image-scan-base.yml │ ├── pipeline.yml │ └── scanning-scc.yml ├── install ├── README.md ├── c1-ocp.myorg.com-base │ └── hosts └── c1-ocp.myorg.com-ldap │ ├── files │ └── my-ldap-ca-bundle.crt │ └── hosts ├── machine-api └── roles │ └── openshift_machineset_config │ ├── .yamllint │ ├── README.adoc │ ├── defaults │ └── main.yml │ ├── tasks │ ├── aws-machineset-group.yml │ ├── aws.yml │ ├── disable-default-worker-machinesets.yml │ ├── enable-cluster-autoscaler.yml │ ├── main.yml │ └── set-facts.yml │ ├── templates │ ├── aws-machineset.yml.j2 │ ├── clusterautoscaler.yml.j2 │ └── machineautoscaler.yml.j2 │ └── vars │ └── main.yml ├── maintenance_cleanup ├── README.md ├── cleanup-bulds.sh ├── delete-dead-pods.sh └── delete-expired-replication-controllers.sh ├── meta └── main.yml ├── networkpolicy ├── .gitignore ├── README.md ├── inventory │ ├── group_vars │ │ └── all.yml │ └── hosts ├── playbooks │ └── apply-network-policies.yml ├── policies │ ├── baseline │ │ ├── allow-from-default-namespace.yml │ │ ├── allow-from-same-namespace.yml │ │ ├── default-deny.yml │ │ └── deny-external-egress.yml │ ├── default │ │ └── docker-registry-ingress.yml │ ├── kube-service-catalog │ │ └── apiserver-ingress.yml │ ├── logging │ │ ├── elasticsearch-nodes-ingress.yml │ │ └── elasticsearch-rest-ingress.yml │ ├── openshift-ansible-service-broker │ │ └── asb-service-broker-ingress.yml │ ├── openshift-infra │ │ ├── cassandra-hawkular-ingress.yml │ │ ├── cassandra-nodes-ingress.yml │ │ └── hawkular-heapster-ingress.yml │ └── openshift-template-service-broker │ │ └── template-service-broker-ingress.yml └── requirements.yml ├── quota-management ├── README.md ├── files │ ├── default-project-template.yaml │ ├── quota-large.yml │ ├── quota-medium.yml │ └── quota-small.yml └── inventory │ ├── group_vars │ └── seed-hosts.yml │ └── hosts ├── satellite ├── README.md └── populate-docker.sh ├── storage ├── README.md ├── create-pvs.sh ├── gluster-health-check ├── pv-migrator │ ├── .gitignore │ ├── README.md │ ├── localhost.ini │ ├── migrate-pvs.yml │ └── private │ │ ├── k8s_migrate_pvs.yml │ │ ├── k8s_migrate_pvs_create_destination_pvs.yml │ │ ├── k8s_migrate_pvs_for_namespace.yml │ │ ├── k8s_migrate_pvs_get_source_pvcs.yml │ │ ├── k8s_migrate_pvs_pods_scale_down.yml │ │ ├── k8s_migrate_pvs_pods_scale_up.yml │ │ ├── k8s_migrate_pvs_pvc_migration.yml │ │ ├── k8s_migrate_pvs_wait_for_pods_ready.yml │ │ └── k8s_run_and_wait_for_migrate_pv_job.yml └── pv-to-path-mapping └── validation ├── README.md ├── __init__.py ├── conftest.py ├── helper_functions.py ├── lib ├── __init__.py └── k8sHelper.py ├── pytest.ini ├── requirements.txt ├── ssh_connection_handling.py ├── test_cluster_install.py └── validate-pre-install.py /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | #### What does this PR do? 2 | Brief explanation of the code or documentation change you have made 3 | 4 | #### How should this be tested? 5 | Automated testing is preferred! Include commands to run your new feature, and also post-run commands to validate that it worked. (please use code blocks to format code samples) 6 | 7 | #### Is there a relevant Issue open for this? 8 | Provide a link to any open issues that describe the problem you are solving. 9 | 10 | Use `resolves #` to auto-close at merge time 11 | 12 | #### Other Relevant info, PRs, etc. 13 | Please provide link to other PRs that may be related (blocking, resolves, etc.) 14 | 15 | #### Who would you like to review this? 16 | cc: @redhat-cop/casl 17 | cc: @redhat-cop/day-in-the-life 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/*.retry 2 | */galaxy/ 3 | **/*.pyc 4 | **/__pycache__ 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The OpenShift Toolkit 2 | 3 | A collection of code samples to help you get started with OpenShift. 4 | 5 | ## Solutions 6 | 7 | * [Custom Autoscaler](./custom-autoscaler/) 8 | * [Syncing a Registry for Bootstrapping Disconnected Installs](./disconnected_registry/) 9 | * [Expanded Health Checks for EFK and Hawkular](./health_check/) 10 | * [Quota Management](./quota-management/) - A sample strategy for managing Quotas 11 | * [NetworkPolicy Management](./networkpolicy/) - A sample strategy for managing Network Policy 12 | * [Pre and Post install OpenShift validation](./validation/) - Automation to ensure clusters are ready for use. 13 | * [Sample Install Inventories](./install/) 14 | * [Capacity Planning Tooling](./capacity-dashboard/) - A Dashboard to help you make capacity decisions. 15 | * [Software Delivery Metrics Dashboard](https://github.com/redhat-cop/mdt-quickstart.git) - A Dashboard to Measure the Flow of Software Features through a Value Stream 16 | * [OpenShift Applier](https://github.com/redhat-cop/openshift-applier.git) - An Automation Framework for Kubernetes 17 | * [Containers Quickstarts](https://github.com/redhat-cop/containers-quickstarts.git) - A set of quickstarts to help you get started building and deploying software on OpenShift 18 | * [Container Pipelines](https://github.com/redhat-cop/container-pipelines.git) - A set of sample CI/CD Piplines for OpenShift 19 | 20 | ## Ansible Playbooks 21 | 22 | * [Create Openshift Custom Login Page with New Logo](./branding/ansible-playbook-openshift-custom-login-page/) 23 | * [Change Openshift Webconsole Logo](./branding/ansible-playbook-openshift-custom-webconsole-logo/) 24 | -------------------------------------------------------------------------------- /backup_restore/README.md: -------------------------------------------------------------------------------- 1 | # OpenShift Admin Bash Scripts 2 | 3 | This folder contains a collection of script to backup certain part of an OpenShift cluster. 4 | 5 | These scripts can be run natively on the superior Linux OS, or MacOS. Windows requires using a bash emulator (GitBash). 6 | 7 | ## Script Overview 8 | 9 | The following table outlines the scripts and purpose. 10 | 11 | Script / Playbook Name | Description | Notes 12 | --- | --- | --- 13 | `ocp-master-cert-backup.sh` | Creates a backup of the master certificates. | Must be run on an OCP master node. 14 | `ocp-project-backup.sh` | Creates a yaml backup of all projects in the OpenShift cluster. | Must be run on an OCP master node. 15 | `ocp-etcd3-pods-backup.sh` | Creates a backup of the etcd (data snapshot & config). Also manages backup retention. | Must be run on the OCP nodes running the etcd static Pods (usually masters). 16 | `master-backup.yaml` | Ansible clone of the bash `ocp-master-cert-backup.sh` script. 17 | -------------------------------------------------------------------------------- /backup_restore/master-backup.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | #perform validation check: ansible-playbook --syntax-check /root/master-backup.yaml 3 | #sample invocation: ansible-playbook -i /etc/ansible/hosts /root/master-backup.yaml 4 | 5 | - hosts: masters 6 | remote_user: root 7 | vars: 8 | #Directory that will be backed up on each master 9 | source_backup_dir: /etc/origin/master 10 | #Prefix of the generated tar.gz file 11 | tar_file_prefix: certs-and-keys 12 | #Temporary location where the tar file will be created. This file will be removed. 13 | dest_backup_dir: /tmp 14 | #Location (on the Ansible host) where the tar will be copied. 15 | #This parameter should be updated to use a location that's backed up on the host. 16 | host_dest_dir: /tmp 17 | 18 | #Run the backup steps, NOTE this was written vs Ansible 2.2, so the archive task does not exist. 19 | tasks: 20 | #So we'll need to copy the files first, then create the tar ... or 21 | #Create the tar then compress the tar 22 | - name: Backup master certs (*.crt) 23 | shell: "tar cf {{ dest_backup_dir }}/{{ tar_file_prefix }}-{{ inventory_hostname }}.tar *.crt" 24 | args: 25 | chdir: "{{ source_backup_dir }}" 26 | - name: Backup master keys (*.key) 27 | shell: "tar rf {{ dest_backup_dir }}/{{ tar_file_prefix }}-{{ inventory_hostname }}.tar *.key" 28 | args: 29 | chdir: "{{ source_backup_dir }}" 30 | - name: Compress tar file to create tar.gz 31 | shell: "gzip {{ dest_backup_dir }}/{{ tar_file_prefix }}-{{ inventory_hostname }}.tar" 32 | args: 33 | chdir: "{{ source_backup_dir }}" 34 | - name: Remove old tar file 35 | file: 36 | path: "{{ dest_backup_dir }}/{{ tar_file_prefix }}-{{ inventory_hostname }}.tar" 37 | state: absent 38 | recurse: no 39 | - name: Copy backup tar.gz back to ansible host machine 40 | fetch: 41 | src: "{{ dest_backup_dir }}/{{ tar_file_prefix }}-{{ inventory_hostname }}.tar.gz" 42 | dest: "{{ host_dest_dir }}" 43 | flat: yes 44 | - name: Remove temp tar.gz file from master 45 | file: 46 | path: "{{ dest_backup_dir }}/{{ tar_file_prefix }}-{{ inventory_hostname }}.tar.gz" 47 | state: absent 48 | recurse: no 49 | -------------------------------------------------------------------------------- /backup_restore/ocp-etcd3-pods-backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Date: 30/09/2018 4 | # 5 | # Script Purpose: Performs a backup of etcd v3 data (snapshot) and config (/etc/etcd/), when etcd running in static Pods (since OCP 3.10). 6 | # Manages backup retention, It also deletes old backup files. 7 | # Version : 1.0 8 | # 9 | 10 | set -euo pipefail 11 | 12 | # Check proper inputs 13 | usage(){ 14 | echo "Backup etcd data (snapshot) and config (/etc/etcd). Creates a tar.gz and save it in a different folder." 15 | echo "Also manages backup retention." 16 | echo "" 17 | echo "Usage:" 18 | echo " $0 dest_folder [retention_days]" 19 | echo "" 20 | echo "Options:" 21 | echo " dest_folder: Destination folder where to save the backup" 22 | echo " [retention_days]: Optional, number of retention days for backup files. Default 30." 23 | echo "" 24 | echo "Examples:" 25 | echo " # Backup etcd node." 26 | echo " $0 /tmp" 27 | echo "" 28 | echo " # Backup etcd node. Retain last 15 days o backup files." 29 | echo " $0 /tmp 15" 30 | echo "" 31 | exit 1 32 | } 33 | 34 | if [[ $# -lt 1 ]] 35 | then 36 | usage 37 | fi 38 | 39 | # Vars 40 | HOSTNAME=$(hostnamectl --static) 41 | TS=$(date +"%Y%m%d-%H%M%S") 42 | 43 | MASTER_EXEC="/usr/local/bin/master-exec" 44 | ETCD_POD_MANIFEST="/etc/origin/node/pods/etcd.yaml" 45 | 46 | ETCD_BCK_DIR="${1}" 47 | ETCD_DATA_BCK_DIR="etcd_data_bck.${TS}" 48 | 49 | ETCD_CONFIG_DIR="/etc/etcd" 50 | ETCD_CONFIG_BCK_DIR="etcd_config_bck.${TS}" 51 | 52 | ETCD_FINAL_BCK_FILE="etcd_bck.${HOSTNAME}.${TS}.tar.gz" 53 | 54 | RETENTION_DAYS=${2-30} 55 | 56 | log(){ 57 | date "+%F %T: ${@}" 58 | } 59 | 60 | die(){ 61 | log "$1" 62 | exit "$2" 63 | } 64 | 65 | ## Pre checks 66 | # Check if the second arg is a number 67 | [[ ! ${RETENTION_DAYS} == ?(-)+([0-9]) ]] \ 68 | && die "Invalid input, is not an integer." 1 69 | 70 | # Check the output dir exists 71 | [ ! -d "${ETCD_BCK_DIR}" ] \ 72 | && die "Invalid output directory, It doesn't exist." 1 73 | 74 | # Check master-exec 75 | [ -z "${MASTER_EXEC}" ] \ 76 | && die "${MASTER_EXEC} not found" 1 77 | 78 | # Check master-exec 79 | [ -z "${ETCD_POD_MANIFEST}" ] \ 80 | && die "${ETCD_POD_MANIFEST} not found" 1 81 | 82 | # Check if the backup destination folder is not full > 95% 83 | USAGE_THRESHOLD=95 84 | USAGE=$(df -h "${ETCD_BCK_DIR}" | grep -v 'Filesystem' | awk '{ print $5}' | cut -d '%' -f 1) 85 | [ "$USAGE" -gt "$USAGE_THRESHOLD" ] \ 86 | && die "${ETCD_BCK_DIR} is almost full ${USAGE}% > ${USAGE_THRESHOLD}%, aborting backup" 1 87 | 88 | 89 | backup_config() { 90 | log "Backing up ETCD config." 91 | cp -a "${ETCD_CONFIG_DIR}" "/tmp/${ETCD_CONFIG_BCK_DIR}" 92 | } 93 | 94 | backup_data() { 95 | 96 | log "Backing up ETCD data, performing snapshot." 97 | # etcd endpoint 98 | ETCD_EP=$(grep https ${ETCD_POD_MANIFEST} | cut -d '/' -f3) 99 | 100 | # snapshot output is /var/lib/etcd/ because is mounted from the host, and we can move it later to another host folder. 101 | # > /usr/local/bin/master-exec etcd etcd /bin/bash -c "ETCDCTL_API=3 /usr/bin/etcdctl \ 102 | # --cert /etc/etcd/peer.crt --key /etc/etcd/peer.key --cacert /etc/etcd/ca.crt --endpoints ${ETCD_EP} snapshot save /var/lib/etcd/snapshot.db" 103 | ${MASTER_EXEC} etcd etcd /bin/sh -c "ETCDCTL_API=3 etcdctl \ 104 | --cert /etc/etcd/peer.crt --key /etc/etcd/peer.key --cacert /etc/etcd/ca.crt \ 105 | --endpoints ${ETCD_EP} snapshot save /var/lib/etcd/snapshot.db" 106 | 107 | log "Backing up ETCD data, validating snapshot." 108 | # Validate the status of the snapshot 109 | # > snapshot status /var/lib/etcd/snapshot.db 110 | ${MASTER_EXEC} etcd etcd /bin/sh -c "ETCDCTL_API=3 etcdctl \ 111 | --cert /etc/etcd/peer.crt --key /etc/etcd/peer.key --cacert /etc/etcd/ca.crt \ 112 | --endpoints ${ETCD_EP} snapshot status /var/lib/etcd/snapshot.db" 113 | 114 | # Check the etcd snapshot 115 | [ "$?" -ne 0 ] \ 116 | && die "/var/lib/etcd/snapshot.db is not a valid etcd backup. Please check the status of your etcd cluster" 1 117 | 118 | # Move the snapshot to the temp bck dir. 119 | mv /var/lib/etcd/snapshot.db /tmp/${ETCD_DATA_BCK_DIR}/snapshot.db 120 | } 121 | 122 | backup(){ 123 | log "Backing up ETCD." 124 | mkdir -p /tmp/${ETCD_DATA_BCK_DIR}/ 125 | 126 | backup_config 127 | backup_data 128 | 129 | log "Creating tar.gz file" 130 | tar cvfz "${ETCD_BCK_DIR}/${ETCD_FINAL_BCK_FILE}" --directory /tmp "${ETCD_CONFIG_BCK_DIR}" "${ETCD_DATA_BCK_DIR}" 131 | 132 | log "Check your backup file on: ${ETCD_BCK_DIR}/${ETCD_FINAL_BCK_FILE}" 133 | 134 | log "Deleting temporary files" 135 | rm -rf "/tmp/${ETCD_CONFIG_BCK_DIR}" "/tmp/${ETCD_DATA_BCK_DIR}" 136 | } 137 | 138 | # Keep the last #RETENTION_DAYS files 139 | purge_old_backups(){ 140 | log "Deleting old backup files...Keeping the last ${RETENTION_DAYS} days" 141 | find "${ETCD_BCK_DIR}"/etcd_bck* -type f -mtime +"${RETENTION_DAYS}" 142 | find "${ETCD_BCK_DIR}"/etcd_bck* -type f -mtime +"${RETENTION_DAYS}" -delete 143 | } 144 | 145 | backup 146 | purge_old_backups 147 | 148 | exit 0 149 | -------------------------------------------------------------------------------- /backup_restore/ocp-master-cert-backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Date: 04/18/2017 4 | # 5 | # Script Purpose: Performs a backup of the certificates on the current master node. 6 | # Version : 1.0 7 | # 8 | 9 | #Directory that will be backed up on each master 10 | readonly SOURCE_BACKUP_DIR="/etc/origin/master" 11 | #Prefix of the generated tar.gz file 12 | readonly TAR_FILE_PREFIX="certs-and-keys" 13 | #Temporary location where the tar file will be created. This file will be removed. 14 | readonly DEST_BACKUP_DIR="/tmp" 15 | #Location (on the master) where the tar will be copied. 16 | #This parameter should be updated to use a location that's backed up on the host. 17 | readonly BACKUP_DEST_DIR="/tmp/ocp-master-backup" 18 | 19 | # Check if executed as root 20 | if [[ $EUID -ne 0 ]]; then 21 | echo "ERROR: This script must be run as root. Aborting." 22 | exit 1 23 | fi 24 | 25 | # Check if executed on OSE master 26 | if ! [ -d $SOURCE_BACKUP_DIR ] ; then 27 | echo "ERROR: This script must be run on an OpenShift master. Aborting." 28 | exit 1 29 | fi 30 | 31 | #Variable to hold path to tar 32 | base_file="${DEST_BACKUP_DIR}/${TAR_FILE_PREFIX}-$HOSTNAME-$(date -u '+%Y%m%d-%H%M%S')" 33 | tar_name="${base_file}.tar" 34 | gz_name="${base_file}.tar.gz" 35 | 36 | #cd into the dir to backup 37 | cd ${SOURCE_BACKUP_DIR} 38 | 39 | #tar the .crt files 40 | echo "Adding all .crt files from ${SOURCE_BACKUP_DIR}" 41 | tar cf ${tar_name} *.crt 42 | 43 | #tar the key files 44 | echo "Adding all .key files from ${SOURCE_BACKUP_DIR}" 45 | tar rf ${tar_name} *.key 46 | 47 | #Compress the tar 48 | echo "Compressing tar file: ${tar_name}" 49 | gzip "${tar_name}" 50 | 51 | #make sure the backup dir exists. 52 | mkdir -p ${BACKUP_DEST_DIR} 53 | 54 | #move the tar.gz to the backup location 55 | echo "Moving: ${gz_name} to: ${BACKUP_DEST_DIR}" 56 | mv ${gz_name} ${BACKUP_DEST_DIR} 57 | 58 | echo "Great Success" 59 | -------------------------------------------------------------------------------- /backup_restore/ocp-project-backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -Exeuo pipefail 4 | 5 | # Date: 08/01/2018 6 | # 7 | # Script Purpose: Create a backup of all projects on the OCP cluster. 8 | # Version : 1.01 9 | # 10 | 11 | #=============================================================================== 12 | #This script must be executed on cluster master node. 13 | #=============================================================================== 14 | 15 | 16 | for pid in $(pidof -x `basename "$0"`); do 17 | [ $pid != $$ ] && echo "[$(date)] : `basename "$0"` : Process is already running with PID $pid" && exit 1 18 | done 19 | 20 | #Update this path to point to the backup location. 21 | readonly ARCHIVE_LOCATION="/opt/backups/master" && [ ! -d $ARCHIVE_LOCATION ] && mkdir -p $ARCHIVE_LOCATION 22 | 23 | # Check if executed as root 24 | if [[ $EUID -ne 0 ]]; then 25 | echo "ERROR: This script must be run as root. Aborting." 26 | exit 1 27 | fi 28 | 29 | # Check if executed on OSE master 30 | if ! systemctl status `systemctl list-units --plain -t service | egrep master-api |cut -d. -f1` >/dev/null 2>&1; then 31 | echo "ERROR: This script must be run on an OpenShift master. Aborting." 32 | exit 1 33 | fi 34 | 35 | 36 | #Login to ocp 37 | login_result=`oc login -u system:admin` 38 | if [[ "$login_result" == *"Login failed"* ]]; then 39 | echo "================================== 40 | Login failed! 41 | ==================================" 42 | exit 1 43 | fi 44 | 45 | #Check the passed in token and validate. If failure occurs, stop. 46 | whoami=`oc whoami` 47 | if [[ "$whoami" != "system:admin" ]]; then 48 | echo "================================== 49 | Script failed user validation! 50 | ==================================" 51 | exit 1 52 | fi 53 | 54 | #Get the hostname of the OCP console 55 | ocp_hostname_with_port=`oc version | grep Server | awk -F/ '{print $3}'` 56 | ocp_hostname=`echo ${ocp_hostname_with_port} | awk -F: '{print $1}'` 57 | echo "OCP Cluster: ${ocp_hostname}" 58 | 59 | #Set the yaml backup path 60 | temp_backup_path="${ocp_hostname}" 61 | 62 | #Check for the project 63 | project_list=`oc projects -q` 64 | #Check to see that we got a response, if not just die. 65 | if [ -z "$project_list" ]; then 66 | 67 | echo "===================================================================== 68 | No projects found! This should not happen, so something bad occured. 69 | =====================================================================" 70 | exit 1 71 | 72 | fi 73 | 74 | #Process the list of projects and export them if they are valid 75 | readarray -t projects <<<"$project_list" 76 | for project in "${projects[@]}" 77 | do 78 | 79 | #check to see if the project is actually validation 80 | project_exists=`oc projects -q | grep ${project}` 81 | if [ -n "$project_exists" ]; then 82 | 83 | echo "Backing up project: ${project}" 84 | 85 | #Create a variable to hold 86 | yaml_dir="${temp_backup_path}/${project}" 87 | 88 | #Make the project directory 89 | mkdir -p ${yaml_dir} 90 | 91 | set +e 92 | #Export all bc,dc,is,route,svc yamls 93 | oc export all -n ${project} -o yaml > ${yaml_dir}/project.yaml 94 | 95 | #Export all the rolebindings 96 | oc get rolebindings -n ${project} -o yaml --export=true > ${yaml_dir}/rolebindings.yaml 97 | 98 | #Export any project serviceaccounts 99 | oc get serviceaccount -n ${project} -o yaml --export=true > ${yaml_dir}/serviceaccount.yaml 100 | 101 | #Export any project secrets 102 | oc get secret -n ${project} -o yaml --export=true > ${yaml_dir}/secret.yaml 103 | 104 | #Export any project pvc 105 | oc get pvc -n ${project} -o yaml --export=true > ${yaml_dir}/pvc.yaml 106 | set -e 107 | else 108 | 109 | #Found an invalid project name, somehting bad may have happened, but we'll 110 | #just continue. 111 | echo "Project: ${project}, is not a valid project." 112 | 113 | fi 114 | 115 | done 116 | 117 | 118 | #Now tar it. 119 | tar_archive="${ARCHIVE_LOCATION}/${temp_backup_path}-$(date -u '+%Y%m%d-%H%M%S').tar.gz" 120 | echo "Creating archive of project backups at: ${tar_archive}" 121 | tar -czf ${tar_archive} ${temp_backup_path} 122 | 123 | echo "Cleanup yamls..." 124 | rm -rf ${temp_backup_path} 125 | 126 | #Yea! 127 | echo "Great Success" 128 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-login-page/.gitignore: -------------------------------------------------------------------------------- 1 | .retry 2 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-login-page/README.md: -------------------------------------------------------------------------------- 1 | Ansible Playbook: Openshift Custom Login Page 2 | ========= 3 | 4 | This playbook contains Jooho.image-resize and Jooho.openshift_custom_login_page. 5 | It resizes an image to fit for logo accordingly for openshift custom login page. 6 | 7 | Refer this [doc](https://goo.gl/2L45bJ) 8 | 9 | Requirements 10 | ------------ 11 | None 12 | 13 | Dependencies 14 | ------------ 15 | 16 | - [Jooho.image-resize](https://galaxy.ansible.com/Jooho/image-resize/) 17 | - [Jooho.openshift-custom-login-page](https://galaxy.ansible.com/Jooho/openshift-custom-login-page/) 18 | 19 | Roles Variables 20 | -------------- 21 | 22 | | Name | Default value | Requird | Description | 23 | |---------------------------|---------------------------------------|----------------------|-----------------------------------------------------------------------------| 24 | | openshift_master_conf_dir | /etc/origin/master | no | Where openshift configuation dir is | 25 | | master_url | http://master1.example.com:8443 | no | API Server URL | 26 | | login_html_dir | /etc/origin/master | no | Where new login html page will locate | 27 | | temp_dir | /tmp | no | Temp directory | 28 | | input_img | sample-openshift-ori.png | no | Original Image InputPath | 29 | | size | 193x144 | no | Resized Image Size | 30 | | overwrite_force | true | no | If true, it overwrite exist resized image 31 | 32 | **NOTE** 33 | If you want to use different vars from default one, you should specify them with -e options 34 | 35 | Example Execute Commands 36 | ----------------------- 37 | 38 | - **Download roles** 39 | ~~~ 40 | cd openshift-toolkit/ansible-playbook-openshift-custom-login-page 41 | 42 | ansible-galaxy install -f -r requirements.yaml -p ./roles 43 | ~~~ 44 | 45 | - **Use default logo** 46 | 47 | *Options:* 48 | - If you want to restart master manually, please add `-e restart_master=false` 49 | - If you want to use your own image, please add `-e input_img=/path/to/logo.png` 50 | 51 | ~~~ 52 | ansible-playbook -i /path/to/hosts ./playbook.yaml 53 | ~~~ 54 | 55 | - **Use your own logo** 56 | ~~~ 57 | ansible-playbook -i /path/to/hosts ./playbook.yaml -e input_img=/path/to/logo.png 58 | ~~~ 59 | 60 | Quick Demo Script 61 | ---------------- 62 | ``` 63 | git clone https://github.com/redhat-cop/openshift-toolkit.git 64 | 65 | cd openshift-toolkit/ansible-playbook-openshift-custom-login-page 66 | 67 | ansible-galaxy install -f -r requirements.yaml -p ./roles 68 | 69 | wget https://i.ytimg.com/vi/iai4v0ocX3w/maxresdefault.jpg -O ./superman.jpg 70 | 71 | ansible-playbook -i ./hosts ./playbook.yaml -e input_img=./superman.jpg 72 | 73 | ``` 74 | 75 | Sample Hosts file 76 | ------------------ 77 | 78 | ``` 79 | [masters] 80 | master1.example.com 81 | 82 | [etcd] 83 | master1.example.com 84 | 85 | [nodes] 86 | master1.example.com openshift_node_labels="{'region': 'mgmt', 'role': 'master'}" 87 | node1.example.com openshift_node_labels="{'region': 'infra', 'role': 'app', 'zone': 'default'}" 88 | node2.example.com openshift_node_labels="{'region': 'infra', 'role': 'app', 'zone': 'default'}" 89 | ``` 90 | 91 | Tip commands 92 | ------------- 93 | Copy ssh key to all nodes 94 | ~~~ 95 | for node in $(cat ./hosts|grep -v '\['|awk '{print $1}'|grep -v '^$'|uniq); do ssh-copy-id -i ~/.ssh/id_rsa.pub $node; done 96 | ~~~ 97 | 98 | Video Clip 99 | ---------- 100 | [![asciicast](https://asciinema.org/a/144986.png)](https://asciinema.org/a/144986) 101 | 102 | 103 | Result Image 104 | ------------ 105 | ![alt Result](./result.png) 106 | 107 | License 108 | ------- 109 | 110 | BSD/MIT 111 | 112 | Author Information 113 | ------------------ 114 | 115 | This role was created in 2016 by [Jooho Lee](http://github.com/jooho). 116 | 117 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-login-page/playbook.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Resize image and create openshift custom login page with the image. 4 | hosts: masters 5 | gather_facts: false 6 | 7 | roles: 8 | - { role: Jooho.image-resize, overwrite_force: true ,when: "inventory_hostname == groups.masters[0]" } 9 | - { role: Jooho.openshift-custom-login-page, logo_img: "{{output_img}}" } 10 | 11 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-login-page/requirements.yaml: -------------------------------------------------------------------------------- 1 | - src: Jooho.image-resize 2 | - src: Jooho.openshift-custom-login-page 3 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-login-page/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-cop/openshift-toolkit/16bf5b054fd5bdfb5018c1f4e06b80470fde716f/branding/ansible-playbook-openshift-custom-login-page/result.png -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-webconsole-logo/.gitignore: -------------------------------------------------------------------------------- 1 | .retry 2 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-webconsole-logo/README.md: -------------------------------------------------------------------------------- 1 | Ansible Playbook: Openshift Custom Webconsole Logo 2 | ========= 3 | 4 | This playbook contains Jooho.image-resize and Jooho.openshift_custom_webconsole_logo. 5 | It resizes an image to fit for logo accordingly for openshift custom webconsole logo. 6 | 7 | Refer this [doc](https://goo.gl/2L45bJ) 8 | 9 | Requirements 10 | ------------ 11 | None 12 | 13 | Dependencies 14 | ------------ 15 | 16 | - [Jooho.image-resize](https://galaxy.ansible.com/Jooho/image-resize) 17 | - [Jooho.openshift-custom-webconsole-logo](https://galaxy.ansible.com/Jooho/openshift-custom-webconsole-logo) 18 | 19 | Roles Variables 20 | -------------- 21 | 22 | | Name | Default value | Requird | Description | 23 | |---------------------------|---------------------------------------|----------------------|-----------------------------------------------------------------------------| 24 | | openshift_master_conf_dir | /etc/origin/master | no | Where openshift configuation dir is | 25 | | master_url | http://master1.example.com:8443 | no | API Server URL | 26 | | stylesheet_base_dir | /etc/origin/master/stylesheet | no | Where new login html page will locate | 27 | | temp_dir | /tmp | no | Temp directory | 28 | | input_img | sample-openshift-ori.png | no | Original Image InputPath | 29 | | size | 193x144 | no | Resized Image Size | 30 | | force | true | no | If true, it overwrite exist resized image/css | 31 | 32 | **NOTE** 33 | If you want to use different vars from default one, you should specify them with -e options 34 | 35 | 36 | Example Execute Commands 37 | ----------------------- 38 | - **Download roles** 39 | ~~~ 40 | cd openshift-toolkit/ansible-playbook-openshift-custom-webconsole-logo 41 | 42 | ansible-galaxy install -f -r requirements.yaml -p ./roles 43 | ~~~ 44 | 45 | - **Use default logo** 46 | 47 | *Options:* 48 | - If you want to restart master manually, please add `-e restart_master=false` 49 | - If you want to use your own image, please add `-e input_img=/path/to/logo.png` 50 | 51 | ~~~ 52 | ansible-playbook -i /path/to/hosts ./playbook.yaml 53 | ~~~ 54 | 55 | - **Use your own logo** 56 | ~~~ 57 | ansible-playbook -i /path/to/hosts ./playbook.yaml -e input_img=/path/to/logo.png 58 | ~~~ 59 | 60 | 61 | Quick Demo Script 62 | ---------------- 63 | ``` 64 | git clone https://github.com/redhat-cop/openshift-toolkit.git 65 | 66 | cd openshift-toolkit/ansible-playbook-openshift-custom-webconsole-logo 67 | 68 | ansible-galaxy install -f -r requirements.yaml -p ./roles 69 | 70 | wget https://i.ytimg.com/vi/iai4v0ocX3w/maxresdefault.jpg -O ./superman.jpg 71 | 72 | ansible-playbook -i ./hosts ./playbook.yaml -e input_img=./superman.jpg 73 | 74 | ``` 75 | 76 | Sample Hosts file 77 | ------------------ 78 | 79 | ``` 80 | [masters] 81 | master1.example.com 82 | 83 | [etcd] 84 | master1.example.com 85 | 86 | [nodes] 87 | master1.example.com openshift_node_labels="{'region': 'mgmt', 'role': 'master'}" 88 | node1.example.com openshift_node_labels="{'region': 'infra', 'role': 'app', 'zone': 'default'}" 89 | node2.example.com openshift_node_labels="{'region': 'infra', 'role': 'app', 'zone': 'default'}" 90 | ``` 91 | 92 | Tip commands 93 | ------------- 94 | Copy ssh key to all nodes 95 | ~~~ 96 | for node in $(cat ./hosts|grep -v '\['|awk '{print $1}'|grep -v '^$'|uniq); do ssh-copy-id -i ~/.ssh/id_rsa.pub $node; done 97 | ~~~ 98 | 99 | Movie Clips 100 | ----------- 101 | [![asciicast](https://asciinema.org/a/144980.png)](https://asciinema.org/a/144980) 102 | 103 | Result Image 104 | ------------ 105 | ![alt Result](./result.png) 106 | 107 | License 108 | ------- 109 | 110 | BSD/MIT 111 | 112 | Author Information 113 | ------------------ 114 | 115 | This role was created in 2017 by [Jooho Lee](http://github.com/jooho). 116 | 117 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-webconsole-logo/playbook.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Resize image and create openshift custom webconsole with the image. 4 | hosts: masters 5 | gather_facts: false 6 | 7 | roles: 8 | - { role: Jooho.image-resize, overwrite_force: true, size: '320x40', when: "inventory_hostname == groups.masters[0]" } 9 | - { role: Jooho.openshift-custom-webconsole-logo, logo_img: "{{output_img}}", master_url: "master1.example.com:8443", stylesheet_base_dir: "/etc/origin/master/stylesheet", overwrite_force: true } 10 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-webconsole-logo/requirements.yaml: -------------------------------------------------------------------------------- 1 | - src: Jooho.image-resize 2 | - src: Jooho.openshift-custom-webconsole-logo 3 | -------------------------------------------------------------------------------- /branding/ansible-playbook-openshift-custom-webconsole-logo/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-cop/openshift-toolkit/16bf5b054fd5bdfb5018c1f4e06b80470fde716f/branding/ansible-playbook-openshift-custom-webconsole-logo/result.png -------------------------------------------------------------------------------- /capacity-dashboard/capacity_management_rules.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: PrometheusRule 3 | metadata: 4 | name: capacity-management-rules 5 | namespace: openshift-monitoring 6 | spec: 7 | groups: 8 | - name: cpu.capacity.rules 9 | rules: 10 | - record: quota_sum:cpu:by_nodegroup 11 | expr: > 12 | sum (openshift_clusterresourcequota_labels * on (name) group_left () openshift_clusterresourcequota_usage{resource="requests.cpu",type="hard"}) by (label_nodegroup) 13 | - record: allocatable_sum:cpu:by_nodegroup 14 | expr: > 15 | sum (kube_node_labels * on (node) group_left () kube_node_status_allocatable_cpu_cores) by (label_nodegroup) 16 | - record: request_sum:cpu:by_nodegroup 17 | expr: > 18 | sum (kube_node_labels * on (node) group_left () sum (kube_pod_container_resource_requests_cpu_cores) by (node)) by (label_nodegroup) 19 | - record: usage_sum:cpu:by_nodegroup 20 | expr: > 21 | sum (kube_node_labels * on (node) group_left () sum (rate (container_cpu_usage_seconds_total{container_name=~".+", container_name!="POD"} [1m])) by (node)) by (label_nodegroup) 22 | - alert: HighCPUQuotaGranted 23 | expr: (quota_sum:cpu:by_nodegroup / allocatable_sum:cpu:by_nodegroup) > 1 24 | for: 1h 25 | labels: 26 | severity: warning 27 | annotations: 28 | message: Node group {{ $labels.label_nodegroup }} is overcommitted ({{ $value }}) with regards to CPU quota 29 | - alert: HighCPURequest 30 | expr: (request_sum:cpu:by_nodegroup / allocatable_sum:cpu:by_nodegroup) > 0.8 31 | for: 1m 32 | labels: 33 | severity: critical 34 | annotations: 35 | message: Pod CPU request for node group {{ $labels.label_nodegroup }} is at {{ $value }}; soon pods will stop being scheduled. 36 | - name: memory.capacity.rules 37 | rules: 38 | - record: quota_sum:memory:by_nodegroup 39 | expr: > 40 | sum (openshift_clusterresourcequota_labels * on (name) group_left () openshift_clusterresourcequota_usage{resource="requests.memory",type="hard"}) by (label_nodegroup) 41 | - record: allocatable_sum:memory:by_nodegroup 42 | expr: > 43 | sum (kube_node_labels * on (node) group_left () kube_node_status_allocatable_memory_bytes) by (label_nodegroup) 44 | - record: request_sum:memory:by_nodegroup 45 | expr: > 46 | sum (kube_node_labels * on (node) group_left () sum (kube_pod_container_resource_requests_memory_bytes) by (node)) by (label_nodegroup) 47 | - record: usage_sum:memory:by_nodegroup 48 | expr: > 49 | sum (kube_node_labels * on (node) group_left () sum(container_memory_usage_bytes{container_name=~".+", container_name!="POD"}) by (node)) by (label_nodegroup) 50 | - alert: HighMemoryQuotaGranted 51 | expr: (quota_sum:memory:by_nodegroup / allocatable_sum:memory:by_nodegroup) > 1 52 | for: 1h 53 | labels: 54 | severity: warning 55 | annotations: 56 | message: Node group {{ $labels.label_nodegroup }} is overcommitted ({{ $value }}) with regards to memory quota 57 | - alert: HighMemoryRequest 58 | expr: (request_sum:memory:by_nodegroup / allocatable_sum:memory:by_nodegroup) > 0.8 59 | for: 1m 60 | labels: 61 | severity: critical 62 | annotations: 63 | message: Pod memory request for node group {{ $labels.label_nodegroup }} is at {{ $value }}; soon pods will stop being scheduled. 64 | -------------------------------------------------------------------------------- /capacity-dashboard/example/README.md: -------------------------------------------------------------------------------- 1 | # Example 2 | 3 | to create an example scenario, label a set of nodes with the label `nodegroup=group1`, then run the following commands: 4 | 5 | ```shell 6 | oc adm new-project p1q1 --node-selector nodegroup=group1 7 | oc label namespace p1q1 quota=quota1 8 | oc adm new-project p2q1 --node-selector nodegroup=group1 9 | oc label namespace p2q1 quota=quota1 10 | oc adm new-project p1q2 --node-selector nodegroup=group1 11 | oc label namespace p1q2 quota=quota2 12 | oc adm new-project p2q2 --node-selector nodegroup=group1 13 | oc label namespace p2q2 quota=quota2 14 | oc apply -f cluster_quota.yaml 15 | ``` 16 | -------------------------------------------------------------------------------- /capacity-dashboard/example/cluster-quota.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: quota.openshift.io/v1 3 | kind: ClusterResourceQuota 4 | metadata: 5 | name: quota1 6 | labels: 7 | nodegroup: group1 8 | spec: 9 | quota: 10 | hard: 11 | glusterfs.storageclass.storage.k8s.io/requests.storage: 10Gi 12 | requests.cpu: "2" 13 | requests.memory: 4Gi 14 | selector: 15 | labels: 16 | matchLabels: 17 | quota: quota1 18 | scopes: 19 | - NotTerminating 20 | --- 21 | apiVersion: quota.openshift.io/v1 22 | kind: ClusterResourceQuota 23 | metadata: 24 | name: quota2 25 | labels: 26 | nodegroup: group1 27 | spec: 28 | quota: 29 | hard: 30 | glusterfs.storageclass.storage.k8s.io/requests.storage: 10Gi 31 | requests.cpu: "2" 32 | requests.memory: 4Gi 33 | selector: 34 | labels: 35 | matchLabels: 36 | quota: quota2 37 | scopes: 38 | - NotTerminating -------------------------------------------------------------------------------- /capacity-dashboard/media/assumptions_setup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-cop/openshift-toolkit/16bf5b054fd5bdfb5018c1f4e06b80470fde716f/capacity-dashboard/media/assumptions_setup.png -------------------------------------------------------------------------------- /capacity-dashboard/media/dashboard-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-cop/openshift-toolkit/16bf5b054fd5bdfb5018c1f4e06b80470fde716f/capacity-dashboard/media/dashboard-screenshot.png -------------------------------------------------------------------------------- /cluster-management/README.md: -------------------------------------------------------------------------------- 1 | # Cluster Management 2 | 3 | This directory contains scripts specific to cluster managment intended to be run after an OCP installation for operationalization purposes. 4 | 5 | ## Garbage collection 6 | 7 | - Sets up spotify docker-gc on hosts. Should be run a standard 'bring your own' OpenShift-ansible inventory file. The playbook will set up garbage collection on all host with the [node] designation. 8 | - View the documentation for spotify docker-gc here: [LINK](https://github.com/spotify/docker-gc) 9 | -------------------------------------------------------------------------------- /cluster-management/garbage-collection/docker-gc-setup.yaml: -------------------------------------------------------------------------------- 1 | #The following playbook creates a recurring docker-gc job. It adds the redhat oce-specific images to an excluder file so that vendor availability will not be a potential point of failure. 2 | 3 | 4 | --- 5 | - name: Setup docker garbage collector 6 | hosts: nodes 7 | become: true 8 | vars: 9 | # Minimum age of containers and images to remove (seconds) 10 | time: 3600 11 | 12 | tasks: 13 | - name: Clone docker-gc repo 14 | git: 15 | repo: 'https://github.com/spotify/docker-gc.git' 16 | dest: /tmp/docker-gc 17 | - name: Copy docker-gc script to proper location 18 | copy: 19 | src: /tmp/docker-gc/docker-gc 20 | dest: /usr/sbin/docker-gc 21 | remote_src: true 22 | - name: Create gc cron job 23 | shell: echo -e "#!/bin/bash\nGRACE_PERIOD_SECONDS={{ time }} /usr/sbin/docker-gc" > /etc/cron.hourly/docker-gc 24 | - name: Delete temp files 25 | file: 26 | state: absent 27 | path: /tmp/docker-gc 28 | - name: Add permissions on cron job 29 | file: 30 | path: /usr/sbin/docker-gc 31 | mode: u=rwX,g=rX,o=rX 32 | - name: Add permissions on cron job 33 | file: 34 | path: /etc/cron.hourly/docker-gc 35 | mode: u=rwX,g=rX,o=rX 36 | - name: Add exclude file 37 | file: 38 | path: /etc/docker-gc-exclude 39 | state: touch 40 | mode: "u=rw,g=r,o=r" 41 | - name: Add excluder for Red Hat Images 42 | blockinfile: 43 | dest: /etc/docker-gc-exclude 44 | block: | 45 | registry.access.redhat.com/openshift3/ose-haproxy-router:* 46 | registry.access.redhat.com/openshift3/ose-deployer:* 47 | registry.access.redhat.com/openshift3/ose-sti-builder:* 48 | registry.access.redhat.com/openshift3/ose-docker-builder:* 49 | registry.access.redhat.com/openshift3/ose-pod:* 50 | registry.access.redhat.com/openshift3/ose-docker-registry:* 51 | registry.access.redhat.com/openshift3/ose-recycler:* 52 | -------------------------------------------------------------------------------- /custom-autoscaler/README.md: -------------------------------------------------------------------------------- 1 | # OpenShift Custom Autoscaler Quickstart 2 | 3 | The following illustrates how one might use OpenShift's [metrics stack](https://docs.openshift.com/container-platform/3.11/install_config/cluster_metrics.html) (Hawkular, Cassandra, Heapster) to develop customized application autoscaling. 4 | 5 | ## Running & Testing 6 | 7 | This script makes a few assumptions about your application 8 | 9 | * The pods are managed by a `deploymentConfig` 10 | * The `deploymentConfig` is named the same as the app. (in this case, `myapp`) 11 | * The `deploymentConfig` has memory limits set on the containers. i.e. 12 | ``` 13 | apiVersion: v1 14 | kind: DeploymentConfig 15 | metadata: 16 | name: myapp 17 | spec: 18 | template: 19 | spec: 20 | container: 21 | - resources: 22 | limits: 23 | memory: 140Mi 24 | 25 | ``` 26 | 27 | Before running the script, you need to export a few variables into your environment. The following represents a minimal setup. A full description of all variables is below. 28 | 29 | ``` 30 | export NAMESPACE=myproject 31 | export TOKEN=`oc whoami -t` 32 | export APP_NAME=myapp 33 | export HAWKULAR_HOSTNAME=hawkular-metrics.apps.ocp-c1.myorg.com 34 | ``` 35 | 36 | From there we can simply run our script. 37 | 38 | ``` 39 | ./scale.py 40 | ``` 41 | 42 | The script will continue to run until exited, either through Ctrl+C or killing the pid. 43 | 44 | ## Environment Variables 45 | 46 | | Variable Name | Default Value | Description | 47 | | --------------| ------------- | ----------- | 48 | | `NAMESPACE` | n/a; Required | Namespace the app is running in. | 49 | | `TOKEN` | n/a; Required | OpenShift User or ServiceAccount token. Can be retrieved using `oc whoami -t` for users or `oc serviceaccount get-token ` for serviceaccounts. | 50 | | `APP_NAME` | n/a; Required | Name of DeploymentConfig. Must match name shown in `oc get dc` | 51 | | `HAWKULAR_HOSTNAME` | n/a; Required | Route hostname for hawkular service. i.e. `oc get route hawkular-metrics -n openshift-infra` | 52 | | `METRIC_PULL_INTERVAL_SECONDS` | 60 | Number of seconds between each scale up check | 53 | | `SCALE_UP_THRESHOLD` | .7 | Floating numeric value 0-1; Percentage of total memory used over which a scale-up will happen | 54 | | `CA_CERT` | False | Path to a CA Certificate file to validate Hawkular certificate. Default value of `False` will turn off certificate checks and flash a warning message. | 55 | 56 | 57 | 58 | ## Issues, Limitations & Future Enhancements 59 | 60 | - Solution should be deployed as a pod in openshift. 61 | - Currently only scales up, not down 62 | - Implementation very specific to memory, but could probably be genericized to pull some other metric. 63 | - Needs better error handling 64 | -------------------------------------------------------------------------------- /custom-autoscaler/scale.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | 3 | import requests, os, json, sys, time 4 | from subprocess import Popen, PIPE 5 | from requests.packages.urllib3.exceptions import InsecureRequestWarning 6 | 7 | 8 | 9 | token = os.environ['TOKEN'] 10 | namespace = os.environ['NAMESPACE'] 11 | app_name = os.environ['APP_NAME'] 12 | hawkular_hostname = os.environ['HAWKULAR_HOSTNAME'] 13 | 14 | if 'METRIC_PULL_INTERVAL_SECONDS' in os.environ: 15 | interval = os.environ['METRIC_PULL_INTERVAL_SECONDS'] 16 | else: 17 | interval = 60 18 | 19 | if 'SCALE_UP_THRESHOLD' in os.environ: 20 | scale_up_threshold = os.environ['SCALE_UP_THRESHOLD'] 21 | else: 22 | scale_up_threshold = .7 23 | 24 | if 'CA_CERT' in os.environ: 25 | ca_cert = os.environ['CA_CERT'] 26 | else: 27 | print "WARNING: Disabling SSL Verification. This should not be done in Production." 28 | print "To get rid of this message, export CA_CERT=/path/to/ca-certificate" 29 | requests.packages.urllib3.disable_warnings(InsecureRequestWarning) 30 | ca_cert = False 31 | 32 | metrics_url='https://{}/hawkular/metrics/gauges/data'.format(hawkular_hostname) 33 | 34 | def pretty_print_POST(req): 35 | """ 36 | At this point it is completely built and ready 37 | to be fired; it is "prepared". 38 | 39 | However pay attention at the formatting used in 40 | this function because it is programmed to be pretty 41 | printed and may differ from the actual request. 42 | """ 43 | print('{}\n{}\n{}'.format( 44 | '-----------START-----------', 45 | req.method + ' ' + req.metrics_url + '?' + '&'.join('{}={}'.format(k, v) for k, v in req.params.items()), 46 | '\n'.join('{}: {}'.format(k, v) for k, v in req.headers.items()), 47 | )) 48 | 49 | def get_pods(deployment_config): 50 | version = Popen(["oc", "-n", "{}".format(namespace), "get", "dc/{}".format(deployment_config), "-o", "jsonpath={ .status.latestVersion }"], stdout=PIPE).stdout.read() 51 | 52 | pods = Popen(["oc", "-n", "{}".format(namespace), "get", "pods", "-l", "deployment={}-{}".format(deployment_config, version), "-o", "jsonpath={ .items[*].metadata.name }"], stdout=PIPE).stdout.read() 53 | 54 | return pods.split() 55 | 56 | def get_app_usage(app_name, pods): 57 | 58 | total_usage = 0 59 | 60 | for pod in pods: 61 | total_usage += get_pod_usage(pod, namespace, token) 62 | 63 | return total_usage 64 | 65 | def get_app_limit(app_name, pods): 66 | 67 | total_limit = 0 68 | 69 | for pod in pods: 70 | total_limit += get_pod_limit(pod, namespace, token) 71 | 72 | return total_limit 73 | 74 | def get_pod_usage(pod_name, namespace, token): 75 | params = { 76 | 'tags': 'descriptor_name:memory/usage,pod_name:{}'.format(pod_name), 77 | 'bucketDuration' : '60000ms', 78 | 'stacked' : 'true', 79 | 'start' : '-1mn' 80 | } 81 | 82 | headers = { 83 | 'Content-Type' : 'application/json', 84 | 'Hawkular-Tenant': namespace, 85 | 'Authorization' : 'Bearer {}'.format(token) 86 | } 87 | 88 | req = requests.Request( 89 | 'GET', 90 | metrics_url, 91 | params=params, 92 | headers=headers 93 | ) 94 | prepared = req.prepare() 95 | s = requests.Session() 96 | r = s.send(prepared, 97 | verify=ca_cert) 98 | 99 | # pretty_print_POST(req) 100 | # print r.text 101 | 102 | usage=r.json()[0] 103 | return usage["avg"] 104 | 105 | def get_pod_limit(pod_name, namespace, token): 106 | params = { 107 | 'tags': 'descriptor_name:memory/limit,pod_name:{}'.format(pod_name), 108 | 'bucketDuration' : '60000ms', 109 | 'stacked' : 'true', 110 | 'start' : '-1mn' 111 | } 112 | 113 | headers = { 114 | 'Content-Type' : 'application/json', 115 | 'Hawkular-Tenant': namespace, 116 | 'Authorization' : 'Bearer {}'.format(token) 117 | } 118 | 119 | req = requests.Request( 120 | 'GET', 121 | metrics_url, 122 | params=params, 123 | headers=headers 124 | ) 125 | prepared = req.prepare() 126 | s = requests.Session() 127 | r = s.send(prepared, 128 | verify=ca_cert) 129 | 130 | # pretty_print_POST(req) 131 | # print r.text 132 | 133 | usage=r.json()[0] 134 | return usage["avg"] 135 | 136 | def scale_up(app_name, pod_count): 137 | print Popen(["oc", "scale", "--replicas={}".format(pod_count+1), "dc/{}".format(app_name)], stdout=PIPE).stdout.read() 138 | 139 | while True: 140 | current_pods = get_pods(app_name) 141 | 142 | usage = get_app_usage(app_name, current_pods) 143 | limit = get_app_limit(app_name, current_pods) 144 | if limit <= 0: 145 | print "ERROR: Cannot scale on apps with no limit defined. Program will exit." 146 | sys.exit(1) 147 | 148 | print "Average usage is {}".format(usage) 149 | print "Averate limit is {}".format(limit) 150 | if usage / limit > scale_up_threshold: 151 | print "scaling up!" 152 | scale_up(app_name, len(current_pods)) 153 | else: 154 | print "staying put" 155 | print "" 156 | time.sleep(interval) 157 | -------------------------------------------------------------------------------- /custom-dashboards/.applier/host_vars/localhost.yml: -------------------------------------------------------------------------------- 1 | ansible_connection: local -------------------------------------------------------------------------------- /custom-dashboards/.applier/hosts: -------------------------------------------------------------------------------- 1 | [seed-hosts] 2 | localhost ansible_connection=local 3 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/dashboards/capacity/capacity-volume-patch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | spec: 3 | template: 4 | spec: 5 | containers: 6 | - name: grafana 7 | volumeMounts: 8 | - mountPath: /grafana-dashboard-definitions/0/capacity 9 | name: grafana-dashboard-capacity 10 | volumes: 11 | - configMap: 12 | defaultMode: 420 13 | name: grafana-dashboard-capacity 14 | name: grafana-dashboard-capacity 15 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/dashboards/cluster-status/master-api-volume-patch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | spec: 3 | template: 4 | spec: 5 | containers: 6 | - name: grafana 7 | volumeMounts: 8 | - mountPath: /grafana-dashboard-definitions/0/master-api 9 | name: grafana-dashboard-master-api 10 | volumes: 11 | - configMap: 12 | defaultMode: 420 13 | name: grafana-dashboard-master-api 14 | name: grafana-dashboard-master-api 15 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/dashboards/cluster-status/pods-volume-patch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | spec: 3 | template: 4 | spec: 5 | containers: 6 | - name: grafana 7 | volumeMounts: 8 | - mountPath: /grafana-dashboard-definitions/0/pods 9 | name: grafana-dashboard-pods 10 | volumes: 11 | - configMap: 12 | defaultMode: 420 13 | name: grafana-dashboard-pods 14 | name: grafana-dashboard-pods 15 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/dashboards/cluster-status/router-volume-patch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | spec: 3 | template: 4 | spec: 5 | containers: 6 | - name: grafana 7 | volumeMounts: 8 | - mountPath: /grafana-dashboard-definitions/0/router 9 | name: grafana-dashboard-router 10 | volumes: 11 | - configMap: 12 | defaultMode: 420 13 | name: grafana-dashboard-router 14 | name: grafana-dashboard-router 15 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/dashboards/cluster-status/traffic-volume-patch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | spec: 3 | template: 4 | spec: 5 | containers: 6 | - name: grafana 7 | volumeMounts: 8 | - mountPath: /grafana-dashboard-definitions/0/traffic 9 | name: grafana-dashboard-traffic 10 | volumes: 11 | - configMap: 12 | defaultMode: 420 13 | name: grafana-dashboard-traffic 14 | name: grafana-dashboard-traffic 15 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifest_templates/grafana-clusterrolebinding.yml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: authorization.openshift.io/v1 2 | groupNames: null 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: grafana-custom 6 | roleRef: 7 | name: grafana 8 | subjects: 9 | - kind: ServiceAccount 10 | name: grafana-custom 11 | namespace: {{ dashboard_namespace }} 12 | userNames: 13 | - system:serviceaccount:{{ dashboard_namespace }}:grafana-custom 14 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/grafana-custom-config-secret.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | grafana.ini: W3VzZXJzXQphbGxvd19zaWduX3VwID0gdHJ1ZQphdXRvX2Fzc2lnbl9vcmcgPSB0cnVlCmF1dG9fYXNzaWduX29yZ19yb2xlID0gQWRtaW4KW2F1dGhdCmRpc2FibGVfbG9naW5fZm9ybSA9IHRydWUKZGlzYWJsZV9zaWdub3V0X21lbnUgPSB0cnVlClthdXRoLmJhc2ljXQplbmFibGVkID0gZmFsc2UKW2F1dGgucHJveHldCmF1dG9fc2lnbl91cCA9IHRydWUKZW5hYmxlZCA9IHRydWUKaGVhZGVyX25hbWUgPSBYLUZvcndhcmRlZC1Vc2VyCltwYXRoc10KZGF0YSA9IC92YXIvbGliL2dyYWZhbmEKbG9ncyA9IC92YXIvbGliL2dyYWZhbmEvbG9ncwpwbHVnaW5zID0gL3Zhci9saWIvZ3JhZmFuYS9wbHVnaW5zCnByb3Zpc2lvbmluZyA9IC9ldGMvZ3JhZmFuYS9wcm92aXNpb25pbmcKW3NlcnZlcl0KaHR0cF9hZGRyID0gMTI3LjAuMC4xCmh0dHBfcG9ydCA9IDMwMDEK 4 | kind: Secret 5 | metadata: 6 | name: grafana-custom-config 7 | type: Opaque 8 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/grafana-dashboards-cm.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | dashboards.yaml: |- 4 | { 5 | "apiVersion": 1, 6 | "providers": [ 7 | { 8 | "folder": "", 9 | "name": "0", 10 | "options": { 11 | "path": "/grafana-dashboard-definitions/0" 12 | }, 13 | "orgId": 1, 14 | "type": "file" 15 | } 16 | ]} 17 | kind: ConfigMap 18 | metadata: 19 | name: grafana-dashboards 20 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/grafana-deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | annotations: 5 | deployment.kubernetes.io/revision: "1" 6 | creationTimestamp: null 7 | generation: 1 8 | labels: 9 | app: grafana-custom 10 | name: grafana-custom 11 | spec: 12 | progressDeadlineSeconds: 600 13 | replicas: 1 14 | revisionHistoryLimit: 10 15 | selector: 16 | matchLabels: 17 | app: grafana-custom 18 | strategy: 19 | rollingUpdate: 20 | maxSurge: 25% 21 | maxUnavailable: 25% 22 | type: RollingUpdate 23 | template: 24 | metadata: 25 | creationTimestamp: null 26 | labels: 27 | app: grafana-custom 28 | spec: 29 | containers: 30 | - args: 31 | - -config=/etc/grafana/grafana.ini 32 | image: registry.redhat.io/openshift3/grafana:v3.11 33 | imagePullPolicy: IfNotPresent 34 | name: grafana 35 | ports: 36 | - containerPort: 3000 37 | name: http 38 | protocol: TCP 39 | resources: 40 | limits: 41 | cpu: 200m 42 | memory: 200Mi 43 | requests: 44 | cpu: 100m 45 | memory: 100Mi 46 | terminationMessagePath: /dev/termination-log 47 | terminationMessagePolicy: File 48 | volumeMounts: 49 | - mountPath: /var/lib/grafana 50 | name: grafana-storage 51 | - mountPath: /etc/grafana/provisioning/datasources 52 | name: grafana-datasources 53 | - mountPath: /etc/grafana/provisioning/dashboards 54 | name: grafana-dashboards 55 | - mountPath: /etc/grafana 56 | name: grafana-config 57 | - args: 58 | - -provider=openshift 59 | - -https-address=:3000 60 | - -http-address= 61 | - -email-domain=* 62 | - -upstream=http://localhost:3001 63 | - '-openshift-sar={"resource": "namespaces", "verb": "get"}' 64 | - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get"}}' 65 | - -tls-cert=/etc/tls/private/tls.crt 66 | - -tls-key=/etc/tls/private/tls.key 67 | - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token 68 | - -cookie-secret-file=/etc/proxy/secrets/session_secret 69 | - -openshift-service-account=grafana-custom 70 | - -openshift-ca=/etc/pki/tls/cert.pem 71 | - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt 72 | - -skip-auth-regex=^/metrics 73 | image: registry.redhat.io/openshift3/oauth-proxy:v3.11 74 | imagePullPolicy: IfNotPresent 75 | name: grafana-proxy 76 | ports: 77 | - containerPort: 3000 78 | name: https 79 | protocol: TCP 80 | resources: {} 81 | terminationMessagePath: /dev/termination-log 82 | terminationMessagePolicy: File 83 | volumeMounts: 84 | - mountPath: /etc/tls/private 85 | name: secret-grafana-tls 86 | - mountPath: /etc/proxy/secrets 87 | name: secret-grafana-proxy 88 | dnsPolicy: ClusterFirst 89 | restartPolicy: Always 90 | schedulerName: default-scheduler 91 | securityContext: {} 92 | serviceAccount: grafana-custom 93 | serviceAccountName: grafana-custom 94 | terminationGracePeriodSeconds: 30 95 | volumes: 96 | - emptyDir: {} 97 | name: grafana-storage 98 | - name: grafana-datasources 99 | secret: 100 | defaultMode: 420 101 | secretName: grafana-datasources 102 | - configMap: 103 | defaultMode: 420 104 | name: grafana-dashboards 105 | name: grafana-dashboards 106 | - name: grafana-config 107 | secret: 108 | defaultMode: 420 109 | secretName: grafana-custom-config 110 | - name: secret-grafana-tls 111 | secret: 112 | defaultMode: 420 113 | secretName: grafana-custom-tls 114 | - name: secret-grafana-proxy 115 | secret: 116 | defaultMode: 420 117 | secretName: grafana-proxy 118 | status: {} 119 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/grafana-route.yml: -------------------------------------------------------------------------------- 1 | apiVersion: route.openshift.io/v1 2 | kind: Route 3 | metadata: 4 | annotations: 5 | openshift.io/host.generated: "true" 6 | creationTimestamp: null 7 | name: grafana-custom 8 | spec: 9 | host: 10 | port: 11 | targetPort: https 12 | tls: 13 | termination: reencrypt 14 | to: 15 | kind: Service 16 | name: grafana-custom 17 | weight: 100 18 | wildcardPolicy: None 19 | status: 20 | ingress: null 21 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/grafana-sa.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | annotations: 5 | serviceaccounts.openshift.io/oauth-redirectreference.grafana: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"grafana-custom"}}' 6 | creationTimestamp: null 7 | name: grafana-custom 8 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/grafana-service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | service.alpha.openshift.io/serving-cert-secret-name: grafana-custom-tls 6 | creationTimestamp: null 7 | name: grafana-custom 8 | spec: 9 | ports: 10 | - name: https 11 | port: 3000 12 | protocol: TCP 13 | targetPort: https 14 | selector: 15 | app: grafana-custom 16 | sessionAffinity: None 17 | type: ClusterIP 18 | status: 19 | loadBalancer: {} 20 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/openshift-state-metrics/openshift-state-metrics-cluster-role-binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: openshift-state-metrics 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: openshift-state-metrics 9 | subjects: 10 | - kind: ServiceAccount 11 | name: openshift-state-metrics 12 | namespace: openshift-monitoring 13 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/openshift-state-metrics/openshift-state-metrics-cluster-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: openshift-state-metrics 5 | rules: 6 | - apiGroups: 7 | - apps.openshift.io 8 | resources: 9 | - deploymentconfigs 10 | verbs: 11 | - list 12 | - watch 13 | - apiGroups: 14 | - build.openshift.io 15 | resources: 16 | - buildconfigs 17 | - builds 18 | verbs: 19 | - list 20 | - watch 21 | - apiGroups: 22 | - quota.openshift.io 23 | resources: 24 | - clusterresourcequotas 25 | verbs: 26 | - list 27 | - watch 28 | - apiGroups: 29 | - route.openshift.io 30 | resources: 31 | - routes 32 | verbs: 33 | - list 34 | - watch 35 | - apiGroups: 36 | - authentication.k8s.io 37 | resources: 38 | - tokenreviews 39 | verbs: 40 | - create 41 | - apiGroups: 42 | - authorization.k8s.io 43 | resources: 44 | - subjectaccessreviews 45 | verbs: 46 | - create 47 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/openshift-state-metrics/openshift-state-metrics-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta2 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | k8s-app: openshift-state-metrics 6 | name: openshift-state-metrics 7 | namespace: openshift-monitoring 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | k8s-app: openshift-state-metrics 13 | template: 14 | metadata: 15 | labels: 16 | k8s-app: openshift-state-metrics 17 | spec: 18 | containers: 19 | - args: 20 | - --logtostderr 21 | - --secure-listen-address=:8443 22 | - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 23 | - --upstream=http://127.0.0.1:8081/ 24 | - --tls-cert-file=/etc/tls/private/tls.crt 25 | - --tls-private-key-file=/etc/tls/private/tls.key 26 | image: quay.io/openshift/origin-kube-rbac-proxy:4.2 27 | name: kube-rbac-proxy-main 28 | ports: 29 | - containerPort: 8443 30 | name: https-main 31 | resources: 32 | limits: 33 | cpu: 20m 34 | memory: 40Mi 35 | requests: 36 | cpu: 10m 37 | memory: 20Mi 38 | volumeMounts: 39 | - mountPath: /etc/tls/private 40 | name: openshift-state-metrics-tls 41 | readOnly: false 42 | - args: 43 | - --logtostderr 44 | - --secure-listen-address=:9443 45 | - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 46 | - --upstream=http://127.0.0.1:8082/ 47 | - --tls-cert-file=/etc/tls/private/tls.crt 48 | - --tls-private-key-file=/etc/tls/private/tls.key 49 | image: quay.io/openshift/origin-kube-rbac-proxy:4.2 50 | name: kube-rbac-proxy-self 51 | ports: 52 | - containerPort: 9443 53 | name: https-self 54 | resources: 55 | limits: 56 | cpu: 20m 57 | memory: 40Mi 58 | requests: 59 | cpu: 10m 60 | memory: 20Mi 61 | volumeMounts: 62 | - mountPath: /etc/tls/private 63 | name: openshift-state-metrics-tls 64 | readOnly: false 65 | - args: 66 | - --host=127.0.0.1 67 | - --port=8081 68 | - --telemetry-host=127.0.0.1 69 | - --telemetry-port=8082 70 | image: quay.io/openshift/origin-openshift-state-metrics:4.2 71 | name: openshift-state-metrics 72 | resources: 73 | limits: 74 | cpu: 100m 75 | memory: 150Mi 76 | requests: 77 | cpu: 100m 78 | memory: 150Mi 79 | nodeSelector: 80 | beta.kubernetes.io/os: linux 81 | priorityClassName: system-cluster-critical 82 | serviceAccountName: openshift-state-metrics 83 | volumes: 84 | - name: openshift-state-metrics-tls 85 | secret: 86 | secretName: openshift-state-metrics-tls 87 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/openshift-state-metrics/openshift-state-metrics-role-binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: openshift-state-metrics 5 | namespace: openshift-monitoring 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: Role 9 | name: openshift-state-metrics 10 | subjects: 11 | - kind: ServiceAccount 12 | name: openshift-state-metrics 13 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/openshift-state-metrics/openshift-state-metrics-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: openshift-state-metrics 5 | namespace: openshift-monitoring 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - pods 11 | verbs: 12 | - get 13 | - apiGroups: 14 | - extensions 15 | resourceNames: 16 | - openshift-state-metrics 17 | resources: 18 | - deployments 19 | verbs: 20 | - get 21 | - update 22 | - apiGroups: 23 | - apps 24 | resourceNames: 25 | - openshift-state-metrics 26 | resources: 27 | - deployments 28 | verbs: 29 | - get 30 | - update 31 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/openshift-state-metrics/openshift-state-metrics-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: openshift-state-metrics 5 | namespace: openshift-monitoring 6 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/openshift-state-metrics/openshift-state-metrics-service-monitor.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: ServiceMonitor 3 | metadata: 4 | labels: 5 | k8s-app: openshift-state-metrics 6 | name: openshift-state-metrics 7 | namespace: openshift-monitoring 8 | spec: 9 | endpoints: 10 | - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 11 | honorLabels: true 12 | interval: 2m 13 | port: https-main 14 | scheme: https 15 | scrapeTimeout: 2m 16 | tlsConfig: 17 | caFile: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt 18 | serverName: openshift-state-metrics.openshift-monitoring.svc 19 | - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 20 | interval: 2m 21 | port: https-self 22 | scheme: https 23 | scrapeTimeout: 2m 24 | tlsConfig: 25 | caFile: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt 26 | serverName: openshift-state-metrics.openshift-monitoring.svc 27 | jobLabel: k8s-app 28 | selector: 29 | matchLabels: 30 | k8s-app: openshift-state-metrics 31 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/manifests/openshift-state-metrics/openshift-state-metrics-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | service.alpha.openshift.io/serving-cert-secret-name: openshift-state-metrics-tls 6 | labels: 7 | k8s-app: openshift-state-metrics 8 | name: openshift-state-metrics 9 | namespace: openshift-monitoring 10 | spec: 11 | ports: 12 | - name: https-main 13 | port: 8443 14 | targetPort: https-main 15 | - name: https-self 16 | port: 9443 17 | targetPort: https-self 18 | selector: 19 | k8s-app: openshift-state-metrics 20 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/namespace/namespace.yml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: {{ dashboard_namespace }} 5 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/patches/openshift-state-metrics-service-monitor.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | spec: 3 | endpoints: 4 | - port: https-main 5 | tlsConfig: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt 6 | - port: https-self 7 | tlsConfig: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt 8 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/secrets/grafana_config.yml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | grafana.ini: {{ grafana_config_grafana_ini }} 4 | kind: Secret 5 | metadata: 6 | name: grafana-config 7 | type: Opaque -------------------------------------------------------------------------------- /custom-dashboards/.openshift/secrets/grafana_datasources.yml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | prometheus.yaml: {{ grafana_datasources_prometheus_yaml }} 4 | kind: Secret 5 | metadata: 6 | name: grafana-datasources 7 | type: Opaque 8 | -------------------------------------------------------------------------------- /custom-dashboards/.openshift/secrets/grafana_proxy.yml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | session_secret: {{ grafana_proxy_session_secret }} 4 | kind: Secret 5 | metadata: 6 | labels: 7 | k8s-app: grafana 8 | name: grafana-proxy 9 | type: Opaque -------------------------------------------------------------------------------- /custom-dashboards/README.md: -------------------------------------------------------------------------------- 1 | # Getting Started with Custom Dashboarding on OpenShift using Grafana 2 | 3 | This repository contains scaffolding and automation for developing a custom dashboarding strategy on OpenShift using the OpenShift Monitoring stack. This includes the following components: 4 | 5 | * **Playbook to Deploy OpenShift Monitoring Stack** 6 | 7 | For clusters that do not already have the monitoring stack installed, the `monitoring.yml` playbook is provided to deploy it. This playbook has been shown to work (though is not supported by Red Hat) as far back as OpenShift 3.9. 8 | * **Custom Grafana for OpenShift 3.11** 9 | 10 | By default OpenShift 3.11 Grafana is a read-only instance. Many organizations may want to add new custom dashboards. This custom grafana will interact with existing Prometheus and will also add all out-of-the-box dashboards plus few more interesting dashboards which may require from day to day operation. Custom Grafana pod uses OpenShift oAuth to authenticate users and assigns "Admin" role to all users so that users can create their own dashboards for additional monitoring. 11 | * **Starter Dashboards** 12 | 13 | Several dashboards are available to be provisioned in your Grafana instance as a starting point for Dashboard Development. See the [dashboards](.openshift/dashboards/) directory for more info. 14 | 15 | ## Deploying the Entire Project 16 | 17 | An [OpenShift Applier](https://github.com/redhat-cop/openshift-applier) inventory has been supplied for deployment convenience. Two playbooks are also provided. The first is to deploy the OpenShift Monitoring stack, using [openshift-ansible](https://github.com/openshift/openshift-ansible), the second deploys the custom Grafana. 18 | 19 | 1. First, you must download the prerequsite repositories, [OpenShift Ansible](https://github.com/openshift/openshift-ansible), and [OpenShift Applier](https://github.com/redhat-cop/openshift-applier). 20 | 21 | ansible-galaxy install -r requirements.yml -p galaxy 22 | 23 | 24 | 2. (Optional) If you are running on an OpenShift Cluster older than 3.11, or your cluster did not have the monitoring stack installed at first install, you can use the following command to install the monitoring stack: 25 | 26 | ansible-playbook -i /path/to/cluster/inventory/ monitoring.yml 27 | 28 | 3. Finally, run the apply.yml playbook to deploy Grafana and all of the Dashboards. 29 | 30 | ansible-playbook -i .applier/ galaxy/openshift-applier/playbooks/openshift-cluster-seed.yml 31 | 32 | ## Selectively Deploying Dashboards 33 | 34 | The above automation deploys everything in this project. If you would like to deploy only specific dashboards of your choosing, you can do so by specifying which applier tags to provision. 35 | 36 | The following command is equivalent to the above: 37 | 38 | ansible-playbook -i .applier/ galaxy/openshift-applier/playbooks/openshift-cluster-seed.yml \ 39 | -e include_tags=infrastructure,dashbard-cluster-status,dashboard-capacity,dashboard-sdm 40 | -------------------------------------------------------------------------------- /custom-dashboards/mdt-secret-discovery-requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - src: https://github.com/redhat-cop/openshift-toolkit.git 4 | version: master 5 | -------------------------------------------------------------------------------- /custom-dashboards/mdt-secret-discovery/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /custom-dashboards/mdt-secret-discovery/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Fetch grafana_config secret 2 | k8s_facts: 3 | api_version: v1 4 | kind: Secret 5 | namespace: openshift-monitoring 6 | name: grafana-config 7 | register: grafana_config_data 8 | 9 | - name: Set grafana_config grafana.ini secret fact 10 | set_fact: 11 | grafana_config_grafana_ini: "{{ grafana_config_data.resources[0]['data']['grafana.ini'] }}" 12 | 13 | - name: Fetch grafana-datasources secret 14 | k8s_facts: 15 | api_version: v1 16 | kind: Secret 17 | namespace: openshift-monitoring 18 | name: grafana-datasources 19 | register: grafana_datasources_data 20 | 21 | - name: Set grafana-config prometheus.yaml secret fact 22 | set_fact: 23 | grafana_datasources_prometheus_yaml: "{{ grafana_datasources_data.resources[0]['data']['prometheus.yaml'] | to_nice_json(indent=2) }}" 24 | 25 | - name: Fetch grafana-proxy secret 26 | k8s_facts: 27 | api_version: v1 28 | kind: Secret 29 | namespace: openshift-monitoring 30 | name: grafana-proxy 31 | register: grafana_proxy_data 32 | 33 | - name: Set grafana-proxy session_secret 34 | set_fact: 35 | grafana_proxy_session_secret: "{{ grafana_proxy_data.resources[0]['data']['session_secret'] }}" 36 | -------------------------------------------------------------------------------- /custom-dashboards/monitoring.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: galaxy/openshift-ansible/playbooks/openshift-monitoring/config.yml 3 | vars: 4 | openshift_release: 3.11 5 | openshift_image_tag: v3.11.104 6 | openshift_cluster_monitoring_operator_install: "{{ dashboard_deploy | default(true) }}" 7 | skip_sanity_checks: True 8 | 9 | - import_playbook: galaxy/openshift-ansible/playbooks/metrics-server/config.yml 10 | vars: 11 | openshift_release: 3.11 12 | openshift_image_tag: v3.11.104 13 | openshift_metrics_server_install: "{{ dashboard_deploy | default(true) }}" 14 | skip_sanity_checks: True 15 | -------------------------------------------------------------------------------- /custom-dashboards/requirements.yml: -------------------------------------------------------------------------------- 1 | - src: https://github.com/redhat-cop/openshift-applier 2 | scm: git 3 | version: v2.1.1 4 | name: openshift-applier 5 | 6 | - src: https://github.com/openshift/openshift-ansible 7 | scm: git 8 | version: release-3.11 9 | name: openshift-ansible 10 | -------------------------------------------------------------------------------- /disconnected_registry/README.md: -------------------------------------------------------------------------------- 1 | # Scripts for Executing Disconnected Installs 2 | 3 | ## Quickstart 4 | 5 | 1. Install a registry server 6 | 7 | ```bash 8 | yum install -y docker docker-distribution python-requests firewalld 9 | 10 | systemctl enable firewalld 11 | systemctl start firewalld 12 | 13 | firewall-cmd --add-port 5000/tcp --permanent 14 | firewall-cmd --reload 15 | 16 | systemctl enable docker-distribution 17 | systemctl start docker-distribution 18 | ``` 19 | 2. We also need to set our internal registry up as an insecure registry. Add the following line to /etc/sysconfig/docker on the box from which you will sync images. 20 | 21 | a. Edit for your server and add to file 22 | 23 | ``` 24 | INSECURE_REGISTRY='--insecure-registry registry.c1-ocp.myorg.com:5000' 25 | ``` 26 | 27 | b. Or edit for your server and run 28 | 29 | ``` 30 | echo "INSECURE_REGISTRY='--insecure-registry registry.c1-ocp.myorg.com:5000'" >> /etc/sysconfig/docker 31 | ``` 32 | 33 | 34 | And then restart docker with: 35 | 36 | ``` 37 | systemctl restart docker 38 | ``` 39 | 40 | 3. Run the `docker-registry-sync.py` script: 41 | 42 | a. To sync Red Hat images to private registry. 43 | 44 | ``` 45 | ./docker-registry-sync.py --from=registry.access.redhat.com --to=:5000 --file=./docker_tags.json --openshift-version=3.10 46 | ``` 47 | 48 | b. To sync Red Hat images to ose-images.tar 49 | 50 | ``` 51 | ./docker-registry-sync.py --from=registry.access.redhat.com --to=tar --file=./docker_tags.json 52 | ``` 53 | 54 | ## Adding Images to Sync List 55 | 56 | The set(s) of images that get synced reside in the `docker_tags.json` file. These lists can be added to or edited in order to change the images that get synced. 57 | -------------------------------------------------------------------------------- /disconnected_registry/docker_tags.json: -------------------------------------------------------------------------------- 1 | { 2 | "core_components": { 3 | "openshift3": [ 4 | "apb-base", 5 | "apb-tools", 6 | "automation-broker-apb", 7 | "csi-attacher", 8 | "csi-driver-registrar", 9 | "csi-livenessprobe", 10 | "csi-provisioner", 11 | "grafana", 12 | "local-storage-provisioner", 13 | "manila-provisioner", 14 | "mariadb-apb", 15 | "mediawiki", 16 | "mediawiki-apb", 17 | "mysql-apb", 18 | "ose-ansible-service-broker", 19 | "ose-cli", 20 | "ose-cluster-autoscaler", 21 | "ose-cluster-capacity", 22 | "ose-cluster-monitoring-operator", 23 | "ose-console", 24 | "ose-configmap-reloader", 25 | "ose-control-plane", 26 | "ose-deployer", 27 | "ose-descheduler", 28 | "ose-docker-builder", 29 | "ose-docker-registry", 30 | "ose-efs-provisioner", 31 | "ose-egress-dns-proxy", 32 | "ose-egress-http-proxy", 33 | "ose-egress-router", 34 | "ose-haproxy-router", 35 | "ose-hyperkube", 36 | "ose-hypershift", 37 | "ose-keepalived-ipfailover", 38 | "ose-kube-rbac-proxy", 39 | "ose-kube-state-metrics", 40 | "ose-metrics-server", 41 | "ose-node", 42 | "ose-node-problem-detector", 43 | "ose-operator-lifecycle-manager", 44 | "ose-ovn-kubernetes", 45 | "ose-pod", 46 | "prometheus-alert-buffer", 47 | "ose-prometheus-config-reloader", 48 | "ose-prometheus-operator", 49 | "ose-recycler", 50 | "ose-service-catalog", 51 | "ose-template-service-broker", 52 | "ose-tests", 53 | "ose-web-console", 54 | "postgresql-apb", 55 | "registry-console", 56 | "snapshot-controller", 57 | "ose-efs-provisioner" 58 | ] 59 | }, 60 | "hosted_components": { 61 | "openshift3": [ 62 | "metrics-cassandra", 63 | "metrics-hawkular-metrics", 64 | "metrics-hawkular-openshift-agent", 65 | "metrics-heapster", 66 | "metrics-schema-installer", 67 | "oauth-proxy", 68 | "ose-logging-curator5", 69 | "ose-logging-elasticsearch5", 70 | "ose-logging-eventrouter", 71 | "ose-logging-fluentd", 72 | "ose-logging-kibana5", 73 | "prometheus", 74 | "prometheus-alertmanager", 75 | "prometheus-node-exporter", 76 | "jenkins-2-rhel7", 77 | "jenkins-agent-maven-35-rhel7", 78 | "jenkins-agent-nodejs-8-rhel7", 79 | "jenkins-slave-base-rhel7", 80 | "jenkins-slave-maven-rhel7", 81 | "jenkins-slave-nodejs-rhel7" 82 | ], 83 | "rhel7": [ 84 | "etcd" 85 | ], 86 | "cloudforms46": [ 87 | "cfme-openshift-postgresql", 88 | "cfme-openshift-memcached", 89 | "cfme-openshift-app-ui", 90 | "cfme-openshift-app", 91 | "cfme-openshift-embedded-ansible", 92 | "cfme-openshift-httpd", 93 | "cfme-httpd-configmap-generator" 94 | ], 95 | "rhgs3": [ 96 | "rhgs-server-rhel7", 97 | "rhgs-volmanager-rhel7", 98 | "rhgs-gluster-block-prov-rhel7", 99 | "rhgs-s3-server-rhel7" 100 | ], 101 | "jboss-amq-6": [ 102 | "amq63-openshift" 103 | ], 104 | "jboss-datagrid-7": [ 105 | "datagrid71-openshift", 106 | "datagrid71-client-openshift" 107 | ], 108 | "jboss-datavirt-6": [ 109 | "datavirt63-openshift", 110 | "datavirt63-driver-openshift" 111 | ], 112 | "jboss-decisionserver-6": [ 113 | "decisionserver64-openshift" 114 | ], 115 | "jboss-processserver-6": [ 116 | "processserver64-openshift" 117 | ], 118 | "jboss-eap-6": [ 119 | "eap64-openshift" 120 | ], 121 | "jboss-eap-7": [ 122 | "eap71-openshift" 123 | ], 124 | "jboss-webserver-3": [ 125 | "webserver31-tomcat7-openshift", 126 | "webserver31-tomcat8-openshift" 127 | ], 128 | "rhscl": [ 129 | "mongodb-32-rhel7", 130 | "mysql-57-rhel7", 131 | "perl-524-rhel7", 132 | "php-56-rhel7", 133 | "postgresql-95-rhel7", 134 | "python-35-rhel7", 135 | "ruby-24-rhel7", 136 | "nodejs-6-rhel7", 137 | "mariadb-101-rhel7" 138 | ], 139 | "redhat-openjdk-18": [ 140 | "openjdk18-openshift" 141 | ], 142 | "redhat-sso-7": [ 143 | "sso71-openshift", 144 | "sso70-openshift" 145 | ] 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /etcd-procedures/README.adoc: -------------------------------------------------------------------------------- 1 | == Different procedures to operate etcd in OCP 2 | 3 | === Project Description 4 | 5 | The aim of this repository is to generate the required tools (ansible playbooks and/or automated scripts) and documentation to operate an etcd cluster for *Openshift Container Platform* and cover the following topics 6 | 7 | - Add new member to an existing Cluster (tested in etcd coming with OCP 3.4, 3.5 and 3.6): link:docs/add_etcd_node.adoc[Document] and link:playbooks/etcd_scaleup.yml[Playbook] 8 | - Remove a member from an existing Cluster 9 | - Replace members from an existing Cluster 10 | - Disaster Recovering 11 | -------------------------------------------------------------------------------- /etcd-procedures/docs/README.adoc: -------------------------------------------------------------------------------- 1 | == How to use these documents 2 | 3 | The following documents can be used to operate an etcd Cluster. Each one contains detailed explanation, commands included step by step, about a particular procedure for an etcd Cluster. 4 | 5 | === Documents 6 | 7 | - link:./add_etcd_node.adoc[add_etcd_node.adoc]: this document can be used to *scale up* an existing etcd Cluster with new member(s). This is only aplicable to containerized etcd Clusters. 8 | -------------------------------------------------------------------------------- /etcd-procedures/docs/add_etcd_node.adoc: -------------------------------------------------------------------------------- 1 | == Introduction 2 | 3 | This procedure will cover the following points in order to scale up an existing etcd Cluster 4 | 5 | - Install etcd as a Docker container in RHEL or RHEL Atomic 6 | - Add etcd Nodes to existing etcd Cluster 7 | 8 | == Requirements 9 | 10 | - A working etcd Cluster with at least 1 Node 11 | - DNS working 12 | - Access to registry.access.redhat.com for image pulling 13 | - SSH access from the existing etcd Nodes to new etcd Nodes 14 | 15 | 16 | === Install etcd as a Docker container 17 | 18 | ==== From one of the existing etcd Nodes, get the etcd container image version we need to run 19 | 20 | Containerized etcd: 21 | # docker ps | grep etcd | awk {'print $2'} 22 | Package etcd: 23 | # etcd --version | grep etcd | awk {'print "registry.access.redhat.com/rhel7/etcd:"$3'} 24 | 25 | ==== Deploy etcd in the new Node 26 | 27 | # export ETCD_IMAGE={output image from previuos step} 28 | # iptables -A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 2379 -j ACCEPT 29 | # iptables -A OS_FIREWALL_ALLOW -p tcp -m state --state NEW -m tcp --dport 2380 -j ACCEPT 30 | 31 | WARNING: Add IPTABLES rules before COMMIT line to /etc/sysconfig/iptables 32 | 33 | # mkdir /etc/etcd 34 | # mkdir /var/lib/etcd 35 | # chcon -Rt svirt_sandbox_file_t /etc/etcd /var/lib/etcd 36 | # restorecon -Rv /etc/etcd /var/lib/etcd 37 | 38 | # src/etcd_container.service.sh $ETCD_IMAGE 39 | # systemctl daemon-reload 40 | # systemctl enable etcd_container 41 | 42 | === Add the new etcd Node to the existing Cluster 43 | 44 | ==== From one of the existing etcd Nodes do the following 45 | 46 | # cd /etc/etcd 47 | # export NEW_ETCD="{new etcd node FQDN hostname}" 48 | # export CN=$NEW_ETCD 49 | # export SAN="IP:$(dig +short A $NEW_ETCD)" 50 | # export PREFIX="./generated_certs/etcd-$CN/" 51 | # mkdir $PREFIX 52 | 53 | # openssl req -new -keyout ${PREFIX}server.key -config ca/openssl.cnf -out ${PREFIX}server.csr -reqexts etcd_v3_req -batch -nodes -subj /CN=$CN 54 | # openssl ca -name etcd_ca -config ca/openssl.cnf -out ${PREFIX}server.crt -in ${PREFIX}server.csr -extensions etcd_v3_ca_server -batch 55 | 56 | # openssl req -new -keyout ${PREFIX}peer.key -config ca/openssl.cnf -out ${PREFIX}peer.csr -reqexts etcd_v3_req -batch -nodes -subj /CN=$CN 57 | # openssl ca -name etcd_ca -config ca/openssl.cnf -out ${PREFIX}peer.crt -in ${PREFIX}peer.csr -extensions etcd_v3_ca_peer -batch 58 | 59 | # cp ca.crt ${PREFIX} 60 | # cp etcd.conf ${PREFIX} 61 | # tar -czvf ${PREFIX}${CN}.tgz -C ${PREFIX} . 62 | 63 | # scp ${PREFIX}${CN}.tgz $CN:/etc/etcd/ 64 | 65 | # export ETCD_CA_HOST="$(hostname)" 66 | # export NEW_ETCD_IP="$(dig +short A $NEW_ETCD)" 67 | # docker exec -it etcd_container etcdctl -C https://${ETCD_CA_HOST}:2379 --ca-file=/etc/etcd/ca.crt --cert-file=/etc/etcd/peer.crt --key-file=/etc/etcd/peer.key member add ${NEW_ETCD} https://${NEW_ETCD_IP}:2380 > /etc/etcd/${NEW_ETCD}.added 68 | # scp /etc/etcd/${NEW_ETCD}.added $CN:/etc/etcd/ 69 | 70 | 71 | ==== From the new etcd Node do the following 72 | 73 | # export NEW_ETCD="{new etcd node hostname}" 74 | # export NEW_ETCD_IP="$(dig +short A $NEW_ETCD)" 75 | 76 | # groupadd etcd 77 | # useradd -c "etcd user" -d /var/lib/etcd -s /sbin/nologin -g etcd etcd 78 | 79 | # tar -xf /etc/etcd/*.tgz -C /etc/etcd/ --overwrite 80 | # chown etcd:etcd /etc/etcd/* 81 | 82 | # src/etcdconf.vars.sh 83 | 84 | # systemctl start etcd_container 85 | 86 | NOTE: As the etcd container must be pulled, review service logs until etcd process start: # journalctl -u etcd_container -f 87 | 88 | ==== From the initial existing etcd Node do the following to check the status 89 | 90 | # docker exec -it etcd_container etcdctl -C https://${ETCD_CA_HOST}:2379 --ca-file=/etc/etcd/ca.crt --cert-file=/etc/etcd/peer.crt --key-file=/etc/etcd/peer.key member list 91 | # docker exec -it etcd_container etcdctl -C https://${ETCD_CA_HOST}:2379 --ca-file=/etc/etcd/ca.crt --cert-file=/etc/etcd/peer.crt --key-file=/etc/etcd/peer.key cluster-health 92 | -------------------------------------------------------------------------------- /etcd-procedures/docs/replace_etcd_nodes.adoc: -------------------------------------------------------------------------------- 1 | == Introduction 2 | 3 | This procedure will cover the following points 4 | 5 | - Replace existing etcd Cluster with new Nodes 6 | -------------------------------------------------------------------------------- /etcd-procedures/playbooks/README.adoc: -------------------------------------------------------------------------------- 1 | == How to use these playbooks 2 | 3 | In order to use these playbooks, an updated Ansible Inventory file for your OCP Cluster is needed. This Inventory file must follow the instructions provided during the https://docs.openshift.com/container-platform/latest/install_config/install/advanced_install.html#configuring-ansible[Advance Install for OCP] 4 | 5 | These playbooks will identify the etcd members to operate by adding the following to the Inventory file: 6 | 7 | - A new entry called *new_etcd* under [OSEv3:children] group 8 | - A new group called *[new_etcd]* 9 | 10 | As an example: 11 | 12 | [source,text] 13 | ---- 14 | [OSEv3:children] 15 | masters 16 | nodes 17 | etcd 18 | new_nodes 19 | new_etcd 20 | 21 | [etcd] 22 | master01.mak.lab openshift_hostname=master01.mak.lab openshift_public_hostname=master01.mak.lab 23 | master02.mak.lab openshift_hostname=master02.mak.lab openshift_public_hostname=master02.mak.lab 24 | master03.mak.lab openshift_hostname=master03.mak.lab openshift_public_hostname=master03.mak.lab 25 | 26 | [new_etcd] 27 | newmaster01.mak.lab openshift_hostname=newmaster01.mak.lab openshift_public_hostname=newmaster01.mak.lab 28 | ---- 29 | 30 | === Playbooks 31 | 32 | * link:playbooks/etcd_scaleup.yml[etcd_scaleup.yml]: automatically scales up an existing etcd cluster with a new member included in [new_etcd] group. This is only aplicable to containerized etcd Clusters. *Once the Cluster has been scaled up, the information about the new member must be added manually to 'etcdClientInfo' section in 'master-config.yaml' file on every Master in order to use this Node as well* 33 | -------------------------------------------------------------------------------- /etcd-procedures/playbooks/templates/etcd_container.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=The Etcd Server container 3 | After=docker.service 4 | Requires=docker.service 5 | PartOf=docker.service 6 | 7 | [Service] 8 | EnvironmentFile=/etc/etcd/etcd.conf 9 | ExecStartPre=-/usr/bin/docker rm -f etcd_container 10 | ExecStart=/usr/bin/docker run --name etcd_container --rm -v /var/lib/etcd:/var/lib/etcd:z -v /etc/etcd:/etc/etcd:z --env-file=/etc/etcd/etcd.conf --net=host --entrypoint=/usr/bin/etcd {{ etcd_version }} 11 | ExecStop=/usr/bin/docker stop etcd_container 12 | SyslogIdentifier=etcd_container 13 | Restart=always 14 | RestartSec=5s 15 | 16 | [Install] 17 | WantedBy=docker.service 18 | -------------------------------------------------------------------------------- /etcd-procedures/src/etcd_container.service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DOCKER_IMAGE=$1 4 | 5 | usage(){ 6 | echo "Usage: etcd_container.service.sh {ETCD_IMAGE}" 7 | exit 2 8 | } 9 | 10 | if [[ "$1" == "" ]];then 11 | usage 12 | else 13 | cat > /etc/systemd/system/etcd_container.service < oc get project -o yaml 9 | ``` 10 | 11 | Look in the `status` of the output to see what needs to be patched. In the example below we'll need to patch `cephclusters.ceph.rook.io` 12 | 13 | ``` 14 | ... 15 | status: 16 | conditions: 17 | - lastTransitionTime: "2020-03-26T10:01:05Z" 18 | message: 'Some content in the namespace has finalizers remaining: cephcluster.ceph.rook.io 19 | in 1 resource instances' 20 | reason: SomeFinalizersRemain 21 | status: "True" 22 | type: NamespaceFinalizersRemaining 23 | ``` 24 | 25 | ### Patch Command to remove finalizer from an object (non-project/namespace) 26 | 27 | ```bash 28 | > oc patch -p '{"metadata":{"finalizers": []}}' --type=merge 29 | ``` 30 | 31 | ### Patch project/namespace script using curl and oc client or certs 32 | 33 | In other cases, however, the project/namespace seems stuck by having a finalizer on the project itself. In that case, the script found here - `fix-terminating.sh` can be utilized to patch the namespace utilizing curl. Note that this script may need to be executed where the necessary certs exists (obtained on the OpenShift master node). 34 | 35 | with certs 36 | 37 | ```bash 38 | > bash fix-terminating.sh 39 | ``` 40 | 41 | or without certs and with the oc client 42 | 43 | ```bash 44 | > bash fix-terminating.sh -c 45 | ``` 46 | -------------------------------------------------------------------------------- /fix-project-terminating/fix-terminating.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | useoc=false 4 | 5 | usage() 6 | { 7 | printf "\nfix-terminating.sh\n\n" 8 | echo "examples" 9 | echo " # Use the oc client to fix my-project 10 | echo " bash fix-terminating.sh -c my-project 11 | echo " # Use certs to fix my-project 12 | echo " bash fix-terminating.sh /path/to/ca.crt /path/to/admin.crt /path/to/admin.key https://cluster:443 my-project 13 | printf "\n" 14 | } 15 | 16 | while (( "$#" )); do 17 | case $1 in 18 | -c | --use-oc) useoc=true 19 | shift 20 | namespace=$1 21 | ;; 22 | -h | --help) usage 23 | exit 1 24 | ;; 25 | esac 26 | shift 27 | done 28 | 29 | data="{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"name\":\"${namespace}\"},\"spec\":{\"finalizers\":[]}}" 30 | 31 | if [ "true" == "$useoc" ] 32 | then 33 | printf "\nusing oc client. Ensure you are logged in\n terminating $namespace \n" 34 | token=$(oc whoami -t) 35 | clusterurl=$(oc whoami --show-server) 36 | 37 | curl -X PUT -H "Content-Type: application/json" \ 38 | --data ${data} \ 39 | -H "Authorization: Bearer $token" \ 40 | ${clusterurl}/api/v1/namespaces/${namespace}/finalize 41 | else 42 | clusterurl=$1 43 | namespace=$2 44 | cacert=${3:-/etc/origin/master/ca.crt} 45 | cert=${4:-/etc/origin/master/admin.crt} 46 | key=${5:-/etc/origin/master/admin.key} 47 | 48 | printf "\nusing certs\n" 49 | curl -X PUT -H "Content-Type: application/json" \ 50 | --data ${data} \ 51 | -cacert ${cacert} \ 52 | --cert ${cert} \ 53 | --key ${key} \ 54 | ${clusterurl}/api/v1/namespaces/${namespace}/finalize 55 | fi 56 | -------------------------------------------------------------------------------- /health_check/README.md: -------------------------------------------------------------------------------- 1 | # OpenShift Admin Bash Scripts 2 | 3 | This folder contains a collection of scripts to perform additional health checks on OpenShift logging and metrics. 4 | 5 | These scripts can be run natively on the superior Linux OS, or MacOS. Windows requires using a bash emulator (GitBash). 6 | 7 | ## Script Overview 8 | 9 | The following table outlines the scripts and purpose. 10 | 11 | Script Name | Description | Notes 12 | --- | --- | --- 13 | `elasticsearch-health-check-ocp33.sh` | Returns the health of the integrated ES cluster. Only works on OCP =< 3.3 | Parameters and auth required, check script doc. 14 | `elasticsearch-health-check-ocp34.sh` | Returns the health of the integrated ES cluster. Only works on OCP >= 3.4 | Parameters and auth required, check script doc, and will not work on GitBash (Windows). 15 | `metrics-health-check.sh` | Checks if the metrics stack is up and running. | Parameters and auth required, check script doc. 16 | -------------------------------------------------------------------------------- /health_check/elasticsearch-health-check-ocp33.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Date: 04/18/2017 4 | # 5 | # Script Purpose: Returns the current status of the elasticsearch cluster. 6 | # This script will only work on OCP 3.3 or earlier 7 | # Version : 1.0 8 | # 9 | 10 | if [ "$#" -lt 1 ]; then 11 | 12 | echo " 13 | Missing input parameters. 14 | 15 | Required Kibana Public URL 16 | 17 | Usage: elasticsearch-health-check-33.sh [Kibana Public URL] [OCP Token] 18 | 19 | Example: ./elasticsearch-health-check-33.sh https://kibana.moos.local djafksdljasdkl;fjads;klfjakls;dfj;kladsjfldksa; 20 | 21 | User token can be retrieved by running 'oc whoami -t' 22 | " 23 | exit 1 24 | fi 25 | 26 | public_url=$1 27 | 28 | #if user supplied token then use it, else get it from CLI 29 | if [ "$#" -eq 2 ]; then 30 | token=$2 31 | else 32 | token=`oc whoami -t` 33 | fi 34 | 35 | #Check the passed in token and validate. If failure occurs, stop. 36 | whoami=`oc whoami -t` 37 | if [[ "$whoami" != "$token" ]]; then 38 | echo "================================== 39 | Script failed token validation! 40 | Check the OCP URL and token. 41 | ==================================" 42 | exit 1 43 | fi 44 | 45 | cluster_status_string=`curl -s --insecure -H "Authorization: Bearer ${token}" -XGET "${public_url}/elasticsearch/_cat/health"` 46 | if [ -z "$cluster_status_string" ]; then 47 | 48 | echo "===================================================================== 49 | cURL returned: ${cluster_status_string}. 50 | =====================================================================" 51 | exit 1 52 | 53 | fi 54 | 55 | cluster_color=`echo -n "${cluster_status_string}" | awk '{print $4}' ` 56 | echo "Cluster Color: ${cluster_color}" 57 | 58 | if [ "$cluster_color" = "green" ]; then 59 | exit 0 60 | else 61 | exit 1 62 | fi 63 | -------------------------------------------------------------------------------- /health_check/elasticsearch-health-check-ocp34.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Date: 04/25/2017 4 | # 5 | # Script Purpose: Returns the current status of the elasticsearch cluster. 6 | # This script will only work on OCP 3.4 or later 7 | # Version : 1.0 8 | # 9 | 10 | if [ "$#" -lt 1 ]; then 11 | 12 | echo " 13 | Missing input parameters. 14 | 15 | Required Kibana Public URL 16 | 17 | Usage: elasticsearch-health-check-34.sh [Kibana Public URL] [OCP Token] 18 | 19 | Example: ./elasticsearch-health-check-34.sh https://kibana.moos.local djafksdljasdkl;fjads;klfjakls;dfj;kladsjfldksa; 20 | 21 | User token can be retrieved by running 'oc whoami -t' 22 | " 23 | exit 1 24 | fi 25 | 26 | public_url=$1 27 | 28 | #if user supplied token then use it, else get it from CLI 29 | if [ "$#" -eq 2 ]; then 30 | token=$2 31 | else 32 | token=`oc whoami -t` 33 | fi 34 | 35 | cluster_color=`curl -sk "${public_url}/api/status" \ 36 | -H "Accept-Encoding: gzip, deflate, sdch, br" \ 37 | -H "Accept-Language: en-US,en;q=0.8" \ 38 | -H "Accept: application/json" \ 39 | -H "User-Agent: health-checker" \ 40 | -H "Authorization: Bearer ${token}" \ 41 | -H "Connection: keep-alive" --compressed | \ 42 | python -c 'import sys, json; print json.dumps(json.load(sys.stdin)["status"]["overall"]["state"], sort_keys=True, indent=4)'` 43 | 44 | 45 | echo "Cluster is ${cluster_color}" 46 | 47 | if [ "$cluster_color" = "\"red\"" ]; then 48 | 49 | exit 1 50 | 51 | fi 52 | 53 | exit 0 54 | -------------------------------------------------------------------------------- /health_check/metrics-health-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Date: 04/18/2017 4 | # 5 | # Script Purpose: Perform a real health check of the metrics service. 6 | # Version : 1.0 7 | # 8 | 9 | 10 | if [ "$#" -lt 1 ]; then 11 | echo " 12 | 13 | Description: This script will actually pull Hawkular stats for an 14 | OpenShift system pod. If the call does not return stats, then we 15 | safely assume the service is down. 16 | 17 | Usage: metrics-health-check.sh [Public URL] [OCP Token] 18 | 19 | Example: metrics-health-check.sh https://hawkular-metrics-openshift-infra.ose34.moos.local asdfsdfasdf8asdifa90sd890fas8df0 20 | 21 | If token is not provided, the script will execute oc whoami -t to get token. 22 | 23 | Requires: cURL, and oc tools if token is not specified. 24 | 25 | " 26 | 27 | exit 1 28 | fi 29 | 30 | public_url=$1 31 | 32 | #if user supplied token then use it, else get it from CLI 33 | if [ "$#" -eq 2 ]; then 34 | token=$2 35 | else 36 | token=`oc whoami -t` 37 | fi 38 | 39 | #Globals 40 | #This is the pod for which metrics will be quried. 41 | #If metrics exist, this pod will exist. 42 | readonly POD_LABEL="hawkular-cassandra" 43 | #Metrics OCP Namespace, this is constant 44 | readonly POD_NAMESPACE="openshift-infra" 45 | #Two connection parameters, these may need to be tweaked 46 | #Depending on what defines a dead service. 47 | readonly MAX_TIME="30" 48 | readonly CONNECT_TIMEOUT="1" 49 | 50 | #cURL parms and headers 51 | #parms passed to cURL 52 | curl_parms="-sk --max-time ${MAX_TIME} --connect-timeout ${CONNECT_TIMEOUT}" 53 | #URL to cURL 54 | url="${public_url}/hawkular/metrics/counters/data?stacked=true&tags=descriptor_name:cpu%2Frx,type:pod,labels:%5E%28%3F%3D.%2A%5Cbmetrics-infra%3A${POD_LABEL}%5Cb%29.%2A%24&bucketDuration=60000ms&start=-2mn" 55 | #Required, Hawkular-Tenant header. 56 | tenant_header="Hawkular-Tenant: ${POD_NAMESPACE}" 57 | #Required, auth token. 58 | token_header="Authorization: Bearer ${token}" 59 | #Not required, just included 60 | accept_header="Accept: application/json" 61 | #Not required, but included for logging. 62 | agent_header="User-Agent: metrics-health-check.sh" 63 | #Required to get cURL to return just the HTTP status code, i.e. 200 64 | output="-o /dev/null" 65 | http_reg="%{http_code}" 66 | 67 | #Uncomment to debug 68 | #echo "cURL url: " ${url} 69 | #echo "tenant: ${tenant_header}" 70 | #echo "token: ${token_header}" 71 | #echo "accept: ${accept_header}" 72 | #echo "agent: ${agent_header}" 73 | 74 | #Perform the cURL, and store the results. 75 | #Anything non 200 specifies an error. 76 | http_resp_code=`curl ${curl_parms} "${url}" \ 77 | -H "${token_header}" \ 78 | -H "${tenant_header}" \ 79 | -H "${accept_header}" \ 80 | -H "${agent_header}" \ 81 | ${output} -w "${http_reg}" ` 82 | 83 | #Echo the response 84 | echo "Server Responded with HTTP Code: $http_resp_code" 85 | 86 | #Perform check and return 0 or -1 87 | if [ "$http_resp_code" = "200" ]; then 88 | echo "Ok" 89 | exit 0 90 | else 91 | echo "Error" 92 | exit 1 93 | fi 94 | -------------------------------------------------------------------------------- /image-management/README.md: -------------------------------------------------------------------------------- 1 | # Tooling for Setting up a Proper Image Management Workflow in OpenShift 2 | 3 | The purpose of this project is to be used as a bootstrap for an custom base image build workflow. 4 | 5 | ## Architecture 6 | 7 | In order to simulate a real world enterprise use case, we define two different _actors_ with different levels of ownership of the process. 8 | 9 | - An _Operations Engineer_ in this case is someone who has admin access to the OpenShift cluster (e.g. a `cluster-admin`) 10 | - An _Image Builder_ is a generic term for the actor in the organization who has ownership of Base Image Builds. This role would typically have restricted rights to the cluster. In this case, we'll say that an _Image Builder_ has `admin` rights on certain projects related to image building. 11 | 12 | The projects here contain the following structures: 13 | 14 | - An [OpenShift Applier](https://github.com/redhat-cop/openshift-applier) inventory for the Operations portion of the automation 15 | - An [OpenShift Applier](https://github.com/redhat-cop/openshift-applier) inventory for the Image Builder portion of the automation. 16 | - Two sample base image builds 17 | - [myorg-openjdk18](./myorg-openjdk18) - A sample showing how an org might customize the Red Hat jdk image to insert their corporate CAs 18 | - [myorg-eap-oraclejdk](./myorg-eap-oraclejdk) - A sample showing an org that has replaced OpenJDK with OracleJDK in the Red Hat image. 19 | 20 | The image build workflow has the following components: 21 | 22 | - An `image-builds` project which is where the custom images will be built 23 | - A `BuildConfig` for each image which defines how the images will be built 24 | - An `ImageStream` which will get created in the `openshift` namespace by default 25 | - A `RoleBinding` to allow the _builder_ `ServiceAccount` to push images into another namespace (e.g. `openshift`) 26 | 27 | The workflow we will deploy looks something like this: 28 | 29 | ![Image Build Workflow](img/workflow.png) 30 | 31 | In this workflow, we use the `image-builds` namespace as the home for running builds. This namespace would be private, accessible only to our _Image Builder_ actor. However, both the Red Hat base images, and the customized `myorg-` images will be hosted in the `openshift` namespace, thus being readable from all other namespaces, and making the resulting content more accessible to the organization. The `buildConfigs` also utilize [_image change triggers_](https://docs.openshift.com/container-platform/3.11/dev_guide/builds/triggering_builds.html#image-change-triggers) to automatically rebuild any time a change is detected in an upstream image. In this way, we ensure that all of our images are always up to date with the latest parent images. 32 | 33 | ## Deploying the Workflow 34 | 35 | | WARNING: The manifests included in this inventory apply cluster-level role bindings and patch _ImageStreams_ in your `openshift` namespace to enable auto-syncing. | 36 | | --- | 37 | 38 | This project uses [Ansible](https://www.ansible.com/) and [Ansible Galaxy](http://docs.ansible.com/ansible/latest/reference_appendices/galaxy.html) to deploy the workflow. 39 | 40 | First thing to do is to install the prerequisites: 41 | ``` 42 | cd image-management/ 43 | ansible-galaxy install -r requirements.yml -p galaxy 44 | ``` 45 | 46 | Next, we can run the _Operator_ inventory to create the projects and map roles 47 | ``` 48 | ansible-playbook -i applier/operator-inventory/ galaxy/openshift-applier/playbooks/openshift-cluster-seed.yml 49 | ``` 50 | 51 | Finally, we can run the _Image Builder_ inventory, which will deploy the Build Configs, and kick off the image builds. 52 | ``` 53 | ansible-playbook -i applier/image-builder-inventory/ galaxy/openshift-applier/playbooks/openshift-cluster-seed.yml 54 | ``` 55 | 56 | Once done, you can track the status of the builds using `oc status -n image-builds` or by looking in the Web Console, under _Builds_. 57 | -------------------------------------------------------------------------------- /image-management/applier/image-builder-inventory/group_vars/seed-hosts.yml: -------------------------------------------------------------------------------- 1 | openshift_cluster_content: 2 | - object: Patch ImageStreams to enable automatic updates 3 | content: 4 | - name: OpenJDK 5 | file: "{{ inventory_dir }}/../manifests/openjdk18-openshift.yml" 6 | params: "{{ inventory_dir }}/../patches/openjdk18-openshift.yml" 7 | action: patch 8 | namespace: openshift 9 | - name: JBoss EAP 10 | file: "{{ inventory_dir }}/../manifests/jboss-eap70-openshift.yml" 11 | params: "{{ inventory_dir }}/../patches/jboss-eap70-openshift.yml" 12 | action: patch 13 | namespace: openshift 14 | - object: Set up Image Builds 15 | content: 16 | - name: Image buildConfigs 17 | template: "{{ inventory_dir }}/../templates/docker-build-git.yml" 18 | params: "{{ inventory_dir }}/../params/image-builds/" 19 | -------------------------------------------------------------------------------- /image-management/applier/image-builder-inventory/hosts: -------------------------------------------------------------------------------- 1 | [seed-hosts] 2 | localhost ansible_connection=local 3 | -------------------------------------------------------------------------------- /image-management/applier/manifests/jboss-eap70-openshift.yml: -------------------------------------------------------------------------------- 1 | apiVersion: image.openshift.io/v1 2 | kind: ImageStream 3 | metadata: 4 | annotations: 5 | openshift.io/display-name: Red Hat JBoss EAP 7.0 6 | openshift.io/image.dockerRepositoryCheck: 2019-02-14T06:50:28Z 7 | openshift.io/provider-display-name: Red Hat, Inc. 8 | version: 1.4.14 9 | creationTimestamp: null 10 | generation: 2 11 | name: jboss-eap70-openshift 12 | spec: 13 | lookupPolicy: 14 | local: false 15 | tags: 16 | -------------------------------------------------------------------------------- /image-management/applier/manifests/openjdk18-openshift.yml: -------------------------------------------------------------------------------- 1 | apiVersion: image.openshift.io/v1 2 | kind: ImageStream 3 | metadata: 4 | annotations: 5 | openshift.io/display-name: Red Hat OpenJDK 8 6 | openshift.io/image.dockerRepositoryCheck: 2019-02-14T06:50:40Z 7 | openshift.io/provider-display-name: Red Hat, Inc. 8 | version: 1.4.14 9 | creationTimestamp: null 10 | generation: 2 11 | name: redhat-openjdk18-openshift 12 | spec: 13 | lookupPolicy: 14 | local: false 15 | tags: 16 | - annotations: 17 | description: Build and run Java applications using Maven and OpenJDK 8. 18 | iconClass: icon-rh-openjdk 19 | openshift.io/display-name: Red Hat OpenJDK 8 20 | sampleContextDir: undertow-servlet 21 | sampleRepo: https://github.com/jboss-openshift/openshift-quickstarts 22 | supports: java:8 23 | tags: builder,java,openjdk,hidden 24 | version: "1.0" 25 | from: 26 | kind: DockerImage 27 | name: docker-registry.default.svc:5000/openshift/redhat-openjdk18-openshift:1.0 28 | generation: 2 29 | importPolicy: {} 30 | name: "1.0" 31 | referencePolicy: 32 | type: Local 33 | - annotations: 34 | description: Build and run Java applications using Maven and OpenJDK 8. 35 | iconClass: icon-rh-openjdk 36 | openshift.io/display-name: Red Hat OpenJDK 8 37 | sampleContextDir: undertow-servlet 38 | sampleRepo: https://github.com/jboss-openshift/openshift-quickstarts 39 | supports: java:8 40 | tags: builder,java,openjdk,hidden 41 | version: "1.1" 42 | from: 43 | kind: DockerImage 44 | name: docker-registry.default.svc:5000/openshift/redhat-openjdk18-openshift:1.1 45 | generation: 2 46 | importPolicy: {} 47 | name: "1.1" 48 | referencePolicy: 49 | type: Local 50 | - annotations: 51 | description: Build and run Java applications using Maven and OpenJDK 8. 52 | iconClass: icon-rh-openjdk 53 | openshift.io/display-name: Red Hat OpenJDK 8 54 | sampleContextDir: undertow-servlet 55 | sampleRepo: https://github.com/jboss-openshift/openshift-quickstarts 56 | supports: java:8 57 | tags: builder,java,openjdk,hidden 58 | version: "1.2" 59 | from: 60 | kind: DockerImage 61 | name: docker-registry.default.svc:5000/openshift/redhat-openjdk18-openshift:1.2 62 | generation: 2 63 | importPolicy: {} 64 | name: "1.2" 65 | referencePolicy: 66 | type: Local 67 | - annotations: 68 | description: Build and run Java applications using Maven and OpenJDK 8. 69 | iconClass: icon-rh-openjdk 70 | openshift.io/display-name: Red Hat OpenJDK 8 71 | sampleContextDir: undertow-servlet 72 | sampleRepo: https://github.com/jboss-openshift/openshift-quickstarts 73 | supports: java:8 74 | tags: builder,java,openjdk,hidden 75 | version: "1.3" 76 | from: 77 | kind: DockerImage 78 | name: docker-registry.default.svc:5000/openshift/redhat-openjdk18-openshift:1.3 79 | generation: 2 80 | importPolicy: {} 81 | name: "1.3" 82 | referencePolicy: 83 | type: Local 84 | - annotations: 85 | description: Build and run Java applications using Maven and OpenJDK 8. 86 | iconClass: icon-rh-openjdk 87 | openshift.io/display-name: Red Hat OpenJDK 8 88 | sampleContextDir: undertow-servlet 89 | sampleRepo: https://github.com/jboss-openshift/openshift-quickstarts 90 | supports: java:8 91 | tags: builder,java,openjdk,hidden 92 | version: "1.4" 93 | from: 94 | kind: DockerImage 95 | name: docker-registry.default.svc:5000/openshift/redhat-openjdk18-openshift:1.4 96 | generation: 2 97 | importPolicy: {} 98 | name: "1.4" 99 | referencePolicy: 100 | type: Local 101 | status: 102 | dockerImageRepository: "" 103 | -------------------------------------------------------------------------------- /image-management/applier/operator-inventory/group_vars/seed-hosts.yml: -------------------------------------------------------------------------------- 1 | openshift_cluster_content: 2 | - object: Set up Image Builds 3 | content: 4 | - name: Create Image Builds Project 5 | template: "{{ inventory_dir }}/../templates/project.yml" 6 | params: "{{ inventory_dir }}/../params/image-management-project" 7 | action: create 8 | - name: Create RoleBinding to images project 9 | template: "{{ inventory_dir }}/../templates/rolebinding.yml" 10 | params: "{{ inventory_dir}}/../params/builder-rolebinding" 11 | -------------------------------------------------------------------------------- /image-management/applier/operator-inventory/hosts: -------------------------------------------------------------------------------- 1 | [seed-hosts] 2 | localhost ansible_connection=local 3 | -------------------------------------------------------------------------------- /image-management/applier/params/builder-rolebinding: -------------------------------------------------------------------------------- 1 | IMAGE_NAMESPACE=openshift 2 | BUILDER_NAMESPACE=image-builds 3 | -------------------------------------------------------------------------------- /image-management/applier/params/image-builds/myorg-eap-oraclejdk: -------------------------------------------------------------------------------- 1 | NAMESPACE=image-builds 2 | IMAGE_NAME=myorg-eap-oraclejdk 3 | BASE_IMAGE_STREAM=jboss-eap70-openshift:1.7 4 | OUTPUT_NAMESPACE=openshift 5 | GIT_URL=https://github.com/redhat-cop/openshift-toolkit 6 | GIT_REF=master 7 | CONTEXT_DIR=image-management/myorg-eap-oraclejdk 8 | -------------------------------------------------------------------------------- /image-management/applier/params/image-builds/myorg-openjdk18: -------------------------------------------------------------------------------- 1 | NAMESPACE=image-builds 2 | IMAGE_NAME=myorg-openjdk18 3 | BASE_IMAGE_STREAM=redhat-openjdk18-openshift:1.4 4 | OUTPUT_NAMESPACE=openshift 5 | GIT_URL=https://github.com/redhat-cop/openshift-toolkit 6 | GIT_REF=master 7 | CONTEXT_DIR=image-management/myorg-openjdk18 8 | -------------------------------------------------------------------------------- /image-management/applier/params/image-management-project: -------------------------------------------------------------------------------- 1 | PROJECT_NAME=image-builds 2 | PROJECT_DISPLAY=Image Management 3 | PROJECT_DESCRIPTION=This is where organizational base images are built and managed 4 | -------------------------------------------------------------------------------- /image-management/applier/patches/jboss-eap70-openshift.yml: -------------------------------------------------------------------------------- 1 | spec: 2 | tags: 3 | - name: "1.7" 4 | importPolicy: 5 | scheduled: true 6 | -------------------------------------------------------------------------------- /image-management/applier/patches/openjdk18-openshift.yml: -------------------------------------------------------------------------------- 1 | spec: 2 | tags: 3 | - name: "1.4" 4 | importPolicy: 5 | scheduled: true 6 | -------------------------------------------------------------------------------- /image-management/applier/templates/docker-build-git.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Template 3 | metadata: 4 | creationTimestamp: null 5 | name: docker-build 6 | objects: 7 | - apiVersion: v1 8 | kind: BuildConfig 9 | metadata: 10 | annotations: 11 | labels: 12 | build: ${IMAGE_NAME} 13 | name: ${IMAGE_NAME} 14 | namespace: ${NAMESPACE} 15 | spec: 16 | failedBuildsHistoryLimit: 5 17 | nodeSelector: null 18 | output: 19 | to: 20 | kind: ImageStreamTag 21 | name: ${IMAGE_NAME}:latest 22 | namespace: ${OUTPUT_NAMESPACE} 23 | postCommit: {} 24 | resources: {} 25 | runPolicy: Serial 26 | source: 27 | contextDir: ${CONTEXT_DIR} 28 | git: 29 | ref: ${GIT_REF} 30 | uri: ${GIT_URL} 31 | type: Git 32 | strategy: 33 | sourceStrategy: 34 | from: 35 | kind: ImageStreamTag 36 | name: ${BASE_IMAGE_STREAM} 37 | namespace: openshift 38 | type: Source 39 | successfulBuildsHistoryLimit: 5 40 | triggers: 41 | - github: 42 | secret: ${GITHUB_WEBHOOK_SECRET} 43 | type: GitHub 44 | - generic: 45 | secret: ${GENERIC_WEBHOOK_SECRET} 46 | type: Generic 47 | - type: ConfigChange 48 | - imageChange: {} 49 | type: ImageChange 50 | status: 51 | lastVersion: 0 52 | - apiVersion: v1 53 | kind: ImageStream 54 | metadata: 55 | annotations: 56 | openshift.io/display-name: ${IS_DISPLAY_NAME} 57 | generation: 1 58 | labels: 59 | build: ${IMAGE_NAME} 60 | name: ${IMAGE_NAME} 61 | namespace: ${OUTPUT_NAMESPACE} 62 | parameters: 63 | - name: IMAGE_NAME 64 | - name: BASE_IMAGE_STREAM 65 | - name: NAMESPACE 66 | - name: OUTPUT_NAMESPACE 67 | value: openshift 68 | - name: CONTEXT_DIR 69 | value: 70 | - name: GIT_URL 71 | value: https://github.com/redhat-cop/openshift-toolkit 72 | - name: GIT_REF 73 | value: master 74 | - name: GITHUB_WEBHOOK_SECRET 75 | value: webhook123 76 | - name: GENERIC_WEBHOOK_SECRET 77 | value: webhook123 78 | - name: IS_DISPLAY_NAME 79 | value: MyOrg.com Sample Image 80 | -------------------------------------------------------------------------------- /image-management/applier/templates/project.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Template 3 | labels: 4 | template: project 5 | metadata: 6 | annotations: 7 | name: project 8 | objects: 9 | - kind: ProjectRequest 10 | apiVersion: v1 11 | metadata: 12 | name: ${PROJECT_NAME} 13 | creationTimestam: null 14 | displayName: ${PROJECT_DISPLAY} 15 | description: ${PROJECT_DESCRIPTION} 16 | parameters: 17 | - description: The Name of the Project 18 | name: PROJECT_NAME 19 | required: true 20 | - description: The Pretty Name for the Project 21 | name: PROJECT_DISPLAY 22 | required: false 23 | - description: An optional project description 24 | name: PROJECT_DESCRIPTION 25 | required: false 26 | -------------------------------------------------------------------------------- /image-management/applier/templates/rolebinding.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Template 3 | labels: 4 | template: builder-rolebinding 5 | metadata: 6 | annotations: 7 | name: builder-rolebinding 8 | objects: 9 | - apiVersion: v1 10 | groupNames: null 11 | kind: RoleBinding 12 | metadata: 13 | creationTimestamp: null 14 | name: myorg-image-builders 15 | namespace: ${IMAGE_NAMESPACE} 16 | roleRef: 17 | name: system:image-builder 18 | subjects: 19 | - kind: ServiceAccount 20 | name: builder 21 | namespace: ${BUILDER_NAMESPACE} 22 | userNames: 23 | - system:serviceaccount:${BUILDER_NAMESPACE}:builder 24 | parameters: 25 | - name: IMAGE_NAMESPACE 26 | value: openshift 27 | - name: BUILDER_NAMESPACE 28 | value: image-builds 29 | -------------------------------------------------------------------------------- /image-management/img/workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-cop/openshift-toolkit/16bf5b054fd5bdfb5018c1f4e06b80470fde716f/image-management/img/workflow.png -------------------------------------------------------------------------------- /image-management/myorg-eap-oraclejdk/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jboss-eap-7/jboss-eap70-openshift:1.5 2 | USER root 3 | 4 | ENV JAVA_HOME="/usr/lib/jvm/jdk1.8.0" \ 5 | JAVA_VENDOR="Oracle" \ 6 | JAVA_VERSION="1.8.0" 7 | 8 | RUN INSTALL_PKGS="java-1.8.0-oracle-devel" && \ 9 | yum install -y $INSTALL_PKGS && \ 10 | yum clean all 11 | 12 | USER 185 13 | -------------------------------------------------------------------------------- /image-management/myorg-openjdk18/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM redhat-openjdk-18/openjdk18-openshift:1.2 2 | 3 | USER root 4 | 5 | RUN update-ca-trust force-enable 6 | ADD certs/myorg*.pem /etc/pki/ca-trust/source/anchors/ 7 | 8 | RUN chmod a+r /etc/pki/ca-trust/source/anchors/myorg*.pem; \ 9 | update-ca-trust 10 | -------------------------------------------------------------------------------- /image-management/myorg-openjdk18/certs/myorg-rootCA.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIF6DCCA9CgAwIBAgIJANmV6i3M+lcyMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYD 3 | VQQGEwJVUzEXMBUGA1UECAwOTm9ydGggQ2Fyb2xpbmExEDAOBgNVBAcMB1JhbGVp 4 | Z2gxFDASBgNVBAoMC015IEZha2UgT3JnMQswCQYDVQQLDAJJVDENMAsGA1UEAwwE 5 | cm9vdDEcMBoGCSqGSIb3DQEJARYNYm9iQG15b3JnLmNvbTAeFw0xODAzMDExOTQ5 6 | MTRaFw0yMDEyMTkxOTQ5MTRaMIGIMQswCQYDVQQGEwJVUzEXMBUGA1UECAwOTm9y 7 | dGggQ2Fyb2xpbmExEDAOBgNVBAcMB1JhbGVpZ2gxFDASBgNVBAoMC015IEZha2Ug 8 | T3JnMQswCQYDVQQLDAJJVDENMAsGA1UEAwwEcm9vdDEcMBoGCSqGSIb3DQEJARYN 9 | Ym9iQG15b3JnLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMY7 10 | XV9F5q1B3HgGE4yfESmDxiifADsdz0QZpDYEjzXdat2zrCZo5/v0UvD6+qNnECin 11 | wPjEoLYu1GemRC24PxyWAh+W6wmj+WX73195hBQLr9XC585QgJRxIGa4fR/61JIn 12 | sFM2IxmtEUVI0x1rej0IuZfST5BgZ7YW6OuKHkid1FDxz6WKHwq8GQZG3m01F5YG 13 | XcSZzQ60n5oiX4ygQsdGc+MBb6ZJz0+Gzv+bG8gId0voSRSWItaT/rH/2w+jhPpD 14 | ci/pbdOfoyvZBOVNRHovdzf0dqpyFPsGZzUkrGzdEFQQBX22jbLZhZClDinQFJ0N 15 | e5LR/hTyR1gQEMnvbp8lsA58HRVEGU1z5HjSqYe9ySEvnl5PIu4IPwEAEf0VVlYg 16 | qcGF+SUwG9yGDINoqBRXMYcy52oqZ5HnhW0/IO4zEDgxcENXdurToTsvopdWX+yv 17 | P/233v6f07flsoD8f/l7uf2La66UZvuLZgSFHqNeZMnBHkJ7P0VZsamV/Z4H2Q3j 18 | uuMbSSNjzu618AKvp5n08ZxJpzauLyxL0gdEFo8UeMbWrr7DUsjuSicD2SAE2VEe 19 | XV6zEZmr3usE9u1QI0TZHYMHgWU9z2RxpiH248QE45U+5M9+rcsZQTpKbJR7TTh6 20 | 0BfWDPl/UxnD4qsccx9+Z8oWhbW6Zk6MMPBVP0gNAgMBAAGjUzBRMB0GA1UdDgQW 21 | BBT97twZS7m7ZPmxO0dlMFebMpATQzAfBgNVHSMEGDAWgBT97twZS7m7ZPmxO0dl 22 | MFebMpATQzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBqz3/0 23 | yDW/uqYXEgxbM6zZ5amyN3RwXNkzDjOMPatJz64fnP9RxH8JwajvZUQzuH7p2EVM 24 | AqS4N6krYKc7R8y7cw7phFNDd2PWQfnu0tDW9c4QEwyDhZoZNccyU2I5l1ALVW6J 25 | Vlock/Yh0XZOnFVyu5xpxeaUb6o1GOflT2cQP4E5n41eEzp42ZXtcxmj2mhAlO8d 26 | FgMqnfv5+4OCWIMZg+XrXfuBwMDeGFKQICg0iqJROg/uMwhEl53rIfMtRhXTeaph 27 | 3ov05h6QD4GJrjsvEKEcziAoe0W2yNOGkszg1xfP6ni0NP/iCFggB/4xjoy7IQ5G 28 | hgEDcgTs7uZavO1HQBj9jgvfWf8LqMV3okFxWI32MS824WlHr8S6acwIjKoVOjJ2 29 | RlRRImMoZhNbWNgdRCX+bAks/PvSLtvsGvlXfC4JrCqJn6Uxxy/wxRpWySvjK/NH 30 | ZFb5GVh7yb+caQ76bQK6Gu3qjskoCZyXfvKI/ofnjuzn6QDcFPCjueBkzpHLEhEK 31 | QpJ5jZRY2Zc9XToQRai5qQNex0wDo+WvtvZOM479KCVPSDNMGFjwBwr3+gtM2Bsd 32 | VYzQHm2azSllqB6hui19UDOjcVxuFVv5nIDdA6d8huIrqyJ0lyNcfbHAR17ghMYo 33 | T7ZlVQ+lasf+FMC9+1eGTOnXByNgfiN1F56n+g== 34 | -----END CERTIFICATE----- 35 | -------------------------------------------------------------------------------- /image-management/requirements.yml: -------------------------------------------------------------------------------- 1 | # This is the Ansible Galaxy requirements file to pull in the correct roles 2 | # to support the operation of CASL provisioning/runs. 3 | 4 | # From 'openshift-applier' 5 | - name: openshift-applier 6 | scm: git 7 | src: https://github.com/redhat-cop/openshift-applier 8 | version: v2.0.9 9 | -------------------------------------------------------------------------------- /image-scanning/README.adoc: -------------------------------------------------------------------------------- 1 | = Add Basic Image Scanning to any Cluster 2 | 3 | This project deploys a Jenkins job that will run an OpenSCAP scan of all images currently being used by `Running` pods. A report will be created for each image and archived by Jenkins into a directory structure as follows: 4 | 5 | [source,bash] 6 | ---- 7 | ./ 8 | / 9 | / 10 | report.html 11 | report-arf.xml 12 | ---- 13 | 14 | The entire report can be downloaded as a ZIP for data processing/archiving purposes. 15 | 16 | == Automated Deployment with Applier 17 | 18 | This capability can be deployed to any cluster using link:https://github.com/redhat-cop/openshift-applier[OpenShift Applier]. 19 | 20 | [source,bash] 21 | ---- 22 | cd image-scanning/ 23 | ansible-galaxy install -r requirements.yml -p galaxy 24 | ansible-playbook -i inventory/ galaxy/openshift-applier/playbooks/openshift-cluster-seed.yml 25 | ---- 26 | 27 | NOTE: This inventory deploys several cluster-level resources such as `ClusterRoleBindings` and a new `SecurityContextConstraint`. Cluster level permissions are required to deploy this, and care should be taken. 28 | 29 | Once deployed, the scan may be run at any time by running: 30 | 31 | [source,bash] 32 | ---- 33 | oc start-build scan-image -n image-scanning 34 | ---- 35 | 36 | Once started, you can follow the job by logging into the Jenkins Console. For each image scanned, you will see a block in the logs that looks like: 37 | 38 | [source,bash] 39 | ---- 40 | Starting scan of image registry.access.redhat.com/openshift3/registry-console@sha256:0a2afe5836ca550e529608a21652a90dc1ec5657e76dee4cab95fdc61e384dc0 41 | 2018/07/12 19:50:50 Pulling image registry.access.redhat.com/openshift3/registry-console@sha256:0a2afe5836ca550e529608a21652a90dc1ec5657e76dee4cab95fdc61e384dc0 42 | 2018/07/12 19:50:53 Finished Downloading Image (10771Kb downloaded) 43 | 2018/07/12 19:50:53 Extracting image registry.access.redhat.com/openshift3/registry-console@sha256:0a2afe5836ca550e529608a21652a90dc1ec5657e76dee4cab95fdc61e384dc0 to /tmp/cve-scan-4lYI9/image-content 44 | 2018/07/12 19:50:56 Writing OpenSCAP results to /tmp/cve-scan-4lYI9/results 45 | Scan of registry.access.redhat.com/openshift3/registry-console@sha256:0a2afe5836ca550e529608a21652a90dc1ec5657e76dee4cab95fdc61e384dc0 complete. 46 | Moving results to /home/jenkins/workspace/image-scanning/image-scanning-scan-image/openshift3/registry-console@sha256:0a2afe5836ca550e529608a21652a90dc1ec5657e76dee4cab95fdc61e384dc0 47 | ---- 48 | 49 | When the job completes, you can download the ZIP file report from the workspace of the `image-scanning/scan-image` Jenkins job. 50 | -------------------------------------------------------------------------------- /image-scanning/inventory/group_vars/seed-hosts.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openshift_cluster_content: 3 | - object: Deploy Pipeline 4 | content: 5 | - name: Create Project 6 | file: "{{ inventory_dir }}/../projects/project.yml" 7 | file_action: create 8 | - name: Image Scan Base Image Build 9 | template: "{{ inventory_dir }}/../templates/image-scan-base.yml" 10 | params: "{{ inventory_dir }}/../params/image-scan-base" 11 | namespace: image-scanning 12 | tags: 13 | - core 14 | - name: Scanner SCC Mapping 15 | template: "{{ inventory_dir }}/../templates/scanning-scc.yml" 16 | params: "{{ inventory_dir }}/../params/scanning-scc-jenkins" 17 | tags: 18 | - core 19 | - name: Pipeline Job Template 20 | template: "{{inventory_dir}}/../templates/pipeline.yml" 21 | params: "{{inventory_dir}}/../params/pipeline" 22 | namespace: image-scanning 23 | tags: 24 | - pipeline 25 | -------------------------------------------------------------------------------- /image-scanning/inventory/hosts: -------------------------------------------------------------------------------- 1 | [seed-hosts] 2 | localhost ansible_connection=local 3 | -------------------------------------------------------------------------------- /image-scanning/params/image-scan-base: -------------------------------------------------------------------------------- 1 | FROM_IMAGE=rhel7:7.6 2 | -------------------------------------------------------------------------------- /image-scanning/params/pipeline: -------------------------------------------------------------------------------- 1 | SCANNER_IMAGE_NAMESPACE=image-scanning 2 | NAMESPACE=image-scanning 3 | -------------------------------------------------------------------------------- /image-scanning/params/scanning-scc-jenkins: -------------------------------------------------------------------------------- 1 | NAMESPACE=image-scanning 2 | SERVICE_ACCOUNT=jenkins 3 | -------------------------------------------------------------------------------- /image-scanning/projects/project.yml: -------------------------------------------------------------------------------- 1 | kind: ProjectRequest 2 | apiVersion: v1 3 | metadata: 4 | name: image-scanning 5 | creationTimestam: null 6 | displayName: Image Scanning Quickstart 7 | -------------------------------------------------------------------------------- /image-scanning/registry-viewer.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ClusterRole 3 | metadata: 4 | annotations: 5 | authorization.openshift.io/system-only: "true" 6 | openshift.io/reconcile-protect: "false" 7 | creationTimestamp: null 8 | name: registry-viewer 9 | rules: 10 | - apiGroups: 11 | - "" 12 | - image.openshift.io 13 | attributeRestrictions: null 14 | resources: 15 | - imagestreamimages 16 | - imagestreammappings 17 | - imagestreams 18 | - imagestreamtags 19 | verbs: 20 | - get 21 | - list 22 | - watch 23 | - apiGroups: 24 | - "" 25 | - image.openshift.io 26 | attributeRestrictions: null 27 | resources: 28 | - imagestreams/layers 29 | verbs: 30 | - get 31 | - apiGroups: 32 | - "" 33 | attributeRestrictions: null 34 | resources: 35 | - namespaces 36 | verbs: 37 | - get 38 | - apiGroups: 39 | - "" 40 | - project.openshift.io 41 | attributeRestrictions: null 42 | resources: 43 | - projects 44 | verbs: 45 | - get 46 | -------------------------------------------------------------------------------- /image-scanning/requirements.yml: -------------------------------------------------------------------------------- 1 | # This is the Ansible Galaxy requirements file to pull in the correct roles 2 | # to support the operation of CASL provisioning/runs. 3 | 4 | # From 'casl-ansible' 5 | - name: openshift-applier 6 | scm: git 7 | src: https://github.com/redhat-cop/openshift-applier 8 | version: v1.0.2 9 | -------------------------------------------------------------------------------- /image-scanning/templates/image-scan-base.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Template 3 | metadata: 4 | name: sign-image-template 5 | annotations: 6 | openshift.io/display-name: Image Signing and Scanning Base 7 | description: Template to create a base image for signing and scanning images 8 | objects: 9 | - apiVersion: v1 10 | kind: ImageStream 11 | metadata: 12 | name: ${NAME} 13 | spec: 14 | lookupPolicy: 15 | local: true 16 | - apiVersion: v1 17 | kind: ImageStream 18 | metadata: 19 | name: ${NAME} 20 | spec: 21 | lookupPolicy: 22 | local: true 23 | - apiVersion: v1 24 | kind: ImageStream 25 | metadata: 26 | name: rhel7 27 | spec: 28 | dockerImageRepository: registry.access.redhat.com/rhel7/rhel 29 | - apiVersion: v1 30 | kind: BuildConfig 31 | metadata: 32 | name: ${NAME} 33 | spec: 34 | output: 35 | to: 36 | kind: ImageStreamTag 37 | name: ${NAME}:latest 38 | source: 39 | dockerfile: |- 40 | FROM registry.access.redhat.com/rhel7/rhel 41 | 42 | RUN yum repolist > /dev/null && \ 43 | yum clean all && \ 44 | yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ 45 | INSTALL_PKGS="docker atomic atomic-openshift-clients image-inspector tar bc jq" && \ 46 | yum install -y --enablerepo=rhel-7-server-ose-${OCP_VERSION}-rpms --enablerepo=rhel-7-server-extras-rpms --enablerepo=rhel-7-server-rpms --setopt=tsflags=nodocs $INSTALL_PKGS && \ 47 | yum clean all 48 | strategy: 49 | dockerStrategy: 50 | env: 51 | - name: OCP_VERSION 52 | value: "${VERSION}" 53 | from: 54 | kind: ImageStreamTag 55 | name: ${FROM_IMAGE} 56 | type: Docker 57 | triggers: 58 | - type: ConfigChange 59 | - imageChange: {} 60 | type: ImageChange 61 | parameters: 62 | - name: NAME 63 | displayName: Name 64 | description: Name Applied to the Resulting ImageStream and BuildConfig 65 | value: image-sign-scan-base 66 | required: true 67 | - description: Image stream tag for the image you'd like to use to build the application 68 | name: FROM_IMAGE 69 | required: true 70 | value: rhel7:latest 71 | - description: OpenShift Major Version 72 | name: VERSION 73 | value: '3.9' 74 | -------------------------------------------------------------------------------- /image-scanning/templates/scanning-scc.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Template 3 | metadata: 4 | creationTimestamp: null 5 | name: scanning-scc 6 | objects: 7 | - kind: SecurityContextConstraints 8 | metadata: 9 | annotations: 10 | kubernetes.io/description: 'Allows access required to scan images across the cluster' 11 | creationTimestamp: null 12 | name: ${SCC_NAME} 13 | allowHostDirVolumePlugin: true 14 | allowHostIPC: true 15 | allowHostNetwork: true 16 | allowHostPID: true 17 | allowHostPorts: true 18 | allowPrivilegedContainer: true 19 | # allowedCapabilities: 20 | # - '*' 21 | allowedFlexVolumes: [] 22 | apiVersion: v1 23 | defaultAddCapabilities: [] 24 | priority: null 25 | readOnlyRootFilesystem: false 26 | requiredDropCapabilities: [] 27 | runAsUser: 28 | type: RunAsAny 29 | seLinuxContext: 30 | type: RunAsAny 31 | # seccompProfiles: 32 | # - '*' 33 | fsGroup: 34 | type: RunAsAny 35 | supplementalGroups: 36 | type: RunAsAny 37 | groups: 38 | - system:cluster-admins 39 | - system:nodes 40 | - system:masters 41 | users: 42 | - system:serviceaccount:${NAMESPACE}:${SERVICE_ACCOUNT} 43 | volumes: 44 | - configMap 45 | - downwardAPI 46 | - emptyDir 47 | - hostPath 48 | - persistentVolumeClaim 49 | - projected 50 | - secret 51 | parameters: 52 | - name: SCC_NAME 53 | value: image-scanning 54 | - name: SERVICE_ACCOUNT 55 | required: true 56 | - name: NAMESPACE 57 | required: true 58 | -------------------------------------------------------------------------------- /install/README.md: -------------------------------------------------------------------------------- 1 | # Sample Inventory Structure for an HA OpenShift Cluster 2 | 3 | This directory represents a sample Infrastructure as Code repo used to manage one or more OpenShift clusters. This is meant to be supporting material for [Installing an HA OpenShift Cluster](http://playbooks-rhtconsulting.rhcloud.com/playbooks/installation/). 4 | 5 | This directory contains several different versions of inventories for a fictional OpenShift cluster, `c1-ocp.myorg.com`. The different directories here depict several iterations one might go through while standing up an OpenShift cluster. Those phases are as follows: 6 | 7 | * [Base Install](./c1-ocp.myorg.com-base): This configuration represents a bare bones install of a Highly Available, disconnected cluster. 8 | * [LDAP Authentication](./c1-ocp.myorg.com-ldap): This configuration shows how LDAP Authentication can be layered on the base install. 9 | * [Metrics] (Coming Soon) 10 | * [Logging] (Coming Soon) 11 | * [Custom Named Certificates] (Coming Soon) 12 | 13 | ## Usage 14 | 15 | Any of the inventories should be fairly runnable out of the box, assuming you've got infrastructure to match. Use it as a base to start your own cluster builds, or build your infrastructure using the [Install Guide](http://playbooks-rhtconsulting.rhcloud.com/playbooks/installation/) and run it directly: 16 | 17 | ``` 18 | ansible-playbook -i ./c1-ocp.myorg.com-base/hosts /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml 19 | ``` 20 | -------------------------------------------------------------------------------- /install/c1-ocp.myorg.com-base/hosts: -------------------------------------------------------------------------------- 1 | [OSEv3:children] 2 | masters 3 | etcd 4 | nodes 5 | 6 | [OSEv3:vars] 7 | openshift_deployment_type=openshift-enterprise 8 | openshift_release=v3.5 9 | 10 | openshift_master_api_port=443 11 | openshift_master_console_port=443 12 | openshift_portal_net=172.30.0.0/16 13 | osm_cluster_network_cidr=10.128.0.0/14 14 | 15 | openshift_master_cluster_method=native 16 | openshift_master_cluster_hostname=console.c1-ocp.myorg.com 17 | openshift_master_cluster_public_hostname=console-internal.c1-ocp.myorg.com 18 | openshift_master_default_subdomain=apps.c1-ocp.myorg.com 19 | 20 | openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/origin/master/htpasswd'}] 21 | openshift_master_htpasswd_users={'admin': '$apr1$6CZ4noKr$IksMFMgsW5e5FL0ioBhkk/', 'developer': '$apr1$AvisAPTG$xrVnJ/J0a83hAYlZcxHVf1'} 22 | 23 | openshift_docker_additional_registries=registry.myorg.com 24 | openshift_docker_insecure_registries=registry.myorg.com 25 | openshift_docker_blocked_registries=registry.access.redhat.com,docker.io 26 | 27 | # configure a pv that mounts "nfs.myorg.com:/exports/registry" 28 | openshift_hosted_registry_storage_kind=nfs 29 | openshift_hosted_registry_storage_access_modes=['ReadWriteMany'] 30 | openshift_hosted_registry_storage_host=nfs.myorg.com 31 | openshift_hosted_registry_storage_nfs_directory=/exports 32 | openshift_hosted_registry_storage_volume_name=registry 33 | openshift_hosted_registry_storage_volume_size=100Gi 34 | 35 | [masters] 36 | openshift-master-[1:3].c1-ocp.myorg.com 37 | 38 | [etcd] 39 | openshift-master-[1:3].c1-ocp.myorg.com 40 | 41 | [nodes] 42 | openshift-master-[1:3].c1-ocp.myorg.com openshift_node_labels="{'region': 'master'}" 43 | openshift-infranode-[1:3].c1-ocp.myorg.com openshift_node_labels="{'region': 'infra'}" 44 | openshift-appnode-1.c1-ocp.myorg.com openshift_node_labels="{'region': 'primary'}" 45 | openshift-appnode-2.c1-ocp.myorg.com openshift_node_labels="{'region': 'primary'}" 46 | -------------------------------------------------------------------------------- /install/c1-ocp.myorg.com-ldap/files/my-ldap-ca-bundle.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | certdatahere 3 | -----END CERTIFICATE----- 4 | -------------------------------------------------------------------------------- /install/c1-ocp.myorg.com-ldap/hosts: -------------------------------------------------------------------------------- 1 | [OSEv3:children] 2 | masters 3 | etcd 4 | nodes 5 | 6 | [OSEv3:vars] 7 | openshift_deployment_type=openshift-enterprise 8 | openshift_release=v3.5 9 | 10 | openshift_master_api_port=443 11 | openshift_master_console_port=443 12 | openshift_portal_net=172.30.0.0/16 13 | osm_cluster_network_cidr=10.128.0.0/14 14 | 15 | openshift_master_cluster_method=native 16 | openshift_master_cluster_hostname=console.c1-ocp.myorg.com 17 | openshift_master_cluster_public_hostname=console-internal.c1-ocp.myorg.com 18 | openshift_master_default_subdomain=apps.c1-ocp.myorg.com 19 | 20 | openshift_master_identity_providers=[{'name': 'my_ldap_provider', 'challenge': 'true', 'login': 'true', 'kind': 'LDAPPasswordIdentityProvider', 'attributes': {'id': ['dn'], 'email': ['mail'], 'name': ['cn'], 'preferredUsername': ['uid']}, 'bindDN': '', 'bindPassword': '', 'ca': 'my-ldap-ca-bundle.crt', 'insecure': 'false', 'url': 'ldap://ldap.myorg.com:389/uid=users,dc=myorg,dc=com?uid'}] 21 | openshift_master_ldap_ca_file="{{ inventory_dir }}files/my-ldap-ca-bundle.crt" 22 | 23 | openshift_docker_additional_registries=registry.myorg.com 24 | openshift_docker_insecure_registries=registry.myorg.com 25 | openshift_docker_blocked_registries=registry.access.redhat.com,docker.io 26 | 27 | # configure a pv that mounts "nfs.myorg.com:/exports/registry" 28 | openshift_hosted_registry_storage_kind=nfs 29 | openshift_hosted_registry_storage_access_modes=['ReadWriteMany'] 30 | openshift_hosted_registry_storage_host=nfs.myorg.com 31 | openshift_hosted_registry_storage_nfs_directory=/exports 32 | openshift_hosted_registry_storage_volume_name=registry 33 | openshift_hosted_registry_storage_volume_size=100Gi 34 | 35 | [masters] 36 | openshift-master-[1:3].c1-ocp.myorg.com 37 | 38 | [etcd] 39 | openshift-master-[1:3].c1-ocp.myorg.com 40 | 41 | [nodes] 42 | openshift-master-[1:3].c1-ocp.myorg.com openshift_node_labels="{'region': 'master'}" 43 | openshift-infranode-[1:3].c1-ocp.myorg.com openshift_node_labels="{'region': 'infra'}" 44 | openshift-appnode-1.c1-ocp.myorg.com openshift_node_labels="{'region': 'primary'}" 45 | openshift-appnode-2.c1-ocp.myorg.com openshift_node_labels="{'region': 'primary'}" 46 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | rules: 3 | line-length: 4 | max: 100 5 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # Required if no default worker machinesets found 4 | #openshift_machineset_config_cloud_provider: ... 5 | #openshift_machineset_config_cluster_infra_id: ... 6 | 7 | openshift_machineset_config_domain: openshift-machineset-config.gpte.redhat.com 8 | openshift_machineset_config_group_label: "{{ openshift_machineset_config_domain }}/machineset-group" 9 | openshift_machineset_config_groups: [] 10 | openshift_machineset_config_disable_default_worker_machinesets: false 11 | 12 | openshift_cluster_autoscaler_spec: 13 | scaleDown: 14 | enabled: true 15 | 16 | # 17 | # AWS configuration 18 | # 19 | 20 | # Default ist of availability zones for machinesets 21 | #openshift_machineset_config_aws_availability_zones 22 | 23 | # JSON output of `aws ec2 describe-subnets --region=$REGION --subnet-ids ...` 24 | #openshift_machineset_config_aws_availability_zone_data: 25 | 26 | # List of availability zones with configuration 27 | #openshift_machineset_config_aws_availability_zones: 28 | 29 | # Must be set globally or on machineset groups as `aws_iam_instance_profile_id` 30 | #openshift_machineset_config_aws_iam_instance_profile_id: ... 31 | 32 | # Default instance type for machinesets 33 | openshift_machineset_config_aws_instance_type: m4.large 34 | 35 | # Default root volume size for machines 36 | openshift_machineset_config_aws_root_volume_size: 120 37 | 38 | # Must be set globally or on machineset groups as `aws_tags` 39 | #openshift_machineset_config_aws_tags: ... 40 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/tasks/aws-machineset-group.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Define {{ machineset_group.name }} machinesets 3 | k8s: 4 | state: present 5 | definition: "{{ lookup('template', 'aws-machineset.yml.j2') | from_yaml }}" 6 | # Iterate through availability zones in reverse order as it makes the math 7 | # easier to scale zone "a" before "b" to match expected behavior. 8 | loop: "{{ availability_zones[::-1] }}" 9 | loop_control: 10 | label: "{{ machineset_name }}" 11 | loop_var: availability_zone 12 | index_var: loop_index 13 | vars: 14 | ami_id: >- 15 | {{ machineset_group.aws_ami_id 16 | | default(aws_coreos_ami_id_by_region[region]) 17 | }} 18 | region: >- 19 | {{ availability_zone.name[:-1] }} 20 | subnet_search_key: >- 21 | {{ availability_zone.subnet_search_key 22 | | default('subnet-id') 23 | }} 24 | subnet_search_value: >- 25 | {{ availability_zone.subnet_search_value }} 26 | instance_type: >- 27 | {{ machineset_group.aws_instance_type | default(default_aws_instance_type) }} 28 | machineset_name: >- 29 | {{ [cluster_infra_id, machineset_group.name, availability_zone.name] | join('-') }} 30 | machineset_node_labels: >- 31 | {{ machineset_group.node_labels 32 | | default({'node-role.kubernetes.io/' + machineset_group.role: ''} 33 | if machineset_group.role|default(False) else {}) 34 | }} 35 | machineset_group_total_replicas: >- 36 | {{ machineset_group.total_replicas 37 | | default(machineset_group.total_replicas_min) 38 | | default(0) 39 | }} 40 | machineset_replicas: >- 41 | {{ ( 42 | (machineset_group_total_replicas|int + loop_index) / availability_zones|count 43 | ) | int }} 44 | root_volume_size: >- 45 | {{ machineset_group.aws_root_volume_size | default(default_aws_root_volume_size) }} 46 | 47 | - name: Define {{ machineset_group.name }} machineautoscalers 48 | k8s: 49 | state: present 50 | definition: "{{ lookup('template', 'machineautoscaler.yml.j2') | from_yaml }}" 51 | # Iterate through availability zones in reverse order as it makes the math 52 | # easier to scale zone "a" before "b" to match expected behavior. 53 | loop: "{{ availability_zones[::-1] }}" 54 | loop_control: 55 | label: "{{ machineset_name }}" 56 | loop_var: availability_zone 57 | index_var: loop_index 58 | vars: 59 | machineset_name: >- 60 | {{ [cluster_infra_id, machineset_group.name, availability_zone.name] | join('-') }} 61 | machineset_min_replicas: >- 62 | {{ ( 63 | (machineset_group.total_replicas_min|default(0) + loop_index) / 64 | availability_zones|count 65 | ) | int }} 66 | machineset_max_replicas: >- 67 | {{ ( 68 | (machineset_group.total_replicas_max|default(100) + loop_index) / 69 | availability_zones|count 70 | ) | int }} 71 | when: machineset_group.autoscale | default(False) | bool 72 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/tasks/aws.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Define custom machinesets 3 | include_tasks: aws-machineset-group.yml 4 | loop: "{{ openshift_machineset_config_groups }}" 5 | loop_control: 6 | label: "{{ machineset_group.name }}" 7 | loop_var: machineset_group 8 | vars: 9 | iam_instance_profile_id: >- 10 | {{ machineset_group.aws_iam_instance_profile_id 11 | | default(openshift_machineset_config_aws_iam_instance_profile_id) 12 | | default(reference_provider_spec_value_iam_instance_profile_id) 13 | }} 14 | security_group_search_key: >- 15 | {{ machineset_group.aws_security_group_search_key 16 | | default(openshift_machineset_config_aws_security_group_search_key) 17 | | default(reference_provider_security_group_search_key) 18 | | default('group-id', True) 19 | }} 20 | security_group_search_value: >- 21 | {{ machineset_group.aws_security_group_search_value 22 | | default(openshift_machineset_config_aws_security_group_search_value) 23 | | default(reference_provider_security_group_search_value) 24 | }} 25 | aws_tags: >- 26 | {{ machineset_group.aws_tags 27 | | default(openshift_machineset_config_aws_tags) 28 | | default(reference_provider_tags) 29 | | combine({'kubernetes.io/cluster/' ~ cluster_infra_id: 'owned'}) 30 | }} 31 | cluster_infra_id: >- 32 | {{ openshift_machineset_config_cluster_infra_id }} 33 | reference_provider_spec_value: >- 34 | {{ default_worker_machinesets 35 | | json_query('[0].spec.template.spec.providerSpec.value') 36 | | default({}, True) 37 | }} 38 | reference_provider_spec_value_iam_instance_profile_id: >- 39 | {% if reference_provider_spec_value -%} 40 | {{ reference_provider_spec_value.iamInstanceProfile.id }} 41 | {%- endif %} 42 | reference_provider_security_group_search_key: >- 43 | {% if reference_provider_spec_value -%} 44 | {{ reference_provider_spec_value.securityGroups[0].filters[0].name }} 45 | {%- endif %} 46 | reference_provider_security_group_search_value: >- 47 | {% if reference_provider_spec_value -%} 48 | {{ reference_provider_spec_value.securityGroups[0].filters[0]['values'][0] }} 49 | {%- endif %} 50 | reference_provider_tags: >- 51 | {{ reference_provider_spec_value.tags 52 | | default([]) 53 | | items2dict(key_name='name', value_name='value') 54 | }} 55 | user_data_secret: >- 56 | {{ machineset_group.aws_user_data_secret 57 | | default('worker-user-data') 58 | }} 59 | availability_zones: >- 60 | {{ machineset_group.aws_availability_zones 61 | | default(default_aws_availability_zones) 62 | | default(default_worker_availability_zones, true) 63 | }} 64 | default_worker_availability_zones: >- 65 | {{ default_worker_machinesets 66 | | json_query(worker_availability_zone_query) 67 | }} 68 | worker_availability_zone_query: >- 69 | [].{ 70 | "name": spec.template.spec.providerSpec.value.placement.availabilityZone, 71 | "subnet_search_key": spec.template.spec.providerSpec.value.subnet.filters[0].name 72 | "subnet_search_value": spec.template.spec.providerSpec.value.subnet.filters[0].values[0] 73 | } 74 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/tasks/disable-default-worker-machinesets.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Scale base worker machinesets to zero 3 | k8s: 4 | state: present 5 | definition: 6 | apiVersion: machine.openshift.io/v1beta1 7 | kind: MachineSet 8 | metadata: 9 | name: "{{ machineset.metadata.name }}" 10 | namespace: openshift-machine-api 11 | spec: 12 | replicas: 0 13 | loop: "{{ default_worker_machinesets }}" 14 | loop_control: 15 | label: "{{ machineset.metadata.name }}" 16 | loop_var: machineset 17 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/tasks/enable-cluster-autoscaler.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Define clusterautoscaler 3 | k8s: 4 | state: present 5 | definition: "{{ lookup('template', 'clusterautoscaler.yml.j2') | from_yaml }}" 6 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set machineset facts 3 | include_tasks: set-facts.yml 4 | 5 | - name: Configure machinesets for cloud provider 6 | include_tasks: "{{ openshift_machineset_config_cloud_provider }}.yml" 7 | 8 | - name: Disable default worker machinesets 9 | include_tasks: disable-default-worker-machinesets.yml 10 | when: disable_default_worker_machinesets|bool 11 | 12 | - name: Enable cluster autoscaler 13 | include_tasks: enable-cluster-autoscaler.yml 14 | when: >- 15 | openshift_machineset_config_groups | json_query('[?autoscale]') 16 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/tasks/set-facts.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Get machinesets 3 | k8s_facts: 4 | api_version: machine.openshift.io/v1beta1 5 | kind: MachineSet 6 | namespace: openshift-machine-api 7 | register: r_get_machinesets 8 | 9 | - name: Set default_worker_machinesets 10 | set_fact: 11 | current_machinesets: >- 12 | {{ r_get_machinesets.resources }} 13 | current_machineset_names: >- 14 | {{ r_get_machinesets.resources 15 | | json_query('[].metadata.name') 16 | }} 17 | default_worker_machinesets: >- 18 | {{ r_get_machinesets.resources 19 | | json_query(default_worker_machineset_json_query) 20 | }} 21 | vars: 22 | # Base worker machinesets will lack machineset group label 23 | default_worker_machineset_json_query: >- 24 | [?!contains(keys(metadata.labels), '{{ machineset_group_label }}')] 25 | 26 | - name: Set cluster facts 27 | set_fact: 28 | openshift_machineset_config_cluster_infra_id: >- 29 | {{ reference_machineset.metadata.labels['machine.openshift.io/cluster-api-cluster'] }} 30 | openshift_machineset_config_cloud_provider: >- 31 | {{ reference_provider_spec_value.apiVersion 32 | | regex_replace('providerconfig\.openshift\.io/v1beta1', '') 33 | }} 34 | vars: 35 | reference_machineset: >- 36 | {{ default_worker_machinesets[0] | default({}) }} 37 | reference_provider_spec_value: >- 38 | {{ reference_machineset 39 | | json_query('spec.template.spec.providerSpec.value') 40 | }} 41 | when: default_worker_machinesets 42 | 43 | - name: Fail if openshift_machineset_config_cloud_provider is undefined 44 | fail: 45 | msg: openshift_machineset_config_cloud_provider is required 46 | when: openshift_machineset_config_cloud_provider is undefined 47 | 48 | - name: Fail if openshift_machineset_config_cluster_infra_id is undefined 49 | fail: 50 | msg: openshift_machineset_config_cluster_infra_id is required 51 | when: openshift_machineset_config_cluster_infra_id is undefined 52 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/templates/aws-machineset.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: machine.openshift.io/v1beta1 3 | kind: MachineSet 4 | metadata: 5 | name: {{ machineset_name | to_json }} 6 | namespace: openshift-machine-api 7 | labels: 8 | {{ machineset_group_label | to_json }}: {{ machineset_group.name | to_json }} 9 | machine.openshift.io/cluster-api-cluster: {{ cluster_infra_id | to_json }} 10 | spec: 11 | 12 | {# Do not set replicas on existing autoscaling machinesets #} 13 | {% if machineset_name not in current_machineset_names 14 | or not machineset_group.autoscale|default(False) 15 | %} 16 | replicas: {{ machineset_replicas | int | to_json }} 17 | {% endif %} 18 | 19 | selector: 20 | matchLabels: 21 | machine.openshift.io/cluster-api-cluster: {{ cluster_infra_id | to_json }} 22 | {% if 'role' in machineset_group %} 23 | machine.openshift.io/cluster-api-machine-role: {{ machineset_group.role | to_json }} 24 | machine.openshift.io/cluster-api-machine-type: {{ machineset_group.role | to_json }} 25 | {% endif %} 26 | machine.openshift.io/cluster-api-machineset: {{ machineset_name | to_json }} 27 | 28 | template: 29 | metadata: 30 | creationTimestamp: null 31 | labels: 32 | {{ machineset_group_label }}: {{ machineset_group.name | to_json }} 33 | machine.openshift.io/cluster-api-cluster: {{ cluster_infra_id | to_json }} 34 | {% if 'role' in machineset_group %} 35 | machine.openshift.io/cluster-api-machine-role: {{ machineset_group.role | to_json }} 36 | machine.openshift.io/cluster-api-machine-type: {{ machineset_group.role | to_json }} 37 | {% endif %} 38 | machine.openshift.io/cluster-api-machineset: {{ machineset_name | to_json }} 39 | spec: 40 | metadata: 41 | creationTimestamp: null 42 | labels: {{ machineset_node_labels | to_json }} 43 | providerSpec: 44 | value: 45 | ami: 46 | id: {{ ami_id | to_json }} 47 | apiVersion: awsproviderconfig.openshift.io/v1beta1 48 | blockDevices: 49 | - ebs: 50 | iops: 0 51 | volumeSize: {{ root_volume_size | int | to_json }} 52 | volumeType: gp2 53 | credentialsSecret: 54 | name: aws-cloud-credentials 55 | deviceIndex: 0 56 | iamInstanceProfile: 57 | id: {{ iam_instance_profile_id | to_json }} 58 | instanceType: {{ instance_type | to_json }} 59 | kind: AWSMachineProviderConfig 60 | metadata: 61 | creationTimestamp: null 62 | placement: 63 | availabilityZone: {{ availability_zone.name | to_json }} 64 | region: {{ region | to_json }} 65 | publicIp: null 66 | securityGroups: 67 | - filters: 68 | - name: {{ security_group_search_key | to_json }} 69 | values: 70 | - {{ security_group_search_value | to_json }} 71 | subnet: 72 | filters: 73 | - name: {{ subnet_search_key | to_json }} 74 | values: 75 | - {{ subnet_search_value | to_json }} 76 | tags: 77 | {% for name, value in aws_tags.items() %} 78 | - name: {{ name | to_json }} 79 | value: {{ value | to_json }} 80 | {% endfor %} 81 | userDataSecret: 82 | name: {{ user_data_secret | to_json }} 83 | versions: 84 | kubelet: "" 85 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/templates/clusterautoscaler.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: autoscaling.openshift.io/v1 3 | kind: ClusterAutoscaler 4 | metadata: 5 | name: default 6 | spec: {{ openshift_cluster_autoscaler_spec | to_json }} 7 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/templates/machineautoscaler.yml.j2: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: autoscaling.openshift.io/v1beta1 3 | kind: MachineAutoscaler 4 | metadata: 5 | name: {{ machineset_name }} 6 | namespace: openshift-machine-api 7 | spec: 8 | minReplicas: {{ machineset_min_replicas }} 9 | maxReplicas: {{ machineset_max_replicas }} 10 | scaleTargetRef: 11 | apiVersion: machine.openshift.io/v1beta1 12 | kind: MachineSet 13 | name: {{ machineset_name }} 14 | -------------------------------------------------------------------------------- /machine-api/roles/openshift_machineset_config/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | config_domain: "{{ openshift_machineset_config_domain }}" 3 | 4 | disable_default_worker_machinesets: >- 5 | {{ openshift_machineset_config_disable_default_worker_machinesets | bool }} 6 | 7 | machineset_group_label: "{{ openshift_machineset_config_group_label }}" 8 | machineset_groups: "{{ openshift_machineset_config_groups }}" 9 | 10 | # 11 | # AWS variables 12 | # 13 | default_aws_instance_type: >- 14 | {{ openshift_machineset_config_aws_instance_type }} 15 | default_aws_root_volume_size: >- 16 | {{ openshift_machineset_config_aws_root_volume_size }} 17 | 18 | # 19 | default_aws_availability_zones: >- 20 | {{ openshift_machineset_config_aws_availability_zones 21 | | default(queried_aws_availability_zones) 22 | }} 23 | 24 | # AWS availability zone information process from aws cli output 25 | queried_aws_availability_zones: >- 26 | {{ openshift_machineset_config_aws_availability_zone_data 27 | | default({}) 28 | | json_query(aws_availability_zones_query) 29 | }} 30 | 31 | # JMESpath query to convert aws cli output to required configuration format 32 | aws_availability_zones_query: >- 33 | Subnets[*].{ 34 | "name": AvailabilityZone, 35 | "subnet_search_key": 'subnet-id', 36 | "subnet_search_value": SubnetId 37 | } 38 | # List of coreos ami ids: 39 | # https://docs.openshift.com/container-platform/4.1/installing/installing_aws_user_infra/installing-aws-user-infra.html#installation-aws-user-infra-rhcos-ami_installing-aws-user-infra 40 | aws_coreos_ami_id_by_region: 41 | ap-northeast-1: ami-0c63b39219b8123e5 42 | ap-northeast-2: ami-073cba0913d2250a4 43 | ap-south-1: ami-0270be11430101040 44 | ap-southeast-1: ami-06eb9d35ede4f08a3 45 | ap-southeast-2: ami-0d980796ce258b5d5 46 | ca-central-1: ami-0f907257d1686e3f7 47 | eu-central-1: ami-02fdd627029c0055b 48 | eu-west-1: ami-0d4839574724ed3fa 49 | eu-west-2: ami-053073b95aa285347 50 | eu-west-3: ami-09deb5deb6567bcd5 51 | sa-east-1: ami-068a2000546e1889d 52 | us-east-1: ami-046fe691f52a953f9 53 | us-east-2: ami-0649fd5d42859bdfc 54 | us-west-1: ami-0c1d2b5606111ac8c 55 | us-west-2: ami-00745fcbb14a863ed 56 | -------------------------------------------------------------------------------- /maintenance_cleanup/README.md: -------------------------------------------------------------------------------- 1 | # OpenShift Admin Bash Scripts 2 | 3 | This folder contains a collection of scripts to clean an OpenShift cluster. 4 | 5 | These scripts can be run natively on the superior Linux OS, or MacOS. Windows requires using a bash emulator (GitBash). 6 | 7 | ## Script Overview 8 | 9 | The following table outlines the scripts and purpose. 10 | 11 | Script Name | Description | Notes 12 | --- | --- | --- 13 | `cleanup-builds.sh` | Deletes builds older than 30 days. | Parameters required, check script doc. 14 | `delete-dead-pods.sh` | Deletes all the dead or completed pods on the cluster. | Parameters and auth required, check script doc. 15 | `delete-expired-replication-controllers.sh` | Deletes all RC's older than 60 days. | Parameters and auth required, check script doc. 16 | -------------------------------------------------------------------------------- /maintenance_cleanup/cleanup-bulds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -gt 1 ]; then 4 | echo " 5 | 6 | Usage: cleanup-builds.sh [Dry Run] 7 | 8 | Example: cleanup-builds.sh false 9 | 10 | NOTE: If not specified, Dry Run is assumed true. 11 | " 12 | 13 | exit 1 14 | fi 15 | 16 | #Globals 17 | LEAVE_BUILDS=30 18 | 19 | dryrun=true 20 | if [ "$#" -gt 0 ]; then 21 | dryrun=$1 22 | fi 23 | 24 | #Check for the project 25 | project_list=`oc projects -q` 26 | #Check to see that we got a response, if not just die. 27 | if [ -z "$project_list" ]; then 28 | 29 | echo "===================================================================== 30 | No projects found! This should not happen, so something bad occured. 31 | =====================================================================" 32 | exit 1 33 | 34 | fi 35 | 36 | #Process the list of projects and export them if they are valid 37 | readarray -t projects <<<"$project_list" 38 | for project in "${projects[@]}" 39 | do 40 | 41 | #check to see if the project is actually validation 42 | project_exists=`oc projects -q | grep ${project}` 43 | if [ -n "$project_exists" ]; then 44 | 45 | echo "Found Project: ${project}" 46 | 47 | build_config_list=`oc get bc -n ${project} --no-headers | awk '{print $1}' ` 48 | if [ -n "$build_config_list" ]; then 49 | 50 | readarray -t build_configs <<<"$build_config_list" 51 | for build_config in "${build_configs[@]}" 52 | do 53 | 54 | echo "Project: ${project}, build config: ${build_config}" 55 | 56 | #Count the number of dashes in the application name, as we'll need to 57 | #adjust the position of the array location after the split 58 | dashCount=`echo $build_config | awk 'BEGIN{FS="-"} {print NF}'` 59 | #echo "$dashCount" 60 | 61 | #Set the array position, but adding a default value of 1 62 | buildNumPos=$(($dashCount + 1)) 63 | #echo "Build number position: $buildNumPos" 64 | 65 | #Get the list of builds 66 | var=`oc get builds -n ${project} --no-headers -l buildconfig=${build_config} | \ 67 | awk -v buildNumPos="$buildNumPos" '{split($1,a,"-"); print a[buildNumPos]}'` 68 | #echo $var 69 | 70 | #Set the max build number 71 | max=0 72 | 73 | #Split the build numbers, to get them as an array 74 | builds=$(echo $var | tr "\r\n" "\n") 75 | 76 | if [ -n "$builds" ]; then 77 | 78 | #Loop the build numbers and find the most recent (max) build 79 | for build in $builds 80 | do 81 | #Perform the check and set the max number 82 | if [ $build -gt $max ]; then 83 | #echo "number: $build" 84 | max=$build 85 | fi 86 | done 87 | 88 | #Echo the max 89 | echo "Most recent build for $build_config is: $max" 90 | 91 | #Now, reloop the numbers and delete all but the max 92 | for build in $builds 93 | do 94 | #echo "build: $build" 95 | if [ $build -lt $(($max - $LEAVE_BUILDS)) ]; then 96 | 97 | if [ "$dryrun" = "true" ]; then 98 | echo "oc delete bulld -n ${project} $build_config-$build" 99 | fi 100 | 101 | if [ "$dryrun" = "false" ]; then 102 | oc delete build -n ${project} $build_config-$build 103 | fi 104 | fi 105 | done 106 | fi 107 | done 108 | fi 109 | fi 110 | done 111 | -------------------------------------------------------------------------------- /maintenance_cleanup/delete-dead-pods.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Date: 04/18/2017 4 | # 5 | # Script Purpose: Cleanup completed or dead pods 6 | # Version : 1.0 7 | # 8 | 9 | 10 | if [ "$#" -gt 1 ]; then 11 | echo " 12 | 13 | Usage: delete-dead-pods.sh [Dry Run] 14 | 15 | Example: delete-dead-pods.sh false 16 | 17 | NOTE: If not specified, Dry Run is assumed true. 18 | " 19 | 20 | exit 1 21 | fi 22 | 23 | dryrun=true 24 | if [ "$#" -gt 0 ]; then 25 | dryrun=$1 26 | fi 27 | 28 | if [ "$dryrun" = "true" ]; then 29 | echo "Executing dry run" 30 | 31 | #Just echo the namespace and pod. 32 | oc get pods --all-namespaces --no-headers | awk '$4 == "Error" \ 33 | || $4 == "Completed" \ 34 | || $4 == "DeadlineExceeded" \ 35 | || $4 == "ContainerCannotRun" \ 36 | || $4 == "Terminating" \ 37 | {system("bash -c '\''echo namespace: "$1", pod: "$2" '\''")}' 38 | 39 | exit 0 40 | fi 41 | 42 | if [ "$dryrun" = "false" ]; then 43 | echo "Executing actual delete" 44 | 45 | #Delete all Error, Completed, DeadlineExceeded, or ContainerCannotRun pods. 46 | oc get pods --all-namespaces --no-headers | awk '$4 == "Error" \ 47 | || $4 == "Completed" \ 48 | || $4 == "DeadlineExceeded" \ 49 | || $4 == "ContainerCannotRun" \ 50 | {system("bash -c '\''oc delete pod -n "$1" "$2" '\''")}' 51 | 52 | #Force kill any hanging pods 53 | oc get pods --no-headers | awk '$4 == "Terminating" \ 54 | {system("bash -c '\''oc delete pod -n "$1" "$2" --grace-period=0 '\''")}' 55 | 56 | exit 0 57 | fi 58 | 59 | echo "Invalid command: $dryrun" 60 | exit 1 61 | -------------------------------------------------------------------------------- /maintenance_cleanup/delete-expired-replication-controllers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Date: 04/19/2017 4 | # 5 | # Script Purpose: Cleanup replication controllers 6 | # This is a massive cleanup script that will remove all RC and deployments 7 | # that have an RC count of 0. 8 | # 9 | # Version : 1.0 10 | # 11 | 12 | if [ "$#" -gt 1 ]; then 13 | echo " 14 | 15 | Usage: delete-expired-replicaiton-controllers.sh [Dry Run] 16 | 17 | Example: delete-expired-replicaiton-controllers.sh false 18 | 19 | NOTE: If not specified, Dry Run is assumed true. 20 | " 21 | 22 | exit 1 23 | fi 24 | 25 | #Globals 26 | MAX_DAYS=60 27 | 28 | dryrun=true 29 | if [ "$#" -gt 0 ]; then 30 | dryrun=$1 31 | fi 32 | 33 | if [ "$dryrun" = "true" ]; then 34 | echo "Executing dry run" 35 | 36 | #Just echo the name 37 | oc get rc --all-namespaces --no-headers | \ 38 | awk '$3 == 0 && \ 39 | $4 == 0 && \ 40 | $5 == 0 && \ 41 | int(substr($6, 1, length($6) - 1 )) > 30 \ 42 | {system("bash -c '\''echo namespace: "$1", dc: "$2", desired: "$3", curr: "$4", age: "$6" '\''")}' 43 | 44 | exit 0 45 | fi 46 | 47 | if [ "$dryrun" = "false" ]; then 48 | echo "Executing actual delete" 49 | 50 | #Delete all RC with a count of 0 51 | oc get rc --all-namespaces --no-headers | \ 52 | awk '$3 == 0 && \ 53 | $4 == 0 && \ 54 | $5 == 0 && \ 55 | int(substr($6, 1, length($6) - 1 )) > 30 \ 56 | {system("bash -c '\''oc delete rc -n "$1" "$2"'\''")}' 57 | 58 | exit 0 59 | fi 60 | 61 | echo "Invalid command: $dryrun" 62 | exit 1 63 | -------------------------------------------------------------------------------- /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | role_name: openshift_machineset_config 4 | author: Johnathan Kupferer 5 | description: Configure OpenShift 4 MachineSets 6 | license: MIT 7 | min_ansible_version: 2.7 8 | platforms: 9 | - name: GenericLinux 10 | versions: 11 | - all 12 | galaxy_tags: 13 | - ocp 14 | - openshift 15 | dependencies: [] 16 | -------------------------------------------------------------------------------- /networkpolicy/.gitignore: -------------------------------------------------------------------------------- 1 | roles 2 | -------------------------------------------------------------------------------- /networkpolicy/inventory/group_vars/all.yml: -------------------------------------------------------------------------------- 1 | --- 2 | openshift_cluster_content: 3 | - object: default-project 4 | content: 5 | - name: docker-registry 6 | file: "{{ inventory_dir }}/../policies/default/docker-registry-ingress.yml" 7 | namespace: default 8 | tags: 9 | - default 10 | - object: service-broker 11 | content: 12 | - name: kube-service-catalog 13 | file: "{{ inventory_dir }}/../policies/kube-service-catalog/" 14 | namespace: kube-service-catalog 15 | tags: 16 | - service-catalog 17 | - name: openshift-template-service-broker 18 | file: "{{ inventory_dir }}/../policies/openshift-template-service-broker/" 19 | namespace: openshift-template-service-broker 20 | tags: 21 | - service-catalog 22 | - name: openshift-ansible-service-broker 23 | file: "{{ inventory_dir }}/../policies/openshift-ansible-service-broker/" 24 | namespace: openshift-ansible-service-broker 25 | tags: 26 | - service-catalog 27 | - object: logging 28 | content: 29 | - name: logging 30 | file: "{{ inventory_dir }}/../policies/logging/" 31 | namespace: logging 32 | tags: 33 | - logging 34 | - object: metrics 35 | content: 36 | - name: metrics 37 | file: "{{ inventory_dir }}/../policies/openshift-infra/" 38 | namespace: openshift-infra 39 | tags: 40 | - metrics -------------------------------------------------------------------------------- /networkpolicy/inventory/hosts: -------------------------------------------------------------------------------- 1 | [seed-hosts] 2 | localhost ansible_connection=local -------------------------------------------------------------------------------- /networkpolicy/playbooks/apply-network-policies.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Run OpenShift applier 4 | import_playbook: roles/casl-ansible/playbooks/openshift-cluster-seed.yml 5 | when: skip_applier is not defined or (skip_applier is defined and skip_applier|bool) 6 | 7 | - hosts: localhost 8 | connection: local 9 | gather_facts: false 10 | vars: 11 | baseline_policies_dir: "{{ inventory_dir }}/../policies/baseline" 12 | apply_default_policies: true 13 | tasks: 14 | - name: Apply Default Policies to All Projects 15 | block: 16 | - name: Get List of OpenShift Projects 17 | command: > 18 | oc get namespaces -o name 19 | register: openshift_namespaces 20 | - name: Apply Default Policies 21 | command: > 22 | oc apply -f {{ baseline_policies_dir }} -n {{ item | basename }} 23 | with_items: "{{ openshift_namespaces.stdout_lines }}" 24 | when: apply_default_policies -------------------------------------------------------------------------------- /networkpolicy/policies/baseline/allow-from-default-namespace.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: allow-from-default-namespace 5 | spec: 6 | podSelector: 7 | ingress: 8 | - from: 9 | - namespaceSelector: 10 | matchLabels: 11 | name: default 12 | -------------------------------------------------------------------------------- /networkpolicy/policies/baseline/allow-from-same-namespace.yml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.k8s.io/v1 2 | kind: NetworkPolicy 3 | metadata: 4 | name: allow-from-same-namespace 5 | spec: 6 | podSelector: 7 | ingress: 8 | - from: 9 | - podSelector: {} 10 | -------------------------------------------------------------------------------- /networkpolicy/policies/baseline/default-deny.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: default-deny 5 | spec: 6 | podSelector: [] 7 | -------------------------------------------------------------------------------- /networkpolicy/policies/baseline/deny-external-egress.yml: -------------------------------------------------------------------------------- 1 | apiVersion: network.openshift.io/v1 2 | kind: EgressNetworkPolicy 3 | metadata: 4 | name: deny-external-egress 5 | spec: 6 | egress: 7 | - type: Deny 8 | to: 9 | cidrSelector: 0.0.0.0/0 10 | -------------------------------------------------------------------------------- /networkpolicy/policies/default/docker-registry-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: registry-allow-other-namespaces 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | docker-registry: default 9 | ingress: 10 | - from: 11 | - namespaceSelector: 12 | matchLabels: {} 13 | ports: 14 | - port: 5000 15 | protocol: TCP 16 | -------------------------------------------------------------------------------- /networkpolicy/policies/kube-service-catalog/apiserver-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: apiserver-ingress 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | app: apiserver 9 | ingress: 10 | - from: 11 | - podSelector: 12 | matchLabels: 13 | app: controller-manager 14 | ports: 15 | - port: 6443 16 | protocol: TCP 17 | -------------------------------------------------------------------------------- /networkpolicy/policies/logging/elasticsearch-nodes-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: elasticsearch-nodes-ingress 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | component: es 9 | ingress: 10 | - from: 11 | - podSelector: 12 | matchLabels: 13 | component: es 14 | -------------------------------------------------------------------------------- /networkpolicy/policies/logging/elasticsearch-rest-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: elasticsearch-rest-ingress 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | component: es 9 | ingress: 10 | - from: 11 | - podSelector: 12 | matchLabels: 13 | component: fluentd 14 | - podSelector: 15 | matchLabels: 16 | component: kibana 17 | - podSelector: 18 | matchLabels: 19 | component: curator 20 | ports: 21 | - port: 9200 22 | protocol: TCP 23 | -------------------------------------------------------------------------------- /networkpolicy/policies/openshift-ansible-service-broker/asb-service-broker-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: asb-service-broker-ingress 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | app: openshift-ansible-service-broker 9 | ingress: 10 | - from: 11 | - namespaceSelector: 12 | matchLabels: 13 | name: kube-service-catalog 14 | ports: 15 | - port: 1338 16 | protocol: TCP 17 | -------------------------------------------------------------------------------- /networkpolicy/policies/openshift-infra/cassandra-hawkular-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: cassandra-hawkular-ingress 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | metrics-infra: hawkular-cassandra 9 | ingress: 10 | - from: 11 | - podSelector: 12 | matchLabels: 13 | metrics-infra: hawkular-metrics 14 | ports: 15 | - port: 9042 16 | protocol: TCP 17 | -------------------------------------------------------------------------------- /networkpolicy/policies/openshift-infra/cassandra-nodes-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: cassandra-nodes-ingress 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | metrics-infra: hawkular-cassandra 9 | ingress: 10 | - from: 11 | - podSelector: 12 | matchLabels: 13 | metrics-infra: hawkular-cassandra 14 | -------------------------------------------------------------------------------- /networkpolicy/policies/openshift-infra/hawkular-heapster-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: hawkular-heapster-ingress 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | metrics-infra: hawkular-metrics 9 | ingress: 10 | - from: 11 | - podSelector: 12 | matchLabels: 13 | metrics-infra: heapster 14 | ports: 15 | - port: 8443 16 | protocol: TCP 17 | -------------------------------------------------------------------------------- /networkpolicy/policies/openshift-template-service-broker/template-service-broker-ingress.yml: -------------------------------------------------------------------------------- 1 | kind: NetworkPolicy 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: template-service-broker-ingress 5 | spec: 6 | podSelector: 7 | matchLabels: 8 | apiserver: "true" 9 | ingress: 10 | - from: 11 | - namespaceSelector: 12 | matchLabels: 13 | name: kube-service-catalog 14 | ports: 15 | - port: 8443 16 | protocol: TCP 17 | -------------------------------------------------------------------------------- /networkpolicy/requirements.yml: -------------------------------------------------------------------------------- 1 | # This is the Ansible Galaxy requirements file to pull in the correct roles 2 | # to support the operation of CASL provisioning/runs. 3 | 4 | # From 'casl-ansible' 5 | - src: https://github.com/redhat-cop/casl-ansible 6 | scm: git 7 | version: v3.7.1 8 | name: casl-ansible -------------------------------------------------------------------------------- /quota-management/files/default-project-template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Template 3 | metadata: 4 | creationTimestamp: null 5 | name: project-request 6 | namespace: default 7 | objects: 8 | - apiVersion: project.openshift.io/v1 9 | kind: Project 10 | metadata: 11 | annotations: 12 | openshift.io/description: ${PROJECT_DESCRIPTION} 13 | openshift.io/display-name: ${PROJECT_DISPLAYNAME} 14 | openshift.io/requester: ${PROJECT_REQUESTING_USER} 15 | creationTimestamp: null 16 | name: ${PROJECT_NAME} 17 | spec: {} 18 | status: {} 19 | - apiVersion: authorization.openshift.io/v1 20 | groupNames: 21 | - system:serviceaccounts:${PROJECT_NAME} 22 | kind: RoleBinding 23 | metadata: 24 | creationTimestamp: null 25 | name: system:image-pullers 26 | namespace: ${PROJECT_NAME} 27 | roleRef: 28 | name: system:image-puller 29 | subjects: 30 | - kind: SystemGroup 31 | name: system:serviceaccounts:${PROJECT_NAME} 32 | userNames: null 33 | - apiVersion: authorization.openshift.io/v1 34 | groupNames: null 35 | kind: RoleBinding 36 | metadata: 37 | creationTimestamp: null 38 | name: system:image-builders 39 | namespace: ${PROJECT_NAME} 40 | roleRef: 41 | name: system:image-builder 42 | subjects: 43 | - kind: ServiceAccount 44 | name: builder 45 | userNames: 46 | - system:serviceaccount:${PROJECT_NAME}:builder 47 | - apiVersion: authorization.openshift.io/v1 48 | groupNames: null 49 | kind: RoleBinding 50 | metadata: 51 | creationTimestamp: null 52 | name: system:deployers 53 | namespace: ${PROJECT_NAME} 54 | roleRef: 55 | name: system:deployer 56 | subjects: 57 | - kind: ServiceAccount 58 | name: deployer 59 | userNames: 60 | - system:serviceaccount:${PROJECT_NAME}:deployer 61 | - apiVersion: authorization.openshift.io/v1 62 | groupNames: null 63 | kind: RoleBinding 64 | metadata: 65 | creationTimestamp: null 66 | name: admin 67 | namespace: ${PROJECT_NAME} 68 | roleRef: 69 | name: admin 70 | subjects: 71 | - kind: User 72 | name: ${PROJECT_ADMIN_USER} 73 | userNames: 74 | - ${PROJECT_ADMIN_USER} 75 | - apiVersion: v1 76 | kind: ResourceQuota 77 | metadata: 78 | annotations: 79 | openshift.io/quota-tier: Small 80 | labels: 81 | quota-tier: Small 82 | name: quota 83 | namespace: ${PROJECT_NAME} 84 | spec: 85 | hard: 86 | cpu: "1" 87 | memory: 6Gi 88 | scopes: 89 | - NotTerminating 90 | - apiVersion: v1 91 | kind: ResourceQuota 92 | metadata: 93 | annotations: 94 | openshift.io/quota-tier: Small 95 | labels: 96 | quota-tier: Small 97 | name: burst-quota 98 | namespace: ${PROJECT_NAME} 99 | spec: 100 | hard: 101 | cpu: "2" 102 | memory: 8Gi 103 | - apiVersion: v1 104 | kind: LimitRange 105 | metadata: 106 | annotations: 107 | openshift.io/quota-tier: Small 108 | labels: 109 | quota-tier: Small 110 | name: limits 111 | namespace: ${PROJECT_NAME} 112 | spec: 113 | limits: 114 | - max: 115 | cpu: 1000m 116 | memory: 1Gi 117 | min: 118 | cpu: 10m 119 | memory: 128Mi 120 | type: Pod 121 | - default: 122 | cpu: 100m 123 | memory: 256Mi 124 | defaultRequest: 125 | cpu: 50m 126 | memory: 256Mi 127 | max: 128 | cpu: 1000m 129 | memory: 1Gi 130 | min: 131 | cpu: 10m 132 | memory: 128Mi 133 | type: Container 134 | parameters: 135 | - name: PROJECT_NAME 136 | - name: PROJECT_DISPLAYNAME 137 | - name: PROJECT_DESCRIPTION 138 | - name: PROJECT_ADMIN_USER 139 | - name: PROJECT_REQUESTING_USER 140 | -------------------------------------------------------------------------------- /quota-management/files/quota-large.yml: -------------------------------------------------------------------------------- 1 | kind: List 2 | metadata: {} 3 | apiVersion: v1 4 | items: 5 | - apiVersion: v1 6 | kind: ResourceQuota 7 | metadata: 8 | annotations: 9 | openshift.io/quota-tier: Large 10 | labels: 11 | quota-tier: Large 12 | name: quota 13 | spec: 14 | hard: 15 | cpu: "4" 16 | memory: 24Gi 17 | scopes: 18 | - NotTerminating 19 | - apiVersion: v1 20 | kind: ResourceQuota 21 | metadata: 22 | annotations: 23 | openshift.io/quota-tier: Large 24 | labels: 25 | quota-tier: Large 26 | name: burst-quota 27 | spec: 28 | hard: 29 | cpu: "8" 30 | memory: 32Gi 31 | - apiVersion: v1 32 | kind: LimitRange 33 | metadata: 34 | annotations: 35 | openshift.io/quota-tier: Large 36 | labels: 37 | quota-tier: Large 38 | name: limits 39 | spec: 40 | limits: 41 | - max: 42 | cpu: 2000m 43 | memory: 2Gi 44 | min: 45 | cpu: 20m 46 | memory: 256Mi 47 | type: Pod 48 | - default: 49 | cpu: 500m 50 | memory: 1024Mi 51 | defaultRequest: 52 | cpu: 100m 53 | memory: 512Mi 54 | max: 55 | cpu: 2000m 56 | memory: 2Gi 57 | min: 58 | cpu: 20m 59 | memory: 256Mi 60 | type: Container 61 | -------------------------------------------------------------------------------- /quota-management/files/quota-medium.yml: -------------------------------------------------------------------------------- 1 | kind: List 2 | metadata: {} 3 | apiVersion: v1 4 | items: 5 | - apiVersion: v1 6 | kind: ResourceQuota 7 | metadata: 8 | annotations: 9 | openshift.io/quota-tier: Medium 10 | labels: 11 | quota-tier: Medium 12 | name: quota 13 | spec: 14 | hard: 15 | cpu: "2" 16 | memory: 12Gi 17 | scopes: 18 | - NotTerminating 19 | - apiVersion: v1 20 | kind: ResourceQuota 21 | metadata: 22 | annotations: 23 | openshift.io/quota-tier: Medium 24 | labels: 25 | quota-tier: Medium 26 | name: burst-quota 27 | spec: 28 | hard: 29 | cpu: "4" 30 | memory: 16Gi 31 | - apiVersion: v1 32 | kind: LimitRange 33 | metadata: 34 | annotations: 35 | openshift.io/quota-tier: Medium 36 | labels: 37 | quota-tier: Medium 38 | name: limits 39 | spec: 40 | limits: 41 | - max: 42 | cpu: 2000m 43 | memory: 2Gi 44 | min: 45 | cpu: 10m 46 | memory: 128Mi 47 | type: Pod 48 | - default: 49 | cpu: 250m 50 | memory: 256Mi 51 | defaultRequest: 52 | cpu: 100m 53 | memory: 256Mi 54 | max: 55 | cpu: 2000m 56 | memory: 2Gi 57 | min: 58 | cpu: 20m 59 | memory: 256Mi 60 | type: Container 61 | -------------------------------------------------------------------------------- /quota-management/files/quota-small.yml: -------------------------------------------------------------------------------- 1 | kind: List 2 | metadata: {} 3 | apiVersion: v1 4 | items: 5 | - apiVersion: v1 6 | kind: ResourceQuota 7 | metadata: 8 | annotations: 9 | openshift.io/quota-tier: Small 10 | labels: 11 | quota-tier: Small 12 | name: quota 13 | spec: 14 | hard: 15 | cpu: "1" 16 | memory: 6Gi 17 | scopes: 18 | - NotTerminating 19 | - apiVersion: v1 20 | kind: ResourceQuota 21 | metadata: 22 | annotations: 23 | openshift.io/quota-tier: Small 24 | labels: 25 | quota-tier: Small 26 | name: burst-quota 27 | spec: 28 | hard: 29 | cpu: "2" 30 | memory: 8Gi 31 | - apiVersion: v1 32 | kind: LimitRange 33 | metadata: 34 | annotations: 35 | openshift.io/quota-tier: Small 36 | labels: 37 | quota-tier: Small 38 | name: limits 39 | spec: 40 | limits: 41 | - max: 42 | cpu: 1000m 43 | memory: 1Gi 44 | min: 45 | cpu: 10m 46 | memory: 128Mi 47 | type: Pod 48 | - default: 49 | cpu: 100m 50 | memory: 256Mi 51 | defaultRequest: 52 | cpu: 50m 53 | memory: 256Mi 54 | max: 55 | cpu: 1000m 56 | memory: 1Gi 57 | min: 58 | cpu: 10m 59 | memory: 128Mi 60 | type: Container 61 | -------------------------------------------------------------------------------- /quota-management/inventory/group_vars/seed-hosts.yml: -------------------------------------------------------------------------------- 1 | openshift_cluster_content: 2 | - object: projectrequest 3 | content: 4 | - name: "myapp-space2-quota" 5 | file: "{{ inventory_dir }}/../files/quota-medium.yml" 6 | namespace: "myapp-space2" 7 | - name: "myapp-space4" 8 | file: "{{ inventory_dir }}/../files/quota-large.yml" 9 | namespace: "myapp-space4" 10 | -------------------------------------------------------------------------------- /quota-management/inventory/hosts: -------------------------------------------------------------------------------- 1 | [masters] 2 | master-[0:2].c1-osp.myorg.com 3 | 4 | [seed-hosts:children] 5 | masters 6 | -------------------------------------------------------------------------------- /satellite/README.md: -------------------------------------------------------------------------------- 1 | In this directory you will find the following scripts. 2 | 3 | * populate-ocp.sh - Create all repositories needed to deploy OpenShift and some additional images 4 | 5 | populate-ocp.sh 6 | --------------- 7 | To use this script, simply modify ORG_ID to match your organization. 8 | -------------------------------------------------------------------------------- /satellite/populate-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ORG_ID=1 4 | PRODUCT_NAME="OCP Docker images" 5 | 6 | upstream_repos=( openshift3/ose-deployer \ 7 | openshift3/ose-docker-registry \ 8 | openshift3/registry-console \ 9 | openshift3/ose-pod \ 10 | openshift3/ose-docker-builder \ 11 | openshift3/ose-sti-builder \ 12 | openshift3/ose-haproxy-router \ 13 | openshift3/logging-elasticsearch \ 14 | openshift3/logging-kibana \ 15 | openshift3/logging-fluentd \ 16 | openshift3/logging-auth-proxy \ 17 | openshift3/metrics-hawkular-metrics \ 18 | openshift3/metrics-cassandra \ 19 | openshift3/metrics-heapster \ 20 | openshift3/ose \ 21 | openshift3/node \ 22 | openshift3/openvswitch \ 23 | rhel7/etcd \ 24 | openshift3/ose-keepalived-ipfailover 25 | ) 26 | 27 | xpaas_images=( redhat-openjdk-18/openjdk18-openshift \ 28 | jboss-webserver-3/webserver30-tomcat8-openshift \ 29 | jboss-eap-7/eap70-openshift \ 30 | redhat-sso-7/sso70-openshift \ 31 | rhscl/postgresql-95-rhel7 \ 32 | rhscl/nodejs-4-rhel7 \ 33 | rhscl/nodejs-6-rhel7 \ 34 | rhscl/python-27-rhel7 \ 35 | rhscl/python-35-rhel7 36 | ) 37 | 38 | jenkins_images=( openshift3/jenkins-2-rhel7 \ 39 | openshift3/jenkins-slave-base-rhel7 \ 40 | openshift3/jenkins-slave-maven-rhel7 \ 41 | openshift3/jenkins-slave-nodejs-rhel7 42 | ) 43 | 44 | hammer product create --name "$PRODUCT_NAME" --organization-id $ORG_ID 45 | 46 | for i in ${upstream_repos[@]}; do 47 | hammer repository create --name "$i" --organization-id $ORG_ID --content-type docker --url "https://registry.access.redhat.com" --docker-upstream-name "$i" --product "$PRODUCT_NAME" 48 | done 49 | 50 | 51 | for i in ${xpaas_images[@]}; do 52 | hammer repository create --name "$i" --organization-id $ORG_ID --content-type docker --url "https://registry.access.redhat.com" --docker-upstream-name "$i" --product "$PRODUCT_NAME" 53 | done 54 | 55 | for i in ${jenkins_images[@]}; do 56 | hammer repository create --name "$i" --organization-id $ORG_ID --content-type docker --url "https://registry.access.redhat.com" --docker-upstream-name "$i" --product "$PRODUCT_NAME" 57 | done 58 | -------------------------------------------------------------------------------- /storage/README.md: -------------------------------------------------------------------------------- 1 | # openshift-toolkit | Storage 2 | 3 | This folder contains helper tools for working with storage in OpenShift. Aka, PVs, PVCs, dynamic storage providers, ect. 4 | 5 | # Scripts 6 | 7 | ## pv-migrator 8 | Used to migrate PVs to a new storage class. See [pv-migrator/README.md](pv-migrator/README.md) for details. 9 | 10 | ## create-pvs.sh 11 | Script to create a bunch of PVs. 12 | 13 | ## gluster-health-check 14 | Script to check the health of OpenShift Container Storage 3 Gluster clusters. 15 | 16 | ## pv-to-path-mapping 17 | Map PVS to storage paths 18 | -------------------------------------------------------------------------------- /storage/create-pvs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Date: 04/18/2017 4 | # 5 | # Script Purpose: Create a bunch of PVs 6 | # Version : 1.0 7 | # 8 | 9 | set -o pipefail 10 | 11 | if [ "$#" -ne 5 ]; then 12 | echo " 13 | 14 | Usage: create-pvs.sh NFS-Server NFS-ROOT-PATH VolumeSize SequenceStart SequenceEnd 15 | 16 | The following command 'create-pvs.sh nfs.moos.local /nfs/root/path 1 1 1' 17 | would create a PV with this json payload 18 | 19 | { 20 | \"apiVersion\": \"v1\", 21 | \"kind\": \"PersistentVolume\", 22 | \"metadata\": { 23 | \"name\": \"pv1g0001\" 24 | }, 25 | \"spec\": { 26 | \"capacity\": { 27 | \"storage\": \"1Gi\" 28 | }, 29 | \"accessModes\": [ 30 | \"ReadWriteOnce\", 31 | \"ReadWriteMany\" 32 | ], 33 | \"nfs\": { 34 | \"path\": \"/nfs/root/path/pv1g0001\", 35 | \"server\": \"nfs.moos.local\" 36 | }, 37 | \"persistentVolumeReclaimPolicy\": \"Recycle\" 38 | } 39 | } 40 | 41 | " 42 | exit 1 43 | fi 44 | 45 | 46 | nfs_server=$1 47 | nfs_path=$2 48 | vol_sz=$3 49 | seq_start=$4 50 | seq_end=$5 51 | 52 | create_persistent_volume() { 53 | vol_nr=$1 54 | vol_size=$2 55 | nfs_server=$3 56 | nfs_path=$4 57 | #For example the PV name will be: pv1g0001 58 | vol_name="pv$2g$(printf %04d $1)" 59 | echo "Creating PV: ${vol_name}" 60 | 61 | # create persistent volume 62 | if ! oc get -n default persistentvolumes pv${vol_nr} >/dev/null 2>&1; then 63 | cat <<-EOF | oc create -n default -f - 64 | { 65 | "apiVersion": "v1", 66 | "kind": "PersistentVolume", 67 | "metadata": { 68 | "name": "${vol_name}" 69 | }, 70 | "spec": { 71 | "capacity": { 72 | "storage": "${vol_size}Gi" 73 | }, 74 | "accessModes": [ 75 | "ReadWriteOnce", 76 | "ReadWriteMany" 77 | ], 78 | "nfs": { 79 | "path": "${nfs_path}/${vol_name}", 80 | "server": "${nfs_server}" 81 | }, 82 | "persistentVolumeReclaimPolicy": "Recycle" 83 | } 84 | } 85 | EOF 86 | else 87 | echo "ERROR: OpenShift persistent volume already exists. This seems wrong. Aborting." 88 | exit 1 89 | fi 90 | } 91 | 92 | # Check if executed as root 93 | if [[ $EUID -ne 0 ]]; then 94 | echo "ERROR: This script must be run as root. Aborting." 95 | exit 1 96 | fi 97 | 98 | # Check if executed on OSE master 99 | if ! systemctl status atomic-openshift-master-api >/dev/null 2>&1; then 100 | echo "ERROR: This script must be run on an OpenShift master. Aborting." 101 | exit 1 102 | fi 103 | 104 | for i in `seq ${seq_start} ${seq_end}`; do 105 | create_persistent_volume $i ${vol_sz} ${nfs_server} ${nfs_path} 106 | done 107 | -------------------------------------------------------------------------------- /storage/gluster-health-check: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | GLUSTER_PODS=$(oc get pods -l glusterfs-node=pod --all-namespaces -o jsonpath='{.items[*].metadata.name}' | tr ' ' "\n") 4 | GLUSTER_POD_NAMESPACES=$(oc get pods -l glusterfs-node=pod --all-namespaces -o jsonpath='{.items[*].metadata.namespace}' | tr ' ' "\n") 5 | GLUSTER_POD_NODES=$(oc get pods -l glusterfs-node=pod --all-namespaces -o jsonpath='{.items[*].spec.nodeName}' | tr ' ' "\n") 6 | 7 | # get health 8 | while read gluster_pod <&3 && read gluster_namespace <&4 && read gluster_node <&5; do 9 | echo 10 | echo 11 | echo "GLUSTER STATUS FOR POD (${gluster_pod}) IN PROJECT (${gluster_namespace}) ON NODE (${gluster_node})" 12 | oc rsh -n ${gluster_namespace} ${gluster_pod} /bin/sh -c 'for vol in $(gluster volume list); do echo $vol; gluster volume heal $vol info | grep -E "Number of entries:" | grep -Ev "Number of entries: [0-]"; gluster volume heal $vol info split-brain | grep -E "Number of entries:" | grep -Ev "Number of entries: [0-]" | gluster volume heal $vol info | grep -E "Status:" | grep -Ev "Status: Connected"; echo; done;' 13 | done 3<<<"${GLUSTER_PODS}" 4<<<"${GLUSTER_POD_NAMESPACES}" 5<<<"${GLUSTER_POD_NODES}" 14 | 15 | -------------------------------------------------------------------------------- /storage/pv-migrator/.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.retry 3 | -------------------------------------------------------------------------------- /storage/pv-migrator/localhost.ini: -------------------------------------------------------------------------------- 1 | localhost ansible_connection=local ansible_python_interpreter="{{ansible_playbook_python}}" 2 | -------------------------------------------------------------------------------- /storage/pv-migrator/migrate-pvs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Migrate PVs 3 | hosts: localhost 4 | module_defaults: 5 | group/k8s: 6 | host: "{{ k8s_host }}" 7 | validate_certs: "{{ k8s_validate_certs | default(true) }}" 8 | tasks: 9 | - include_tasks: private/k8s_migrate_pvs.yml 10 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_migrate_pvs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Authenticed with K8s" 3 | block: 4 | 5 | - name: "K8s Migrate PVs | Get and Set k8s API key" 6 | block: 7 | - name: "K8s Migrate PVs | Get K8s access token" 8 | k8s_auth: 9 | username: "{{ k8s_username }}" 10 | password: "{{ k8s_password }}" 11 | register: k8s_auth_results 12 | 13 | - name: "K8s Migrate PVs | Set k8s_api_key" 14 | set_fact: 15 | k8s_api_key: "{{ k8s_auth_results.k8s_auth.api_key }}" 16 | when: (k8s_api_key is not defined) and ((k8s_username is defined) and (k8s_password is defined)) 17 | 18 | - name: "K8s Migrate PVs | Set defaults" 19 | block: 20 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_pre_pod_shutdown_stage_only" 21 | set_fact: 22 | k8s_pv_migrator_pre_pod_shutdown_stage_only: False 23 | when: k8s_pv_migrator_pre_pod_shutdown_stage_only is not defined 24 | 25 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_pvc_label_selectors" 26 | set_fact: 27 | k8s_pv_migrator_pvc_label_selectors: [] 28 | when: k8s_pv_migrator_pvc_label_selectors is not defined 29 | 30 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_job_wait_timeout" 31 | set_fact: 32 | k8s_pv_migrator_job_wait_timeout: 3600 33 | when: k8s_pv_migrator_job_wait_timeout is not defined 34 | 35 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_temp_destination_pvc_postfix" 36 | set_fact: 37 | k8s_pv_migrator_temp_destination_pvc_postfix: '-new' 38 | when: k8s_pv_migrator_temp_destination_pvc_postfix is not defined 39 | 40 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_temp_destination_pvc_bind_wait_retries" 41 | set_fact: 42 | k8s_pv_migrator_temp_destination_pvc_bind_wait_retries: 60 43 | when: k8s_pv_migrator_temp_destination_pvc_bind_wait_retries is not defined 44 | 45 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_temp_destination_pvc_bind_wait_delay" 46 | set_fact: 47 | k8s_pv_migrator_temp_destination_pvc_bind_wait_delay: 1 48 | when: k8s_pv_migrator_temp_destination_pvc_bind_wait_delay is not defined 49 | 50 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_pods_shutdown_wait_retries" 51 | set_fact: 52 | k8s_pv_migrator_pods_shutdown_wait_retries: 120 53 | when: k8s_pv_migrator_pods_shutdown_wait_retries is not defined 54 | 55 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_pods_shutdown_wait_delay" 56 | set_fact: 57 | k8s_pv_migrator_pods_shutdown_wait_delay: 1 58 | when: k8s_pv_migrator_pods_shutdown_wait_delay is not defined 59 | 60 | - name: "K8s Migrate PVs | Set defaults | k8s_pv_migrator_pods_start_wait_timeout" 61 | set_fact: 62 | k8s_pv_migrator_pods_start_wait_timeout: 600 63 | when: k8s_pv_migrator_pods_start_wait_timeout is not defined 64 | 65 | - name: "K8s Migrate PVs | Loop namespaces to migrate PVS per namespace" 66 | include_tasks: private/k8s_migrate_pvs_for_namespace.yml 67 | loop: "{{ k8s_pv_migrator_namespaces }}" 68 | loop_control: 69 | loop_var: _k8s_pv_migrator_namespace 70 | 71 | always: 72 | - name: "K8s Migrate PVs | Revoke K8s access token" 73 | k8s_auth: 74 | state: absent 75 | api_key: "{{ k8s_api_key }}" 76 | when: ((k8s_username is defined) and (k8s_password is defined)) 77 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_migrate_pvs_create_destination_pvs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Create temporary destination PVCs using new storage class | {{ _k8s_pv_migrator_namespace }}" 3 | k8s: 4 | api_key: "{{ k8s_api_key }}" 5 | state: present 6 | definition: 7 | apiVersion: v1 8 | kind: PersistentVolumeClaim 9 | metadata: 10 | name: "{{ pvc.metadata.name }}{{ k8s_pv_migrator_temp_destination_pvc_postfix }}" 11 | namespace: "{{ _k8s_pv_migrator_namespace }}" 12 | spec: 13 | accessModes: "{{ pvc.spec.accessModes }}" 14 | resources: "{{ pvc.spec.resources }}" 15 | storageClassName: "{{ k8s_pv_migrator_destination_storageclass }}" 16 | loop: "{{ pvcs_sources }}" 17 | loop_control: 18 | loop_var: pvc 19 | register: pvcs_destinations_creation_results 20 | 21 | - name: "K8s Migrate PVs | Wait for temporary destination PVCs to be bound to new destination PV | {{ _k8s_pv_migrator_namespace }}" 22 | k8s_info: 23 | api_key: "{{ k8s_api_key }}" 24 | kind: PersistentVolumeClaim 25 | namespace: "{{ _k8s_pv_migrator_namespace }}" 26 | name: "{{ pvc_destination.metadata.name }}" 27 | until: pvcs_destinations_results.resources | rejectattr('status.phase', 'regex', 'Bound') | list | length == 0 28 | retries: "{{ k8s_pv_migrator_temp_destination_pvc_bind_wait_retries }}" 29 | delay: "{{ k8s_pv_migrator_temp_destination_pvc_bind_wait_delay }}" 30 | register: pvcs_destinations_results 31 | loop: "{{ pvcs_destinations_creation_results.results | map(attribute='result') | list }}" 32 | loop_control: 33 | loop_var: pvc_destination 34 | 35 | - name: "K8s Migrate PVs | Get temporary PVC destiantions | {{ _k8s_pv_migrator_namespace }}" 36 | set_fact: 37 | pvcs_destinations: "{{ pvcs_destinations_results.results | map(attribute='resources') | list | flatten }}" 38 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_migrate_pvs_for_namespace.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Wait for pods to be ready pre-migration | {{ _k8s_pv_migrator_namespace }}" 3 | include_tasks: k8s_migrate_pvs_wait_for_pods_ready.yml 4 | 5 | - name: "K8s Migrate PVs | Get source PVCs | {{ _k8s_pv_migrator_namespace }}" 6 | include_tasks: k8s_migrate_pvs_get_source_pvcs.yml 7 | 8 | - block: 9 | - name: "K8s Migrate PVs | Create destination PVs | {{ _k8s_pv_migrator_namespace }}" 10 | include_tasks: k8s_migrate_pvs_create_destination_pvs.yml 11 | 12 | - name: "K8s Migrate PVs | Perform pre pod scale-down PV rsync | {{ _k8s_pv_migrator_namespace }}" 13 | include_tasks: k8s_run_and_wait_for_migrate_pv_job.yml 14 | vars: 15 | pv_migrator_job_name: 'pre-sync' 16 | 17 | - block: 18 | - name: "K8s Migrate PVs | Pods Scale-down | {{ _k8s_pv_migrator_namespace }}" 19 | include_tasks: k8s_migrate_pvs_pods_scale_down.yml 20 | 21 | - name: "K8s Migrate PVs | Perform post pod scale-down PV rsync | {{ _k8s_pv_migrator_namespace }}" 22 | include_tasks: k8s_run_and_wait_for_migrate_pv_job.yml 23 | vars: 24 | pv_migrator_job_name: 'post-sync' 25 | 26 | - name: "K8s Migrate PVs | Perform PVC migration | {{ _k8s_pv_migrator_namespace }}" 27 | include_tasks: k8s_migrate_pvs_pvc_migration.yml 28 | 29 | - name: "K8s Migrate PVs | Pods Scale-up | {{ _k8s_pv_migrator_namespace }}" 30 | include_tasks: k8s_migrate_pvs_pods_scale_up.yml 31 | when: not k8s_pv_migrator_pre_pod_shutdown_stage_only 32 | when: pvcs_sources | length > 0 33 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_migrate_pvs_get_source_pvcs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Get PVCs to migrate in namespace with correct label(s) | {{ _k8s_pv_migrator_namespace }}" 3 | k8s_info: 4 | api_key: "{{ k8s_api_key }}" 5 | kind: PersistentVolumeClaim 6 | namespace: "{{ _k8s_pv_migrator_namespace }}" 7 | label_selectors: "{{ k8s_pv_migrator_pvc_label_selectors }}" 8 | register: persistentvolumeclaims 9 | 10 | - name: "K8s Migrate PVs | Filter out PVCs that are already on destiation storage class | {{ _k8s_pv_migrator_namespace }}" 11 | set_fact: 12 | pvcs_sources: "{{ persistentvolumeclaims.resources | rejectattr('spec.storageClassName', 'match', k8s_pv_migrator_destination_storageclass) | list}}" 13 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_migrate_pvs_pods_scale_down.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Get all scaleable resources | {{ _k8s_pv_migrator_namespace }}" 3 | k8s_info: 4 | api_key: "{{ k8s_api_key }}" 5 | kind: "{{ item }}" 6 | namespace: "{{ _k8s_pv_migrator_namespace }}" 7 | register: pre_scaledown_scaleable_resources 8 | loop: 9 | - DeploymentConfig 10 | - Deployment 11 | - StatefulSet 12 | - ReplicaSet 13 | - ReplicationController 14 | 15 | # NOTE: this does not work due to: https://github.com/ansible/ansible/pull/65312 16 | #- name: "Scale down all scaleable resources to 0 | namespace={{ _k8s_pv_migrator_namespace }}" 17 | # k8s_scale: 18 | # resource_definition: "{{ item.1 }}" 19 | # replicas: 0 20 | # wait: yes 21 | # loop: "{{ pre_scaledown_scaleable_resources.results | subelements('resources') }}" 22 | 23 | - name: "K8s Migrate PVs | Scale down all scaleable resources to 0 | {{ _k8s_pv_migrator_namespace }}" 24 | k8s: 25 | api_key: "{{ k8s_api_key }}" 26 | kind: "{{ item.1.kind }}" 27 | namespace: "{{ _k8s_pv_migrator_namespace }}" 28 | name: "{{ item.1.metadata.name }}" 29 | definition: 30 | spec: 31 | replicas: 0 32 | loop: "{{ pre_scaledown_scaleable_resources.results | subelements('resources') }}" 33 | 34 | - block: 35 | - name: "K8s Migrate PVs | Wait for all pods to be stopped | {{ _k8s_pv_migrator_namespace }}" 36 | k8s_info: 37 | api_key: "{{ k8s_api_key }}" 38 | kind: Pod 39 | namespace: "{{ _k8s_pv_migrator_namespace }}" 40 | register: pods 41 | until: pods.resources | rejectattr('status.phase', 'regex', 'Succeeded|Failed') | list | length == 0 42 | retries: "{{ k8s_pv_migrator_pods_shutdown_wait_retries }}" 43 | delay: "{{ k8s_pv_migrator_pods_shutdown_wait_delay }}" 44 | rescue: 45 | - name: "K8s Migrate PVs | Pods that are not in 'Succeeded' or 'Failed' state | {{ _k8s_pv_migrator_namespace }}" 46 | debug: 47 | msg: "{{ pods.resources | rejectattr('status.phase', 'regex', 'Succeeded|Failed') | list | map(attribute='metadata.name') | list }}" 48 | 49 | - name: "K8s Migrate PVs | Allow user time to manually kill stuck pods | {{ _k8s_pv_migrator_namespace }}" 50 | pause: 51 | prompt: "Not all pods in namespace ({{ _k8s_pv_migrator_namespace }}) stopped before timeout ({{ k8s_pv_migrator_pods_shutdown_wait_retries * k8s_pv_migrator_pods_shutdown_wait_delay }}) was reached. Now would be a good time to see if any pods are stuck in Termminiating state and manually destroy them. Do you want to continue with the PV migration now? (Yes/No)" 52 | echo: yes 53 | register: pod_shutdown_failed_pause 54 | 55 | - name: "K8s Migrate PVs | Not all pods could be verified as stopped and user opted to stop migration | {{ _k8s_pv_migrator_namespace }}" 56 | fail: 57 | msg: "User selected not to continue migration namespace ({{ _k8s_pv_migrator_namespace }} when prompted based on Pods failing to terminate" 58 | when: pod_shutdown_failed_pause is not regex("Y|y|Yes|yes") 59 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_migrate_pvs_pods_scale_up.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Scale up all scaleable resources to origional replica count | {{ _k8s_pv_migrator_namespace }}" 3 | k8s: 4 | api_key: "{{ k8s_api_key }}" 5 | kind: "{{ item.1.kind }}" 6 | namespace: "{{ _k8s_pv_migrator_namespace }}" 7 | name: "{{ item.1.metadata.name }}" 8 | definition: 9 | spec: 10 | replicas: "{{ item.1.spec.replicas }}" 11 | loop: "{{ pre_scaledown_scaleable_resources.results | subelements('resources') }}" 12 | 13 | - name: "K8s Migrate PVs | Wait for pods to be ready post-migration | {{ _k8s_pv_migrator_namespace }}" 14 | include_tasks: k8s_migrate_pvs_wait_for_pods_ready.yml 15 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_migrate_pvs_pvc_migration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Set destination PVs to 'Retain' so they do not delete when their temporary PVC is deleted | {{ _k8s_pv_migrator_namespace }}" 3 | k8s: 4 | api_key: "{{ k8s_api_key }}" 5 | kind: PersistentVolume 6 | namespace: "{{ _k8s_pv_migrator_namespace }}" 7 | name: "{{ pvc_destination.spec.volumeName }}" 8 | definition: 9 | spec: 10 | persistentVolumeReclaimPolicy: 'Retain' 11 | loop: "{{ pvcs_destinations }}" 12 | loop_control: 13 | loop_var: pvc_destination 14 | register: pv_destinations_updates 15 | 16 | - name: "K8s Migrate PVs | Delete temporary destination PVCs | {{ _k8s_pv_migrator_namespace }}" 17 | k8s: 18 | api_key: "{{ k8s_api_key }}" 19 | kind: PersistentVolumeClaim 20 | namespace: "{{ _k8s_pv_migrator_namespace }}" 21 | name: "{{ pvc_destination.metadata.name }}" 22 | state: absent 23 | loop: "{{ pvcs_destinations }}" 24 | loop_control: 25 | loop_var: pvc_destination 26 | 27 | - name: "K8s Migrate PVs | Set source PVs to 'Retain' and label | {{ _k8s_pv_migrator_namespace }}" 28 | k8s: 29 | api_key: "{{ k8s_api_key }}" 30 | kind: PersistentVolume 31 | name: "{{ pvc_source_and_destination.0.spec.volumeName }}" 32 | definition: 33 | metadata: 34 | labels: 35 | migrated: "true" 36 | migrated-to-pv: "{{ pvc_source_and_destination.1.spec.volumeName }}" 37 | spec: 38 | persistentVolumeReclaimPolicy: 'Retain' 39 | loop: "{{ pvcs_sources | zip(pvcs_destinations) | list }}" 40 | loop_control: 41 | loop_var: pvc_source_and_destination 42 | 43 | - name: "K8s Migrate PVs | Delete source PVCs | {{ _k8s_pv_migrator_namespace }}" 44 | k8s: 45 | api_key: "{{ k8s_api_key }}" 46 | kind: PersistentVolumeClaim 47 | namespace: "{{ _k8s_pv_migrator_namespace }}" 48 | name: "{{ pvc_source.metadata.name }}" 49 | state: absent 50 | loop: "{{ pvcs_sources }}" 51 | loop_control: 52 | loop_var: pvc_source 53 | 54 | - name: "K8s Migrate PVs | Remove claimRef from destination PVs and update labels | {{ _k8s_pv_migrator_namespace }}" 55 | k8s: 56 | api_key: "{{ k8s_api_key }}" 57 | kind: PersistentVolume 58 | namespace: "{{ _k8s_pv_migrator_namespace }}" 59 | name: "{{ pvc_source_and_destination.1.spec.volumeName }}" 60 | definition: 61 | metadata: 62 | labels: 63 | migrated-from-pv: "{{ pvc_source_and_destination.0.spec.volumeName }}" 64 | spec: 65 | claimRef: Null 66 | loop: "{{ pvcs_sources | zip(pvcs_destinations) | list }}" 67 | loop_control: 68 | loop_var: pvc_source_and_destination 69 | 70 | - name: "K8s Migrate PVs | Create new PVC with origional source PVC name pre-bound to new destination PV created in new storage class | {{ _k8s_pv_migrator_namespace }}" 71 | k8s: 72 | api_key: "{{ k8s_api_key }}" 73 | kind: PersistentVolumeClaim 74 | namespace: "{{ _k8s_pv_migrator_namespace }}" 75 | state: present 76 | apply: yes 77 | resource_definition: "{{ pvc_source_and_destination.0 | combine({'spec':{'storageClassName': pvc_source_and_destination.1.spec.storageClassName }}, {'spec':{'volumeName': pvc_source_and_destination.1.spec.volumeName}}, {'metadata':{'labels':{'migrated-from-pv':pvc_source_and_destination.0.spec.volumeName}}}, {'metadata':{'resourceVersion': None}}, recursive=True) }}" 78 | loop: "{{ pvcs_sources | zip(pvcs_destinations) | list }}" 79 | loop_control: 80 | loop_var: pvc_source_and_destination 81 | 82 | - name: "K8s Migrate PVs | Set destination PVs back to their orgional reclaim policy | {{ _k8s_pv_migrator_namespace }}" 83 | k8s: 84 | api_key: "{{ k8s_api_key }}" 85 | kind: PersistentVolume 86 | name: "{{ pv_destination_update.result.metadata.name }}" 87 | definition: 88 | spec: 89 | persistentVolumeReclaimPolicy: "{{ pv_destination_update.diff.before.spec.persistentVolumeReclaimPolicy | default('Retain') }}" 90 | loop: "{{ pv_destinations_updates.results }}" 91 | loop_control: 92 | loop_var: pv_destination_update 93 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_migrate_pvs_wait_for_pods_ready.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Get all pods | {{ _k8s_pv_migrator_namespace }}" 3 | k8s_info: 4 | api_key: "{{ k8s_api_key }}" 5 | kind: Pod 6 | namespace: "{{ _k8s_pv_migrator_namespace }}" 7 | register: k8s_pods 8 | 9 | - name: "K8s Migrate PVs | Wait for all pods to be available | {{ _k8s_pv_migrator_namespace }}" 10 | k8s: 11 | api_key: "{{ k8s_api_key }}" 12 | kind: Pod 13 | name: "{{ pod.metadata.name }}" 14 | namespace: "{{ _k8s_pv_migrator_namespace }}" 15 | wait: yes 16 | wait_condition: 17 | type: Ready 18 | status: "True" 19 | wait_timeout : "{{ k8s_pv_migrator_pods_start_wait_timeout }}" 20 | loop: "{{ k8s_pods.resources | rejectattr('status.phase', 'regex', 'Succeeded') | list }}" 21 | loop_control: 22 | loop_var: pod 23 | -------------------------------------------------------------------------------- /storage/pv-migrator/private/k8s_run_and_wait_for_migrate_pv_job.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "K8s Migrate PVs | Create pv-migrator-{{ pv_migrator_job_name }} job per PVC | {{ _k8s_pv_migrator_namespace }}" 3 | k8s: 4 | api_key: "{{ k8s_api_key }}" 5 | state: present 6 | wait: yes 7 | definition: 8 | apiVersion: v1 9 | kind: Job 10 | metadata: 11 | name: "pv-migrator-{{ pv_migrator_job_name }}-{{ pvc_source_and_destination.0.metadata.name }}" 12 | namespace: "{{ _k8s_pv_migrator_namespace }}" 13 | spec: 14 | parallelism: 1 15 | completions: 1 16 | template: 17 | metadata: 18 | name: "pv-migrator-{{ pv_migrator_job_name }}-{{ pvc_source_and_destination.0.metadata.name }}" 19 | spec: 20 | restartPolicy: Never 21 | containers: 22 | - name: pv-migrator 23 | image: registry.redhat.io/rhel7/rhel-tools 24 | volumeMounts: 25 | - name: migration-source 26 | mountPath: "/migration-source" 27 | - name: migration-destination 28 | mountPath: "/migration-destination" 29 | command: 30 | - 'sh' 31 | - '-c' 32 | - 'rsync -avxHAX --ignore-missing-args --progress /migration-source/* /migration-destination' 33 | volumes: 34 | - name: migration-source 35 | persistentVolumeClaim: 36 | claimName: "{{ pvc_source_and_destination.0.metadata.name }}" 37 | - name: migration-destination 38 | persistentVolumeClaim: 39 | claimName: "{{ pvc_source_and_destination.1.metadata.name }}" 40 | loop: "{{ pvcs_sources | zip(pvcs_destinations) | list }}" 41 | loop_control: 42 | loop_var: pvc_source_and_destination 43 | 44 | - name: "K8s Migrate PVs | Wait for pv-migrator-{{ pv_migrator_job_name }} jobs to complete | {{ _k8s_pv_migrator_namespace }}" 45 | k8s: 46 | api_key: "{{ k8s_api_key }}" 47 | kind: Job 48 | namespace: "{{ _k8s_pv_migrator_namespace }}" 49 | name: "pv-migrator-{{ pv_migrator_job_name }}-{{ pvc.metadata.name }}" 50 | wait: yes 51 | wait_condition: 52 | type: Complete 53 | status: "True" 54 | wait_timeout: "{{ k8s_pv_migrator_job_wait_timeout | default(3600) }}" 55 | loop: "{{ pvcs_sources }}" 56 | loop_control: 57 | loop_var: pvc 58 | -------------------------------------------------------------------------------- /storage/pv-to-path-mapping: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # map PVS to storage paths 4 | printf "%-40s\t%-40s\t%s\n" "PATH" "PV" "CLAIM" 5 | PVS=$(oc get pv -o jsonpath='{.items[*].metadata.name}' | tr ' ' "\n") 6 | declare -A PATH_TO_PV_MAP 7 | while read pv <&3; do 8 | path=$(oc get pv ${pv} -o jsonpath='{.spec..path}') 9 | claim=$(oc get pv ${pv} -o jsonpath='{.spec.claimRef.name}') 10 | 11 | printf "%-40s\t%-40s\t%s\n" "${path}" "${pv}" "${claim}" 12 | done 3<<<"${PVS}" 13 | 14 | -------------------------------------------------------------------------------- /validation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-cop/openshift-toolkit/16bf5b054fd5bdfb5018c1f4e06b80470fde716f/validation/__init__.py -------------------------------------------------------------------------------- /validation/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def pytest_addoption(parser): 5 | # Master option section 6 | parser.addoption( 7 | "--master-node-count", action="store", default=3, help="Master node count." 8 | ) 9 | parser.addoption( 10 | "--etcd-node-count", action="store", default=3, help="ectd node count." 11 | ) 12 | # Infra option section 13 | parser.addoption( 14 | "--router-node-count", action="store", default=3, help="Router node count." 15 | ) 16 | parser.addoption( 17 | "--registry-pod-count", action="store", default=3, help="Registry pod count." 18 | ) 19 | # Logging option section 20 | parser.addoption( 21 | "--es-pod-count", action="store", default=3, help="Elasticsearch pod count." 22 | ) 23 | parser.addoption( 24 | "--kibana-pod-count", action="store", default=1, help="Kibana pod count." 25 | ) 26 | # Monitoring option section 27 | parser.addoption( 28 | "--prom-pod-count", action="store", default=2, help="Prometheus pod count." 29 | ) 30 | parser.addoption( 31 | "--alertmanager-pod-count", action="store", default=2, help="Alertmanager pod count." 32 | ) 33 | parser.addoption( 34 | "--grafana-pod-count", action="store", default=1, help="Grafana pod count." 35 | ) 36 | parser.addoption( 37 | "--kube-state-metrics-pod-count", action="store", default=1, help="kube-state-metrics pod count." 38 | ) 39 | 40 | 41 | # Master fixtures 42 | @pytest.fixture 43 | def master_node_count(request): 44 | return request.config.getoption("--master-node-count") 45 | 46 | 47 | @pytest.fixture 48 | def etcd_node_count(request): 49 | return request.config.getoption("--etcd-node-count") 50 | 51 | 52 | # Infra fixtures 53 | @pytest.fixture 54 | def router_node_count(request): 55 | return request.config.getoption("--router-node-count") 56 | 57 | 58 | @pytest.fixture 59 | def registry_pod_count(request): 60 | return request.config.getoption("--registry-pod-count") 61 | 62 | 63 | # Logging fixtures 64 | @pytest.fixture 65 | def es_pod_count(request): 66 | return request.config.getoption("--es-pod-count") 67 | 68 | 69 | @pytest.fixture 70 | def kibana_pod_count(request): 71 | return request.config.getoption("--kibana-pod-count") 72 | 73 | 74 | # Monitoring fixtures 75 | @pytest.fixture 76 | def prom_pod_count(request): 77 | return request.config.getoption("--prom-pod-count") 78 | 79 | 80 | @pytest.fixture 81 | def alertmanager_pod_count(request): 82 | return request.config.getoption("--alertmanager-pod-count") 83 | 84 | 85 | @pytest.fixture 86 | def grafana_pod_count(request): 87 | return request.config.getoption("--grafana-pod-count") 88 | 89 | 90 | @pytest.fixture 91 | def kube_state_metrics_pod_count(request): 92 | return request.config.getoption("--kube-state-metrics-pod-count") -------------------------------------------------------------------------------- /validation/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-cop/openshift-toolkit/16bf5b054fd5bdfb5018c1f4e06b80470fde716f/validation/lib/__init__.py -------------------------------------------------------------------------------- /validation/lib/k8sHelper.py: -------------------------------------------------------------------------------- 1 | # Creator: Muhammad Aizuddin Bin Zali < mzali@redhat.com> 2 | # Date Created: 7th April 2019 3 | # Primary Function: k8s helper library for pytest validation. 4 | 5 | from kubernetes import client, config 6 | 7 | 8 | class k8sHelper: 9 | def __init__(self): 10 | config.load_kube_config() 11 | 12 | # Make v1 and cv1 as global pointer that can be called from anywhere when this class instantiated. 13 | global v1 14 | global cv1 15 | v1 = client.CoreV1Api() 16 | cv1 = client.CustomObjectsApi() 17 | 18 | def get_running_pods_by_label(self, namespace, label): 19 | self.ret = v1.list_namespaced_pod( 20 | namespace, field_selector='status.phase=Running', label_selector=label) 21 | return len(self.ret.items) 22 | 23 | def get_route_by_name(self, namespace, name): 24 | self.routes = cv1.list_namespaced_custom_object('route.openshift.io', 'v1', namespace, 'routes') 25 | jsonpath_expr = parse('items[?metadata.name == {}].$'.format(name)) 26 | return [match.value for match in jsonpath_expr.find(self.routes)][0] 27 | 28 | def get_node_count(self): 29 | self.ret = v1.list_node() 30 | return len(self.ret.items) 31 | 32 | def get_node_by_label(self, label): 33 | self.ret = v1.list_node(selector=label) 34 | return self.ret 35 | -------------------------------------------------------------------------------- /validation/pytest.ini: -------------------------------------------------------------------------------- 1 | # Run `pytest --help` to get more information on using `-k EXPRESSION` argument. 2 | 3 | [pytest] 4 | # 1. Run all tests by default. 5 | addopts = --etcd-node-count=3 --router-node-count=2 --registry-pod-count=3 --master-node-count=3 --prom-pod-count=2 --alertmanager-pod-count=3 --grafana-pod-count=1 --kube-state-metrics-pod-count=1 --es-pod-count=3 --kibana-pod-count=1 6 | 7 | # 2. Deselect test using marker. E.g skip logging test. 8 | #addopts = --etcd-node-count=3 --router-node-count=2 --registry-pod-count=3 --master-node-count=3 -k "not logging" 9 | 10 | # 3. Run only test what marked with master or infra keyword. 11 | #addopts = addopts = --etcd-node-count=3 --router-node-count=2 --registry-pod-count=3 --master-node-count=3 -k "master" 12 | 13 | # 4. Debug purposes. To see which test will be selected to run 14 | #addopts = --etcd-node-count=3 --router-node-count=2 --registry-pod-count=3 --master-node-count=3 --collect-only -------------------------------------------------------------------------------- /validation/requirements.txt: -------------------------------------------------------------------------------- 1 | kubernetes 2 | pytest 3 | jsonpath_ng 4 | -------------------------------------------------------------------------------- /validation/ssh_connection_handling.py: -------------------------------------------------------------------------------- 1 | # Owner: Steve Ovens 2 | # Date Created: July 2015 3 | # Primary Function: This is an ssh connection handler to be imported by any script requiring remote access. 4 | # It will do nothing if called directly. 5 | # Dependencies: helper_functions.py 6 | 7 | class HandleSSHConnections: 8 | """This class allows for easier multiple connections. It also handles running remote commands """ 9 | from helper_functions import ImportHelper 10 | ImportHelper.import_error_handling("paramiko", globals()) 11 | ImportHelper.import_error_handling("time", globals()) 12 | 13 | def __init__(self): 14 | self.ssh = paramiko.SSHClient() 15 | 16 | def open_ssh(self, server, user_name): 17 | if not self.ssh_is_connected(): 18 | self.ssh.load_system_host_keys() 19 | self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 20 | self.ssh.connect(server, username=user_name, timeout=120) 21 | self.transport = self.ssh.get_transport() 22 | self.psuedo_tty = self.transport.open_session() 23 | self.psuedo_tty.get_pty() 24 | self.read_tty = self.psuedo_tty.makefile() 25 | 26 | def close_ssh(self): 27 | if self.ssh_is_connected(): 28 | self.read_tty.close() 29 | self.psuedo_tty.close() 30 | self.ssh.close() 31 | time.sleep(2) 32 | 33 | def ssh_is_connected(self): 34 | transport = self.ssh.get_transport() if self.ssh else None 35 | return transport and transport.is_active() 36 | 37 | @staticmethod 38 | def run_remote_commands(ssh_object, command): 39 | stdin, stdout, stderr = ssh_object.ssh.exec_command(command) 40 | temp_list = stdout.readlines() 41 | return(temp_list) -------------------------------------------------------------------------------- /validation/test_cluster_install.py: -------------------------------------------------------------------------------- 1 | from .lib import k8sHelper 2 | import pytest 3 | 4 | # Instantiate k8s_helper class from k8s_helper library. 5 | k8s_client = k8sHelper.k8sHelper() 6 | 7 | 8 | # Master Test Section # 9 | @pytest.mark.master 10 | def test_master_controllers(master_node_count): 11 | assert k8s_client.get_running_pods_by_label( 12 | 'kube-system', 'openshift.io/component=controllers') == int(master_node_count), \ 13 | "Should have {} master controller pods".format(master_node_count) 14 | 15 | 16 | @pytest.mark.master 17 | def test_master_api(master_node_count): 18 | assert k8s_client.get_running_pods_by_label( 19 | 'kube-system', 'openshift.io/component=api') == int(master_node_count), \ 20 | "Should have {} master api pods".format(master_node_count) 21 | 22 | 23 | @pytest.mark.master 24 | def test_master_etcd(etcd_node_count): 25 | assert k8s_client.get_running_pods_by_label( 26 | 'kube-system', 'openshift.io/component=etcd') == int(etcd_node_count), \ 27 | "Should have {} etcd pods".format(etcd_node_count) 28 | 29 | 30 | # Infra Test Section # 31 | @pytest.mark.infra 32 | def test_infra_router(router_node_count): 33 | assert k8s_client.get_running_pods_by_label( 34 | 'default', 'deploymentconfig=router') == int(router_node_count), \ 35 | "Should have {} router pods".format(router_node_count) 36 | 37 | 38 | @pytest.mark.infra 39 | def test_infra_registry(registry_pod_count): 40 | assert k8s_client.get_running_pods_by_label( 41 | 'default', 'deploymentconfig=docker-registry') == int(registry_pod_count), \ 42 | "Should have {} registry pods".format(registry_pod_count) 43 | 44 | 45 | # Logging Test Section # 46 | @pytest.mark.logging 47 | def test_logging_fluentd(): 48 | assert k8s_client.get_running_pods_by_label( 49 | 'openshift-logging', 'component=fluentd') == k8s_client.get_node_count(), \ 50 | "Should have one fluentd pod for every node in the cluster" 51 | 52 | 53 | @pytest.mark.logging 54 | def test_logging_elasticsearch(es_pod_count): 55 | assert k8s_client.get_running_pods_by_label( 56 | 'openshift-logging', 'component=es') >= int(es_pod_count), \ 57 | "Should have {} Elasticsearch pod in the cluster".format(es_pod_count) 58 | 59 | 60 | @pytest.mark.logging 61 | def test_logging_kibana(kibana_pod_count): 62 | assert k8s_client.get_running_pods_by_label( 63 | 'openshift-logging', 'component=kibana') == int(kibana_pod_count), \ 64 | "Should have {} Kibana pod in the cluster".format(kibana_pod_count) 65 | 66 | 67 | # Monitoring Test Section # 68 | @pytest.mark.monitoring 69 | def test_monitoring_node_exporter(): 70 | assert k8s_client.get_running_pods_by_label( 71 | 'openshift-monitoring', 'app=node-exporter') == k8s_client.get_node_count(), \ 72 | "Should have {} fluentd pod in the cluster as total number of node is {}.".format(k8s_client.get_node_count()) 73 | 74 | 75 | @pytest.mark.monitoring 76 | def test_monitoring_prometheus(prom_pod_count): 77 | assert k8s_client.get_running_pods_by_label( 78 | 'openshift-monitoring', 'app=prometheus') == int(prom_pod_count), \ 79 | "Should have {} prometheus pod in the cluster".format(prom_pod_count) 80 | 81 | 82 | @pytest.mark.monitoring 83 | def test_monitoring_alertmanager(alertmanager_pod_count): 84 | assert k8s_client.get_running_pods_by_label( 85 | 'openshift-monitoring', 'app=alertmanager') == int(alertmanager_pod_count), \ 86 | "Should have {} alertmanager pod in the cluster".format(alertmanager_pod_count) 87 | 88 | 89 | @pytest.mark.monitoring 90 | def test_monitoring_grafana(grafana_pod_count): 91 | assert k8s_client.get_running_pods_by_label( 92 | 'openshift-monitoring', 'app=grafana') == int(grafana_pod_count), \ 93 | "Should have {} grafana pod in the cluster".format(grafana_pod_count) 94 | 95 | 96 | @pytest.mark.monitoring 97 | def test_monitoring_kube_state_metrics(kube_state_metrics_pod_count): 98 | assert k8s_client.get_running_pods_by_label( 99 | 'openshift-monitoring', 'app=kube-state-metrics') == int(kube_state_metrics_pod_count), \ 100 | "Should have {} kube-state-metrics pod in the cluster".format(kube_state_metrics_pod_count) 101 | --------------------------------------------------------------------------------