├── .gitreview ├── LICENSE ├── README.md ├── charts ├── att-comdev │ ├── Jenkinsfile │ └── seed.groovy └── openstack │ └── helm-toolkit-compatibility │ └── Jenkinsfile ├── cicd ├── CH-SuperSeed │ ├── seed.groovy │ └── superseed.sh ├── JumpHost │ ├── Jenkinsfile.groovy │ └── seed.groovy ├── NodeCleanup │ ├── Jenkinsfile │ └── seed.groovy └── SuperSeed │ ├── seed.groovy │ └── superseed.Jenkinsfile ├── code-review ├── Jenkinsfile └── seed.groovy ├── community └── .gitignore ├── dev ├── Jenkinsfile └── seed.groovy ├── eric-cicd ├── CAPD │ └── jenkins_capd ├── CAPG │ └── jenkins_capg.groovy ├── CAPO │ └── jenkins_capo.groovy ├── CAPZ │ ├── jenkins_capz │ ├── jenkins_capz-new.groovy │ ├── jenkins_capz_eric_nrf_cnf │ ├── jenkins_capz_eric_nrf_cnf_clusterctl │ ├── jenkins_capz_merged.groovy │ └── jenkins_capz_vpp_cnf ├── METAL3 │ └── jenkins_m3 └── seed.groovy ├── images ├── airship │ ├── JenkinsfilePromenade │ ├── promenade │ │ └── resiliency │ │ │ └── Jenkinsfile │ └── seed.groovy ├── att-comdev │ ├── minimirror │ │ ├── Jenkinsfile │ │ └── seed.groovy │ └── promenade │ │ ├── JenkinsfilePromenade │ │ └── JenkinsfileResiliency ├── baseimages │ ├── Jenkinsfile │ └── seed.groovy ├── code-review-image-build │ ├── Jenkinsfile │ └── seed.groovy ├── coredns │ ├── Jenkinsfile │ └── seed.groovy ├── jenkins │ ├── Jenkinsfile │ └── seed.groovy ├── kubernetes │ ├── Jenkinsfile │ └── seed.groovy ├── nc │ └── ro-helm │ │ ├── Jenkinsfile │ │ └── seed.groovy ├── openstack │ ├── helm-images │ │ ├── Jenkinsfile │ │ └── seed.groovy │ ├── kolla │ │ └── Jenkinsfile │ └── loci │ │ ├── JenkinsfileCommunity │ │ ├── JenkinsfileMos │ │ ├── mos-review │ │ ├── JenkinsfileCodeReview.groovy │ │ ├── JenkinsfileDebug.groovy │ │ ├── JenkinsfileGeneric.groovy │ │ ├── JenkinsfileGenericImageBuild.groovy │ │ ├── JenkinsfileReleaseNightly.groovy │ │ ├── JenkinsfileTestDeployment.groovy │ │ ├── JenkinsfileTox.groovy │ │ ├── JenkinsfileUpdateMirrors.groovy │ │ ├── JenkinsfileUplift.groovy │ │ └── seed.groovy │ │ └── seed.groovy └── ubuntu-base-image │ ├── Jenkinsfile │ └── seed.groovy ├── integration ├── airship │ ├── airship-in-a-bottle │ │ ├── Jenkinsfile │ │ └── seed.groovy │ └── gate-multinode │ │ ├── Jenkinsfile │ │ └── seed.groovy ├── funcs.groovy ├── genesis-full │ └── Jenkinsfile └── site-update │ └── Jenkinsfile ├── nc ├── Airshipctl │ └── airshipctl.groovy ├── Maintenance │ ├── AirshipCtlBot.groovy │ ├── AirshipctlStatusReport.groovy │ ├── CleanupOldBuildLogs.groovy │ ├── VinoBot.groovy │ └── WorkerMaintenance.groovy ├── Treasuremap │ └── Treasuremapv2-AirshipctlKnownState.groovy └── seed.groovy ├── osh ├── dockerfiles │ └── Dockerfile ├── openstack │ ├── images │ │ └── kolla │ │ │ └── Dockerfile │ ├── openstack-addons │ │ └── .gitignore │ ├── openstack-helm │ │ └── Jenkinsfile │ └── openstack-infra │ │ └── .gitignore └── seedjobs │ ├── chartCISeedJob │ └── seedJob.groovy.unused │ └── chartUpdateSeedJob │ └── seedJob.groovy.unused ├── packages ├── pip-pkg │ ├── Jenkinsfile │ └── seed.groovy └── ubuntu │ ├── Jenkinsfile │ └── seed.groovy ├── resources ├── cicd │ └── remove_tests.sh └── heat │ ├── glance │ └── ubuntu.image.template.yaml │ ├── nova │ └── flavor.template.yaml │ ├── stack │ ├── bootstrap.sh │ ├── cloud-config │ ├── loci-bootstrap.sh │ ├── ubuntu.armada.cluster.stack.template.yaml │ ├── ubuntu.basic.stack.template.yaml │ ├── ubuntu.flat.stack.template.yaml │ └── ubuntu.flat.volume.stack.template.yaml │ ├── ubuntu1604.m1.large.yaml │ ├── ubuntu1604.m1.medium.yaml │ ├── ubuntu1604.m1.small.yaml │ ├── ubuntu1604.m1.xlarge.yaml │ └── ubuntu1804.m1.large.yaml ├── src ├── att │ └── comdev │ │ └── cicd │ │ └── config │ │ └── conf.groovy └── test │ └── groovy │ ├── CicdHelperTest.groovy │ └── build │ └── build.gradle ├── test ├── Jenkinsfile └── seed.groovy ├── ucp ├── .gitignore └── charts │ └── build_charts.sh └── vars ├── artifactory.groovy ├── cicd_helper.groovy ├── creds.groovy ├── email.groovy ├── gerrit.groovy ├── gerrithub.groovy ├── heat.groovy ├── image.groovy ├── jenkins.groovy ├── keystone.groovy ├── keystone2.groovy ├── keystone3.groovy ├── loci.groovy ├── logs.groovy ├── notify.groovy ├── osh.groovy ├── pegleg.groovy ├── pegleg2.groovy ├── pip.groovy ├── publish.groovy ├── redfish.groovy ├── shipyard.groovy ├── shipyard2.groovy ├── shipyard3.groovy ├── ssh.groovy ├── ssh_pwd.groovy ├── utils.groovy ├── vm.groovy └── vm2.groovy /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.gerrithub.io 3 | port=29418 4 | project=att-comdev/cicd.git 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CI/CD 2 | 3 | This repository is a collection of various scripts and tools that will be used to create some basic pipelines to test and deploy OSH. It is what AT&T may use for CI in the Dev environment. More to come in the README.md of steps to take to get your Jenkins server to work with the scripts provided. 4 | 5 | Sincerely, 6 | 7 | AT&T Integrated Cloud Community Development Team 8 | 9 | 10 | # Shared Libraries 11 | 12 | Folder `vars` is used for shared functions across pipelines. 13 | 14 | See more information for this plugin [here](https://jenkins.io/doc/book/pipeline/shared-libraries). 15 | 16 | -------------------------------------------------------------------------------- /charts/att-comdev/Jenkinsfile: -------------------------------------------------------------------------------- 1 | import att.comdev.cicd.config.conf 2 | 3 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 4 | 5 | ARTF_REPO="att-comdev-helm-local/charts" 6 | CHARTS_URL = "https://review.gerrithub.io/att-comdev/charts" 7 | HELM_REPO = conf.HELM_REPO 8 | GIT_COMMIT = "" 9 | HELM_TOOLKIT_COMMIT = "" 10 | CHARTS_HOME = "" 11 | 12 | FLAVOR = 'm1.xlarge' 13 | 14 | vm(flavor: FLAVOR) { 15 | stage('Setup environment'){ 16 | vm.setproxy() 17 | } 18 | 19 | stage('Deploy Kubernetes AIO'){ 20 | CHARTS_HOME = "${WORKSPACE}/charts" 21 | HELM_TOOLKIT_COMMIT = osh.deployK8sAIO() 22 | 23 | // allow s3 admin user and bucket creation 24 | def file_location = "${WORKSPACE}/openstack-helm/ceph-rgw/values.yaml" 25 | def chartOverride = readYaml file: file_location 26 | chartOverride.conf.rgw_s3.enabled = true 27 | sh "sudo rm -rf ${file_location}" 28 | writeYaml file: file_location, data: chartOverride 29 | } 30 | 31 | stage('Project Checkout'){ 32 | if (JOB_BASE_NAME == "openstack-helm"){ 33 | gerrit.cloneProject(CHARTS_URL, 34 | GERRIT_BRANCH, 35 | "refs/heads/${GERRIT_BRANCH}", 36 | CHARTS_HOME) 37 | } else if (JOB_BASE_NAME != "openstack-helm" && 38 | (GERRIT_EVENT_TYPE == "patchset-created" || 39 | GERRIT_EVENT_TYPE == "comment-added")){ 40 | gerrit.cloneToBranch(CHARTS_URL, 41 | GERRIT_PATCHSET_REVISION, 42 | CHARTS_HOME) 43 | } else { 44 | gerrit.cloneToBranch(CHARTS_URL, 45 | GERRIT_NEWREV, 46 | CHARTS_HOME) 47 | } 48 | dir(CHARTS_HOME){ 49 | GIT_COMMIT = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() 50 | } 51 | } 52 | 53 | dir(CHARTS_HOME){ 54 | stage('Build & Package'){ 55 | HELM_REPO.each { entry -> sh "helm repo add $entry.key $entry.value" } 56 | 57 | // build all charts for openstack-helm 58 | def cmd = (JOB_BASE_NAME == "openstack-helm") ? "make all" : "make ${JOB_BASE_NAME}" 59 | def status = sh(returnStatus: true, script: cmd) 60 | if (status != 0) { 61 | currentBuild.result = 'FAILED' 62 | error "Charts build failed for ${GIT_COMMIT}!" 63 | }else{ 64 | print "Charts were built for ${GIT_COMMIT}" 65 | } 66 | } 67 | 68 | stage('Rename tarballs'){ 69 | def cmd = 'rename "s/[0-9].[0-9].[0-9]/'+GIT_COMMIT+'.'+HELM_TOOLKIT_COMMIT+'/g" *.tgz' 70 | echo (cmd) 71 | sh(returnStatus:false, script: cmd) 72 | } 73 | 74 | stage('Deploy Openstack Infra'){ 75 | // unset Openstack infra OS_* defaults 76 | // disable network policy until charts in att-comdev/charts support it 77 | withEnv(['OS_REGION_NAME=', 78 | 'OS_USERNAME=', 79 | 'OS_PASSWORD=', 80 | 'OS_PROJECT_NAME=', 81 | 'OS_PROJECT_DOMAIN_NAME=', 82 | 'OS_USER_DOMAIN_NAME=', 83 | 'OS_AUTH_URL=', 84 | "OSH_EXTRA_HELM_ARGS=--values=${WORKSPACE}/openstack-helm/tools/overrides/releases/ocata/loci.yaml --set manifests.network_policy=false"]) { 85 | osh.installOSHLite() 86 | } 87 | } 88 | 89 | stage('Deploy Chart(s)'){ 90 | // Deploy the built chart(s) 91 | charts = sh(returnStdout: true, script: "ls *${GIT_COMMIT}*.tgz").trim() 92 | print "charts is ${charts}" 93 | for (chart in charts.split()) { 94 | chart_name = chart.split("-${GIT_COMMIT}")[0] 95 | 96 | // deploy in openstack namespace having keystone 97 | sh "helm install --name ${chart_name} --namespace openstack ${chart_name}*.tgz" 98 | 99 | // wait 30 min for pods to come up 100 | sh "${WORKSPACE}/openstack-helm/tools/deployment/common/wait-for-pods.sh openstack 1800" 101 | 102 | sh "helm status ${chart_name}" 103 | 104 | // delete chart to free up resources prior to installing another chart 105 | sh "helm delete --purge ${chart_name}" 106 | 107 | // wait 5 min for pods to terminate 108 | sh "${WORKSPACE}/openstack-helm/tools/deployment/common/wait-for-pods.sh openstack 300" 109 | } 110 | } 111 | 112 | stage('Publish'){ 113 | if("${GERRIT_EVENT_TYPE}" != "change-merged"){ 114 | ARTF_REPO="${ARTF_REPO}/test" 115 | } 116 | publish.putArtifacts("*${GIT_COMMIT}.*.tgz", "${ARTF_REPO}/") 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /charts/att-comdev/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_FOLDER="charts/att-comdev" 2 | folder(JOB_FOLDER) 3 | 4 | def projects = ['jenkins', 5 | 'artifactory-ha', 6 | 'distribution', 7 | 'mission-control', 8 | 'xray'] 9 | 10 | projects.each { project_name -> 11 | JOB_BASE_NAME=project_name 12 | pipelineJob("${JOB_FOLDER}/${JOB_BASE_NAME}") { 13 | description("This job builds the helm charts for ${JOB_BASE_NAME}") 14 | logRotator { 15 | daysToKeep(90) 16 | } 17 | parameters { 18 | stringParam { 19 | name ('GERRIT_PROJECT') 20 | defaultValue("att-comdev/charts") 21 | description('Gerrit refspec or branch') 22 | trim(true) 23 | } 24 | stringParam { 25 | name ('GERRIT_REFSPEC') 26 | defaultValue('master') 27 | description('Gerrit refspec or branch') 28 | trim(true) 29 | } 30 | stringParam { 31 | name ('GERRIT_CHANGE_NUMBER') 32 | defaultValue('0') 33 | description('patchset number') 34 | trim(true) 35 | } 36 | stringParam { 37 | name ('GERRIT_EVENT_TYPE') 38 | defaultValue('patchset-created') 39 | description('patchset-created or change-merged') 40 | trim(true) 41 | } 42 | } 43 | triggers { 44 | gerritTrigger { 45 | serverName('Gerrithub-jenkins-temp') 46 | gerritProjects { 47 | gerritProject { 48 | compareType('PLAIN') 49 | pattern("att-comdev/charts") 50 | branches { 51 | branch { 52 | compareType("ANT") 53 | pattern("**") 54 | } 55 | } 56 | filePaths { 57 | filePath { 58 | compareType("ANT") 59 | pattern("${project_name}/**") 60 | } 61 | } 62 | disableStrictForbiddenFileVerification(false) 63 | } 64 | } 65 | triggerOnEvents { 66 | changeMerged() 67 | patchsetCreated { 68 | excludeDrafts(true) 69 | excludeTrivialRebase(false) 70 | excludeNoCodeChange(false) 71 | } 72 | commentAddedContains { 73 | commentAddedCommentContains('recheck') 74 | } 75 | } 76 | } 77 | definition { 78 | cps { 79 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 80 | sandbox(false) 81 | } 82 | } 83 | } 84 | } 85 | } 86 | 87 | // This pipeline builds the charts in att-comdev/charts 88 | // for any changes in helm-toolkit to ensure charts are not broken 89 | JOB_BASE_NAME="openstack-helm" 90 | pipelineJob("${JOB_FOLDER}/${JOB_BASE_NAME}") { 91 | description('This job builds the helm charts of all components in att-comdev/charts \n' + 92 | 'for helm-toolkit changes in openstack/openstack-helm') 93 | logRotator { 94 | daysToKeep(90) 95 | } 96 | triggers { 97 | gerritTrigger { 98 | silentMode(true) 99 | serverName('OS-CommunityGerrit') 100 | gerritProjects { 101 | gerritProject { 102 | compareType('PLAIN') 103 | pattern("openstack/openstack-helm") 104 | branches { 105 | branch { 106 | compareType("ANT") 107 | pattern("**") 108 | } 109 | } 110 | filePaths { 111 | filePath { 112 | compareType("ANT") 113 | pattern("helm-toolkit/**") 114 | } 115 | } 116 | disableStrictForbiddenFileVerification(false) 117 | } 118 | } 119 | triggerOnEvents { 120 | changeMerged() 121 | } 122 | } 123 | definition { 124 | cps { 125 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 126 | sandbox(false) 127 | } 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /charts/openstack/helm-toolkit-compatibility/Jenkinsfile: -------------------------------------------------------------------------------- 1 | NODE_NAME = "charts-helm-compat-${BUILD_NUMBER}" 2 | NODE_TMPL = "charts/ubuntu.m1.medium.yaml" 3 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 4 | //manual run will need these variables: 5 | if (GERRIT_CHANGE_NUMBER == 'none'){ 6 | GERRIT_CHANGE_NUMBER = "manual run of ${GERRIT_REFSPEC}" 7 | } 8 | 9 | def build_charts = ''' 10 | 11 | set -e 12 | 13 | #List of projects to work with is a job parameter: 14 | #PROJECT_LIST="armada deckhand drydock promenade shipyard" 15 | #See seed.groovy job parameters. 16 | 17 | record_fail(){ 18 | echo -e "\n ERROR: Project ${1} failed! " | tee -a ${failed_log} 19 | grep -i 'error' ${1}.log >> ${failed_log} 20 | } 21 | 22 | prepare_env(){ 23 | git clone -q --depth 1 https://git.openstack.org/openstack/openstack-helm.git 24 | cd openstack-helm 25 | if [ ! ${GERRIT_REFSPEC} = "master" ]; then 26 | git fetch https://git.openstack.org/openstack/openstack-helm.git ${GERRIT_REFSPEC} 27 | git checkout FETCH_HEAD 28 | fi 29 | source tools/gate/vars.sh 30 | source tools/gate/funcs/helm.sh 31 | which helm || helm_install 32 | helm_serve 33 | helm_plugin_template_install ||: 34 | make helm-toolkit 35 | } 36 | 37 | clone_projects(){ 38 | cd ${WDIR}/build 39 | for project in ${PROJECT_LIST}; do 40 | git clone -q --depth 1 https://review.gerrithub.io/att-comdev/${project} 41 | done 42 | } 43 | 44 | make_charts(){ 45 | set -xe 46 | for project in $PROJECT_LIST; do 47 | cd ${WDIR}/build/${project} 48 | #Assume we have Makefile in every project we test. 49 | make charts 2>&1 | tee ${project}.log || record_fail ${project} 50 | done 51 | } 52 | 53 | BUILD_DIR='build' 54 | WDIR=`pwd` 55 | failed_log=${WDIR}/build/failed.log 56 | 57 | #----- MAIN -----# 58 | mkdir ${BUILD_DIR} 59 | cd ${BUILD_DIR} 60 | #helm installation process is muted: 61 | prepare_env &> /dev/null 62 | clone_projects &> /dev/null 63 | make_charts 64 | echo "Done!" 65 | 66 | if [ -f ${failed_log} ]; then 67 | cat ${failed_log} 68 | exit 1 69 | fi 70 | ''' 71 | 72 | vm(NODE_NAME,NODE_TMPL){ 73 | stage('Make Charts'){ 74 | def status = sh(returnStatus: true, script: build_charts) 75 | if (status != 0) { 76 | currentBuild.result = 'FAILED' 77 | def warning_msg = "WARNING: Check failed for ${GERRIT_CHANGE_NUMBER}!\n" 78 | def error_msg = readFile "${WORKSPACE}/build/failed.log" 79 | def urls_msg = "\n${GERRIT_CHANGE_URL} \n${BUILD_URL}" 80 | notify.msg(warning_msg + error_msg + urls_msg) 81 | //also we'll notify #jenkins channel 82 | notify.msg(warning_msg + error_msg + urls_msg, '#jenkins') 83 | }else{ 84 | notify.msg("${GERRIT_CHANGE_NUMBER} is OK") 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /cicd/CH-SuperSeed/seed.groovy: -------------------------------------------------------------------------------- 1 | release_branch = "v1.4" 2 | base_path = "cicd" 3 | job_path = "${base_path}/CH-SuperSeed" 4 | folder(base_path) 5 | 6 | // needs to be a freestyle job to be able to process DSLs 7 | freeStyleJob(job_path) { 8 | logRotator{ 9 | daysToKeep(90) 10 | } 11 | label('master') 12 | parameters { 13 | stringParam{ 14 | name ('RELEASE_FILE_PATH') 15 | defaultValue('src/release_file') 16 | description("File that points to a list of seed.groovy's to execute for a Cloudharbor site") 17 | trim(true) 18 | } 19 | stringParam { 20 | name ('GERRIT_REFSPEC') 21 | defaultValue('origin/master') 22 | description('Gerrit refspec') 23 | trim(true) 24 | } 25 | stringParam { 26 | name ('GERRIT_PROJECT') 27 | defaultValue('nc-cicd') 28 | description('Project on Gerrit') 29 | trim(true) 30 | } 31 | } 32 | 33 | triggers { 34 | gerritTrigger { 35 | gerritProjects { 36 | gerritProject { 37 | compareType('REG_EXP') 38 | pattern("^cicd\$") 39 | branches { 40 | branch { 41 | compareType("ANT") 42 | pattern(release_branch) 43 | } 44 | } 45 | disableStrictForbiddenFileVerification(false) 46 | } 47 | gerritProject { 48 | compareType('REG_EXP') 49 | pattern("^nc-cicd\$") 50 | branches { 51 | branch { 52 | compareType("ANT") 53 | pattern(release_branch) 54 | } 55 | } 56 | disableStrictForbiddenFileVerification(false) 57 | } 58 | } 59 | triggerOnEvents { 60 | /// PatchsetCreated trigger should be manually enabled on staging: 61 | //patchsetCreated { 62 | // excludeDrafts(true) 63 | // excludeTrivialRebase(false) 64 | // excludeNoCodeChange(false) 65 | //} 66 | 67 | /// changeMerged trigger for production: 68 | changeMerged() 69 | } 70 | } 71 | } 72 | steps { 73 | //Wipe the workspace: 74 | wrappers { 75 | preBuildCleanup() 76 | credentialsBinding { 77 | usernamePassword('JENKINS_USER', 'JENKINS_TOKEN', 'jenkins-token') 78 | } 79 | } 80 | shell(readFileFromWorkspace("${job_path}/superseed.sh")) 81 | jobDsl { 82 | targets('${BUILD_NUMBER}/**/seed*.groovy') 83 | // Add ignoreMissingFiles to ignore when seeds are not copied for patchsets 84 | ignoreMissingFiles(true) 85 | //ignoreExisting(true) 86 | //removeAction('DISABLE') 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /cicd/CH-SuperSeed/superseed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | export http_proxy=$HTTP_PROXY 6 | export https_proxy=$http_proxy 7 | 8 | git_clone(){ 9 | 10 | local project_name=$1 11 | local local_path=$2 12 | local refspec=$3 13 | local gerrit_url=$INTERNAL_GERRIT_SSH 14 | 15 | if [[ "${local_path}" =~ ^[\/\.]*$ ]]; then 16 | echo "ERROR: Bad local path '${local_path}'" 17 | exit 1 18 | fi 19 | 20 | if [ -z "${refspec}" ]; then 21 | echo "ERROR: Empty refspec given" 22 | exit 1 23 | fi 24 | 25 | git clone ${gerrit_url}/${project_name} ${local_path} 26 | 27 | pushd ${local_path} 28 | if [[ "${refspec}" =~ ^refs\/ ]]; then 29 | git fetch ${gerrit_url}/${project_name} ${refspec} 30 | git checkout FETCH_HEAD 31 | else 32 | git checkout ${refspec} 33 | fi 34 | popd 35 | } 36 | 37 | create_seed_list(){ 38 | # Create a list of seeds from the release file 39 | # which is located under the src/ directory 40 | 41 | local release_file=$1 42 | 43 | if [ ! -f ${release_file} ]; then 44 | echo "ERROR: Release file not found!" 45 | exit 1 46 | fi 47 | 48 | while read -r line || [[ -n "$line" ]]; do 49 | line+="," 50 | release_list=$release_list$line 51 | echo "INFO: Text read from release file: $line" 52 | done < ${release_file} 53 | 54 | export RELEASE_LIST=$release_list 55 | echo "INFO: RELEASE_LIST is [${RELEASE_LIST}]" 56 | 57 | } 58 | 59 | copy_seed(){ 60 | # Split comma separated seed list 61 | # copy all seed.groovy files with prefixed dir name 62 | # param - comma separated relative paths of seed.groovy 63 | 64 | # "${BUILD_NUMBER}/seed_.groovy" is a hardcoded path 65 | # for 'Process Job DSLs' part of the job. 66 | # See cicd/CH-SuperSeed/seed.groovy file. 67 | 68 | mkdir -p ${WORKSPACE}/${BUILD_NUMBER} 69 | seed_list=$(echo $1 | tr "," "\n") 70 | 71 | for seed in $seed_list; do 72 | seed_file="${WORKSPACE}/${seed}" 73 | if [ -f ${seed_file} ]; then 74 | 75 | # copy dependency file(s) 76 | grep -E "evaluate\(.+\.groovy.?\)" "${seed_file}" | while read -r match ; do 77 | eval dep_file_target_loc=$(echo "$match" | cut -d '"' -f2) 78 | dep_file_name=$(basename "$dep_file_target_loc") 79 | dep_file_loc=$(find ${WORKSPACE} -name "$dep_file_name") 80 | cp -a $dep_file_loc $dep_file_target_loc 81 | done 82 | 83 | # rename the seed groovy as an unique yet readable filename 84 | # this allows for multiple seed in same dir 85 | random_string=$(head /dev/urandom | tr -dc A-Za-z | head -c 6) 86 | # convert '-' to '_' to avoid dsl script name error 87 | seed_dir=$(dirname ${seed} | awk -F '/' '{print $NF}' | tr '-' '_') 88 | filename_only=$(basename ${seed} | cut -d. -f1 | tr '-' '_') 89 | cp -a ${seed_file} ${WORKSPACE}/${BUILD_NUMBER}/seed_${seed_dir}_${filename_only}_${random_string}.groovy 90 | else 91 | # Fail the build if file doesn't exists: 92 | echo "ERROR: ${seed_file} not found" 93 | exit 1 94 | fi 95 | done 96 | } 97 | 98 | 99 | ######MAIN##### 100 | git_clone ${GERRIT_PROJECT} ${WORKSPACE} ${GERRIT_REFSPEC} 101 | 102 | create_seed_list ${RELEASE_FILE_PATH} 103 | 104 | 105 | # Skip applying the seed files for patchsets 106 | if [[ ${GERRIT_EVENT_TYPE} == "patchset-created" ]]; then 107 | echo "INFO: Not applying seeds for patchsets, Seeds are applied only after merge" 108 | exit 0 109 | fi 110 | 111 | if [[ ! ${RELEASE_LIST} =~ ^tests/ ]]; then 112 | copy_seed ${RELEASE_LIST} 113 | else 114 | echo "Not copying seed(s), because it is in tests/ directory" 115 | fi 116 | 117 | # Empty space for DSL script debug information: 118 | echo -e "=================================================\n" 119 | -------------------------------------------------------------------------------- /cicd/JumpHost/Jenkinsfile.groovy: -------------------------------------------------------------------------------- 1 | CONFIG_HEADER = """ 2 | global 3 | daemon 4 | maxconn 10000 5 | 6 | defaults 7 | timeout connect 500s 8 | timeout client 5000s 9 | timeout server 1h 10 | """ 11 | 12 | def getConfigSection(Integer octet) { 13 | def frontend = "frontend-${octet}" 14 | def backend = "backend-${octet}" 15 | def server = "server-${octet}" 16 | def ip = "10.0.0.${octet}" 17 | def port = 10000 + octet 18 | return """ 19 | frontend ${frontend} 20 | bind *:${port} 21 | default_backend ${backend} 22 | timeout client 1h 23 | 24 | backend ${backend} 25 | mode tcp 26 | server ${server} ${ip}:22 27 | """ 28 | } 29 | 30 | 31 | def compileConfig(configHeader) { 32 | def config = configHeader 33 | for (it in 2..254) { 34 | config += getConfigSection(it) 35 | } 36 | writeFile file: "myhaproxy.cfg", text: config 37 | } 38 | 39 | 40 | vm (doNotDeleteNode: true, useJumphost: false) { 41 | compileConfig(CONFIG_HEADER) 42 | sh ''' 43 | export DEBIAN_FRONTEND=noninteractive 44 | sudo apt-get update 45 | sudo apt-get install haproxy iptables-persistent -y'" 46 | sudo cp myhaproxy.cfg /etc/haproxy/haproxy.cfg" 47 | sudo service haproxy restart" 48 | 49 | echo "====== Setup insterfaces ======" 50 | netfile=$(find /etc/network/interfaces.d -name "*.cfg") 51 | for interface in $(ls -1 /sys/class/net | grep ens); do 52 | if [ $interface != "ens3" ];then 53 | sudo bash -c "echo 'auto $interface' >> ${netfile}" 54 | sudo bash -c "echo 'iface $interface inet dhcp' >> ${netfile}" 55 | sudo ifdown $interface 56 | sudo ifup $interface 57 | fi 58 | done 59 | 60 | sudo iptables -A FORWARD -i ens4 -o ens3 -j ACCEPT 61 | sudo iptables -A FORWARD -i ens3 -o ens4 -m state --state ESTABLISHED,RELATED -j ACCEPT 62 | sudo iptables -t nat -A POSTROUTING -o ens3 -j MASQUERADE 63 | sudo iptables-save > /etc/iptables/rules.v4 64 | ''' 65 | } 66 | -------------------------------------------------------------------------------- /cicd/JumpHost/seed.groovy: -------------------------------------------------------------------------------- 1 | path = "cicd/JumpHost" 2 | 3 | pipelineJob(path) { 4 | description("This job creates a jumphost to access jenkins agents run in Openstack VMs") 5 | logRotator { 6 | daysToKeep(30) 7 | } 8 | definition { 9 | cps { 10 | script(readFileFromWorkspace("${path}/Jenkinsfile.groovy")) 11 | sandbox(false) 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /cicd/NodeCleanup/Jenkinsfile: -------------------------------------------------------------------------------- 1 | import att.comdev.cicd.config.conf 2 | 3 | // node used for launching VMs in AF 4 | def launch_node = 'jenkins-node-launch' 5 | 6 | def EXCLUDE_NODE_LIST = conf.EXCLUDE_NODES.split() 7 | 8 | stage ('Release Jenkins slave') { 9 | 10 | println("DELETE_NODENAME: " + DELETE_NODENAME) 11 | println("EXCLUDE_NODE_LIST: " + EXCLUDE_NODE_LIST) 12 | 13 | for (exclude_node in EXCLUDE_NODE_LIST) { 14 | if (DELETE_NODENAME.equalsIgnoreCase(exclude_node)) { 15 | println "$DELETE_NODENAME belongs to list of critical nodes excluded from deletion" 16 | error "Cannot delete node $DELETE_NODENAME" 17 | } 18 | } 19 | 20 | // in AF Jenkins slaves are operated from 'jenkins-node-launch' node, not from 'master' 21 | node(launch_node) { 22 | jenkins.node_delete(DELETE_NODENAME) 23 | } 24 | node('master') { 25 | jenkins.node_delete(DELETE_NODENAME) 26 | } 27 | } 28 | stage ('Delete stack') { 29 | node(launch_node) { 30 | heat.stack_delete(DELETE_NODENAME) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /cicd/NodeCleanup/seed.groovy: -------------------------------------------------------------------------------- 1 | base_path = "cicd" 2 | job_path = "${base_path}/NodeCleanup" 3 | folder("${base_path}") 4 | 5 | pipelineJob(job_path) { 6 | description("This job deletes the jenkins node and its underlying stack") 7 | logRotator { 8 | daysToKeep(30) 9 | } 10 | parameters { 11 | stringParam { 12 | name ('DELETE_NODENAME') 13 | defaultValue('') 14 | description('Node to be deleted') 15 | trim(true) 16 | } 17 | } 18 | definition { 19 | cps { 20 | script(readFileFromWorkspace("${job_path}/Jenkinsfile")) 21 | sandbox(false) 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /cicd/SuperSeed/seed.groovy: -------------------------------------------------------------------------------- 1 | base_path = 'cicd' 2 | job_path = "${base_path}/SuperSeed" 3 | folder("${base_path}") 4 | 5 | pipelineJob("${job_path}") { 6 | description('This Job is used create or update other jobs from seed scripts. ' + 7 | 'It can use files from various locations including github.') 8 | logRotator { 9 | daysToKeep(90) 10 | } 11 | parameters { 12 | stringParam('SEED_PATH', '', 'Comma delimited Seed path \n' + 13 | 'Example: cicd/SuperSeed/seed.groovy,cicd/NodeCleanup/seed.groovy') 14 | stringParam('RELEASE_FILE_PATH', '', 'File with a list of seed.groovy files') 15 | stringParam('GERRIT_REFSPEC', 'origin/master', "Gerrit Refspec") 16 | 17 | // this parameter is not needed for the actual SuperSeed job 18 | // all the interactions with git are done using refspec 19 | // but existing jobs sometimes use GERRIT_REFSPEC variable during seeding 20 | // so this parameter is kept here for backward compatibility. 21 | stringParam('GERRIT_BRANCH', 'master',"Branch for provided GERRIT_REFSPEC") 22 | 23 | stringParam('GERRIT_HOST', 'review.gerrithub.io', 'Gerrit Host') 24 | stringParam('GERRIT_PROJECT', 'att-comdev/cicd', 'Project on Gerrit') 25 | } 26 | 27 | properties { 28 | pipelineTriggers { 29 | triggers { 30 | gerritTrigger { 31 | gerritProjects { 32 | gerritProject { 33 | compareType('PLAIN') 34 | pattern('att-comdev/cicd') 35 | branches { 36 | branch { 37 | compareType('ANT') 38 | pattern('**') 39 | } 40 | } 41 | disableStrictForbiddenFileVerification(false) 42 | } 43 | gerritProject { 44 | compareType('PLAIN') 45 | pattern("nc-cicd") 46 | branches { 47 | branch { 48 | compareType('ANT') 49 | pattern('**') 50 | } 51 | } 52 | disableStrictForbiddenFileVerification(false) 53 | } 54 | } 55 | triggerOnEvents { 56 | /// PatchsetCreated trigger should be manually enabled on staging: 57 | patchsetCreated { 58 | excludeDrafts(true) 59 | excludeTrivialRebase(false) 60 | excludeNoCodeChange(false) 61 | } 62 | 63 | commentAddedContains { 64 | commentAddedCommentContains('(?i)recheck|reverify|rebuild') 65 | } 66 | 67 | /// changeMerged trigger for production: 68 | changeMerged() 69 | } 70 | } 71 | definition { 72 | cps { 73 | script(readFileFromWorkspace("cicd/SuperSeed/superseed.Jenkinsfile")) 74 | } 75 | } 76 | } 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /code-review/seed.groovy: -------------------------------------------------------------------------------- 1 | pipelineJob("code-review") { 2 | description("This job is for python applications to run pep8, bandit, unit tests, and code coverage") 3 | logRotator{ 4 | daysToKeep(90) 5 | } 6 | parameters { 7 | stringParam { 8 | defaultValue("") 9 | description('Internal project name for manual build.') 10 | name ('PROJECT_NAME') 11 | trim(true) 12 | } 13 | stringParam { 14 | defaultValue("") 15 | description('Reference for manual build of internal project.\n\n' + 16 | 'Branch or gerrit refspec is supported.') 17 | name ('PROJECT_REF') 18 | trim(true) 19 | } 20 | stringParam { 21 | defaultValue("") 22 | description('Branch for manual build of internal project.\n\n') 23 | name ('PROJECT_BRANCH') 24 | trim(true) 25 | } 26 | } 27 | triggers { 28 | definition { 29 | cps { 30 | script(readFileFromWorkspace("code-review/Jenkinsfile")) 31 | sandbox() 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /community/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/att-comdev/cicd/8e92dce5a7bcb048638ebf6454a2de28f7b94332/community/.gitignore -------------------------------------------------------------------------------- /dev/Jenkinsfile: -------------------------------------------------------------------------------- 1 | 2 | def (fip, port) = vm (initScript: NODE_DATA, image: NODE_IMAGE, 3 | flavor: NODE_FLAVOR, nodePostfix: NODE_POSTFIX, 4 | buildType: 'basic', doNotDelete: true) { 5 | sh ('hostname') 6 | sh("echo ${SSH_KEY} >> /home/ubuntu/.ssh/authorized_keys") 7 | } 8 | 9 | currentBuild.displayName = "#${BUILD_NUMBER} VM ${NODE_POSTFIX} ${fip}:${port}" 10 | print "Access VM on Floating IP => ssh ubuntu@${fip} -p ${port}" 11 | -------------------------------------------------------------------------------- /dev/seed.groovy: -------------------------------------------------------------------------------- 1 | 2 | JOB_FOLDER='dev' 3 | JOB_NAME='vms' 4 | 5 | folder(JOB_FOLDER) 6 | 7 | pipelineJob("${JOB_FOLDER}/${JOB_NAME}") { 8 | 9 | description("This job allows to create a VM on demand, and keep it running") 10 | 11 | logRotator { 12 | daysToKeep(30) 13 | } 14 | 15 | parameters { 16 | 17 | stringParam { 18 | name ('NODE_POSTFIX') 19 | defaultValue('user-vm1') 20 | description('Unique name to identify heat stack to be created') 21 | trim(true) 22 | } 23 | 24 | stringParam { 25 | name ('SSH_KEY') 26 | defaultValue('') 27 | description('Public SSH key to access the VM') 28 | trim(true) 29 | } 30 | 31 | choiceParam('NODE_DATA', ['bootstrap.sh'], 32 | 'Select user data based on resources/heat/stack') 33 | 34 | choiceParam('NODE_IMAGE', ['cicd-ubuntu-16.04-server-cloudimg-amd64', 35 | 'cicd-ubuntu-18.04-server-cloudimg-amd64'], 36 | 'Select image for the VM') 37 | 38 | choiceParam('NODE_FLAVOR', ['m1.medium', 'm1.large', 'm1.xlarge'], 39 | 'Select flavor for the VM') 40 | } 41 | 42 | definition { 43 | cps { 44 | sandbox() 45 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 46 | } 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /eric-cicd/CAPD/jenkins_capd: -------------------------------------------------------------------------------- 1 | def AIRSHIPCTL 2 | def AIRSHIP_SRC 3 | def KUBECONFIG 4 | def WORKERS_COUNT 5 | def CONTROLPLANE_COUNT 6 | def TEST_SITE 7 | def TARGET_CLUSTER_NAME 8 | def KIND_EXPERIMENTAL_DOCKER_NETWORK 9 | def PROVIDER 10 | node(targetVM) { 11 | withEnv(['AIRSHIP_SRC=/tmp/airship', 12 | 'KUBECONFIG=$HOME/.airship/kubeconfig', 13 | 'WORKERS_COUNT=2', 14 | 'CONTROLPLANE_COUNT=1', 15 | 'TEST_SITE=docker-test-site', 16 | 'TARGET_CLUSTER_NAME=target-cluster', 17 | 'KIND_EXPERIMENTAL_DOCKER_NETWORK=bridge', 18 | 'AIRSHIPCTL=/tmp/airship/airshipctl', 19 | 'PROVIDER=default' ]){ 20 | try{ 21 | AIRSHIPCTL = sh(script: """echo /tmp/airship/airshipctl""", returnStdout:true).trim() 22 | stage("Clone Airshipctl") { 23 | writeFile file: 'clone.sh', text: '''#!/bin/bash 24 | sudo swapoff -a 25 | sudo rm -rf ~/.airship 26 | sudo rm -rf /tmp/airship 27 | mkdir /tmp/airship 28 | cd /tmp/airship 29 | git clone https://opendev.org/airship/airshipctl.git $AIRSHIPCTL 30 | cd $AIRSHIPCTL 31 | git fetch https://review.opendev.org/airship/airshipctl '''+ env.REF_SPEC_SCRIPTS +''' && git checkout FETCH_HEAD''' 32 | sh 'bash ./clone.sh' 33 | } 34 | stage("Install Kustomize, Kind and KubeCtl") { 35 | writeFile file: 'install.sh', text: '''#!/bin/bash 36 | cd $AIRSHIPCTL && ./tools/gate/00_setup.sh 37 | curl -sSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.8.5/kustomize_v3.8.5_linux_amd64.tar.gz | tar -C /tmp -xzf - 38 | sudo install /tmp/kustomize /usr/local/bin 39 | kustomize version 40 | ./tools/deployment/provider_common/01_install_kind.sh 41 | ./tools/deployment/01_install_kubectl.sh''' 42 | sh 'bash ./install.sh' 43 | } 44 | stage("install Airshipctl") { 45 | writeFile file: 'airshipctl.sh', text: '''#!/bin/bash 46 | cd $AIRSHIPCTL && ./tools/deployment/21_systemwide_executable.sh''' 47 | sh 'bash ./airshipctl.sh' 48 | } 49 | stage("Initialize Kind Cluster"){ 50 | writeFile file: 'kind.sh', text: '''#!/bin/bash 51 | echo "deleting existing clusters" && kind delete cluster --name ephemeral-cluster && kind delete cluster --name target-cluster 52 | cd $AIRSHIPCTL && CLUSTER=ephemeral-cluster KIND_CONFIG=./tools/deployment/templates/kind-cluster-with-extramounts ./tools/document/start_kind.sh 53 | ./tools/deployment/provider_common/03-init-airship-config.sh''' 54 | sh 'bash ./kind.sh' 55 | } 56 | stage("Deploy Ephermal"){ 57 | writeFile file: 'epheremal.sh', text: '''#!/bin/bash 58 | cd $AIRSHIPCTL && PROVIDER=default TEST_SITE=docker-test-site PROVIDER_MANIFEST=docker_manifest ./tools/deployment/26_deploy_capi_ephemeral_node.sh''' 59 | sh 'bash ./epheremal.sh' 60 | } 61 | stage("Deploy Control Plane"){ 62 | writeFile file: 'control-plane.sh', text: '''#!/bin/bash 63 | cd $AIRSHIPCTL && CONTROLPLANE_COUNT=1 TEST_SITE=docker-test-site ./tools/deployment/provider_common/30_deploy_controlplane.sh''' 64 | sh 'bash ./control-plane.sh' 65 | } 66 | stage("Cluster Move"){ 67 | writeFile file: 'cluster-init.sh', text: '''#!/bin/bash 68 | cd $AIRSHIPCTL && KUBECONFIG=/tmp/target-cluster.kubeconfig ./tools/deployment/provider_common/32_cluster_init_target_node.sh && ./tools/deployment/provider_common/33_cluster_move_target_node.sh''' 69 | sh 'bash ./cluster-init.sh' 70 | } 71 | stage("Deploy Worker Node"){ 72 | writeFile file: 'worker-node.sh', text: '''#!/bin/bash 73 | cd $AIRSHIPCTL && WORKERS_COUNT="2" KUBECONFIG="/tmp/target-cluster.kubeconfig" TEST_SITE="docker-test-site" ./tools/deployment/provider_common/34_deploy_worker_node.sh''' 74 | sh 'bash ./worker-node.sh' 75 | } 76 | }catch (err) { 77 | echo "Failed: ${err}" 78 | currentBuild.result = 'FAILURE' 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /eric-cicd/CAPO/jenkins_capo.groovy: -------------------------------------------------------------------------------- 1 | def AIRSHIPCTL 2 | def AIRSHIP_SRC 3 | def KUBECONFIG 4 | def WORKERS_COUNT 5 | def CONTROLPLANE_COUNT 6 | def TEST_SITE 7 | def TARGET_CLUSTER_NAME 8 | def KIND_EXPERIMENTAL_DOCKER_NETWORK 9 | def PROVIDER 10 | node(targetVM) { 11 | withEnv(['AIRSHIP_SRC=/tmp/airship', 12 | 'KUBECONFIG=/home/capo/.airship/kubeconfig', 13 | 'WORKERS_COUNT=2', 14 | 'CONTROLPLANE_COUNT=1', 15 | 'TEST_SITE=openstack-test-site', 16 | 'TARGET_CLUSTER_NAME=target-cluster', 17 | 'EPHEMERAL_CLUSTER_NAME=ephemeral-cluster', 18 | 'KIND_EXPERIMENTAL_DOCKER_NETWORK=bridge', 19 | 'AIRSHIPCTL=/tmp/airship/airshipctl', 20 | 'PROVIDER_MANIFEST=openstack_manifest', 21 | 'PROVIDER=default' ]){ 22 | try{ 23 | AIRSHIPCTL = sh(script: """echo /tmp/airship/airshipctl""", returnStdout:true).trim() 24 | stage("Clone Airshipctl") { 25 | writeFile file: 'clone.sh', text: '''#!/bin/bash 26 | sudo swapoff -a 27 | sudo rm -rf ~/.airship 28 | sudo rm -rf /tmp/airship 29 | mkdir /tmp/airship 30 | cd /tmp/airship 31 | git clone https://opendev.org/airship/airshipctl.git $AIRSHIPCTL 32 | cd $AIRSHIPCTL 33 | git fetch https://review.opendev.org/airship/airshipctl '''+ env.REF_SPEC_SCRIPTS +''' && git checkout FETCH_HEAD 34 | sed -i 's/"value":.*/"value": "6f02d568-b46d-44ac-86e2-fc88b167fa90"}/g' manifests/site/openstack-test-site/ephemeral/controlplane/external_network_id.json 35 | sed -i 's/clouds.yaml:.*/clouds.yaml: '''+ env.CLOUDS_YAML_B64 +'''/g' manifests/site/openstack-test-site/ephemeral/controlplane/cluster_clouds_yaml_patch.yaml 36 | sed -i 's/clouds.yaml:.*/clouds.yaml: '''+ env.CLOUDS_YAML_B64 +'''/g' manifests/site/openstack-test-site/target/workers/cluster_clouds_yaml_patch.yaml 37 | sed -i 's/content:.*/content: '''+ env.CLOUDS_CONF_B64 +'''/' manifests/site/openstack-test-site/ephemeral/controlplane/control_plane_config_patch.yaml 38 | sed -i 's/content:.*/content: '''+ env.CLOUDS_CONF_B64 +'''/' manifests/site/openstack-test-site/target/workers/workers_cloud_conf_patch.yaml 39 | sed -i 's#ssh-rsa:.*#ssh-rsa: '''+ env.SSH_PUB_KEY +'''#g' manifests/site/openstack-test-site/ephemeral/controlplane/ssh_key_patch.yaml 40 | sed -i 's#ssh-rsa:.*#ssh-rsa: '''+ env.SSH_PUB_KEY +'''#g' manifests/site/openstack-test-site/target/workers/workers_ssh_key_patch.yaml 41 | ''' 42 | sh 'bash ./clone.sh' 43 | } 44 | stage("Install Kustomize, Kind and KubeCtl") { 45 | writeFile file: 'install.sh', text: '''#!/bin/bash 46 | cd $AIRSHIPCTL && ./tools/gate/00_setup.sh 47 | curl -sSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.8.5/kustomize_v3.8.5_linux_amd64.tar.gz | tar -C /tmp -xzf - 48 | sudo install /tmp/kustomize /usr/local/bin 49 | kustomize version 50 | ./tools/deployment/provider_common/01_install_kind.sh 51 | ./tools/deployment/01_install_kubectl.sh''' 52 | sh 'bash ./install.sh' 53 | } 54 | stage("install Airshipctl") { 55 | writeFile file: 'airshipctl.sh', text: '''#!/bin/bash 56 | cd $AIRSHIPCTL && ./tools/deployment/21_systemwide_executable.sh''' 57 | sh 'bash ./airshipctl.sh' 58 | } 59 | stage("Generate Airship Config File"){ 60 | writeFile file: 'config.sh', text: '''#!/bin/bash 61 | cd $AIRSHIPCTL && ./tools/deployment/provider_common/03-init-airship-config.sh''' 62 | sh 'bash ./config.sh' 63 | } 64 | stage("Initialize Kind Cluster"){ 65 | writeFile file: 'kind.sh', text: '''#!/bin/bash 66 | echo "deleting existing clusters" && kind delete clusters --all 67 | cd $AIRSHIPCTL && CLUSTER=ephemeral-cluster KIND_CONFIG=./tools/deployment/templates/kind-cluster-with-extramounts ./tools/document/start_kind.sh 68 | # cd $AIRSHIPCTL && AIRSHIP_CONFIG_METADATA_PATH=manifests/site/openstack-test-site/metadata.yaml SITE=openstack-test-site EXTERNAL_KUBECONFIG="true" ./tools/deployment/22_test_configs.sh''' 69 | sh 'bash ./kind.sh' 70 | } 71 | stage("Deploy Ephermal"){ 72 | writeFile file: 'epheremal.sh', text: '''#!/bin/bash 73 | cd $AIRSHIPCTL && PROVIDER=default TEST_SITE=openstack-test-site PROVIDER_MANIFEST=openstack_manifest ./tools/deployment/26_deploy_capi_ephemeral_node.sh''' 74 | sh 'bash ./epheremal.sh' 75 | } 76 | stage("Deploy Control Plane"){ 77 | writeFile file: 'control-plane.sh', text: '''#!/bin/bash 78 | cd $AIRSHIPCTL && CONTROLPLANE_COUNT=1 TEST_SITE=openstack-test-site ./tools/deployment/provider_common/30_deploy_controlplane.sh''' 79 | sh 'bash ./control-plane.sh' 80 | } 81 | stage("Clusterctl Init Target"){ 82 | writeFile file: 'cluster-init.sh', text: '''#!/bin/bash 83 | cd $AIRSHIPCTL && KUBECONFIG=/tmp/target-cluster.kubeconfig ./tools/deployment/provider_common/32_cluster_init_target_node.sh''' 84 | sh 'bash ./cluster-init.sh' 85 | } 86 | stage("Clusterctl Move"){ 87 | writeFile file: 'cluster-move.sh', text: '''#!/bin/bash 88 | cd $AIRSHIPCTL && ./tools/deployment/provider_common/33_cluster_move_target_node.sh''' 89 | sh 'bash ./cluster-move.sh' 90 | } 91 | stage("Deploy Worker Node"){ 92 | writeFile file: 'worker-node.sh', text: '''#!/bin/bash 93 | cd $AIRSHIPCTL && WORKERS_COUNT="2" KUBECONFIG="/tmp/target-cluster.kubeconfig" SITE="openstack-test-site" ./tools/deployment/provider_common/34_deploy_worker_node.sh''' 94 | sh 'bash ./worker-node.sh' 95 | } 96 | stage("Cleanup Openstack Target CAPI Resources"){ 97 | writeFile file: 'cleanup_openstack_resources.sh', text: '''#!/bin/bash 98 | cd $AIRSHIPCTL && ./tools/deployment/provider_common/35_cleanup_capi_resources.sh''' 99 | sh 'bash ./cleanup_openstack_resources.sh' 100 | } 101 | }catch (err) { 102 | echo "Failed: ${err}" 103 | currentBuild.result = 'FAILURE' 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /eric-cicd/METAL3/jenkins_m3: -------------------------------------------------------------------------------- 1 | def AIRSHIPCTL 2 | def HOME_DIR 3 | def BUILDGATEYML 4 | def KUBEADMCONFIGTEMPLATE 5 | 6 | node(targetVM) { 7 | AIRSHIPCTL = sh(script: """echo ~/airshipctl/""", returnStdout:true).trim() 8 | HOME_DIR = sh(script: """echo ~""", returnStdout:true).trim() 9 | BUILDGATEYML = sh(script: """echo ~/airshipctl/playbooks/airship-airshipctl-build-gate.yaml""", returnStdout:true).trim() 10 | KUBEADMCONFIGTEMPLATE = sh(script: """echo ~/airshipctl/manifests/site/test-site/target/workers/kubeadmconfigtemplate.yaml""", returnStdout:true).trim() 11 | stage("cloning Airshipctl") { 12 | sh label: '', script: """git clone https://opendev.org/airship/airshipctl.git ${AIRSHIPCTL}""" 13 | } 14 | stage("Modify Config && SSH key generation"){ 15 | sh label: '', script: """[ -f ~/.ssh/id_rsa ] && echo "ssh key exists" || ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -P "" && \\ 16 | sed -i "s/\\(\\s*worker_disk_size\\):\\s*[0-9]*/\\1: 120/" ${BUILDGATEYML} && \\ 17 | sed -i "s/\\(\\s*worker_vm_memory_mb\\):\\s*[0-9]*/\\1: 35840/" ${BUILDGATEYML} && \\ 18 | sed -i "s/\\(\\s*worker_vm_vcpus\\):\\s*[0-9]*/\\1: 20/" ${BUILDGATEYML} && \\ 19 | ssh_key=`cat ~/.ssh/id_rsa.pub` && \\ 20 | sed -i "s|\\(ssh-rsa\\).*|\$ssh_key|" ${KUBEADMCONFIGTEMPLATE}""" 21 | } 22 | stage("Install setup.sh") { 23 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/gate/00_setup.sh""" 24 | } 25 | stage("Install build_gate.sh") { 26 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/gate/10_build_gate.sh""" 27 | } 28 | stage("Install kubectl.sh") { 29 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/01_install_kubectl.sh""" 30 | } 31 | stage("Install clusterctl.sh") { 32 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/02_install_clusterctl.sh""" 33 | } 34 | stage("Install systemwide_executable") { 35 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/21_systemwide_executable.sh""" 36 | } 37 | stage("Modify test_configs.sh") { 38 | sh label: '', script: """cd $AIRSHIPCTL && sudo sed -i "s|/tmp/airship|${HOME_DIR}|g" ./tools/deployment/22_test_configs.sh""" 39 | } 40 | stage("Install test_configs.sh") { 41 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/22_test_configs.sh""" 42 | } 43 | stage("Build Ephermal Node") { 44 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/24_build_ephemeral_iso.sh""" 45 | } 46 | stage("Deploy Ephermal Node") { 47 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/25_deploy_ephemeral_node.sh""" 48 | } 49 | stage("Deploy metal3_capi_ephemeral_node") { 50 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/26_deploy_metal3_capi_ephemeral_node.sh""" 51 | } 52 | stage("Install Control Plane") { 53 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/30_deploy_controlplane.sh""" 54 | } 55 | stage("Deploy initinfra_target_node") { 56 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/31_deploy_initinfra_target_node.sh""" 57 | } 58 | stage("Install cluster_init_target_node") { 59 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/32_cluster_init_target_node.sh""" 60 | } 61 | stage("Move Target Node") { 62 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/33_cluster_move_target_node.sh""" 63 | } 64 | stage("Deploy Worker Node") { 65 | sh label: '', script: """cd $AIRSHIPCTL && sudo -S ./tools/deployment/34_deploy_worker_node.sh""" 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /images/airship/promenade/resiliency/Jenkinsfile: -------------------------------------------------------------------------------- 1 | 2 | import groovy.json.JsonSlurperClassic 3 | 4 | 5 | NODE_NAME = "promenade-resiliency-${BUILD_NUMBER}" 6 | NODE_TMPL = 'promenade/promenade-virsh.yaml' 7 | 8 | BASE_IMAGE = "${ARTF_WEB_URL}/ubuntu-images/ubuntu-16.04-server-cloudimg-amd64-disk1.img" 9 | 10 | if (!env.GERRIT_REFSPEC) { 11 | GERRIT_REFSPEC = 'master' 12 | GERRIT_EVENT_TYPE = "manual (${GERRIT_REFSPEC})" 13 | } 14 | 15 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 16 | 17 | 18 | // handle generic Promenade stage 19 | def promenade_stage = { item -> 20 | try { 21 | def cmd = "${WORKSPACE}/tools/g2/stages/${item.script}" 22 | if (item.arguments) { 23 | def args = item.arguments.collect {e -> return "'${e}'"} 24 | cmd = cmd + " " + args.join(" ") 25 | } 26 | timeout (30) { 27 | ansiColor('xterm') { 28 | withEnv(["BASE_IMAGE_URL=${BASE_IMAGE}"]) { 29 | sh cmd 30 | } 31 | } 32 | } 33 | 34 | } catch (err) { 35 | if (item.on_error) { 36 | sh "${WORKSPACE}/tools/g2/on_error/${item.on_error}" 37 | } 38 | error(err) 39 | 40 | } finally { 41 | if (item.publish) { 42 | if (item.publish.junit) { 43 | item.publish.junit.each { 44 | junit it 45 | } 46 | } 47 | } 48 | } 49 | } 50 | 51 | 52 | //// main flow 53 | 54 | vm(NODE_NAME, NODE_TMPL) { 55 | 56 | stage('Prepare') { 57 | gerrit.clone("https://review.opendev.org/airship/promenade", GERRIT_REFSPEC) 58 | 59 | ansiColor('xterm') { 60 | sh "${WORKSPACE}/tools/setup_gate.sh" 61 | } 62 | } 63 | 64 | sh 'mkdir -p tmp' 65 | 66 | def mfpath = "${WORKSPACE}/tools/g2/manifests/resiliency.json" 67 | def mf = readFile mfpath 68 | def manifest = new JsonSlurperClassic().parseText(mf) 69 | 70 | def env = ["GATE_MANIFEST=${mfpath}", 71 | "GATE_UTILS=${WORKSPACE}/tools/g2/lib/all.sh", 72 | "GATE_DEBUG=0", 73 | "TEMP_DIR=${WORKSPACE}/tmp"] 74 | 75 | // given manifest contains stage names and related scripts to enable 76 | // pipeline to be written in a generic way calling predefined steps. 77 | // The loop executes stages in the defined order 78 | manifest.stages.each { 79 | stage(it.name) { 80 | withEnv(env) { 81 | promenade_stage(it) 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /images/airship/seed.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurper 2 | folder("images") 3 | folder("images/airship") 4 | folder("images/airship/patchset") 5 | 6 | imagesJson = '''{ "github":[{ 7 | "repo":"https://review.opendev.org/airship/promenade", 8 | "directory":"images/airship/patchset/promenade", 9 | "image":"multi-node-promenade", 10 | "name":"promenade", 11 | "jenkinsfile_loc":"JenkinsfilePromenade" 12 | }]}''' 13 | 14 | jsonSlurper = new JsonSlurper() 15 | object = jsonSlurper.parseText(imagesJson) 16 | 17 | for (entry in object.github) { 18 | folder("${entry.directory}") 19 | pipelineJob("${entry.directory}/${entry.image}") { 20 | logRotator{ 21 | daysToKeep(90) 22 | } 23 | parameters { 24 | stringParam { 25 | defaultValue("${entry.repo}") 26 | description('Name of repo in airship to build') 27 | name ('GIT_REPO') 28 | trim(true) 29 | } 30 | stringParam { 31 | defaultValue("1.0.0") 32 | description('Put RC version here') 33 | name('VERSION') 34 | trim(true) 35 | } 36 | } 37 | configure { 38 | node -> node / 'properties' / 'jenkins.branch.RateLimitBranchProperty_-JobPropertyImpl'{ 39 | durationName 'hour' 40 | count '10' 41 | } 42 | } 43 | triggers { 44 | gerritTrigger { 45 | silentMode(true) 46 | serverName('ATT-airship-CI') 47 | gerritProjects { 48 | gerritProject { 49 | compareType('PLAIN') 50 | pattern("airship/${entry.name}") 51 | branches { 52 | branch { 53 | compareType("ANT") 54 | pattern("**") 55 | } 56 | } 57 | disableStrictForbiddenFileVerification(false) 58 | } 59 | } 60 | triggerOnEvents { 61 | patchsetCreated { 62 | excludeDrafts(false) 63 | excludeTrivialRebase(false) 64 | excludeNoCodeChange(false) 65 | } 66 | commentAddedContains { 67 | commentAddedCommentContains('recheck') 68 | } 69 | } 70 | } 71 | 72 | definition { 73 | cps { 74 | script(readFileFromWorkspace("images/airship/${entry.jenkinsfile_loc}")) 75 | sandbox(false) 76 | } 77 | } 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /images/att-comdev/minimirror/Jenkinsfile: -------------------------------------------------------------------------------- 1 | import com.att.nccicd.config.conf as config 2 | conf = new config(env).CONF 3 | 4 | import groovy.json.JsonSlurperClassic 5 | import groovy.json.JsonOutput 6 | 7 | IMAGE_PREFIX="att-comdev" 8 | VERSION=1.0 9 | GERRIT_CLONE_URL = "ssh://" + conf.SSH_REPO_USER + "@" + conf.GERRIT_URL 10 | REFSPEC = "refs/changes/*:refs/changes/*" 11 | ARTF_PATH = "minimirror/gpgkey-private" 12 | 13 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 14 | 15 | IMAGE="${ARTF_DOCKER_URL}/${IMAGE_PREFIX}/mini-mirror:${GERRIT_PATCHSET_REVISION}-ubuntu_xenial" 16 | IMAGE_LATEST="${ARTF_DOCKER_URL}/${IMAGE_PREFIX}/mini-mirror:latest" 17 | IMAGE_DOCKER="${ARTF_DOCKER_URL}/${IMAGE_PREFIX}/mini-mirror:${GERRIT_PATCHSET_REVISION}-docker-ubuntu" 18 | IMAGE_DOCKER_LATEST="${ARTF_DOCKER_URL}/${IMAGE_PREFIX}/mini-mirror:latest" 19 | 20 | def label = "worker-${UUID.randomUUID().toString()}" 21 | try { 22 | podTemplate(label: label,yaml: """ 23 | apiVersion: v1 24 | kind: Pod 25 | spec: 26 | securityContext: 27 | runAsUser: 0 28 | nodeSelector: 29 | dind-node: enabled 30 | """, containers: [ 31 | containerTemplate(name: "ubuntu", 32 | image: "${ARTF_DOCKER_URL}/ubuntu_copy/18.04/nc-ubuntu-18.04@sha256:a08437dd42b2d095cef653d485414fe351401c55b480f16908d5911e4ba75c0f", 33 | command: "cat", 34 | ttyEnabled: true)], 35 | volumes: [hostPathVolume(hostPath: '/var/run/dindproxy/docker.sock', mountPath: '/var/run/docker.sock')]) { 36 | node(label){ 37 | container("ubuntu"){ 38 | 39 | stage('Setup environment'){ 40 | sh "apt-get install sudo make docker.io curl -y" 41 | } 42 | stage("checkout"){ 43 | if("${GERRIT_PROJECT}" == "${MINIMIRROR_PROJECT}" && env.GERRIT_NEWREV){ 44 | echo ("${GERRIT_NEWREV} is being used to override refspec: ${GERRIT_REFSPEC}") 45 | gerrit.cloneToBranch("https://opendev.org/openstack/openstack-helm-images", "${GERRIT_NEWREV}", "${JOB_BASE_NAME}") 46 | } 47 | else if("${GERRIT_PROJECT}" == "${MINIMIRROR_PROJECT}" && GERRIT_EVENT_TYPE != 'change-merged') { 48 | gerrit.cloneToBranch("https://opendev.org/openstack/openstack-helm-images", "${GERRIT_PATCHSET_REVISION}", "${JOB_BASE_NAME}") 49 | } 50 | else { 51 | sh "mkdir -p ${JOB_BASE_NAME}" 52 | dir("${WORKSPACE}/${JOB_BASE_NAME}") { 53 | git "https://opendev.org/openstack/openstack-helm-images" 54 | sh "git checkout master" 55 | } 56 | } 57 | dir ("minimirror/mini-mirror") { 58 | if("${GERRIT_PROJECT}" != "${MINIMIRROR_PROJECT}"){ 59 | if(GERRIT_EVENT_TYPE == 'change-merged') { 60 | gerrit.cloneProject(GERRIT_CLONE_URL + "/" + conf.GLOBAL_REPO, GERRIT_NEWREV, REFSPEC, conf.GLOBAL_REPO, conf.JENKINS_GERRIT_MTN5_CRED_ID) 61 | } else { 62 | gerrit.cloneProject(GERRIT_CLONE_URL + "/" + conf.GLOBAL_REPO, GERRIT_REFSPEC, REFSPEC, conf.GLOBAL_REPO, conf.JENKINS_GERRIT_MTN5_CRED_ID) 63 | } 64 | }else { 65 | gerrit.cloneProject(GERRIT_CLONE_URL + "/" + conf.GLOBAL_REPO, CLCP_MANIFESTS, REFSPEC, conf.GLOBAL_REPO, conf.JENKINS_GERRIT_MTN5_CRED_ID) 66 | } 67 | artifactory.download("${ARTF_PATH}", "gpgkey-private") 68 | } 69 | } 70 | stage('Build') { 71 | dir ("minimirror/mini-mirror"){ 72 | withCredentials([string(credentialsId: 'mini-mirror-pass', variable: 'MINI_PASS')]) { 73 | sh ''' 74 | export NO_PROXY="${NO_PROXY}" 75 | export no_proxy="${NO_PROXY}" 76 | REGISTRY_URI="${ARTF_DOCKER_URL}/att-comdev/" \ 77 | VERSION="${GERRIT_PATCHSET_REVISION}" DISTRO=ubuntu_xenial \ 78 | HTTP_PROXY=${HTTP_PROXY} HTTPS_PROXY=${HTTP_PROXY} \ 79 | APTLY_CONFIG_PATH=/aic-clcp-manifests/tools/mini-mirror/etc/aptly.conf \ 80 | MIRROR_SOURCE_DIR=/aic-clcp-manifests/tools/mini-mirror/sources \ 81 | RELEASE_SIGN_KEY_PATH=/gpgkey-private RELEASE_SIGN_KEY_PASSPHRASE=${MINI_PASS} \ 82 | bash build.sh 83 | ''' 84 | } 85 | sh "docker images" 86 | } 87 | } 88 | stage('Image Publish'){ 89 | publish.artifactory (IMAGE, "${IMAGE_PREFIX}/${JOB_BASE_NAME}:${GERRIT_PATCHSET_REVISION}-ubuntu.${BUILD_TIMESTAMP}") 90 | if (GERRIT_EVENT_TYPE == 'change-merged') { 91 | publish.artifactory (IMAGE, IMAGE_LATEST) 92 | } 93 | } 94 | } 95 | } 96 | } 97 | // catch both errors and exceptions 98 | } catch (Throwable err) { 99 | currentBuild.result = 'FAILURE' 100 | if (env.GERRIT_EVENT_TYPE == 'change-merged') { 101 | email.sendMail(recipientProviders: [developers(), requestor()], 102 | to: env.EMAIL_LIST) 103 | } 104 | throw err 105 | } 106 | -------------------------------------------------------------------------------- /images/att-comdev/minimirror/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_FOLDER="images/att-comdev/minimirror" 2 | folder("images/att-comdev") 3 | folder("images/att-comdev/minimirror") 4 | pipelineJob("${JOB_FOLDER}/minimirror") { 5 | logRotator{ 6 | daysToKeep(90) 7 | } 8 | configure { 9 | node -> node / 'properties' / 'jenkins.branch.RateLimitBranchProperty_-JobPropertyImpl'{ 10 | durationName 'hour' 11 | count '3' 12 | } 13 | } 14 | parameters { 15 | stringParam('MINIMIRROR_PROJECT',"openstack/openstack-helm-images") 16 | stringParam('CLCP_MANIFESTS',"master") 17 | } 18 | triggers { 19 | gerritTrigger { 20 | silentMode(false) 21 | serverName('Any Server') 22 | gerritProjects { 23 | gerritProject { 24 | compareType('PLAIN') 25 | pattern("openstack/openstack-helm-images") 26 | branches { 27 | branch { 28 | compareType('ANT') 29 | pattern("**/master") 30 | } 31 | } 32 | filePaths { 33 | filePath { 34 | compareType('ANT') 35 | pattern("mini-mirror/**") 36 | } 37 | } 38 | disableStrictForbiddenFileVerification(false) 39 | } 40 | gerritProject { 41 | compareType('PLAIN') 42 | pattern("aic-clcp-manifests") 43 | branches { 44 | branch { 45 | compareType('ANT') 46 | pattern("**") 47 | } 48 | } 49 | filePaths { 50 | filePath { 51 | compareType('ANT') 52 | pattern("tools/mini*/**") 53 | } 54 | } 55 | disableStrictForbiddenFileVerification(false) 56 | } 57 | } 58 | triggerOnEvents { 59 | patchsetCreated { 60 | excludeDrafts(true) 61 | excludeTrivialRebase(false) 62 | excludeNoCodeChange(true) 63 | } 64 | changeMerged() 65 | commentAddedContains { 66 | commentAddedCommentContains('recheck') 67 | } 68 | } 69 | } 70 | } 71 | definition { 72 | cps { 73 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 74 | sandbox(false) 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /images/att-comdev/promenade/JenkinsfileResiliency: -------------------------------------------------------------------------------- 1 | 2 | import groovy.json.JsonSlurperClassic 3 | 4 | 5 | NODE_NAME = "promenade-resiliency-${BUILD_NUMBER}" 6 | NODE_TMPL = 'promenade/promenade-virsh.yaml' 7 | 8 | BASE_IMAGE = "${ARTF_WEB_URL}/ubuntu-images/ubuntu-16.04-server-cloudimg-amd64-disk1.img" 9 | 10 | if (!env.GERRIT_REFSPEC) { 11 | GERRIT_REFSPEC = 'master' 12 | GERRIT_EVENT_TYPE = "manual (${GERRIT_REFSPEC})" 13 | } 14 | 15 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 16 | 17 | 18 | // handle generic Promenade stage 19 | def promenade_stage = { item -> 20 | try { 21 | def cmd = "${WORKSPACE}/tools/g2/stages/${item.script}" 22 | if (item.arguments) { 23 | def args = item.arguments.collect {e -> return "'${e}'"} 24 | cmd = cmd + " " + args.join(" ") 25 | } 26 | timeout (30) { 27 | ansiColor('xterm') { 28 | withEnv(["BASE_IMAGE_URL=${BASE_IMAGE}"]) { 29 | sh cmd 30 | } 31 | } 32 | } 33 | 34 | } catch (err) { 35 | if (item.on_error) { 36 | sh "${WORKSPACE}/tools/g2/on_error/${item.on_error}" 37 | } 38 | error(err) 39 | 40 | } finally { 41 | if (item.publish) { 42 | if (item.publish.junit) { 43 | item.publish.junit.each { 44 | junit it 45 | } 46 | } 47 | } 48 | } 49 | } 50 | 51 | 52 | //// main flow 53 | 54 | vm(NODE_NAME, NODE_TMPL) { 55 | 56 | stage('Prepare') { 57 | gerrithub.clone("att-comdev/promenade", GERRIT_REFSPEC) 58 | 59 | ansiColor('xterm') { 60 | sh "${WORKSPACE}/tools/setup_gate.sh" 61 | } 62 | } 63 | 64 | sh 'mkdir -p tmp' 65 | 66 | def mfpath = "${WORKSPACE}/tools/g2/manifests/resiliency.json" 67 | def mf = readFile mfpath 68 | def manifest = new JsonSlurperClassic().parseText(mf) 69 | 70 | def env = ["GATE_MANIFEST=${mfpath}", 71 | "GATE_UTILS=${WORKSPACE}/tools/g2/lib/all.sh", 72 | "GATE_DEBUG=0", 73 | "TEMP_DIR=${WORKSPACE}/tmp"] 74 | 75 | // given manifest contains stage names and related scripts to enable 76 | // pipeline to be written in a generic way calling predefined steps. 77 | // The loop executes stages in the defined order 78 | manifest.stages.each { 79 | stage(it.name) { 80 | withEnv(env) { 81 | promenade_stage(it) 82 | } 83 | } 84 | } 85 | } -------------------------------------------------------------------------------- /images/baseimages/seed.groovy: -------------------------------------------------------------------------------- 1 | folder("images/base-images") 2 | 3 | pipelineJob("images/base-images/create-ubuntu-base-image") { 4 | configure { 5 | node -> node / 'properties' / 'jenkins.branch.RateLimitBranchProperty_-JobPropertyImpl'{ 6 | durationName 'hour' 7 | count '10' 8 | } 9 | } 10 | triggers { 11 | gerritTrigger { 12 | silentMode(true) 13 | serverName('Gerrithub-jenkins-temp') 14 | gerritProjects { 15 | gerritProject { 16 | compareType('PLAIN') 17 | pattern("att-comdev/dockerfiles") 18 | branches { 19 | branch { 20 | compareType("ANT") 21 | pattern("**") 22 | } 23 | } 24 | filePaths { 25 | filePath { 26 | compareType('REG_EXP') 27 | pattern('base-images/.*') 28 | } 29 | } 30 | disableStrictForbiddenFileVerification(false) 31 | } 32 | } 33 | triggerOnEvents { 34 | patchsetCreated { 35 | excludeDrafts(false) 36 | excludeTrivialRebase(false) 37 | excludeNoCodeChange(false) 38 | } 39 | changeMerged() 40 | commentAddedContains { 41 | commentAddedCommentContains('recheck') 42 | } 43 | } 44 | } 45 | definition { 46 | cps { 47 | script(readFileFromWorkspace("images/baseimages/Jenkinsfile")) 48 | sandbox(false) 49 | } 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /images/code-review-image-build/Jenkinsfile: -------------------------------------------------------------------------------- 1 | import com.att.nccicd.config.conf 2 | 3 | IMAGE_TAG = BUILD_TIMESTAMP 4 | DOCKER_REGISTRY = ARTF_DOCKER_URL 5 | IMAGE_PREFIX = "code-review-image" 6 | BASE_IMAGE = (env.BASE_IMAGE) ? BASE_IMAGE : conf.POD_IMAGE_1604 7 | IMAGE_NAME = BASE_IMAGE.split('/')[-1].split('@|:')[0] 8 | IMAGE_PATH = "${IMAGE_PREFIX}/${IMAGE_NAME}-addons:${IMAGE_TAG}" 9 | IMAGE = "${DOCKER_REGISTRY}/${IMAGE_PATH}" 10 | VERSION = 1.0 11 | 12 | currentBuild.displayName = "#${BUILD_NUMBER} - MANUAL" 13 | 14 | def label = "worker-${UUID.randomUUID().toString()}" 15 | podTemplate(label: label, 16 | yaml: """ 17 | apiVersion: v1 18 | kind: Pod 19 | spec: 20 | securityContext: 21 | runAsUser: 0 22 | nodeSelector: 23 | jenkins-node: enabled 24 | """, 25 | containers: [containerTemplate(name: "ubuntu", 26 | image: BASE_IMAGE, 27 | command: "cat", 28 | ttyEnabled: true)], 29 | volumes: [hostPathVolume(hostPath: '/var/run/dindproxy/docker.sock', 30 | mountPath: '/var/run/docker.sock')]) { 31 | node(label){ 32 | container("ubuntu"){ 33 | 34 | stage('Setup environment'){ 35 | sh "apt-get update" 36 | sh "apt-get install sudo docker.io -y" 37 | } 38 | 39 | stage('Create Dockerfile') { 40 | sh """cat << EOF | sudo tee -a Dockerfile-code-review 41 | FROM ${BASE_IMAGE} 42 | RUN apt-get update && \\ 43 | apt-get install -y \\ 44 | sudo \\ 45 | git \\ 46 | make \\ 47 | gcc \\ 48 | python \\ 49 | build-essential \\ 50 | python3-minimal \\ 51 | python3-setuptools \\ 52 | python-pip \\ 53 | python-dev \\ 54 | python-flake8 \\ 55 | python3-pip \\ 56 | libffi-dev \\ 57 | gettext \\ 58 | bandit \\ 59 | mongodb \\ 60 | libpq-dev \\ 61 | libldap2-dev \\ 62 | libsasl2-dev \\ 63 | libssl-dev \\ 64 | libcurl4-openssl-dev \\ 65 | strace 66 | RUN pip install --upgrade pip 67 | RUN pip install --no-cache-dir tox 68 | EOF""" 69 | sh 'cat Dockerfile-code-review' 70 | } 71 | stage('Build image'){ 72 | sh "sudo docker pull ${BASE_IMAGE}" 73 | def cmd="sudo docker inspect --format='{{index .RepoDigests 0}}' ${BASE_IMAGE}" 74 | def base_sha256 = sh(returnStdout: true, script: cmd).trim() 75 | sh """sudo docker build --force-rm --no-cache \ 76 | --build-arg http_proxy=${HTTP_PROXY} \ 77 | --build-arg https_proxy=${HTTPS_PROXY} \ 78 | --build-arg NO_PROXY=${NO_PROXY} \ 79 | --label org.opencontainers.image.base-image=${base_sha256} \ 80 | --label org.opencontainers.image.revision=${VERSION}-${BUILD_NUMBER} \ 81 | --label org.opencontainers.image.event="MANUAL" \ 82 | -f ./Dockerfile-code-review \ 83 | -t ${IMAGE} .""" 84 | } 85 | stage('Image Publish'){ 86 | publish.artifactory(IMAGE, IMAGE_PATH) 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /images/code-review-image-build/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_FOLDER="images" 2 | folder(JOB_FOLDER) 3 | 4 | JOB_BASE_NAME="code-review-image-build" 5 | pipelineJob("${JOB_FOLDER}/${JOB_BASE_NAME}") { 6 | description("This job builds the image used in code-review pipeline") 7 | logRotator { 8 | daysToKeep(90) 9 | } 10 | parameters { 11 | stringParam { 12 | name ('BASE_IMAGE') 13 | defaultValue('') 14 | description('Base image defaults to com.att.nccicd.config.conf.POD_IMAGE_1604') 15 | trim(true) 16 | } 17 | } 18 | triggers { 19 | definition { 20 | cps { 21 | script(readFileFromWorkspace("${JOB_FOLDER}/${JOB_BASE_NAME}/Jenkinsfile")) 22 | sandbox(false) 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /images/coredns/Jenkinsfile: -------------------------------------------------------------------------------- 1 | NODE_NAME="${JOB_BASE_NAME}-${BUILD_NUMBER}" 2 | NODE_TMPL = "docker/ubuntu.m1.medium.yaml" 3 | currentBuild.displayName = "#${BUILD_NUMBER}" 4 | 5 | vm(NODE_NAME,NODE_TMPL) { 6 | /* stage("Env Setup") { 7 | sh ''' 8 | export GO_VERSION="1.9.3" OS="linux" ARCH="amd64" 9 | gopkg="go${GO_VERSION}.${OS}-${ARCH}.tar.gz" 10 | 11 | wget https://redirector.gvt1.com/edgedl/go/${gopkg} 12 | sudo tar -C /usr/local -xzf ${gopkg} 13 | echo 'export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin' | sudo tee -a /etc/profile 14 | echo 'export GOPATH=${HOME}/go' | sudo tee -a /etc/profile 15 | 16 | ''' 17 | } 18 | */ 19 | stage("Checkout") { 20 | gerrit.cloneRepository url: 'https://github.com/coredns/coredns.git', 21 | refspec: 'master', 22 | targetDirectory: '.' 23 | } 24 | 25 | stage("Build Img") { 26 | def build_binary = ''' 27 | docker run --rm -v ${PWD}:/go/src/github.com/coredns/coredns \ 28 | -w /go/src/github.com/coredns/coredns golang:1.9 make 29 | ''' 30 | def stage_status = sh(returnStatus: true, script: build_binary ) 31 | if (stage_status != 0) { 32 | currentBuild.result = 'FAILED' 33 | notify.msg("CoreDNS: Build failed!") 34 | } 35 | sh 'docker build . -t coredns:latest' 36 | sh 'docker run -it coredns:latest -version' 37 | } 38 | 39 | stage('Upload') { 40 | publish.artifactory('coredns:latest','coredns:latest') 41 | } 42 | } 43 | 44 | -------------------------------------------------------------------------------- /images/coredns/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_FOLDER='images' 2 | JOB_NAME='coredns' 3 | 4 | folder(JOB_FOLDER) 5 | pipelineJob("${JOB_FOLDER}/${JOB_NAME}") { 6 | logRotator{ 7 | daysToKeep(90) 8 | } 9 | parameters { 10 | stringParam { 11 | name ('GO_VERSION') 12 | defaultValue('1.9.3') 13 | description('GoLang version') 14 | } 15 | } 16 | scm { 17 | github('coredns/coredns', 'master') 18 | } 19 | triggers { 20 | scm 'H * * * *' 21 | } 22 | definition { 23 | cps { 24 | script(readFileFromWorkspace("${JOB_FOLDER}/${JOB_NAME}/Jenkinsfile")) 25 | sandbox() 26 | } 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /images/jenkins/Jenkinsfile: -------------------------------------------------------------------------------- 1 | import com.att.nccicd.config.conf as config 2 | conf = new config(env).CONF 3 | 4 | currentBuild.displayName = "#${BUILD_NUMBER}-${GERRIT_EVENT_TYPE}" 5 | GIT_REPO="att-comdev/dockerfiles" 6 | IMAGE_TAG="${GERRIT_PATCHSET_REVISION}" 7 | IMAGE_PREFIX="jenkins" 8 | IMAGE_LATEST = "${IMAGE_PREFIX}/${JOB_BASE_NAME}:latest" 9 | IMAGE="${ARTF_DOCKER_URL}/${IMAGE_PREFIX}" 10 | DEFAULT_NAMESPACE="${ARTF_DOCKER_URL}" 11 | 12 | def label = "worker-${UUID.randomUUID().toString()}" 13 | try { 14 | podTemplate(label: label, 15 | yaml: cicd_helper.podExecutorConfig(conf.JNLP_IMAGE, "0", "jenkins-nodes", "jenkins-nodes"), 16 | containers: [containerTemplate(name: "ubuntu", 17 | image: conf.POD_IMAGE_1804, 18 | command: "cat", 19 | ttyEnabled: true)], 20 | volumes: [hostPathVolume(hostPath: '/var/run/dindproxy/docker.sock', 21 | mountPath: '/var/run/docker.sock')]) { 22 | node(label){ 23 | container("ubuntu"){ 24 | stage('Build & Checkout'){ 25 | if (env.GERRIT_NEWREV) { 26 | echo ("${GERRIT_NEWREV} is being used to override refspec: ${GERRIT_REFSPEC}") 27 | IMAGE_TAG="${GERRIT_NEWREV}" 28 | } 29 | 30 | gerrithub.cloneToBranch("${GIT_REPO}", "${IMAGE_TAG}","") 31 | if(GERRIT_EVENT_TYPE != 'change-merged') { 32 | gerrit.rebase() 33 | } 34 | 35 | sh "apt-get update -y" 36 | sh "apt-get install make sudo docker.io -y" 37 | 38 | sh "docker pull docker.io/${IMAGE_PREFIX}/${JOB_BASE_NAME}:${DEFAULT_TAG}" 39 | def cmd="docker inspect --format='{{index .RepoDigests 0}}' \ 40 | ${IMAGE_PREFIX}/${JOB_BASE_NAME}:${DEFAULT_TAG}" 41 | def base_sha256 = sh(returnStdout: true, script: cmd).trim() 42 | 43 | sh "sudo make build:${IMAGE_PREFIX}:${DEFAULT_TAG} \ 44 | DEFAULT_NAMESPACE=${DEFAULT_NAMESPACE} \ 45 | EXTRA_BUILD_ARGS=\"--build-arg FROM=${IMAGE_PREFIX}/${IMAGE_PREFIX}:${DEFAULT_TAG} \ 46 | --label org.opencontainers.image.revision=${IMAGE_TAG} \ 47 | --label org.opencontainers.image.source=${GERRIT_CHANGE_URL} \ 48 | --label org.opencontainers.image.base-image=${base_sha256} \ 49 | --label org.opencontainers.image.revision=${DEFAULT_TAG}.${BUILD_NUMBER} \ 50 | --label org.opencontainers.image.event=${GERRIT_EVENT_TYPE} \ 51 | --build-arg http_proxy=$HTTP_PROXY \ 52 | --build-arg https_proxy=$HTTPS_PROXY \ 53 | --build-arg no_proxy=$NO_PROXY \ 54 | --force-rm \ 55 | --no-cache\" \ 56 | DEFAULT_TAG=${DEFAULT_TAG}" 57 | } 58 | 59 | stage('Publish'){ 60 | if (GERRIT_EVENT_TYPE == 'change-merged') { 61 | //Only going to store images on merged into this directory 62 | publish.artifactory("${IMAGE}:${DEFAULT_TAG}","${IMAGE_PREFIX}/${JOB_BASE_NAME}:${IMAGE_TAG}.${BUILD_TIMESTAMP}") 63 | publish.artifactory("${IMAGE}:${DEFAULT_TAG}",IMAGE_LATEST) 64 | } else { 65 | publish.artifactory("${IMAGE}:${DEFAULT_TAG}","test/${IMAGE_PREFIX}/${JOB_BASE_NAME}:${IMAGE_TAG}.${BUILD_TIMESTAMP}") 66 | } 67 | } 68 | } 69 | } 70 | } 71 | // catch both errors and exceptions 72 | } catch (Throwable err) { 73 | if (env.GERRIT_EVENT_TYPE == 'change-merged') { 74 | currentBuild.result = 'FAILURE' 75 | email.sendMail(recipientProviders: [developers(), requestor()], 76 | to: env.EMAIL_LIST) 77 | throw err 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /images/jenkins/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_FOLDER="images/jenkins" 2 | def project_name = "jenkins" 3 | pipelineJob("${JOB_FOLDER}") { 4 | description("This job builds the image for ${project_name}") 5 | logRotator { 6 | daysToKeep(90) 7 | } 8 | parameters { 9 | stringParam { 10 | name ('GERRIT_REFSPEC') 11 | defaultValue('master') 12 | description('Gerrit refspec or branch') 13 | trim(true) 14 | } 15 | stringParam { 16 | name ('GERRIT_PATCHSET_REVISION') 17 | defaultValue('0') 18 | description('patchset revision') 19 | trim(true) 20 | } 21 | stringParam { 22 | name ('GERRIT_EVENT_TYPE') 23 | defaultValue('patchset-created') 24 | description('patchset-created or change-merged') 25 | trim(true) 26 | } 27 | stringParam { 28 | name ('DEFAULT_TAG') 29 | defaultValue('lts') 30 | description('Tag of the upstream image that needs to be built') 31 | trim(true) 32 | } 33 | } 34 | triggers { 35 | gerritTrigger { 36 | silentMode(true) 37 | serverName('Gerrithub-jenkins') 38 | gerritProjects { 39 | gerritProject { 40 | compareType('PLAIN') 41 | pattern("att-comdev/charts") 42 | branches { 43 | branch { 44 | compareType("ANT") 45 | pattern("**") 46 | } 47 | } 48 | filePaths { 49 | filePath { 50 | compareType("ANT") 51 | pattern("${project_name}/**") 52 | } 53 | } 54 | disableStrictForbiddenFileVerification(false) 55 | } 56 | gerritProject { 57 | compareType('PLAIN') 58 | pattern("att-comdev/dockerfiles") 59 | branches { 60 | branch { 61 | compareType("ANT") 62 | pattern("**") 63 | } 64 | } 65 | filePaths { 66 | filePath { 67 | compareType("ANT") 68 | pattern("${project_name}/**") 69 | } 70 | } 71 | disableStrictForbiddenFileVerification(false) 72 | } 73 | 74 | } 75 | triggerOnEvents { 76 | changeMerged() 77 | patchsetCreated { 78 | excludeDrafts(true) 79 | excludeTrivialRebase(false) 80 | excludeNoCodeChange(false) 81 | } 82 | commentAddedContains { 83 | commentAddedCommentContains('recheck') 84 | } 85 | } 86 | } 87 | definition { 88 | cps { 89 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 90 | sandbox(false) 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /images/kubernetes/Jenkinsfile: -------------------------------------------------------------------------------- 1 | NODE_NAME="${JOB_BASE_NAME}-${BUILD_NUMBER}" 2 | NODE_TMPL = "kube-node/ubuntu.m1.xlarge.yaml" 3 | currentBuild.displayName = "#${BUILD_NUMBER} ${RELEASE_BRANCH}" 4 | 5 | vm(NODE_NAME,NODE_TMPL) { 6 | stage("Checkout ${RELEASE_BRANCH}") { 7 | gerrit.cloneRepository url: 'https://github.com/kubernetes/kubernetes.git', 8 | refspec: RELEASE_BRANCH, 9 | targetDirectory: '.' 10 | } 11 | 12 | stage("Build ${RELEASE_BRANCH}") { 13 | // 'make quick-release' builds images only for the current platform and skip tests. 14 | // 'make release' will test the code then build images for all supported platforms. 15 | // See: https://github.com/kubernetes/kubernetes/blob/master/README.md for details. 16 | def build_release = 'make quick-release' 17 | def stage_status = sh(returnStatus: true, script: build_release ) 18 | if (stage_status != 0) { 19 | currentBuild.result = 'FAILED' 20 | notify.msg("${RELEASE_BRANCH}: Build failed!") 21 | } 22 | } 23 | 24 | stage('Upload') { 25 | def uploadScript = ''' 26 | #!/bin/bash 27 | set -xe 28 | 29 | #I want to be sure we have correct tags in docker repo: 30 | if [ -z ${RELEASE_BRANCH} ] || [ -z ${BUILD_NUMBER} ]; then 31 | echo "ERROR: no RELEASE_BRANCH or BUILD_NUMBER passed." 32 | echo "====================env========================" 33 | env 34 | exit 1 35 | fi 36 | 37 | RELEASE_DIR="_output/release-images/amd64/" 38 | [ ! -z "${RELEASE_DIR}" ] && cd ${RELEASE_DIR} || exit 1 39 | 40 | #importing tars to docker and pushing to local repo: 41 | for i in `ls *.tar | cut -f1 -d'.'`; do 42 | RELEASE_TAG="${ARTF_DOCKER_URL}/ucp/kube-${RELEASE_BRANCH}/${i}:build-${BUILD_NUMBER}" 43 | LATEST_TAG="${ARTF_DOCKER_URL}/ucp/kube-${RELEASE_BRANCH}/${i}:latest" 44 | docker import ${i}.tar ${RELEASE_TAG} 45 | docker import ${i}.tar ${LATEST_TAG} 46 | docker push ${RELEASE_TAG} 47 | docker push ${LATEST_TAG} 48 | done 49 | ''' 50 | withCredentials([usernamePassword( 51 | credentialsId: 'jenkins-artifactory', 52 | usernameVariable: 'ARTIFACTORY_USER', 53 | passwordVariable: 'ARTIFACTORY_PASSWORD')]) { 54 | opts = '-u $ARTIFACTORY_USER -p $ARTIFACTORY_PASSWORD' 55 | sh "docker login ${opts} ${env.ARTF_DOCKER_URL}" 56 | } 57 | 58 | def build_status = sh(returnStatus: true, script: uploadScript ) 59 | if (build_status != 0) { 60 | currentBuild.result = 'FAILED' 61 | notify.msg("Kubernetes ${RELEASE_BRANCH}: Upload failed!") 62 | }else{ 63 | notify.msg("Kubernetes: new ${RELEASE_BRANCH} available!") 64 | } 65 | } 66 | } 67 | 68 | -------------------------------------------------------------------------------- /images/kubernetes/seed.groovy: -------------------------------------------------------------------------------- 1 | 2 | JOB_FOLDER="images/kubernetes" 3 | folder(JOB_FOLDER) 4 | 5 | //add new release pipelines : branches here: 6 | def jobs = ['kubernetes-1.8':'release-1.8', 7 | 'kubernetes-1.9':'release-1.9', 8 | 'kubernetes-master':'master'] 9 | 10 | jobs.each { job_name, branch -> 11 | JOB_NAME=job_name 12 | RELEASE_BRANCH=branch 13 | pipelineJob("${JOB_FOLDER}/${JOB_NAME}") { 14 | logRotator{ 15 | daysToKeep(90) 16 | } 17 | parameters { 18 | stringParam { 19 | name ('RELEASE_BRANCH') 20 | defaultValue(RELEASE_BRANCH) 21 | description('Kube release branch here') 22 | } 23 | } 24 | scm { 25 | github('kubernetes/kubernetes', RELEASE_BRANCH) 26 | } 27 | triggers { 28 | scm 'H * * * *' 29 | } 30 | definition { 31 | cps { 32 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 33 | sandbox() 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /images/nc/ro-helm/Jenkinsfile: -------------------------------------------------------------------------------- 1 | DOCKER_REGISTRY="${ARTF_DOCKER_URL}" 2 | IMAGE_PREFIX="nc" 3 | IMAGE_NAME="ro" 4 | IMAGE_VERSION="1.0.0" 5 | IMAGE_TAG="${GERRIT_PATCHSET_REVISION}" 6 | IMAGE_LATEST="${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:latest" 7 | 8 | vm () { 9 | stage('Checkout') { 10 | cleanWs() 11 | if (GERRIT_EVENT_TYPE == 'change-merged') { 12 | IMAGE_TAG="${GERRIT_NEWREV}" 13 | } 14 | gerrit.cloneDownstream(repo: 'ro-helm', refspec: IMAGE_TAG) 15 | gerrit.cloneDownstream(repo: 'ro-clcp-inventory', refspec: 'master') 16 | } 17 | stage('Init') { 18 | dir("${WORKSPACE}/ro-helm/scripts"){ 19 | sh('sudo bash init.sh') 20 | } 21 | } 22 | stage('Build') { 23 | echo 'Building RO Image file.' 24 | dir("${WORKSPACE}/ro-helm"){ 25 | sh('make images IMAGE_PREFIX=${IMAGE_PREFIX} \ 26 | IMAGE_NAME=${IMAGE_NAME} \ 27 | DOCKER_REGISTRY=${ARTF_DOCKER_URL} \ 28 | LABEL="org.label-schema.vcs-ref=${IMAGE_TAG} \ 29 | --label org.label-schema.vcs-url=${GERRIT_CHANGE_URL} \ 30 | --label org.label-schema.version=${IMAGE_VERSION}-${BUILD_NUMBER}" \ 31 | IMAGE_TAG=${IMAGE_TAG}') 32 | } 33 | } 34 | stage('Publish') { 35 | echo "publish image to artifactory" 36 | publish.artifactory ("${IMAGE_NAME}:${IMAGE_TAG}", "${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}.${BUILD_TIMESTAMP}") 37 | if (GERRIT_EVENT_TYPE == 'change-merged') { 38 | publish.artifactory ("${IMAGE_NAME}:${IMAGE_TAG}", IMAGE_LATEST) 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /images/nc/ro-helm/seed.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurper 2 | 3 | def imagesJson = '''{ "nc":[{ 4 | "repo":"nc", 5 | "images":[ 6 | "ro-helm"] 7 | }]}''' 8 | 9 | def jsonSlurper = new JsonSlurper() 10 | def object = jsonSlurper.parseText(imagesJson) 11 | folder("images/nc") 12 | folder("images/nc/ro-helm") 13 | for (entry in object.nc) { 14 | for (image in entry.images) { 15 | pipelineJob("images/${entry.repo}/${image}/${image}") { 16 | logRotator{ 17 | daysToKeep(90) 18 | } 19 | configure { 20 | node -> node / 'properties' / 'jenkins.branch.RateLimitBranchProperty_-JobPropertyImpl'{ 21 | durationName 'hour' 22 | count '4' 23 | } 24 | } 25 | triggers { 26 | gerritTrigger { 27 | serverName('mtn5-gerrit') 28 | gerritProjects { 29 | gerritProject { 30 | compareType('PLAIN') 31 | pattern("${image}") 32 | branches { 33 | branch { 34 | compareType("ANT") 35 | pattern("**") 36 | } 37 | } 38 | disableStrictForbiddenFileVerification(false) 39 | } 40 | } 41 | triggerOnEvents { 42 | patchsetCreated { 43 | excludeDrafts(false) 44 | excludeTrivialRebase(false) 45 | excludeNoCodeChange(false) 46 | } 47 | changeMerged() 48 | commentAddedContains { 49 | commentAddedCommentContains('recheck') 50 | } 51 | } 52 | } 53 | 54 | definition { 55 | cps { 56 | script(readFileFromWorkspace("images/${entry.repo}/${image}/Jenkinsfile")) 57 | sandbox() 58 | } 59 | } 60 | } 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /images/openstack/helm-images/Jenkinsfile: -------------------------------------------------------------------------------- 1 | import com.att.nccicd.config.conf as config 2 | conf = new config(env).CONF 3 | 4 | IMAGE_TAG = "${GERRIT_PATCHSET_REVISION}" 5 | DOCKER_REGISTRY = "${ARTF_DOCKER_URL}" 6 | IMAGE_PREFIX = "openstack-helm-images" 7 | IMAGE_LATEST = "${IMAGE_PREFIX}/${JOB_BASE_NAME}:latest" 8 | VERSION = 1.0 9 | HELM_IMAGES_URL = "https://review.opendev.org/openstack/openstack-helm-images" 10 | HELM_IMAGES_HOME = "" 11 | BUILD_ARGS = [] 12 | EXTRA_BUILD_ARGS = [] 13 | BASE_IMAGE = conf.UBUNTU_JAMMY_BASE_IMAGE 14 | SUFFIX = env.UBUNTU_VERSION ?: "ubuntu_jammy" 15 | BUILD_DIR_NAME = env.DIR_NAME ?: JOB_BASE_NAME 16 | 17 | if (SUFFIX.contains('focal')) { 18 | BASE_IMAGE = conf.UBUNTU_FOCAL_BASE_IMAGE 19 | } 20 | 21 | if ("${JOB_BASE_NAME}" == 'calicoctl-utility') { 22 | BASE_IMAGE = "calico/ctl:${CALICOCTL_VERSION}" 23 | SUFFIX = "alpine" 24 | BUILD_ARGS += [ 25 | "--build-arg CALICOCTL_VERSION=${CALICOCTL_VERSION}",] 26 | EXTRA_BUILD_ARGS += [ 27 | "--label calicoctl.version='${CALICOCTL_VERSION}'",] 28 | } 29 | 30 | if (JOB_BASE_NAME.contains('ceph-')) { 31 | RELEASE_NAME = 'release_jammy' 32 | RELEASE_TAG = 'tag_jammy' 33 | CEPH_REPO = conf.CEPH_REPO['repo_jammy'] 34 | CEPH_REPO_KEY = conf.CEPH_REPO['key_jammy'] 35 | if(SUFFIX.contains('focal')) { 36 | RELEASE_NAME = 'release_focal' 37 | RELEASE_TAG = 'tag_focal' 38 | CEPH_REPO = conf.CEPH_REPO['repo_focal'] 39 | CEPH_REPO_KEY = conf.CEPH_REPO['key_focal'] 40 | } 41 | BUILD_ARGS += [ 42 | "--build-arg CEPH_RELEASE=${conf.CEPH_RELEASE[RELEASE_NAME]}", 43 | "--build-arg CEPH_RELEASE_TAG=${conf.CEPH_RELEASE[RELEASE_TAG]}", 44 | "--build-arg CEPH_REPO=${CEPH_REPO}", 45 | "--build-arg CEPH_KEY=${CEPH_REPO_KEY}",] 46 | } 47 | 48 | if ("${JOB_BASE_NAME}" == 'elasticsearch-s3') { 49 | SUFFIX = "7_1_0" 50 | } 51 | 52 | if ("${JOB_BASE_NAME}" == 'fluentd') { 53 | SUFFIX = "debian" 54 | } 55 | 56 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 57 | if(env.GERRIT_NEWREV){ 58 | echo ("${GERRIT_NEWREV} is being used to override refspec: ${GERRIT_REFSPEC}") 59 | IMAGE_TAG = env.GERRIT_NEWREV 60 | } 61 | 62 | BUILD_ARGS += [ 63 | "--build-arg FROM=${BASE_IMAGE}", 64 | "--build-arg http_proxy=${HTTP_PROXY}", 65 | "--build-arg https_proxy=${HTTPS_PROXY}", 66 | "--build-arg NO_PROXY=${NO_PROXY}", 67 | "--build-arg no_proxy=${NO_PROXY}", 68 | ] 69 | 70 | EXTRA_BUILD_ARGS += [ 71 | "--label org.opencontainers.image.revision=${IMAGE_TAG}", 72 | "--label org.opencontainers.image.source=${GERRIT_CHANGE_URL}", 73 | "--label org.opencontainers.image.project=${GERRIT_PROJECT}", 74 | "--label org.opencontainers.image.event=${GERRIT_EVENT_TYPE}", 75 | "--build-arg DOCKER_REGISTRY=${DOCKER_REGISTRY}", 76 | ] 77 | 78 | def label = "worker-${UUID.randomUUID().toString()}" 79 | try { 80 | podTemplate(label: label, 81 | yaml: cicd_helper.podExecutorConfig(conf.JNLP_IMAGE, "0", "jenkins-nodes", "jenkins-nodes"), 82 | containers: [containerTemplate(name: "ubuntu", 83 | image: conf.POD_IMAGE_1804, 84 | command: "cat", 85 | ttyEnabled: true)], 86 | volumes: [hostPathVolume(hostPath: '/var/run/dindproxy/docker.sock', 87 | mountPath: '/var/run/docker.sock')]) { 88 | node(label){ 89 | container("ubuntu"){ 90 | HELM_IMAGES_HOME = "${WORKSPACE}/openstack-helm-images" 91 | 92 | stage("Checkout"){ 93 | gerrit.cloneToBranch(HELM_IMAGES_URL, 94 | IMAGE_TAG, 95 | HELM_IMAGES_HOME) 96 | } 97 | 98 | stage('Setup environment'){ 99 | sh "apt-get update" 100 | sh "apt-get install make sudo docker.io -y" 101 | } 102 | 103 | stage('Build') { 104 | dir ("${HELM_IMAGES_HOME}/${BUILD_DIR_NAME}") { 105 | UPDATED_IMAGE_TAG = "${IMAGE_TAG}-${SUFFIX}.${BUILD_TIMESTAMP}" 106 | IMAGE = "${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${JOB_BASE_NAME}:${UPDATED_IMAGE_TAG}" 107 | 108 | sh """docker build -f Dockerfile.${SUFFIX} --network host \ 109 | ${BUILD_ARGS.join(' ')} ${EXTRA_BUILD_ARGS.join(' ')} \ 110 | -t ${IMAGE} .""" 111 | } 112 | } 113 | stage('Image Publish'){ 114 | publish.artifactory (IMAGE, "${IMAGE_PREFIX}/${JOB_BASE_NAME}:${UPDATED_IMAGE_TAG}") 115 | 116 | if (GERRIT_EVENT_TYPE == 'change-merged') { 117 | publish.artifactory (IMAGE, IMAGE_LATEST) 118 | } 119 | } 120 | } 121 | } 122 | } 123 | // catch both errors and exceptions 124 | } catch (Throwable err) { 125 | currentBuild.result = 'FAILURE' 126 | if (env.GERRIT_EVENT_TYPE == 'change-merged') { 127 | email.sendMail(recipientProviders: [developers(), requestor()], 128 | to: env.EMAIL_LIST) 129 | } 130 | throw err 131 | } 132 | -------------------------------------------------------------------------------- /images/openstack/loci/mos-review/JenkinsfileCodeReview.groovy: -------------------------------------------------------------------------------- 1 | import com.att.nccicd.config.conf as config 2 | import groovy.json.JsonSlurperClassic 3 | 4 | conf = new config(env).CONF 5 | 6 | json = new JsonSlurperClassic() 7 | projectList = json.parseText(PROJECT_LIST) 8 | RELEASE_BRANCH_MAP = json.parseText(RELEASE_BRANCH_MAP) 9 | 10 | def keyForValue(map, value) { 11 | map.find { it.value == value }?.key 12 | } 13 | 14 | RELEASE = keyForValue(RELEASE_BRANCH_MAP, GERRIT_BRANCH) 15 | 16 | if ( params.GERRIT_TOPIC =~ /^debug.*/ ) { 17 | error ("Pipeline is disabled for changes with debug topic") 18 | } 19 | 20 | ut_params = [ 21 | stringParam(name: 'PROJECT_NAME', value: GERRIT_PROJECT), 22 | stringParam(name: 'PROJECT_REF', value: GERRIT_REFSPEC), 23 | stringParam(name: 'PROJECT_BRANCH', value: GERRIT_BRANCH), 24 | ] 25 | 26 | runningSet = [:] 27 | 28 | runningSet['codeReview'] = { 29 | stage("Code-review") { 30 | if (GERRIT_EVENT_TYPE != 'change-merged' && 31 | !conf.UT_SKIP_LIST.contains(GERRIT_PROJECT)) { 32 | print "Building review job with ${ut_params}" 33 | job = utils.runBuild("${JOB_BASE}/code-review", ut_params) 34 | } else { 35 | print "Skipping checks" 36 | } 37 | } 38 | } 39 | 40 | def getRefParamName(String project) { 41 | project.toUpperCase().replace('-', '_') + "_REF" 42 | } 43 | 44 | def compileDependencies(Map dependencyMap) { 45 | res = [] 46 | dependencyMap.each { name, ref -> res.add("${name}:${ref}") } 47 | res.join(" ") 48 | } 49 | 50 | if (projectList.contains(GERRIT_PROJECT)) { 51 | parameters = [ 52 | stringParam(name: getRefParamName(GERRIT_PROJECT), 53 | value: GERRIT_REFSPEC), 54 | ] 55 | } else { 56 | parameters = [ 57 | stringParam( 58 | name: 'DEPENDENCY_LIST', 59 | value: compileDependencies([(GERRIT_PROJECT): GERRIT_REFSPEC]) 60 | ), 61 | ] 62 | } 63 | parameters.add(stringParam(name: 'EVENT_TYPE', value: GERRIT_EVENT_TYPE)) 64 | parameters.add(stringParam(name: 'RELEASE', value: RELEASE)) 65 | 66 | runningSet['genericPipeline'] = { 67 | print "Building Generic Pipeline with ${parameters}" 68 | stage("Generic Pipeline") { 69 | job = utils.runBuild("${JOB_BASE}/GenericPipeline", parameters) 70 | currentBuild.description = job.getBuildVariables()["IMAGES"] 71 | } 72 | } 73 | 74 | parallel runningSet 75 | -------------------------------------------------------------------------------- /images/openstack/loci/mos-review/JenkinsfileTox.groovy: -------------------------------------------------------------------------------- 1 | import com.att.nccicd.config.conf as config 2 | conf = new config(env).CONF 3 | 4 | REFS = "+refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/*" 5 | REQUIREMENT_REPO = "mos-requirements" 6 | 7 | currentBuild.displayName = "#${BUILD_NUMBER} ${PROJECT_NAME}" 8 | 9 | IMAGE = "cicd-ubuntu-18.04-server-cloudimg-amd64" 10 | TOX_CHECK = 'OS_LOG_PATH=.; tox' 11 | if ( [conf.YOGA_BRANCH, conf.WALLABY_BRANCH, conf.XENA_BRANCH].contains(PROJECT_BRANCH) ) { 12 | IMAGE = "cicd-ubuntu-20.04-server-cloudimg-amd64" 13 | } 14 | if ( [conf.ANTELOPE_BRANCH, conf.CARACAL_BRANCH].contains(PROJECT_BRANCH) ) { 15 | IMAGE = "cicd-ubuntu-22.04-server-cloudimg-amd64" 16 | } 17 | 18 | def compileSshData() { 19 | sshConfig = "" 20 | keys = [] 21 | parseSshData().each { entry -> 22 | sshConfig += "Host $entry.value.resource\n" + 23 | "User $entry.value.user\n" 24 | keys.add(entry.key) 25 | } 26 | return [keys, sshConfig] 27 | } 28 | 29 | // Parse json containing 30 | // {'': {'user': '', 'resource': ''}}, ...} 31 | // The source of data what credentials to use in ssh-agent with what user 32 | // to what resource 33 | def parseSshData() { 34 | return new groovy.json.JsonSlurper().parseText(SSH_DATA) 35 | } 36 | 37 | // Compile ssh-agent key names and ssh config from SSH_DATA to be used 38 | // for fetching projects to internal mirror 39 | (KEY_NAMES, SSH_CONFIG) = compileSshData() 40 | 41 | 42 | vm (image: IMAGE, flavor: 'm1.large') { 43 | stage("Setup Node") { 44 | writeFile file: "${HOME}/.ssh/config", text: SSH_CONFIG 45 | sh "sudo apt-get update" 46 | sh "sudo bash -c 'export DEBIAN_FRONTEND=noninteractive; " + 47 | "apt-get -y install python3-pip gettext libpq-dev " + 48 | "libssl-dev libsasl2-dev libldap2-dev bandit " + 49 | "qemu-utils'" 50 | sh "sudo pip3 install --index-url ${ARTF_PIP_INDEX_URL} virtualenv " + 51 | "'tox<4.0.0.a2'" 52 | sh "sudo bash -c 'mkdir -p /opt/stack; chown ubuntu:ubuntu /opt/stack'" 53 | } 54 | stage('Project Checkout') { 55 | withEnv(["SHALLOW_CLONE=False"]) { 56 | gerrit.cloneToBranch("${INTERNAL_GERRIT_SSH}/${PROJECT_NAME}", 57 | PROJECT_REF, 58 | "test-repo", 59 | INTERNAL_GERRIT_KEY, 60 | REFS) 61 | } 62 | gerrit.cloneProject("${INTERNAL_GERRIT_SSH}/${REQUIREMENT_REPO}", 63 | PROJECT_BRANCH, 64 | "refs/heads/${PROJECT_BRANCH}", 65 | REQUIREMENT_REPO, 66 | INTERNAL_GERRIT_KEY) 67 | 68 | } 69 | def ucPath = "${WORKSPACE}/${REQUIREMENT_REPO}/upper-constraints.txt" 70 | stage('Tox') { 71 | dir("${WORKSPACE}/${REQUIREMENT_REPO}") { 72 | sshagent([INTERNAL_GERRIT_KEY]) { 73 | sh ('''mkdir /tmp/wheels; 74 | virtualenv .venv; 75 | . .venv/bin/activate; 76 | pip install pkginfo; 77 | for item in $(grep '^git+' upper-constraints.txt); do 78 | pip wheel --no-deps --wheel-dir /tmp/wheels ${item} 79 | done; 80 | sed -i '/^git+/d' upper-constraints.txt; 81 | for wheel in $(ls /tmp/wheels/*.whl); do 82 | cmd=\"import pkginfo; \ 83 | wheel = pkginfo.Wheel('${wheel}'); \ 84 | msg = '{}==={}'.format(wheel.name, \ 85 | wheel.version); \ 86 | print(msg)\" 87 | python -c "${cmd}" >> upper-constraints.txt 88 | done;''') 89 | } 90 | } 91 | dir("${WORKSPACE}/test-repo") { 92 | withEnv(["UPPER_CONSTRAINTS_FILE=${ucPath}", 93 | "TOX_CONSTRAINTS_FILE=${ucPath}", 94 | "PIP_INDEX_URL=${ARTF_PIP_INDEX_URL}", 95 | "PIP_FIND_LINKS=/tmp/wheels", 96 | "HTTPS_PROXY=", "HTTP_PROXY="]) { 97 | sshagent([INTERNAL_GERRIT_KEY]) { 98 | sh TOX_CHECK 99 | } 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /images/openstack/loci/mos-review/JenkinsfileUplift.groovy: -------------------------------------------------------------------------------- 1 | import com.att.nccicd.config.conf as config 2 | import groovy.json.JsonSlurperClassic 3 | 4 | conf = new config(env).CONF 5 | 6 | json = new JsonSlurperClassic() 7 | 8 | NET_RETRY_COUNT = env.NET_RETRY_COUNT.toInteger() 9 | 10 | IMAGES_VERSIONS = json.parseText(params.IMAGES) 11 | 12 | MANIFESTS_PROJECT_NAME = conf.GLOBAL_REPO 13 | VERSIONS_PATH = conf.VERSIONS_PATH 14 | MANIFESTS_BRANCH = conf.OS_RELEASE_MANIFESTS_BRANCH_MAP[RELEASE] 15 | RELEASES_REGEX = "(${json.parseText(env.SUPPORTED_RELEASES).join("|")})" 16 | if (TARGET_RELEASE_ONLY.toBoolean()) { 17 | RELEASES_REGEX = RELEASE 18 | } 19 | 20 | COMMIT_ARGS = ("-c 'user.name=Jenkins (${INTERNAL_GERRIT_USER})' " + 21 | "-c 'user.email=${INTERNAL_GERRIT_USER}@att.com'") 22 | 23 | def compileSshData() { 24 | sshConfig = "" 25 | keys = [] 26 | json.parseText(SSH_DATA).each { entry -> 27 | sshConfig += "Host ${entry.value.resource}\n" + 28 | "User ${entry.value.user}\n" 29 | keys.add(entry.key) 30 | } 31 | return [keys, sshConfig] 32 | } 33 | 34 | def getProjectRepoUrl(prj) { 35 | return prj.contains("ssh://") ? prj : "${INTERNAL_GERRIT_SSH}/${prj}" 36 | } 37 | 38 | 39 | // Compile ssh-agent key names and ssh config from SSH_DATA to be used 40 | // for fetching projects to internal mirror 41 | (KEY_NAMES, SSH_CONFIG) = compileSshData() 42 | 43 | 44 | vm (initScript: 'loci-bootstrap.sh', 45 | image: 'cicd-ubuntu-18.04-server-cloudimg-amd64', 46 | flavor: 'm1.medium', 47 | nodePostfix: '', 48 | doNotDeleteNode: false) { 49 | // Create ssh config on slave to control what login is used for 50 | // what resource 51 | writeFile file: "${HOME}/.ssh/config", text: SSH_CONFIG 52 | sh "sudo bash -c 'echo \"nameserver ${DNS_SERVER_TWO}\" > /etc/resolv.conf'" 53 | stage("Replacing images in versions.yaml") { 54 | env.GERRIT_TOPIC = TOPIC 55 | def changeId = null 56 | 57 | changes = gerrit.getTopicCommitInfo( 58 | MANIFESTS_PROJECT_NAME, INTERNAL_GERRIT_URL, 59 | INTERNAL_GERRIT_PORT, INTERNAL_GERRIT_KEY 60 | ) 61 | 62 | if (changes.size() > 1) { 63 | error("Something wrong occurred. There should be either one or " + 64 | "none open changes for topic ${TOPIC}.\n" + 65 | "Please close extra changes.") 66 | } 67 | 68 | if (changes) { 69 | changeId = changes[0]["id"] 70 | } 71 | 72 | utils.retrier (NET_RETRY_COUNT) { 73 | gerrit.cloneToBranch( 74 | getProjectRepoUrl(MANIFESTS_PROJECT_NAME), 75 | MANIFESTS_BRANCH, 76 | MANIFESTS_PROJECT_NAME, 77 | INTERNAL_GERRIT_KEY, 78 | null 79 | ) 80 | } 81 | dir(MANIFESTS_PROJECT_NAME) { 82 | versions = readFile VERSIONS_PATH 83 | versions_updated = versions 84 | IMAGES_VERSIONS.each { _, data -> 85 | image = data['url'] 86 | comment = " ${data['vcsRef']}.${data['date']}" 87 | (_, replace_to, pattern) = ((image =~ /.*?\/((.*?)[@:].*)/)[0]) 88 | // For pattern replace actual release name by regex matching any release 89 | imagePattern = pattern.replaceAll(RELEASES_REGEX, RELEASES_REGEX) + '[@:].*' 90 | imagePattern = imagePattern.replaceAll('(openstack|openstack-patchset)/', 91 | '(openstack|openstack-patchset)/') 92 | versions_updated = versions_updated.replaceAll( 93 | imagePattern, replace_to) 94 | commentPattern = "(?<=#).*\\.\\d{4}(-\\d{2}){2}_\\d{2}(-\\d{2}){2}.*(?=\n.{0,100}${replace_to})" 95 | versions_updated = versions_updated.replaceAll( 96 | commentPattern, comment) 97 | } 98 | if(versions_updated == versions) { 99 | print "No new changes. Versions.yaml is up to date." 100 | } else { 101 | writeFile file: VERSIONS_PATH, text: versions_updated 102 | sshagent([INTERNAL_GERRIT_KEY]) { 103 | sh ("scp -p -P ${INTERNAL_GERRIT_PORT} " + 104 | "${INTERNAL_GERRIT_URL}:hooks/commit-msg " + 105 | ".git/hooks/") 106 | msg = params.COMMIT_MESSAGE + "\n\n${env.BUILD_URL}" 107 | if (changeId) { 108 | msg += "\n\nChange-Id: ${changeId}" 109 | } 110 | sh "git ${COMMIT_ARGS} commit -a -m '${msg}'" 111 | sh "git show HEAD" 112 | sh "git push origin HEAD:refs/for/${MANIFESTS_BRANCH}%topic=${TOPIC}" 113 | } 114 | } 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /images/ubuntu-base-image/Jenkinsfile: -------------------------------------------------------------------------------- 1 | import com.att.nccicd.config.conf as config 2 | conf = new config(env).CONF 3 | 4 | DOCKER_REGISTRY = "${ARTF_SECURE_DOCKER_URL}" 5 | Ubuntu_Repo = "${ARTF_UBUNTU_REPO}" 6 | IMAGE_TAG = "${IMAGETAG}" 7 | VERSION = 1.0 8 | 9 | currentBuild.displayName = "#${BUILD_NUMBER} Build on ${BUILD_TIMESTAMP}" 10 | 11 | vm () { 12 | cleanWs() 13 | sh ('hostname') 14 | stage('Setup environment') { 15 | sh "sudo apt-get update" 16 | sh "sudo apt-get install -y multistrap" 17 | } 18 | stage('Build') { 19 | sh """cat << EOF | sudo tee ~/xenial.conf 20 | [General] 21 | arch=amd64 22 | directory=ubuntu_xenial 23 | cleanup=true 24 | noauth=false 25 | explicitsuite=false 26 | unpack=true 27 | bootstrap=xenial xenial-updates xenial-security 28 | aptsources=xenial xenial-updates xenial-security 29 | 30 | [xenial] 31 | packages=apt apt-transport-https ca-certificates libssl1.0.0 openssl 32 | source=https://artifacts-nc.auk3.cci.att.com/artifactory/archive-ubuntu 33 | keyring=ubuntu-keyring 34 | suite=xenial 35 | components=main multiverse restricted universe 36 | 37 | [xenial-updates] 38 | source=https://artifacts-nc.auk3.cci.att.com/artifactory/archive-ubuntu 39 | keyring=ubuntu-keyring 40 | suite=xenial-updates 41 | components=main multiverse restricted universe 42 | 43 | [xenial-security] 44 | source=https://artifacts-nc.auk3.cci.att.com/artifactory/archive-ubuntu 45 | keyring=ubuntu-keyring 46 | suite=xenial-security 47 | components=main multiverse restricted universe 48 | EOF""" 49 | sh """cat << EOF | sudo tee ~/bionic.conf 50 | [General] 51 | arch=amd64 52 | directory=ubuntu_bionic 53 | cleanup=true 54 | noauth=false 55 | explicitsuite=false 56 | unpack=true 57 | bootstrap=bionic bionic-updates bionic-security 58 | aptsources=bionic bionic-updates bionic-security 59 | 60 | [bionic] 61 | packages=apt ca-certificates 62 | source=https://artifacts-nc.auk3.cci.att.com/artifactory/archive-ubuntu 63 | keyring=ubuntu-keyring 64 | suite=bionic 65 | components=main multiverse restricted universe 66 | 67 | [bionic-updates] 68 | source=https://artifacts-nc.auk3.cci.att.com/artifactory/archive-ubuntu 69 | keyring=ubuntu-keyring 70 | suite=bionic-updates 71 | components=main multiverse restricted universe 72 | 73 | [bionic-security] 74 | source=https://artifacts-nc.auk3.cci.att.com/artifactory/archive-ubuntu 75 | keyring=ubuntu-keyring 76 | suite=bionic-security 77 | components=main multiverse restricted universe 78 | EOF""" 79 | sh "sudo multistrap -f ~/${IMAGE_TAG}.conf" 80 | sh "sudo chmod 644 ubuntu_${IMAGE_TAG}/etc/apt/trusted.gpg.d/*.gpg" 81 | sh "sudo tar -C ubuntu_${IMAGE_TAG} -c . | sudo docker import - ubuntu:${IMAGE_TAG}" 82 | IMAGE = "${DOCKER_REGISTRY}/ubuntu:${IMAGE_TAG}-${BUILD_TIMESTAMP}" 83 | sh "sudo docker tag ubuntu:${IMAGE_TAG} ${IMAGE}" 84 | } 85 | stage('Image Publish') { 86 | publish.artifactory (IMAGE, "${JOB_BASE_NAME}:${IMAGE_TAG}-${BUILD_TIMESTAMP}") 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /images/ubuntu-base-image/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_FOLDER="images/ubuntu-base-image" 2 | folder(JOB_FOLDER) 3 | 4 | def projects = ['ubuntu'] 5 | 6 | projects.each { project_name -> 7 | JOB_BASE_NAME=project_name 8 | pipelineJob("${JOB_FOLDER}/${JOB_BASE_NAME}") { 9 | description("This job builds the Ubuntu Base image") 10 | logRotator { 11 | daysToKeep(90) 12 | } 13 | parameters { 14 | stringParam("IMAGETAG", "xenial", "Ubuntu Release Name") 15 | } 16 | triggers { 17 | definition { 18 | cps { 19 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 20 | sandbox(false) 21 | } 22 | } 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /integration/airship/airship-in-a-bottle/Jenkinsfile: -------------------------------------------------------------------------------- 1 | // pipeline for airship-in-a-bottle, more info at 2 | // http://www.airshipit.org/ 3 | 4 | ARTF_BASE = "cicd/logs/${JOB_NAME}/${BUILD_ID}" 5 | 6 | REPO_URL = 'https://review.opendev.org/airship/in-a-bottle' 7 | REPO_HOME = '/root/deploy/in-a-bottle' 8 | 9 | publicNet = 'public' 10 | if (env.AIRSHIP_VM_PUBLIC_NET) { 11 | publicNet = env.AIRSHIP_VM_PUBLIC_NET 12 | } 13 | 14 | vm(flavor: 'm1.xxlarge', artifactoryLogs: true, publicNet: publicNet) { 15 | 16 | stage('Checkout'){ 17 | 18 | sh 'sudo mkdir /root/deploy' 19 | sh 'sudo setfacl -m u:ubuntu:rwx /root/deploy' 20 | sh 'sudo setfacl -m u:ubuntu:rwx /root' 21 | 22 | gerrit.cloneToBranch(REPO_URL, GERRIT_PATCHSET_REVISION, REPO_HOME) 23 | 24 | if(GERRIT_EVENT_TYPE != 'change-merged') { 25 | dir(REPO_HOME) { 26 | gerrit.rebase() 27 | } 28 | } 29 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 30 | } 31 | 32 | stage('Run Airship') { 33 | dir("${REPO_HOME}/manifests/dev_single_node"){ 34 | try { 35 | timeout(120){ 36 | sh 'sudo ./airship-in-a-bottle.sh -y' 37 | } 38 | } catch (err) { 39 | try { 40 | sh 'sudo debug-report.sh' 41 | def hostname = sh (script: 'hostname', returnStdout: true).trim() 42 | artifactory.upload("debug-${hostname}.tgz", 43 | "${ARTF_BASE}/debug-${hostname}.tgz") 44 | } catch (lerr){ 45 | // pipeline may not yet deployed basic k8s 46 | // cause of the issue will be found in Jenkins console log 47 | print "Failed to publish debug report: ${lerr}" 48 | } 49 | error(err) 50 | } 51 | } 52 | } 53 | } 54 | 55 | -------------------------------------------------------------------------------- /integration/airship/airship-in-a-bottle/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_BASE = 'integration/airship' 2 | folder('integration/airship') 3 | 4 | pipelineJob("${JOB_BASE}/airship-in-a-bottle") { 5 | 6 | logRotator{ 7 | daysToKeep(90) 8 | } 9 | 10 | configure { 11 | node -> node / 'properties' / 'jenkins.branch.RateLimitBranchProperty_-JobPropertyImpl'{ 12 | durationName 'hour' 13 | count '3' 14 | } 15 | } 16 | 17 | triggers { 18 | gerritTrigger { 19 | serverName('ATT-airship-CI') 20 | gerritProjects { 21 | gerritProject { 22 | compareType('PLAIN') 23 | pattern("airship/in-a-bottle") 24 | branches { 25 | branch { 26 | compareType("ANT") 27 | pattern("**") 28 | } 29 | } 30 | disableStrictForbiddenFileVerification(false) 31 | } 32 | } 33 | triggerOnEvents { 34 | patchsetCreated { 35 | excludeDrafts(false) 36 | excludeTrivialRebase(false) 37 | excludeNoCodeChange(false) 38 | } 39 | changeMerged() 40 | commentAddedContains { 41 | commentAddedCommentContains('^recheck\$') 42 | } 43 | } 44 | } 45 | 46 | definition { 47 | cps { 48 | script(readFileFromWorkspace("${JOB_BASE}/airship-in-a-bottle/Jenkinsfile")) 49 | sandbox(false) 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /integration/airship/gate-multinode/Jenkinsfile: -------------------------------------------------------------------------------- 1 | // pipeline for virtual multi-node gate 2 | 3 | ARTF_BASE = "cicd/logs/${JOB_NAME}/${BUILD_ID}" 4 | 5 | REPO_URL = 'https://review.opendev.org/airship/in-a-bottle' 6 | REPO_HOME = '/root/deploy/in-a-bottle' 7 | 8 | publicNet = 'public' 9 | if (env.AIRSHIP_VM_PUBLIC_NET) { 10 | publicNet = env.AIRSHIP_VM_PUBLIC_NET 11 | } 12 | 13 | image = 'ubuntu-18.04' 14 | if (env.AIRSHIP_MULTINODE_VM_IMAGE) { 15 | image = env.AIRSHIP_MULTINODE_VM_IMAGE 16 | } 17 | 18 | vm(flavor: 'airship.seaworthy', 19 | image: image, 20 | artifactoryLogs: true, 21 | doNotDeleteNode: false, 22 | timeout: 300, 23 | publicNet: publicNet) { 24 | 25 | stage('Checkout'){ 26 | 27 | sh 'sudo mkdir /root/deploy' 28 | sh 'sudo setfacl -m u:ubuntu:rwx /root/deploy' 29 | sh 'sudo setfacl -m u:ubuntu:rwx /root' 30 | 31 | // fixme: this should be fixed in gate setup scripts 32 | sh 'sudo groupadd libvirtd' 33 | sh 'sudo useradd virtmgr' 34 | sh 'sudo usermod -a -G libvirtd virtmgr' 35 | 36 | gerrit.cloneToBranch(REPO_URL, GERRIT_PATCHSET_REVISION, REPO_HOME) 37 | 38 | if(GERRIT_EVENT_TYPE != 'change-merged') { 39 | dir(REPO_HOME) { 40 | gerrit.rebase() 41 | } 42 | } 43 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 44 | } 45 | 46 | stage('Setup Gate') { 47 | dir("${REPO_HOME}/tools/multi_nodes_gate"){ 48 | timeout(20){ 49 | ansiColor('xterm') { 50 | sh 'sudo ./setup_gate.sh' 51 | } 52 | } 53 | } 54 | } 55 | 56 | stage('Run Airship') { 57 | dir("${REPO_HOME}/tools/multi_nodes_gate"){ 58 | timeout(240){ 59 | ansiColor('xterm') { 60 | 61 | // use Westmere for nested VMs 62 | def opts = ["VIRSH_CPU_OPTS=Westmere"] 63 | 64 | withEnv(opts) { 65 | sh 'sudo -E ./gate.sh' 66 | } 67 | } 68 | } 69 | } 70 | } 71 | } 72 | 73 | -------------------------------------------------------------------------------- /integration/airship/gate-multinode/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_BASE = 'integration/airship' 2 | folder('integration/airship') 3 | 4 | pipelineJob("${JOB_BASE}/airship-multinode") { 5 | 6 | logRotator{ 7 | daysToKeep(90) 8 | } 9 | 10 | configure { 11 | node -> node / 'properties' / 'jenkins.branch.RateLimitBranchProperty_-JobPropertyImpl'{ 12 | durationName 'hour' 13 | count '3' 14 | } 15 | } 16 | 17 | triggers { 18 | gerritTrigger { 19 | serverName('ATT-airship-CI') 20 | gerritProjects { 21 | gerritProject { 22 | compareType('PLAIN') 23 | pattern("airship/in-a-bottle") 24 | branches { 25 | branch { 26 | compareType("ANT") 27 | pattern("**") 28 | } 29 | } 30 | disableStrictForbiddenFileVerification(false) 31 | } 32 | } 33 | triggerOnEvents { 34 | patchsetCreated { 35 | excludeDrafts(false) 36 | excludeTrivialRebase(false) 37 | excludeNoCodeChange(false) 38 | } 39 | changeMerged() 40 | commentAddedContains { 41 | commentAddedCommentContains('^recheck\$') 42 | } 43 | } 44 | } 45 | 46 | definition { 47 | cps { 48 | script(readFileFromWorkspace("${JOB_BASE}/gate-multinode/Jenkinsfile")) 49 | sandbox(false) 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /integration/site-update/Jenkinsfile: -------------------------------------------------------------------------------- 1 | import org.yaml.snakeyaml.Yaml 2 | import org.yaml.snakeyaml.DumperOptions; 3 | 4 | import groovy.json.JsonSlurperClassic 5 | import groovy.json.JsonOutput 6 | 7 | 8 | JENKINS_SLAVE_BUILDER = 'genesis-builder' 9 | 10 | PROM_BUILD_NODE_NAME = "prom-build" 11 | PROM_BUILD_NODE_IP = '10.24.20.99' 12 | GENESIS_NODE_NAME = "genesis" 13 | GENESIS_NODE_IP = '10.24.20.100' 14 | VMX_NODE_NAME = "vmx" 15 | VMX_NODE_IP = '10.24.20.101' 16 | 17 | 18 | PROMENADE_IMAGE='artifacts-aic.atlantafoundry.com/att-comdev/promenade@sha256:e08b077c5d75f725120dfa1900dffbde790661b8815af90083ff19c6edcfbc2d' 19 | PEGLEG_IMAGE='artifacts-aic.atlantafoundry.com/att-comdev/pegleg:b5aed4df775a2dd665f16e8a0674a0bab869431a' 20 | 21 | MANIFEST_PREFIX='region/atl-lab1' 22 | 23 | SONOBUOY_CFG="${MANIFEST_PREFIX}/test/sonobuoy.yaml" 24 | 25 | ARTF_BASE="clcp-integration/${JOB_BASE_NAME}" 26 | 27 | DECKHAND_URL='http://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0' 28 | 29 | 30 | def prom_build_prepare = { 31 | stage("Build Prepare") { 32 | gerrit.cloneUpstream(repo: 'att-comdev/treasuremap', refspec: GERRIT_PATCHSET_REVISION, targetDirectory: '.') 33 | gerrit.cloneRepo url: 'ssh://jenkins-attcomdev@10.24.20.18:29418/clcp-integration', 34 | creds: 'jenkins-stage-master', 35 | targetDirectory: 'clcp-integration', 36 | refspec: 'master' 37 | 38 | sh "cp ${WORKSPACE}/clcp-integration/secrets/artifactory_docker_key.yaml ${WORKSPACE}/deployment_files/site/atl-lab1/secrets/passphrases/" 39 | } 40 | } 41 | 42 | //// artf utils 43 | 44 | def artf = Artifactory.server 'artifactory' 45 | 46 | def artf_spec = { pattern, target -> 47 | spec = ['files': [['pattern': pattern, 48 | 'target': target, 49 | 'flat': true]]] 50 | return new JsonOutput().toJson(spec) 51 | } 52 | 53 | def artf_publish = { pattern, target -> 54 | info = artf.upload(artf_spec(pattern, target)) 55 | artf.publishBuildInfo(info) 56 | } 57 | 58 | def artf_download = { pattern, target -> 59 | artf.download(artf_spec(pattern, target)) 60 | } 61 | 62 | 63 | 64 | def pegleg_site_collect = { 65 | stage('pegleg Site Collect') { 66 | sh "mkdir ${WORKSPACE}/atl-lab1_yaml-${BUILD_NUMBER}" 67 | sh "sudo docker run --rm -t -v \$(pwd):/target ${PEGLEG_IMAGE} pegleg site -p /target/deployment_files collect atl-lab1 -s /target/atl-lab1_yaml-${BUILD_NUMBER}" 68 | } 69 | } 70 | 71 | def site_config_publish = { 72 | stage('Site Config Publish') { 73 | sh "tar czf site-config.tar.gz-${BUILD_NUMBER} atl-lab1_yaml-${BUILD_NUMBER}" 74 | artf_publish('site-config.tar.gz-${BUILD_NUMBER}', "${ARTF_BASE}/configs/") 75 | } 76 | } 77 | 78 | // def stable_site_config_publish = { 79 | // stage('Stable Site Config Publish') { 80 | // artf_publish('site-config.tar.gz', "${ARTF_BASE}/stable/configs/") 81 | // } 82 | // } 83 | 84 | //// deckhand utils 85 | def deckhand_load = { 86 | stage('Deckhand Load') { 87 | artf_download("${ARTF_BASE}/configs/site-config.tar.gz-${BUILD_NUMBER}", "") 88 | sh "tar xzf site-config.tar.gz-${BUILD_NUMBER}" 89 | sh "cp /home/ubuntu/jenkins/workspace/integration/genesis-full/atl-lab1_yaml/certificates.yaml ${WORKSPACE}/atl-lab1_yaml-${BUILD_NUMBER}" 90 | 91 | dir ("/home/ubuntu/jenkins/workspace/integration/genesis-full/shipyard/tools") { 92 | sh "./deckhand_load_yaml.sh atl_lab1 ${WORKSPACE}/atl-lab1_yaml-${BUILD_NUMBER}" 93 | } 94 | } 95 | } 96 | 97 | 98 | ////Site Update 99 | 100 | def site_update = { 101 | stage ('Shipyard Site Update') { 102 | 103 | dir ("/home/ubuntu/jenkins/workspace/integration/genesis-full/shipyard/tools") { 104 | sh "bash update_site.sh" 105 | } 106 | } 107 | } 108 | 109 | 110 | def sonobuoy_run = { 111 | if (SONOBUOY_ENABLED.toBoolean()) { 112 | stage('Sonobuoy E2E (v0.9.0)') { 113 | sh 'mkdir -p /tmp/sonobuoy' // test results 114 | 115 | sh "cat ${SONOBUOY_CFG} |sudo kubectl apply -f -" 116 | 117 | timeout (12) { 118 | cmd = 'sudo kubectl get pods -n heptio-sonobuoy |grep 1/1' 119 | while (sh(returnStatus: true, script: cmd)) sleep 30 120 | } 121 | 122 | timeout (120) { 123 | cmd = 'sudo kubectl get pods -n heptio-sonobuoy |grep 1/1' 124 | while (!sh(returnStatus: true, script: cmd)) sleep 300 125 | } 126 | } 127 | 128 | stage('Sonobuoy Publish') { 129 | 130 | artf_publish('/tmp/sonobuoy/*.tar.gz', "${ARTF_BASE}/sonobuoy/") 131 | 132 | sh 'mkdir -p results' 133 | sh 'tar xf /tmp/sonobuoy/*.tar.gz -C results' 134 | 135 | junit 'results/plugins/e2e/results/junit_01.xml' 136 | } 137 | } 138 | } 139 | def debug_report = { 140 | stage('Debug Report'){ 141 | sh "sudo bash /usr/local/bin/debug-report.sh" 142 | artf_publish('debug-genesis.tgz', "${ARTF_BASE}/logs/${BUILD_NUMBER}") 143 | } 144 | } 145 | 146 | 147 | //// main flow 148 | node(PROM_BUILD_NODE_NAME) { 149 | prom_build_prepare() 150 | pegleg_site_collect() 151 | site_config_publish() 152 | } 153 | 154 | node(GENESIS_NODE_NAME) { 155 | 156 | try { 157 | // upload site YAMLs into Deckhand 158 | deckhand_load() 159 | 160 | // Update_site 161 | site_update() 162 | 163 | // e2e kubernetes conformance tests (optional) 164 | sonobuoy_run() 165 | } finally{ 166 | debug_report() 167 | } 168 | } -------------------------------------------------------------------------------- /nc/Airshipctl/airshipctl.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurperClassic 2 | import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException 3 | import hudson.triggers.TimerTrigger 4 | import hudson.model.Cause.UserIdCause 5 | 6 | // Update Build Title to make it more readable and search to filter builds 7 | if (!env.GERRIT_BRANCH) { 8 | GERRIT_EVENT_TYPE = "Manual" 9 | } 10 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 11 | if (currentBuild.rawBuild.getCause(hudson.triggers.TimerTrigger.TimerTriggerCause)) { 12 | currentBuild.displayName = "#${BUILD_NUMBER} Timer (${GERRIT_REFSPEC})" 13 | } 14 | 15 | def cloneref = { 16 | sh "sudo rm -rf airshipctl || true" 17 | sh "git clone https://review.opendev.org/airship/airshipctl" 18 | envVars = env.getEnvironment() 19 | println envVars.containsKey("GERRIT_REFSPEC") 20 | if (envVars.containsKey("GERRIT_REFSPEC")) { 21 | sh "cd airshipctl && git fetch https://review.opendev.org/airship/airshipctl $GERRIT_REFSPEC && git checkout FETCH_HEAD && git checkout -b gerrit_current" 22 | } 23 | } 24 | 25 | def collect_log = { 26 | println("Colelcting Log") 27 | sh "cd airshipctl && tools/gate/99_collect_logs.sh" 28 | sh "sudo tar -cvzf /tmp/airshipctl_build_${BUILD_NUMBER}-logs.tgz -C /tmp/logs ." 29 | sh "sudo apt install -y sshpass" 30 | 31 | withCredentials([usernamePassword(credentialsId: "jenkins_master", 32 | usernameVariable: "USER", 33 | passwordVariable: "PASSWORD")]) { 34 | sh "sshpass -p ${PASSWORD} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/airshipctl_build_${BUILD_NUMBER}-logs.tgz ${USER}@10.254.125.160:/tmp" 35 | } 36 | 37 | withCredentials([usernamePassword(credentialsId: "jenkins_master", 38 | usernameVariable: "USER", 39 | passwordVariable: "PASSWORD")]) { 40 | sh "sshpass -p ${PASSWORD} ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${USER}@10.254.125.160 sudo cp /tmp/airshipctl_build_${BUILD_NUMBER}-logs.tgz /mnt/jenkins-data/userContent/" 41 | } 42 | println "The log location: https://jenkins.nc.opensource.att.com/userContent/airshipctl_build_${BUILD_NUMBER}-logs.tgz" 43 | } 44 | 45 | def deploy_airship2 = { 46 | 47 | stage('Clone Code') { 48 | cloneref() 49 | } 50 | 51 | stage('Cleanup'){ 52 | println("Cleanup packages, images") 53 | sh "cd airshipctl && tools/deployment/clean.sh || true " 54 | } 55 | stage('Setup') { 56 | println("Running setup") 57 | withEnv (["AIRSHIPCTL_WS=${WORKSPACE}/airshipctl"]) { 58 | sh " airshipctl/tools/gate/00_setup.sh" 59 | } 60 | } 61 | stage('build') { 62 | println("Running build") 63 | sh "cd airshipctl && tools/gate/10_build_gate.sh" 64 | } 65 | 66 | stage('install') { 67 | println("Running Deployment") 68 | try { 69 | sh "cd airshipctl && tools/gate/20_run_gate_runner.sh" 70 | try { 71 | print "Build Succeded. Start Log collection" 72 | collect_log() 73 | } catch (error) { 74 | print "Log collection Failed : ${error.getMessage()}" 75 | } 76 | } catch (FlowInterruptedException err) { 77 | print "Skip Log collect for aborted Jobs" 78 | throw err 79 | } catch (err){ 80 | try { 81 | print "Build Failed : ${err.getMessage()}. Start Log collection" 82 | collect_log() 83 | } catch (error) { 84 | print "Log collection Failed : ${error.getMessage()}" 85 | } 86 | throw err 87 | } 88 | } 89 | } 90 | 91 | //// main flow 92 | try { 93 | node (label: NODE_LABEL){ 94 | cleanWs() 95 | deploy_airship2() 96 | } 97 | } catch (error) { 98 | print "Build failed: ${error.getMessage()}" 99 | currentBuild.result = 'FAILURE' 100 | } 101 | -------------------------------------------------------------------------------- /nc/Maintenance/AirshipCtlBot.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurperClassic 2 | import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException 3 | import hudson.triggers.TimerTrigger 4 | import hudson.model.Cause.UserIdCause 5 | 6 | run_airship_bot = { 7 | SCRIPT_PATH = "sudo -i -u ubuntu /home/ubuntu/bot/gerrit-to-github-bot/run_airshipctl_bot.sh" 8 | println("Run Airshipctl BOT. Fetch gerrit changes from last $SYNC_TIME_WINDOW") 9 | sh "$SCRIPT_PATH $SYNC_TIME_WINDOW" 10 | } 11 | 12 | //// main flow 13 | try { 14 | node (label: NODE_LABEL){ 15 | cleanWs() 16 | run_airship_bot() 17 | } 18 | } catch (error) { 19 | print "Build failed: ${error.getMessage()}" 20 | currentBuild.result = 'FAILURE' 21 | } -------------------------------------------------------------------------------- /nc/Maintenance/AirshipctlStatusReport.groovy: -------------------------------------------------------------------------------- 1 | import jenkins.model.* 2 | import hudson.plugins.promoted_builds.* 3 | import groovy.json.JsonOutput 4 | 5 | def FILE_HEADER = ['JOB Name','Build', 'Slave', 'Description', 'Branch', 'Result', 'Date'] 6 | def fileName = 'StatusReport.csv' 7 | def allRecords = [FILE_HEADER] 8 | def msg = '' 9 | 10 | 11 | def status_report = { 12 | print "started status report" 13 | def docker_limit_fails = [:] 14 | def job_names = ['Airshipctl'] 15 | print job_names 16 | def status_map = [TOTAL: 0, SUCCESS: 0, FAILURE: 0, ABORTED: 0, RUNNING: 0, UNSTABLE: 0, NOT_BUILT: 0] 17 | def slave_status = [:] 18 | for(String job_name in job_names){ 19 | def job = Jenkins.instance.getItemByFullName( "development/"+job_name ) 20 | def cutOfDate = System.currentTimeMillis() - 1000L * 60 * 60 * 24 * NO_OF_DAYS.toInteger() 21 | for ( build in job.getBuilds().limit(LIMIT.toInteger()) ) { 22 | if ( build != null && build.getTimeInMillis() > cutOfDate) { 23 | def display_name = build.displayName 24 | def branch = display_name.substring(display_name.lastIndexOf(" ")+1) 25 | // Get console log in Plain text format to get away with wierd links 26 | def log = '' 27 | log = build.getLog() 28 | def slave = log =~ /Running on (.*) in/ 29 | def desc = log =~ /Triggered by Gerrit: (.*)|Retriggered by user (.*)|Started by (.*)|Manually triggered by (.*)/ 30 | slave_str = 'UNKNOWN' 31 | if (slave.size()) { 32 | slave_str = slave[0][1] 33 | } 34 | slave_str = slave_str.toString() 35 | build_number = build.getNumber() 36 | build_url = build.getAbsoluteUrl() 37 | if (desc[0][1]) { 38 | desc_str = desc[0][1] 39 | } else { 40 | desc_str = desc[0][0] 41 | } 42 | desc_str = desc_str.toString() 43 | time = build.getTime().toString() 44 | 45 | result = build.getResult().toString() 46 | if (result == 'null') { 47 | result = 'RUNNING' 48 | } 49 | def count = slave_status.find{ it.key == slave_str }?.value 50 | if (count) { 51 | slave_status.put(slave_str, slave_status.get(slave_str) + 1) 52 | } else { 53 | slave_status.put(slave_str, 1) 54 | } 55 | if (result == 'FAILURE') { 56 | def limit_issue = log =~ /You have reached your pull rate limit./ 57 | if (limit_issue.size()) { 58 | docker_limit_fails.put(build_number, slave_str) 59 | } 60 | } 61 | status_map.put('TOTAL', status_map.get('TOTAL') + 1) 62 | status_map.put(result, status_map.get(result) + 1) 63 | allRecords.add([job_name, build_url, slave_str, desc_str, branch, result, time]) 64 | //print job_name + " " + build_url + " " + " " + slave_str + " " + desc_str + " " + branch + " " + result + " " + time 65 | } 66 | } 67 | } 68 | timer_count = allRecords.count { it[3].contains('timer')} 69 | timer_success = allRecords.count { it[3].contains('timer') && it[5].contains('SUCCESS')} 70 | timer_failure = allRecords.count { it[3].contains('timer') && it[5].contains('FAILURE')} 71 | msg = ''' 72 | ---------------------------------------------------------------------------------------------------- 73 | Status: ''' + status_map + ''' 74 | Workers: ''' + slave_status + ''' 75 | Timers: Total: ''' + timer_count + ''' SUCCESS: ''' + timer_success + ''' FAILURE: ''' + timer_failure + ''' 76 | Failures due to Docker download limit : ''' + docker_limit_fails.size() + ''' 77 | ---------------------------------------------------------------------------------------------------- 78 | ''' 79 | return msg 80 | } 81 | 82 | node (label: NODE_LABEL){ 83 | msg = status_report() 84 | print msg 85 | } 86 | -------------------------------------------------------------------------------- /nc/Maintenance/CleanupOldBuildLogs.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurperClassic 2 | import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException 3 | import hudson.triggers.TimerTrigger 4 | import hudson.model.Cause.UserIdCause 5 | 6 | cleanup_log = { 7 | COMMAND = "sudo find /mnt/jenkins-data/userContent/ -mtime +${NO_OF_DAYS_TO_CLEANUP} -type f -delete" 8 | print "Command: ${COMMAND}" 9 | sh "sudo apt install -y sshpass" 10 | withCredentials([usernamePassword(credentialsId: "jenkins_master", 11 | usernameVariable: "USER", 12 | passwordVariable: "PASSWORD")]) { 13 | sh "sshpass -p ${PASSWORD} ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${USER}@${HOST} ${COMMAND}" 14 | } 15 | } 16 | 17 | //// main flow 18 | try { 19 | node (label: NODE_LABEL){ 20 | cleanWs() 21 | cleanup_log() 22 | } 23 | } catch (error) { 24 | print "Build failed: ${error.getMessage()}" 25 | currentBuild.result = 'FAILURE' 26 | } -------------------------------------------------------------------------------- /nc/Maintenance/VinoBot.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurperClassic 2 | import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException 3 | import hudson.triggers.TimerTrigger 4 | import hudson.model.Cause.UserIdCause 5 | 6 | run_airship_bot = { 7 | SCRIPT_PATH = "sudo -i -u ubuntu /home/ubuntu/bot/gerrit-to-github-bot/run_vino_bot.sh" 8 | println("Run Airshipctl BOT. Fetch gerrit changes from last $SYNC_TIME_WINDOW") 9 | sh "$SCRIPT_PATH $SYNC_TIME_WINDOW" 10 | } 11 | 12 | //// main flow 13 | try { 14 | node (label: NODE_LABEL){ 15 | cleanWs() 16 | run_airship_bot() 17 | } 18 | } catch (error) { 19 | print "Build failed: ${error.getMessage()}" 20 | currentBuild.result = 'FAILURE' 21 | } -------------------------------------------------------------------------------- /nc/Maintenance/WorkerMaintenance.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurperClassic 2 | import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException 3 | import hudson.triggers.TimerTrigger 4 | import hudson.model.Cause.UserIdCause 5 | 6 | def run_on_slave = { 7 | try { 8 | cleanWs() 9 | print "COMMANDS to RUN on IDLE NODES FOR MAINTENANCE" 10 | sh "df -h" 11 | sh "free -m" 12 | 13 | sh "sudo ls -lrth /var/lib/libvirt/images/ | true" 14 | sh "sudo ls -lrth /tmp/ | true" 15 | 16 | print "Apt autoremove and Clean" 17 | sh "sudo apt-get autoremove -y --purge && sudo apt autoclean && sudo apt clean | true" 18 | 19 | print "Delete tmp dirs" 20 | sh "sudo rm -rf /tmp/*" 21 | sh "sudo rm -rf /var/tmp/*" 22 | 23 | print "Docker Cleanup" 24 | sh "sudo docker system prune -af | true" 25 | sh "sudo docker volume prune -f | true" 26 | 27 | print "Clean Virsh console logs" 28 | sh "sudo rm -rf /var/log/libvirt-consoles/air-worker-1-console.log" 29 | sh "sudo rm -rf /var/log/libvirt-consoles/air-target-1-console.log" 30 | sh "sudo rm -rf /var/log/libvirt-consoles/air-ephemeral-console.log" 31 | 32 | print " Clean journalctl logs " 33 | sh "sudo journalctl --vacuum-size=500M" 34 | 35 | print "clear apache old logs" 36 | sh "sudo rm -rf /var/log/apache2/*.gz" 37 | 38 | } catch(Exception error) { 39 | print "Build failed: ${error.getMessage()}" 40 | } 41 | } 42 | 43 | //// main flow 44 | try { 45 | def slaves = [] 46 | // Get the list of idle slaves 47 | hudson.model.Hudson.instance.slaves.each { 48 | if ( !it.getComputer().isOffline() && it.getComputer().countBusy() == 0 && 49 | it.getLabelString() == WORKER_LABEL) { 50 | slaves << it.name 51 | } 52 | } 53 | 54 | println('*'*120) 55 | print "Cleanup scripts will be executed on these worker VMs" 56 | print(slaves) 57 | println('*'*120) 58 | 59 | slaves.each { 60 | print("Running on ${it} as it seems to be idle") 61 | node(it) { 62 | run_on_slave() 63 | } 64 | } 65 | } catch (error) { 66 | print "Build failed: ${error.getMessage()}" 67 | currentBuild.result = 'FAILURE' 68 | } -------------------------------------------------------------------------------- /nc/Treasuremap/Treasuremapv2-AirshipctlKnownState.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurperClassic 2 | import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException 3 | import hudson.triggers.TimerTrigger 4 | import hudson.model.Cause.UserIdCause 5 | 6 | if (!env.GERRIT_BRANCH) { 7 | GERRIT_EVENT_TYPE = "Manual" 8 | } 9 | currentBuild.displayName = "#${BUILD_NUMBER} ${GERRIT_EVENT_TYPE}" 10 | if (currentBuild.rawBuild.getCause(hudson.triggers.TimerTrigger.TimerTriggerCause)) { 11 | currentBuild.displayName = "#${BUILD_NUMBER} Timer (${GERRIT_REFSPEC})" 12 | } 13 | 14 | def cloneref = { 15 | sh "sudo rm -rf airshipctl || true" 16 | sh "git clone https://review.opendev.org/airship/airshipctl" 17 | sh "git clone -b v2.0 https://review.opendev.org/airship/treasuremap" 18 | envVars = env.getEnvironment() 19 | println envVars.containsKey("GERRIT_REFSPEC") 20 | if (envVars.containsKey("GERRIT_REFSPEC")) { 21 | sh "cd treasuremap && git fetch https://review.opendev.org/airship/treasuremap $GERRIT_REFSPEC && git checkout FETCH_HEAD && git checkout -b gerrit_current" 22 | } 23 | if(envVars.containsKey("AIRSHIPCTL_REF")) { 24 | sh "cd airshipctl && git fetch https://review.opendev.org/airship/airshipctl $AIRSHIPCTL_REF && git checkout FETCH_HEAD && git checkout -b gerrit_current" 25 | } 26 | } 27 | 28 | def collect_log = { 29 | println("Colelcting Log") 30 | } 31 | 32 | def trigger = { 33 | stage('Clone Code') { 34 | sh "sudo rm -rf treasuremap || true" 35 | cloneref() 36 | } 37 | stage('Cleanup'){ 38 | println("Cleanup packages, images") 39 | sh "sudo rm -rf ~/.airship || true" 40 | sh "sudo rm -rf /tmp/* || true" 41 | sh "cd airshipctl && tools/deployment/clean.sh || true " 42 | } 43 | stage('Setup') { 44 | println("Running setup") 45 | sh "cd treasuremap && tools/gate/00_setup.sh" 46 | } 47 | stage('build') { 48 | println("Running build") 49 | sh "cd treasuremap && tools/gate/10_build_gate.sh" 50 | } 51 | stage('install') { 52 | println("Running Deployment") 53 | try { 54 | sh "cd treasuremap && tools/gate/20_run_gate_runner.sh" 55 | try { 56 | print "Build Succeded. Start Log collection" 57 | collect_log() 58 | } catch (error) { 59 | print "Log collection Failed : ${error.getMessage()}" 60 | } 61 | } catch (FlowInterruptedException err) { 62 | print "Skip Log collect for aborted Jobs" 63 | throw err 64 | } catch (err){ 65 | try { 66 | print "Build Failed : ${err.getMessage()}. Start Log collection" 67 | collect_log() 68 | } catch (error) { 69 | print "Log collection Failed : ${error.getMessage()}" 70 | } 71 | throw err 72 | } 73 | } 74 | } 75 | 76 | //// main flow 77 | 78 | try { 79 | node (label: NODE_LABEL){ 80 | cleanWs() 81 | trigger() 82 | } 83 | } catch (error) { 84 | print "Build failed: ${error.getMessage()}" 85 | currentBuild.result = 'FAILURE' 86 | } -------------------------------------------------------------------------------- /osh/dockerfiles/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/ubuntu:16.04 2 | ENV KOLLA_VERSION="3.0.3" \ 3 | KOLLA_IMAGE_TAG="newton" \ 4 | KOLLA_IMAGE_MAINTAINER="Pete Birley (pete@port.direct)" \ 5 | OPENSTACK_SOURCE="git" \ 6 | OPENSTACK_SOURCE_ROOT="https://git.openstack.org/openstack" \ 7 | OPENSTACK_VERSION="stable/newton" 8 | RUN set -x \ 9 | && apt-get update \ 10 | && apt-get dist-upgrade -y \ 11 | && apt-get install -y --no-install-recommends \ 12 | git \ 13 | python-pip \ 14 | ca-certificates \ 15 | docker.io \ 16 | build-essential \ 17 | python-dev \ 18 | python-pip \ 19 | python-virtualenv \ 20 | libssl-dev \ 21 | && pip install -U pip \ 22 | && pip install -U wheel setuptools \ 23 | && pip install tox \ 24 | && git clone http://git.openstack.org/openstack/kolla.git /opt/kolla \ 25 | && cd /opt/kolla \ 26 | && git checkout tags/$KOLLA_VERSION \ 27 | && tox -egenconfig \ 28 | && mkdir -p /etc/kolla \ 29 | && mv etc/kolla/kolla-build.conf /etc/kolla/ \ 30 | && rm -rf .tox \ 31 | && pip install . -r /opt/kolla/requirements.txt \ 32 | && pip install crudini 33 | RUN set -x \ 34 | && KOLLA_CONFIG="/etc/kolla/kolla-build.conf" \ 35 | && crudini --set ${KOLLA_CONFIG} DEFAULT base "ubuntu" \ 36 | && crudini --set ${KOLLA_CONFIG} DEFAULT base_image "docker.io/ubuntu" \ 37 | && crudini --set ${KOLLA_CONFIG} DEFAULT base_tag "16.04" \ 38 | && crudini --set ${KOLLA_CONFIG} DEFAULT base_arch "x86_64" \ 39 | && crudini --set ${KOLLA_CONFIG} DEFAULT namespace "kolla" \ 40 | && crudini --set ${KOLLA_CONFIG} DEFAULT install_type "source" \ 41 | && crudini --set ${KOLLA_CONFIG} DEFAULT threads "1" \ 42 | && crudini --set ${KOLLA_CONFIG} DEFAULT tag "${KOLLA_IMAGE_TAG}" \ 43 | && mkdir -p /var/lib/kolla \ 44 | && crudini --set ${KOLLA_CONFIG} DEFAULT work_dir "/var/lib/kolla" \ 45 | && crudini --set ${KOLLA_CONFIG} DEFAULT maintainer "${KOLLA_IMAGE_MAINTAINER}" \ 46 | && for PARAM in type location reference; do \ 47 | sed -i "s/^#${PARAM}/${PARAM}/g" ${KOLLA_CONFIG}; \ 48 | done \ 49 | && crudini --get ${KOLLA_CONFIG} | while read SECTION; do \ 50 | (LOCATION=$(crudini --get /etc/kolla/kolla-build.conf ${SECTION} location 2> /dev/null ) && \ 51 | echo $LOCATION | grep -q ^\$tarballs_base/ && ( \ 52 | crudini --set ${KOLLA_CONFIG} ${SECTION} type ${OPENSTACK_SOURCE}; \ 53 | LOCATION=${LOCATION#\$tarballs_base/}; \ 54 | crudini --set ${KOLLA_CONFIG} ${SECTION} location "${OPENSTACK_SOURCE_ROOT}/${LOCATION%/*}"; \ 55 | crudini --set ${KOLLA_CONFIG} ${SECTION} reference "${OPENSTACK_VERSION}"; \ 56 | ) || true) \ 57 | done 58 | -------------------------------------------------------------------------------- /osh/openstack/images/kolla/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/att-comdev/cicd/8e92dce5a7bcb048638ebf6454a2de28f7b94332/osh/openstack/images/kolla/Dockerfile -------------------------------------------------------------------------------- /osh/openstack/openstack-addons/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/att-comdev/cicd/8e92dce5a7bcb048638ebf6454a2de28f7b94332/osh/openstack/openstack-addons/.gitignore -------------------------------------------------------------------------------- /osh/openstack/openstack-helm/Jenkinsfile: -------------------------------------------------------------------------------- 1 | JENKINS_VM_LAUNCH = 'local-vm-launch' 2 | NODE_NAME="openstack-helm-${env.BUILD_NUMBER}" 3 | NODE_TMPL = "openstack-helm/openstack-helm.yaml" 4 | ARTF_URL = env.ARTF_WEB_URL 5 | OSH_VERSION = "0.1.0.${env.BUILD_NUMBER}" 6 | OSH_PS_ARTF_REPO = "openstack-patchset/openstack-helm/"+OSH_VERSION 7 | OSH_ARTF_REPO = "openstack/openstack-helm/"+OSH_VERSION 8 | def funcs 9 | 10 | try{ 11 | stage('Create Jenkins Node'){ 12 | node(JENKINS_VM_LAUNCH) { 13 | gerrit.cloneUpstream(repo: 'att-comdev/cicd', refspec: 'master', targetDirectory: '.') 14 | 15 | funcs = load "${WORKSPACE}/integration/funcs.groovy" 16 | funcs.jenkins_slave_launch(NODE_NAME, "${env.HOME}/${NODE_TMPL}") 17 | } 18 | } 19 | stage('Nodes Wait'){ 20 | timeout (10) { 21 | node (NODE_NAME) { 22 | echo "Verifying that Jenkins node comes up." 23 | } 24 | } 25 | } 26 | node(NODE_NAME) { 27 | ansiColor('xterm') { 28 | stage('Checkout') { 29 | gerrit.cloneRepository url: 'https://git.openstack.org/openstack/openstack-helm', 30 | refspec: GERRIT_REFSPEC, 31 | targetDirectory: '.' 32 | } 33 | stage('Kubeadm-AIO') { 34 | try { 35 | timeout(45) { 36 | withEnv(["INTEGRATION=aio", 37 | "INTEGRATION_TYPE=basic", 38 | "POD_START_TIMEOUT=720"]) { 39 | sh "./tools/gate/setup_gate.sh" 40 | } 41 | } 42 | def server = Artifactory.server 'artifactory' 43 | if ( "${GERRIT_EVENT_TYPE}" == 'change-merged' || "$GERRIT_EVENT_TYPE" =='change-merged' ){ 44 | withCredentials([usernamePassword(credentialsId: 'jenkins-artifactory', 45 | usernameVariable: 'ARTIFACTORY_USER', 46 | passwordVariable: 'ARTIFACTORY_PASSWORD')]) { 47 | def uploadSpec = """{"files": [{"pattern": "*-0.1.0.tgz", 48 | "target": "$OSH_ARTF_REPO/" 49 | }]}""" 50 | server.upload(uploadSpec) 51 | } 52 | } else { 53 | withCredentials([usernamePassword(credentialsId: 'jenkins-artifactory', 54 | usernameVariable: 'ARTIFACTORY_USER', 55 | passwordVariable: 'ARTIFACTORY_PASSWORD')]) { 56 | def uploadSpec = """{"files": [{"pattern": "*-0.1.0.tgz", 57 | "target": "$OSH_PS_ARTF_REPO/" 58 | }]}""" 59 | server.upload(uploadSpec) 60 | } 61 | } 62 | } finally { 63 | stage('Publish Logs'){ 64 | node(NODE_NAME){ 65 | try { 66 | sh 'mkdir -p logs' 67 | sh 'tar -cf logs-basic-$BUILD_NUMBER.tgz logs' 68 | nexusArtifactUploader artifacts: [[ artifactId: 'org.openstack.helm', 69 | classifier: '', 70 | file: 'logs-basic-$BUILD_NUMBER.tgz', 71 | type: 'x-gtar']], 72 | credentialsId: 'nexus3', 73 | groupId: 'openstack-helm', 74 | nexusUrl: '$NEXUS3_URL', 75 | nexusVersion: 'nexus3', 76 | protocol: 'http', 77 | repository: 'att-comdev-jenkins-logs', 78 | version: '$BUILD_NUMBER' 79 | } catch (Exception e) { 80 | // ignore artifactory errors for now 81 | } 82 | } 83 | } 84 | } 85 | } 86 | } 87 | } 88 | } finally { 89 | stage('Delete Jenkins Node'){ 90 | node(JENKINS_VM_LAUNCH) { 91 | funcs.jenkins_slave_destroy(NODE_NAME) 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /osh/openstack/openstack-infra/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/att-comdev/cicd/8e92dce5a7bcb048638ebf6454a2de28f7b94332/osh/openstack/openstack-infra/.gitignore -------------------------------------------------------------------------------- /packages/pip-pkg/Jenkinsfile: -------------------------------------------------------------------------------- 1 | GERRIT_URL = "${INTERNAL_GERRIT_SSH}/${GERRIT_PROJECT}" 2 | DISPLAY_PREFIX = (env.GERRIT_EVENT_TYPE) ? GERRIT_EVENT_TYPE : "manual" 3 | currentBuild.displayName = "#${BUILD_NUMBER} ${DISPLAY_PREFIX} ${GERRIT_BRANCH}" 4 | 5 | /** 6 | * Override default package versioning to remove "dev" and add commit hash 7 | 8 | * @param gerritProject gerrit project for OpenStack client 9 | * @param commitID commit hash of the patchset 10 | */ 11 | def set_version(String gerritProject, String commitID ){ 12 | dir (gerritProject){ 13 | prefix = (env.GERRIT_EVENT_TYPE == 'patchset-created') ? 'ps.' : '' 14 | // install pbr to avoid warnings 15 | pip.installPackages(['pbr']) 16 | devversion = sh (returnStdout: true, 17 | script: 'python setup.py --version').trim() 18 | return devversion.tokenize('dev')[0] + 19 | devversion.tokenize('dev')[1] + 20 | '+' + prefix + commitID 21 | } 22 | } 23 | 24 | vm (initScript: 'loci-bootstrap.sh') { 25 | 26 | stage('Clone OpenStack client'){ 27 | if (env.GERRIT_PATCHSET_REVISION){ 28 | COMMIT_ID = GERRIT_PATCHSET_REVISION 29 | gerrit.cloneToBranch(GERRIT_URL, 30 | COMMIT_ID, 31 | GERRIT_PROJECT, 32 | INTERNAL_GERRIT_KEY) 33 | } else{ 34 | COMMIT_ID = gerrit.getVersion(GERRIT_URL, 35 | GERRIT_BRANCH, 36 | INTERNAL_GERRIT_KEY) 37 | gerrit.cloneProject(GERRIT_URL, 38 | GERRIT_BRANCH, 39 | COMMIT_ID, 40 | GERRIT_PROJECT, 41 | INTERNAL_GERRIT_KEY) 42 | } 43 | } 44 | 45 | stage('Updating host'){ 46 | pip.updateHost() 47 | } 48 | 49 | stage('Create pypirc file for pkg upload'){ 50 | pip.createPypirc('jenkins-artifactory') 51 | } 52 | 53 | stage('Build pip package and upload'){ 54 | version = set_version(GERRIT_PROJECT, COMMIT_ID) 55 | withEnv(["PBR_VERSION=$version"]){ 56 | pip.buildPackageAndUpload(GERRIT_PROJECT) 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /packages/pip-pkg/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_BASE='packages' 2 | 3 | folder("${JOB_BASE}/pip-pkg") 4 | 5 | MOS_PROJECTS = ['mos-keystoneclient', 6 | 'mos-heatclient', 7 | 'mos-glanceclient', 8 | 'mos-cinderclient', 9 | 'mos-neutronclient', 10 | 'mos-novaclient', 11 | 'mos-ceilometerclient', 12 | 'mos-swiftclient', 13 | 'mos-openstackclient'] 14 | 15 | MOS_PROJECTS.each { project -> 16 | pipelineJob("${JOB_BASE}/pip-pkg/${project}") { 17 | logRotator{ 18 | daysToKeep(90) 19 | } 20 | // limit surge of patchsets 21 | configure { 22 | node -> node / 'properties' / 'jenkins.branch.RateLimitBranchProperty_-JobPropertyImpl'{ 23 | durationName 'hour' 24 | count '3' 25 | } 26 | } 27 | 28 | parameters { 29 | stringParam { 30 | name ('GERRIT_PROJECT') 31 | defaultValue(project) 32 | description('Gerrit project') 33 | } 34 | stringParam { 35 | name ('GERRIT_BRANCH') 36 | defaultValue('master') 37 | description('Gerrit project') 38 | } 39 | stringParam { 40 | name ('ARTF_LOCAL_PYPI_URL') 41 | defaultValue("https://${ARTF_WEB_URL}/api/pypi/pypi-local") 42 | description('Artifactory repo url') 43 | } 44 | } 45 | 46 | triggers { 47 | gerritTrigger { 48 | serverName('mtn5-gerrit') 49 | gerritProjects { 50 | gerritProject { 51 | compareType('PLAIN') 52 | pattern(project) 53 | branches { 54 | branch { 55 | compareType("ANT") 56 | pattern("master") 57 | } 58 | } 59 | disableStrictForbiddenFileVerification(false) 60 | } 61 | } 62 | 63 | triggerOnEvents { 64 | patchsetCreated { 65 | excludeDrafts(false) 66 | excludeTrivialRebase(false) 67 | excludeNoCodeChange(false) 68 | } 69 | changeMerged() 70 | } 71 | } 72 | 73 | definition { 74 | cps { 75 | script(readFileFromWorkspace("${JOB_BASE}/pip-pkg/Jenkinsfile")) 76 | sandbox() 77 | } 78 | } 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /packages/ubuntu/seed.groovy: -------------------------------------------------------------------------------- 1 | JOB_FOLDER="packages/ubuntu" 2 | JOB_NAME="BuildPackages" 3 | 4 | folder(JOB_FOLDER) 5 | 6 | pipelineJob("${JOB_FOLDER}/${JOB_NAME}") { 7 | 8 | displayName('Build Packages') 9 | description('\nThis job is supposed to build (backport/re-build) custom '+ 10 | 'Ubuntu packages and upload them to Artifactory.\nList of '+ 11 | 'packages is in Jenkinsfile, submit a change to amend it.') 12 | 13 | logRotator{ 14 | daysToKeep(90) 15 | } 16 | 17 | parameters { 18 | choiceParam('UPLOAD_PACKAGES', ['false', 'true'], 'Upload packages to repository') 19 | choiceParam('FAKE_GPG_KEY', ['false', 'true'], 'Use fake GPG key') 20 | } 21 | 22 | definition { 23 | cps { 24 | sandbox() 25 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 26 | } 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /resources/cicd/remove_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | remove_tests_by_pattern() { 6 | pattern="$1" 7 | for exclude_dir in $(find ./ -name "$pattern" -type d | grep -Ev "contrib|etc|doc|tempest|meta|compat" | sed 's/.\///') ; do 8 | if [ -d "$exclude_dir" ]; then 9 | echo "Deleting tests: '$exclude_dir'" 10 | rm -rf "$exclude_dir"; 11 | echo "Add 'prune $exclude_dir into MANIFEST.in" 12 | echo "prune $exclude_dir" >> MANIFEST.in 13 | fi 14 | done 15 | } 16 | 17 | remove_tests() { 18 | echo "$1" 19 | if [ "$PROJECT" != "horizon" ]; then 20 | pushd "$1" 21 | remove_tests_by_pattern "tests" 22 | remove_tests_by_pattern "test" 23 | popd 24 | fi 25 | } 26 | remove_tests "/tmp/$PROJECT" 27 | -------------------------------------------------------------------------------- /resources/heat/glance/ubuntu.image.template.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: ocata 2 | 3 | parameters: 4 | cloudImage: 5 | type: string 6 | default: https://artifacts-aic.atlantafoundry.com/artifactory/ubuntu-images/bionic/current/bionic-server-cloudimg-amd64.img 7 | imageName: 8 | type: string 9 | default: Ubuntu-18.04 10 | resources: 11 | image: 12 | type: OS::Glance::Image 13 | properties: 14 | container_format: bare 15 | disk_format: qcow2 16 | location: { get_param: cloudImage } 17 | min_disk: 10 18 | min_ram: 1024 19 | name: { get_param: imageName } 20 | os_distro: ubuntu -------------------------------------------------------------------------------- /resources/heat/nova/flavor.template.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The OpenStack-Helm Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # openstack stack create --wait -t osh-node-flavor.yaml osh-node-flavor 16 | heat_template_version: ocata 17 | 18 | description: Template that allows for a creation of a flavor 19 | 20 | parameters: 21 | flavorName: 22 | type: string 23 | default: m1.large 24 | disk: 25 | type: string 26 | default: 64 27 | ram: 28 | type: string 29 | default: 32768 30 | vcpus: 31 | type: string 32 | default: 8 33 | resources: 34 | flavor: 35 | type: OS::Nova::Flavor 36 | properties: 37 | name: {get_param: flavorName} 38 | disk: {get_param: disk} 39 | ram: {get_param: ram} 40 | vcpus: {get_param: vcpus} 41 | 42 | -------------------------------------------------------------------------------- /resources/heat/stack/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "====== Installing JAVA packages ======" 3 | apt-get update 4 | apt-get install -y default-jre-headless 5 | 6 | echo "====== Updating /etc/hosts ======" 7 | cmd='/127.0.1.1/!p;$a127.0.1.1' 8 | cmd="${cmd} $(hostname)" 9 | sudo sed -i -n -e "${cmd}" /etc/hosts 10 | -------------------------------------------------------------------------------- /resources/heat/stack/cloud-config: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | package_update: false 4 | apt_preserve_sources_list: true 5 | 6 | manage_resolv_conf: true 7 | resolv_conf: 8 | nameservers: 9 | - '8.8.8.8' 10 | - '8.8.4.4' 11 | 12 | ntp: 13 | enabled: true 14 | ntp_client: ntp 15 | servers: 16 | - 0.ubuntu.pool.ntp.org 17 | - 1.ubuntu.pool.ntp.org 18 | 19 | packages: 20 | - fio 21 | - htop 22 | - default-jre-headless 23 | 24 | users: 25 | - ubuntu: 26 | groups: [docker, libvirt] 27 | - name: virtmgr 28 | groups: [libvirt, libvirtd] 29 | 30 | groups: 31 | - libvirt 32 | - libvirtd 33 | - docker 34 | 35 | runcmd: 36 | - chmod 644 /etc/cron.daily/apt-compat 37 | - systemctl disable apt-daily-upgrade.timer apt-daily.timer 38 | - systemctl stop apt-daily-upgrade.timer apt-daily.timer 39 | -------------------------------------------------------------------------------- /resources/heat/stack/loci-bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo "127.0.1.1 $(hostname)" >> /etc/hosts 6 | 7 | # required for docker within docker 8 | mkdir -p /etc/docker 9 | cat << EOF > /etc/docker/daemon.json 10 | { 11 | "storage-driver": "overlay2" 12 | } 13 | EOF 14 | 15 | # git-ssh wrapper 16 | cat << EOF > /usr/bin/git-ssh-wrapper 17 | #!/bin/bash 18 | ssh -i "\$SSH_KEY" "\$@" 19 | EOF 20 | 21 | chmod a+x /usr/bin/git-ssh-wrapper 22 | -------------------------------------------------------------------------------- /resources/heat/stack/ubuntu.basic.stack.template.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: newton 2 | 3 | description: This template brings up a stack, setting flavor, size of VM, port & floating IP 4 | 5 | parameters: 6 | public_net: 7 | type: string 8 | default: public 9 | private_net: 10 | type: string 11 | default: private 12 | image: 13 | type: string 14 | default: cicd-ubuntu-16.04-server-cloudimg-amd64 15 | flavor: 16 | type: string 17 | default: m1.large 18 | 19 | conditions: 20 | add_routable_ip: 21 | not: 22 | equals: 23 | - get_param: public_net 24 | - '' 25 | 26 | resources: 27 | server: 28 | type: OS::Nova::Server 29 | properties: 30 | config_drive: true 31 | image: { get_param: image } 32 | flavor: { get_param: flavor } 33 | key_name: jenkins-slave-keypair 34 | networks: 35 | if: 36 | - add_routable_ip 37 | - [port: { get_resource: routable_port }] 38 | - [port: { get_resource: private_port }] 39 | 40 | user_data: 41 | str_replace: 42 | template: 43 | get_file: cloud-config 44 | params: 45 | $message: Hello World! 46 | 47 | private_port: 48 | type: OS::Neutron::Port 49 | properties: 50 | network_id: { get_param: private_net } 51 | security_groups: [jenkins-security] 52 | 53 | routable_port: 54 | condition: add_routable_ip 55 | type: OS::Neutron::Port 56 | properties: 57 | network_id: { get_param: public_net } 58 | security_groups: [jenkins-security] 59 | 60 | outputs: 61 | routable_ip: 62 | description: The external IP associated to the server 63 | value: 64 | if: 65 | - add_routable_ip 66 | - get_attr: 67 | - routable_port 68 | - fixed_ips 69 | - 0 70 | - ip_address 71 | - get_attr: 72 | - private_port 73 | - fixed_ips 74 | - 0 75 | - ip_address 76 | -------------------------------------------------------------------------------- /resources/heat/stack/ubuntu.flat.stack.template.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: ocata 2 | 3 | description: Generic template for launching Ubuntu 4 | 5 | parameters: 6 | public_net: 7 | type: string 8 | default: public 9 | private_net: 10 | type: string 11 | default: public 12 | image: 13 | type: string 14 | default: bionic-server-cloudimg-amd64 15 | flavor: 16 | type: string 17 | default: m1.large 18 | key: 19 | type: string 20 | default: airship-cicd 21 | 22 | resources: 23 | server: 24 | type: OS::Nova::Server 25 | properties: 26 | config_drive: true 27 | image: { get_param: image } 28 | flavor: { get_param: flavor } 29 | key_name: { get_param: key } 30 | 31 | networks: 32 | - port: { get_resource: server_port } 33 | 34 | user_data_format: RAW 35 | user_data: {get_file: cloud-config} 36 | 37 | server_port: 38 | type: OS::Neutron::Port 39 | properties: 40 | network_id: 41 | get_param: public_net 42 | security_groups: [airship-cicd] 43 | 44 | outputs: 45 | floating_ip: 46 | description: The external IP associated to the server 47 | value: { get_attr: [server, first_address ] } 48 | -------------------------------------------------------------------------------- /resources/heat/stack/ubuntu.flat.volume.stack.template.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: ocata 2 | 3 | description: Generic template for launching VM from volume 4 | 5 | parameters: 6 | public_net: 7 | type: string 8 | default: public 9 | private_net: 10 | type: string 11 | default: private 12 | image: 13 | type: string 14 | default: bionic-server-cloudimg-amd64 15 | flavor: 16 | type: string 17 | default: m1.large 18 | key: 19 | type: string 20 | default: jenkins-slave-keypair 21 | 22 | resources: 23 | server: 24 | type: OS::Nova::Server 25 | properties: 26 | config_drive: true 27 | flavor: { get_param: flavor } 28 | key_name: { get_param: key } 29 | networks: 30 | - port: { get_resource: server_port } 31 | block_device_mapping: 32 | - device_name: vda 33 | volume_id: { get_resource: cinder_volume } 34 | delete_on_termination: true 35 | user_data_format: RAW 36 | user_data: {get_file: cloud-config} 37 | 38 | cinder_volume: 39 | type: OS::Cinder::Volume 40 | properties: 41 | size: 300 42 | image: { get_param: image } 43 | 44 | server_port: 45 | type: OS::Neutron::Port 46 | properties: 47 | network_id: 48 | get_param: public_net 49 | security_groups: [jenkins-security] 50 | 51 | outputs: 52 | floating_ip: 53 | description: The external IP associated to the server 54 | value: { get_attr: [server, first_address ] } 55 | -------------------------------------------------------------------------------- /resources/heat/ubuntu1604.m1.large.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: newton 2 | 3 | description: Large instance. 4 | 5 | resources: 6 | server: 7 | type: OS::Nova::Server 8 | properties: 9 | image: cicd-ubuntu-16.04-server-cloudimg-amd64 10 | flavor: m1.large 11 | key_name: jenkins-slave-keypair 12 | networks: 13 | - port: { get_resource: server_port } 14 | 15 | user_data: 16 | str_replace: 17 | template: {get_file: bootstrap.sh} 18 | params: 19 | $message: Hello World! 20 | 21 | server_port: 22 | type: OS::Neutron::Port 23 | properties: 24 | network_id: private 25 | security_groups: [jenkins-security] 26 | 27 | server_floating_ip: 28 | type: OS::Neutron::FloatingIP 29 | properties: 30 | floating_network_id: public 31 | port_id: { get_resource: server_port } 32 | 33 | outputs: 34 | floating_ip: 35 | description: The external IP associated to the server 36 | value: { get_attr: [server_floating_ip, floating_ip_address] } 37 | 38 | -------------------------------------------------------------------------------- /resources/heat/ubuntu1604.m1.medium.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: newton 2 | 3 | description: Medium instance. 4 | 5 | resources: 6 | server: 7 | type: OS::Nova::Server 8 | properties: 9 | image: cicd-ubuntu-16.04-server-cloudimg-amd64 10 | flavor: m1.medium 11 | key_name: jenkins-slave-keypair 12 | networks: 13 | - port: { get_resource: server_port } 14 | 15 | user_data: 16 | str_replace: 17 | template: {get_file: bootstrap.sh} 18 | params: 19 | $message: Hello World! 20 | 21 | server_port: 22 | type: OS::Neutron::Port 23 | properties: 24 | network_id: private 25 | security_groups: [jenkins-security] 26 | 27 | server_floating_ip: 28 | type: OS::Neutron::FloatingIP 29 | properties: 30 | floating_network_id: public 31 | port_id: { get_resource: server_port } 32 | 33 | outputs: 34 | floating_ip: 35 | description: The external IP associated to the server 36 | value: { get_attr: [server_floating_ip, floating_ip_address] } 37 | 38 | -------------------------------------------------------------------------------- /resources/heat/ubuntu1604.m1.small.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: newton 2 | 3 | description: Small instance. 4 | 5 | resources: 6 | server: 7 | type: OS::Nova::Server 8 | properties: 9 | image: cicd-ubuntu-16.04-server-cloudimg-amd64 10 | flavor: m1.small 11 | key_name: jenkins-slave-keypair 12 | networks: 13 | - port: { get_resource: server_port } 14 | 15 | user_data: 16 | str_replace: 17 | template: {get_file: bootstrap.sh} 18 | params: 19 | $message: Hello World! 20 | 21 | server_port: 22 | type: OS::Neutron::Port 23 | properties: 24 | network_id: private 25 | security_groups: [jenkins-security] 26 | 27 | server_floating_ip: 28 | type: OS::Neutron::FloatingIP 29 | properties: 30 | floating_network_id: public 31 | port_id: { get_resource: server_port } 32 | 33 | outputs: 34 | floating_ip: 35 | description: The external IP associated to the server 36 | value: { get_attr: [server_floating_ip, floating_ip_address] } 37 | 38 | -------------------------------------------------------------------------------- /resources/heat/ubuntu1604.m1.xlarge.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: newton 2 | 3 | description: XLarge instance. 4 | 5 | resources: 6 | server: 7 | type: OS::Nova::Server 8 | properties: 9 | image: cicd-ubuntu-16.04-server-cloudimg-amd64 10 | flavor: m1.xlarge 11 | key_name: jenkins-slave-keypair 12 | networks: 13 | - port: { get_resource: server_port } 14 | 15 | user_data: 16 | str_replace: 17 | template: {get_file: bootstrap.sh} 18 | params: 19 | $message: Hello World! 20 | 21 | server_port: 22 | type: OS::Neutron::Port 23 | properties: 24 | network_id: private 25 | security_groups: [jenkins-security] 26 | 27 | server_floating_ip: 28 | type: OS::Neutron::FloatingIP 29 | properties: 30 | floating_network_id: public 31 | port_id: { get_resource: server_port } 32 | 33 | outputs: 34 | floating_ip: 35 | description: The external IP associated to the server 36 | value: { get_attr: [server_floating_ip, floating_ip_address] } 37 | 38 | -------------------------------------------------------------------------------- /resources/heat/ubuntu1804.m1.large.yaml: -------------------------------------------------------------------------------- 1 | heat_template_version: newton 2 | 3 | description: Large instance. 4 | 5 | resources: 6 | server: 7 | type: OS::Nova::Server 8 | properties: 9 | image: cicd-ubuntu-18.04-server-cloudimg-amd64 10 | flavor: m1.large 11 | key_name: jenkins-slave-keypair 12 | networks: 13 | - port: { get_resource: server_port } 14 | 15 | user_data: 16 | str_replace: 17 | template: {get_file: bootstrap.sh} 18 | params: 19 | $message: Hello World! 20 | 21 | server_port: 22 | type: OS::Neutron::Port 23 | properties: 24 | network_id: private 25 | security_groups: [jenkins-security] 26 | 27 | server_floating_ip: 28 | type: OS::Neutron::FloatingIP 29 | properties: 30 | floating_network_id: public 31 | port_id: { get_resource: server_port } 32 | 33 | outputs: 34 | floating_ip: 35 | description: The external IP associated to the server 36 | value: { get_attr: [server_floating_ip, floating_ip_address] } 37 | 38 | -------------------------------------------------------------------------------- /src/att/comdev/cicd/config/conf.groovy: -------------------------------------------------------------------------------- 1 | package att.comdev.cicd.config 2 | 3 | class conf { 4 | 5 | // Static Images 6 | public static final String PEGLEG_IMAGE = "artifacts-aic.atlantafoundry.com/att-comdev/pegleg:ef47933903047339bd63fcfa265dfe4296e8a322" 7 | public static final String PROMENADE_IMAGE = "artifacts-aic.atlantafoundry.com/att-comdev/promenade@sha256:e8a6d6e671fa330e63a91b978757c3cde241aad81f2166aebc9a0880702c0f7c" 8 | 9 | // Artifactory 10 | public static final String ARTF_SERVER_ID = "artifactory" 11 | public static final String ARTF_WEB_URL = "https://artifacts-aic.atlantafoundry.com/artifactory" 12 | 13 | //docker-in-docker image 14 | public static final String DIND_IMAGE = "docker:17.07.0-dind" 15 | 16 | //nginx image 17 | public static final String NGINX_IMAGE = "nginx" 18 | 19 | //Helm Repo lists 20 | public static final Map HELM_REPO = ['stable' : 'https://kubernetes-charts.storage.googleapis.com/', 21 | 'incubator' : 'https://kubernetes-charts-incubator.storage.googleapis.com/'] 22 | 23 | // Other 24 | public static final String EXCLUDE_NODES = "master jenkins-node-launch 10.24.20.18-slave 10.24.20.19-slave MyCodeReviewDev att-comdev-charts-dev cab24-r820-18 cab24-r820-19 openstack-helm-charts genesis clcp-seaworthy-genesis 5ec-seaworthy-genesis" 25 | 26 | // GitHub 27 | public static final String GITHUB_URL = "github.com" 28 | public static final String JENKINS_GITHUB_CRED_ID = "github-key" 29 | public static final String GITHUB_REPO_PATH_PREFIX = "" 30 | public static final String GITHUB_REPO_PREFIX = "" 31 | } 32 | -------------------------------------------------------------------------------- /src/test/groovy/build/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id "groovy" 3 | id "jacoco" 4 | } 5 | 6 | group = "att.comdev" 7 | version = "1.0.0" 8 | 9 | sourceSets { 10 | main { 11 | groovy { 12 | srcDirs = ['vars', 13 | 'src/att/comdev'] 14 | // todo: research how to fix these, they have compilation issues, impacting coverage being seen 15 | exclude '**/creds.groovy', 16 | '**/osh.groovy', 17 | '**/cicd_helper.groovy' 18 | } 19 | } 20 | } 21 | 22 | test { 23 | // delete old test reports 24 | dependsOn cleanTest 25 | 26 | // don't ignore/ignore failures flag 27 | ignoreFailures = false 28 | 29 | // minimize logging 30 | testLogging.maxGranularity = 0 31 | 32 | // show stdout from tests 33 | onOutput { dest, event -> print event.message } 34 | 35 | // show test results 36 | def results = [] 37 | afterTest { desc, result -> println "${desc.className.split("\\.")[-1]}: " + 38 | "${desc.name}: ${result.resultType}" } 39 | afterSuite { desc, result -> if (desc.className) { results << result } } 40 | 41 | // show summary 42 | doLast { 43 | println "Tests: ${results.sum { it.testCount }}" + 44 | ", Failures: ${results.sum { it.failedTestCount }}" + 45 | ", Errors: ${results.sum { it.exceptions.size() }}" + 46 | ", Skipped: ${results.sum { it.skippedTestCount }}" 47 | } 48 | 49 | finalizedBy jacocoTestReport 50 | } 51 | 52 | jacocoTestCoverageVerification { 53 | violationRules { 54 | rule { 55 | limit { 56 | counter = 'LINE' 57 | value = 'COVEREDRATIO' 58 | // todo: up this when we want to make coverage gate 59 | minimum = 0.00 60 | } 61 | } 62 | } 63 | } 64 | 65 | // code coverage enforcement 66 | check.dependsOn jacocoTestCoverageVerification 67 | 68 | repositories { 69 | mavenCentral() 70 | maven { 71 | url 'https://repo.jenkins-ci.org/public' 72 | } 73 | } 74 | 75 | dependencies { 76 | compile group: 'org.jenkins-ci.plugins.workflow', name: 'workflow-step-api', version: '2.7', ext: 'jar' 77 | compile "org.codehaus.groovy:groovy-all:2.5.0" 78 | testCompile "org.yaml:snakeyaml:1.24" 79 | testCompile "com.lesfurets:jenkins-pipeline-unit:1.1" 80 | testCompile "junit:junit:4.12" 81 | } -------------------------------------------------------------------------------- /test/Jenkinsfile: -------------------------------------------------------------------------------- 1 | JENKINS_VM_LAUNCH='local-vm-launch' 2 | NODE_NAME="${JOB_BASE_NAME}-${BUILD_NUMBER}" 3 | NODE_TMPL="docker/ubuntu.m1.medium.yaml" 4 | 5 | vm(NODE_NAME, NODE_TMPL) { 6 | stage('Test setProperty'){ 7 | sh "echo URL: " + params.url 8 | sh "echo Properties: " + params.prop 9 | publish.setProperty(params.creds, params.url, params.prop) 10 | } 11 | stage('Test getImageDigest'){ 12 | sh "echo " + params.image 13 | sh "echo Digest: " + image.getImageDigest(params.image) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /test/seed.groovy: -------------------------------------------------------------------------------- 1 | /** 2 | * Testing library functions 3 | */ 4 | 5 | JOB_FOLDER='test' 6 | JOB_NAME='LibraryTest' 7 | 8 | folder(JOB_FOLDER) 9 | 10 | pipelineJob("${JOB_FOLDER}/${JOB_NAME}") { 11 | 12 | displayName('Testing library in vars/') 13 | description('This job is supposed to call methods from libraries located in vars/ to test if they work properly') 14 | 15 | logRotator{ 16 | daysToKeep(90) 17 | } 18 | 19 | parameters { 20 | credentials { 21 | name ('creds') 22 | credentialType('usernamePassword') 23 | defaultValue('jenkins-artifactory') 24 | description('Artifactory credentials to use') 25 | required true 26 | } 27 | stringParam { 28 | name ('url') 29 | defaultValue('clcp-manifests') 30 | description('URL of artifact onto which we would set properties') 31 | } 32 | stringParam { 33 | name ('prop') 34 | defaultValue('key1=value1;key2=value2') 35 | description('Properties we are setting onto the artifact') 36 | } 37 | stringParam { 38 | name ('image') 39 | defaultValue('${IMAGE}') 40 | description('Docker image to use when getting it\'s Digest') 41 | } 42 | } 43 | 44 | definition { 45 | cps { 46 | sandbox() 47 | script(readFileFromWorkspace("${JOB_FOLDER}/Jenkinsfile")) 48 | } 49 | } 50 | 51 | } 52 | 53 | -------------------------------------------------------------------------------- /ucp/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/att-comdev/cicd/8e92dce5a7bcb048638ebf6454a2de28f7b94332/ucp/.gitignore -------------------------------------------------------------------------------- /ucp/charts/build_charts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | #List of projects to work with is a job parameter: 6 | #PROJECT_LIST="armada deckhand drydock promenade shipyard" 7 | #See seed.groovy job parameters. 8 | 9 | record_fail(){ 10 | echo -e "\n project $1 failed: " | tee -a ${failed_log} 11 | cat ${1}.log >> ${failed_log} 12 | #Fail logged, continue building charts manually: 13 | helm_pkg 14 | } 15 | 16 | cleanup(){ 17 | rm -rf build && mkdir build 18 | } 19 | 20 | prepare_env(){ 21 | git clone -q --depth 1 https://git.openstack.org/openstack/openstack-helm.git 22 | cd openstack-helm 23 | if [ ! ${GERRIT_REFSPEC} = "master" ]; then 24 | git fetch https://git.openstack.org/openstack/openstack-helm.git ${GERRIT_REFSPEC} 25 | git checkout FETCH_HEAD 26 | fi 27 | source tools/gate/vars.sh 28 | source tools/gate/funcs/helm.sh 29 | which helm || helm_install 30 | helm_serve 31 | helm_plugin_template_install ||: 32 | make helm-toolkit 33 | } 34 | 35 | clone_projects(){ 36 | cd ${WDIR}/build 37 | for project in ${PROJECT_LIST}; do 38 | git clone -q --depth 1 https://review.gerrithub.io/att-comdev/${project} 39 | done 40 | } 41 | 42 | helm_pkg(){ 43 | #Some projects fail to create charts with make or don't have Makefile, 44 | #I want to create and upload their charts anyway: 45 | for i in `ls charts`; do 46 | helm dep up charts/$i 47 | helm package charts/$i 48 | done 49 | } 50 | 51 | make_charts(){ 52 | set -xe 53 | for project in $PROJECT_LIST; do 54 | cd ${WDIR}/build/${project} 55 | if [ -f Makefile ]; then 56 | make charts &> ${project}.log || record_fail ${project} 57 | else 58 | helm_pkg &> ${project}.log || record_fail ${project} 59 | fi 60 | done 61 | } 62 | 63 | WDIR=`pwd` 64 | failed_log=${WDIR}/build/failed.log 65 | 66 | ####MAIN#### 67 | cleanup && cd build 68 | prepare_env 69 | clone_projects 70 | make_charts 71 | echo "Done!" 72 | if [ -f $failed_log ]; then 73 | cat $failed_log 74 | exit 1 75 | fi 76 | -------------------------------------------------------------------------------- /vars/artifactory.groovy: -------------------------------------------------------------------------------- 1 | import att.comdev.cicd.config.conf 2 | 3 | import groovy.json.JsonOutput 4 | 5 | /** 6 | * Helper method to JSON-ify the request we send off to Artifactory 7 | * for file upload/download 8 | * 9 | * @param file The file we're interested in uploading/downloading 10 | * @param target Where, in Artifactory, we want to grab/place this file 11 | * @return spec Translation into JSON of the file to upload/download and its path 12 | */ 13 | private def spec(file, target) { 14 | spec = ["files": [["pattern": file, 15 | "target": target, 16 | "flat": true]]] 17 | return new JsonOutput().toJson(spec) 18 | } 19 | 20 | /** 21 | * Perform the upload to Artifactory, given the file and 22 | * target/path. 23 | * 24 | * @param file The file we're interested in uploading 25 | * @param target Where, in Artifactory, we want to place this file 26 | */ 27 | def upload(file, target) { 28 | Artifactory.server(conf.ARTF_SERVER_ID).upload(spec(file, target)) 29 | } 30 | 31 | /** 32 | * Perform the upload to Artifactory, given the file and 33 | * target/path, as well as publish the build info. 34 | * 35 | * @param file The file we're interested in uploading 36 | * @param target Where, in Artifactory, we want to place this file 37 | */ 38 | def uploadWithBuildInfo(file, target) { 39 | info = Artifactory.server(conf.ARTF_SERVER_ID).upload(spec(file, target)) 40 | Artifactory.server(conf.ARTF_SERVER_ID).publishBuildInfo(info) 41 | } 42 | 43 | /** 44 | * Perform the download from Artifactory, given the file and 45 | * target/path. 46 | * 47 | * @param file The file we're interested in downloading 48 | * @param target Where, in Artifactory, we want to grab this file from 49 | */ 50 | def download(file, target) { 51 | Artifactory.server(conf.ARTF_SERVER_ID).download(spec(file, target)) 52 | } 53 | -------------------------------------------------------------------------------- /vars/creds.groovy: -------------------------------------------------------------------------------- 1 | import hudson.plugins.sshslaves.*; 2 | import com.cloudbees.jenkins.plugins.sshcredentials.impl.* 3 | import com.cloudbees.plugins.credentials.impl.*; 4 | import com.cloudbees.plugins.credentials.*; 5 | import com.cloudbees.plugins.credentials.domains.*; 6 | 7 | /** 8 | * Return a collection of Global credentials defined within Jenkins 9 | */ 10 | def getGlobalCreds() { 11 | return SystemCredentialsProvider.getInstance().getStore().getCredentials(Domain.global()) 12 | } 13 | 14 | /** 15 | * Return a list of Global credential IDs defined within Jenkins 16 | */ 17 | def getGlobalCredIds() { 18 | def globalCreds = getGlobalCreds() 19 | def credIds = [] 20 | for(cred in globalCreds) { 21 | credIds.add(cred.id.toString()) 22 | } 23 | return credIds 24 | } 25 | 26 | /** 27 | * Create a Global user/password credential within Jenkins 28 | * 29 | * @param id the ID you wish the global credential to have 30 | * @param description the description you wish the global credential to have 31 | * @param user the username of the global credential you're creating 32 | * @param pass the password of the global credential you're creating 33 | */ 34 | def createGlobalCred(id, description, user, pass) { 35 | Credentials c = (Credentials) new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, id, description, user, pass) 36 | SystemCredentialsProvider.getInstance().getStore().addCredentials(Domain.global(), c) 37 | } 38 | 39 | /** 40 | * Create a Global SSH user/key credential within Jenkins 41 | * 42 | * @param id the ID you wish the global credential to have 43 | * @param description the description you wish the global credential to have 44 | * @param user the username of the global credential you're creating 45 | * @param key the private key of the global credential you're creating 46 | */ 47 | def createGlobalSshCred(id, description, user, key) { 48 | def privateKeySource = new BasicSSHUserPrivateKey.DirectEntryPrivateKeySource(key) 49 | def secret = new BasicSSHUserPrivateKey(CredentialsScope.GLOBAL, id, user, privateKeySource, "", id) 50 | SystemCredentialsProvider.getInstance().getStore().addCredentials(Domain.global(), secret) 51 | } 52 | 53 | /** 54 | * Delete a Global SSH user/key credential within Jenkins 55 | * 56 | * @param id the ID you wish the global credential to have 57 | */ 58 | def deleteGlobalSshCred(id) { 59 | def globalCreds = getGlobalCreds() 60 | def c = globalCreds.findResult { it.id == id ? it : null } 61 | if (c) { 62 | SystemCredentialsProvider.getInstance().getStore().removeCredentials(Domain.global(), c) 63 | } 64 | } 65 | 66 | /** 67 | * Delete a Global user/password credential within Jenkins 68 | * 69 | * @param id the ID of the established global credential 70 | * @param description the description of the established global credential 71 | * @param user the username of the global credential you're deleting 72 | * @param pass the password of the global credential you're deleting 73 | */ 74 | def deleteGlobalCred(id, description, user, pass) { 75 | Credentials c = (Credentials) new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, id, description, user, pass) 76 | SystemCredentialsProvider.getInstance().getStore().removeCredentials(Domain.global(), c) 77 | } 78 | 79 | /** 80 | * Recreate a Global user/password credential within Jenkins (delete -> create) 81 | * Note: delete should not fail if the credential does not already exist 82 | * 83 | * @param id the ID of the global credential 84 | * @param description the description you wish the global credential 85 | * @param user the username of the global credential you're deleting/creating 86 | * @param pass the password of the global credential you're deleting/creating 87 | */ 88 | def recreateGlobalCred(id, description, user, pass) { 89 | deleteGlobalCred(id, description, user, pass) 90 | createGlobalCred(id, description, user, pass) 91 | } 92 | 93 | /** Helper for update ssh key for jenkins slave 94 | * 95 | * @param node String Slave's name 96 | * @param key String New cred id 97 | */ 98 | def updateSshKeySlave(node, key) { 99 | def slave = Hudson.instance.slaves.find({it.name == node}); 100 | def slaveLauncher = slave.getLauncher() 101 | slaveLauncher.credentialsId = key 102 | } 103 | -------------------------------------------------------------------------------- /vars/email.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Send email 3 | * 4 | * @param email Map of email details 5 | */ 6 | def sendMail(Map email=[:]) { 7 | // ENABLE_EMAIL as a string allows for configuring as both global env var or pipeline parameter 8 | if(env.ENABLE_EMAIL == "true") { 9 | jenkins_instance = JENKINS_URL.split("://")[1].split('/')[0] 10 | subject = email.find{ it.key == "subject" }?.value ?: "$jenkins_instance ### ${BUILD_TAG} ### ${currentBuild.currentResult}" 11 | body = email.find{ it.key == "body" }?.value ?: """The job status is [${currentBuild.currentResult}] 12 | For more details see the build logs at ${BUILD_URL} 13 | 14 | 15 | """ 16 | to = email.find{ it.key == "to" }?.value ?: "" 17 | from = email.find{ it.key == "from" }?.value ?: "" 18 | attachLog = email.find{ it.key == "attachLog" }?.value ?: false 19 | attachmentsPattern = email.find{ it.key == "attachmentsPattern" }?.value ?: "" 20 | recipientProviders = email.find{ it.key == "recipientProviders" }?.value ?: [] 21 | 22 | // Usage example: sendMail(attachLog: true, attachmentsPattern: "*.html", to: "mymailid") 23 | // sendMail(attachLog: true, attachmentsPattern: "myfile.txt", to: "mymailid,yourmailid") 24 | // sendMail(subject:"Build failed", body: "Build failed, check logs", recipientProviders: [culprit()]) 25 | // sendMail(subject:"Build success", body: "Build succeeded", recipientProviders: [developers(), requestor()]) 26 | // sendMail(recipientProviders: [requestor()], to: "mymailid") 27 | // see https://jenkins.io/doc/pipeline/steps/email-ext/#emailext-extended-email for more details 28 | emailext body: body, 29 | subject: subject, 30 | to: to, 31 | from: from, 32 | attachLog: attachLog, 33 | attachmentsPattern: attachmentsPattern, 34 | recipientProviders: recipientProviders 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /vars/gerrithub.groovy: -------------------------------------------------------------------------------- 1 | /** 2 | * This is a wrapper library for vars/gerrit.groovy for use with gerrithub 3 | * Deprecated: please use 'gerrit.cloneUpstream' instead 4 | * TODO: remove it completely 5 | */ 6 | 7 | @Deprecated 8 | def clone(String project, String refspec){ 9 | // Usage example: gerrithub.clone("att-comdev/cicd", "origin/master") 10 | // clone refspec: gerrithub.clone("att-comdev/cicd", "${env.GERRIT_REFSPEC}") 11 | gerrit.clone("https://review.gerrithub.io/" + project, refspec) 12 | } 13 | 14 | @Deprecated 15 | def cloneToBranch(String project, String refspec, String targetDirectory){ 16 | //This method is used so that we can checkout the patchset to a local 17 | //branch and then rebase it locally with the current master before we build and test 18 | gerrit.cloneToBranch("https://review.gerrithub.io/" + project, refspec, targetDirectory) 19 | } 20 | 21 | @Deprecated 22 | def cloneProject(String project, String branch, String refspec, String targetDirectory){ 23 | //This method is used so that we can checkout different project 24 | //from any patchset in different pipelines 25 | // Usage example: gerrithub.cloneProject("att-comdev/cicd", "*/master", "refs/XX/XX/XX" "cicd") 26 | gerrit.cloneProject("https://review.gerrithub.io/" + project, branch, refspec, targetDirectory) 27 | } 28 | -------------------------------------------------------------------------------- /vars/heat.groovy: -------------------------------------------------------------------------------- 1 | // Required credentials names 2 | // - jenkins-openstack 3 | // - jenkins-token 4 | // - jenkins-slave-ssh 5 | 6 | 7 | // Jenkins global env variables (: examples) 8 | // - JENKINS_URL 9 | // - JENKINS_CLI 10 | 11 | 12 | def openstack_cmd(String cmd, String mount = "", Boolean useHeatContainer = true) { 13 | openstack_credentials = [ 14 | 'OS_AUTH_URL': OS_AUTH_URL, 15 | 'OS_PROJECT_DOMAIN_NAME': 'default', 16 | 'OS_USER_DOMAIN_NAME': 'default', 17 | 'OS_PROJECT_NAME': OS_PROJECT_NAME, 18 | 'OS_REGION_NAME': OS_REGION_NAME, 19 | 'OS_USERNAME': OS_USERNAME, 20 | 'OS_PASSWORD': OS_PASSWORD, 21 | 'OS_IDENTITY_API_VERSION':'3' 22 | ] 23 | 24 | if (useHeatContainer == false) { 25 | return "${openstack_credentials.collect { "${it.key}=${it.value}" }.join(' ')} $cmd" 26 | } 27 | 28 | docker_env = openstack_credentials.collect { "-e ${it.key}=${it.value}" }.join(' ') 29 | docker_opts = "--rm --net=host" 30 | 31 | if (mount) { 32 | docker_opts += " -v ${mount}:/target" 33 | } 34 | 35 | return "sudo docker run ${docker_opts} ${docker_env} ${OS_KEYSTONE_IMAGE} ${cmd}" 36 | } 37 | 38 | //example of parameters --parameter 'cloudImage=${cloudImage}' --parameter ... 39 | def stack_create(String name, String tmpl, String parameters, Boolean useHeatContainer = true) { 40 | 41 | withCredentials([usernamePassword(credentialsId: 'jenkins-openstack-18', 42 | usernameVariable: 'OS_USERNAME', 43 | passwordVariable: 'OS_PASSWORD')]) { 44 | String cmd 45 | if (useHeatContainer) { 46 | cmd = openstack_cmd("openstack stack create -t '/target/\$(basename ${tmpl})' '${name}' ${parameters} --wait", "\$(dirname ${tmpl})") 47 | } else { 48 | cmd = openstack_cmd("openstack stack create -t '${tmpl}' '${name}' ${parameters} --wait", null, false) 49 | } 50 | code = sh (script: cmd, returnStatus: true) 51 | if (!code) { 52 | // todo: improve timeouts to more user friendly look 53 | timeout = 300 54 | for (i = 0; i < timeout; i=i+10) { 55 | cmd = openstack_cmd("openstack stack show -f value -c stack_status ${name}", null, useHeatContainer) 56 | ret = sh (script: cmd, returnStdout: true).trim() 57 | if (ret == "CREATE_COMPLETE") { 58 | print "Stack ${name} created!" 59 | return 60 | } else if (ret != "CREATE_IN_PROGRESS") { 61 | cmd = openstack_cmd("openstack stack show ${name}", null, useHeatContainer) 62 | ret = sh (script: cmd, returnStdout: true) 63 | print "Stack status:\n${ret}" 64 | print "Heat stack error ${name}" 65 | sh "exit 1" 66 | } 67 | sleep 30 68 | } 69 | } 70 | print "Failed to create stack ${name}" 71 | sh "exit 1" 72 | } 73 | } 74 | 75 | 76 | def stack_delete(String name, Boolean useHeatContainer = true) { 77 | withCredentials([usernamePassword(credentialsId: 'jenkins-openstack-18', 78 | usernameVariable: 'OS_USERNAME', 79 | passwordVariable: 'OS_PASSWORD')]) { 80 | cmd = openstack_cmd("openstack stack delete --wait --yes ${name}", null, useHeatContainer) 81 | code = sh (script: cmd, returnStatus: true) 82 | if (!code) { 83 | cmd = openstack_cmd("openstack stack list", null, useHeatContainer) 84 | ret = sh (script: cmd, returnStdout: true) 85 | if (!ret.contains(name)) { 86 | print "Stack ${name} deleted!" 87 | return 88 | } 89 | print "Failed to delete stack ${name}" 90 | sh "exit 1" // 91 | } else { 92 | print "Likely stack ${name} did not exist! It's OK." 93 | } 94 | } 95 | } 96 | 97 | 98 | def stack_output(String name, String output, Boolean useHeatContainer = true) { 99 | withCredentials([usernamePassword(credentialsId: 'jenkins-openstack-18', 100 | usernameVariable: 'OS_USERNAME', 101 | passwordVariable: 'OS_PASSWORD')]) { 102 | cmd = openstack_cmd("openstack stack output show -f value -c output_value ${name} ${output}", null, useHeatContainer) 103 | return sh(returnStdout: true, script: cmd).trim() 104 | } 105 | } 106 | 107 | def stack_status(String name, Boolean useHeatContainer = true) { 108 | cmd = openstack_cmd("openstack stack show -f value -c stack_status ${name}", null, useHeatContainer) 109 | ret = sh (script: cmd, returnStdout: true).trim() 110 | if (ret != "CREATE_COMPLETE") { 111 | print "Failed to create stack ${name}" 112 | sh "exit 1" 113 | } 114 | } -------------------------------------------------------------------------------- /vars/image.groovy: -------------------------------------------------------------------------------- 1 | //Use this method to build images in a repeatable fashion 2 | def makeImages(){ 3 | sh ''' 4 | sudo make images \ 5 | IMAGE_PREFIX=${IMAGE_PREFIX} \ 6 | IMAGE_NAME=\${JOB_BASE_NAME} \ 7 | DOCKER_REGISTRY=\${ARTF_DOCKER_URL} \ 8 | LABEL='org.label-schema.vcs-ref=\${IMAGE_TAG} \ 9 | --label org.label-schema.vcs-url=\${GERRIT_CHANGE_URL} \ 10 | --label org.label-schema.version=0.1.0-\${BUILD_NUMBER}' \ 11 | IMAGE_TAG=\${IMAGE_TAG} 12 | ''' 13 | } 14 | 15 | /** 16 | * Provides docker image sha256 digest 17 | * 18 | * @param image Docker image, e.g. ${IMAGE} 19 | * @return docker-registry/docker-image@sha256 20 | * Example artifacts-aic.atlantafoundry.com/docker-repo/internal-images@sha256:xxxxx 21 | */ 22 | String getImageDigest (String image) { 23 | // Usage example: 24 | // def imageDigest=image.getImageDigest(IMAGE) 25 | // publish.setProperty ('jenkins-artifactory', \ 26 | // 'https://artifacts-aic.atlantafoundry.com/artifactory/api/storage/clcp-manifests', \ 27 | // "${RELEASE_CURRENT_KEY}=${imageDigest}") 28 | sh "sudo docker inspect --format='{{index .RepoDigests 0}}' ${image}" 29 | } 30 | -------------------------------------------------------------------------------- /vars/jenkins.groovy: -------------------------------------------------------------------------------- 1 | //This groovy file is used for Jenkins node methods. 2 | 3 | def node_create(String name, String host, String port = '22', 4 | String key = 'jenkins-slave-ssh', Number numOfExecutors = 2) { 5 | config = node_config(name, host, key, numOfExecutors, port) 6 | withCredentials([usernamePassword(credentialsId: 'jenkins-token', 7 | usernameVariable: 'JENKINS_USER', 8 | passwordVariable: 'JENKINS_TOKEN')]) { 9 | 10 | if ( JENKINS_CLI_URL =~ "http[s://|://].*" ) { 11 | opts = "-s \$JENKINS_CLI_URL -auth \$JENKINS_USER:\$JENKINS_TOKEN" 12 | java_args = "-Djavax.net.ssl.trustStore=/var/jenkins_home/JenkinsKeystore -Djavax.net.ssl.trustStorePassword=changeit -jar" 13 | } else { 14 | opts = "-s http://\$JENKINS_CLI_URL -auth \$JENKINS_USER:\$JENKINS_TOKEN" 15 | java_args = "-jar" 16 | } 17 | cmd = "echo '${config}' | java ${java_args} \$JENKINS_CLI ${opts} create-node ${name}" 18 | sh (script: cmd, returnStatus: true) 19 | } 20 | } 21 | 22 | 23 | def node_delete(String name) { 24 | withCredentials([usernamePassword(credentialsId: 'jenkins-token', 25 | usernameVariable: 'JENKINS_USER', 26 | passwordVariable: 'JENKINS_TOKEN')]) { 27 | 28 | if ( JENKINS_CLI_URL =~ "http[s://|://].*" ) { 29 | opts = "-s \$JENKINS_CLI_URL -auth \$JENKINS_USER:\$JENKINS_TOKEN" 30 | java_args = "-Djavax.net.ssl.trustStore=/var/jenkins_home/JenkinsKeystore -Djavax.net.ssl.trustStorePassword=changeit -jar" 31 | } else { 32 | opts = "-s http://\$JENKINS_CLI_URL -auth \$JENKINS_USER:\$JENKINS_TOKEN" 33 | java_args = "-jar" 34 | } 35 | cmd = "java ${java_args} \$JENKINS_CLI $opts delete-node $name" 36 | code = sh (script: cmd , returnStatus: true) 37 | // todo: handle exit code properly 38 | } 39 | } 40 | 41 | //jenkins-slave-ssh is already in use for the foundry. We need to standardize to something not in use. 42 | def node_config(String name, String host, String key, Number numOfExecutors, String port) { 43 | config = """ 44 | ${name} 45 | 46 | /home/ubuntu/jenkins 47 | ${numOfExecutors} 48 | EXCLUSIVE 49 | 50 | 51 | ${host} 52 | ${port} 53 | ${key} 54 | 0 55 | 0 56 | 57 | 58 | 59 | 60 | ubuntu 61 | """ 62 | return config 63 | } 64 | -------------------------------------------------------------------------------- /vars/keystone.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonOutput 2 | 3 | /** 4 | * Retrieve a token from a site's Keystone that can potentially be used to talk to and perform actions with the 5 | * other k8s cluster components. 6 | * 7 | * @param keystoneCreds Credentials (user+pass) established within Jenkins to authenticate against a site's Keystone. 8 | * @param keystoneUrl The IAM URL of the site you are authenticating against. 9 | * @param withCreds Boolean. Flag for using jenkins configuration to get keystone credentials. 10 | * In case of disable flag will use keystoneCredId as a password and username as user for keystone request. 11 | * Default value is True. 12 | * @return res The response supplied by the Keystone upon successful authentication 13 | */ 14 | def retrieveToken(keystoneCreds, keystoneUrl, withCreds=true, username='shipyard') { 15 | 16 | if( !(keystoneCreds instanceof List)) { 17 | keystoneCreds = [keystoneCreds] 18 | } 19 | 20 | def res 21 | keystoneCreds.any { 22 | def password = it 23 | if (withCreds) { 24 | withCredentials([[$class: "UsernamePasswordMultiBinding", 25 | credentialsId: it, 26 | usernameVariable: "user", 27 | passwordVariable: "pass"]]) { 28 | username = user 29 | password = pass 30 | } 31 | } 32 | 33 | def req = ["auth": [ 34 | "identity": [ 35 | "methods": ["password"], 36 | "password": [ 37 | "user": ["name": username, 38 | "domain": ["id": "default"], 39 | "password": password]]]]] 40 | 41 | def jreq = new JsonOutput().toJson(req) 42 | 43 | res = httpRequest(url: keystoneUrl + "/v3/auth/tokens", 44 | contentType: "APPLICATION_JSON", 45 | httpMode: "POST", 46 | quiet: true, 47 | validResponseCodes: '200:504', 48 | requestBody: jreq) 49 | 50 | if(res) { 51 | if(res.status == 201) { 52 | // this is like a for loop "break", get out of collection iterating 53 | print "Keystone token request succeesful: ${res.status}" 54 | true 55 | } else if(res.status == 401 && it != keystoneCreds.last()) { 56 | // this is like a for loop "continue", move to the next item in the collection 57 | print "Unauthorized exception. Check next creds." 58 | return 59 | // In case of keystone is not accessible or has some issues repeat with the same creds 60 | } else { 61 | retry(6) { 62 | try { 63 | res = httpRequest(url: keystoneUrl + "/v3/auth/tokens", 64 | contentType: "APPLICATION_JSON", 65 | httpMode: "POST", 66 | quiet: true, 67 | validResponseCodes: '200:504', 68 | requestBody: jreq) 69 | if(res.status == 201) { 70 | print "Keystone token request succeesful: ${res.status}" 71 | return true 72 | } else if(res.status == 401 && it != keystoneCreds.last()) { 73 | // this is like a for loop "continue", move to the next item in the collection 74 | print "Unauthorized exception. Check next creds." 75 | return 76 | } 77 | error("Unexpected return code for token request ${res.status}.") 78 | } catch(err) { 79 | sleep 120 80 | error(err.getMessage()) 81 | } 82 | } 83 | } 84 | } 85 | } 86 | return res 87 | } 88 | -------------------------------------------------------------------------------- /vars/logs.groovy: -------------------------------------------------------------------------------- 1 | //Methods that can be used to retrieve logs from Jenkins jobs 2 | 3 | 4 | /** 5 | * getJenkinsConsoleOutput - This method will curl the Jenkins console output and place it into 6 | * log file that will be published to Artifactory. We do not want this to error a pipeline but 7 | * we do want to be notified when it is not publishing anything. 8 | * 9 | **/ 10 | def getJenkinsConsoleOutput() { 11 | logFile = "" 12 | sh 'mkdir -p jenkinsConsole' 13 | if("x${GERRIT_CHANGE_NUMBER}" != "x" && "x${GERRIT_PATCHSET_NUMBER}" != "x") { 14 | logFile = "${GERRIT_CHANGE_NUMBER}-${GERRIT_PATCHSET_NUMBER}" 15 | } else if("x${BUILD_URL}" != "x" && "x${JOB_BASE_NAME}" != "x") { 16 | logFile = "${JOB_BASE_NAME}-${BUILD_NUMBER}" 17 | } 18 | if("x${logFile}" != "x"){ 19 | cmd = "curl -s -o ./jenkinsConsole/${logFile}.log ${BUILD_URL}consoleText" 20 | sh (script: cmd, returnStatus: true) 21 | return "./jenkinsConsole/${logFile}.log" 22 | } else { 23 | return "" 24 | } 25 | } -------------------------------------------------------------------------------- /vars/notify.groovy: -------------------------------------------------------------------------------- 1 | 2 | // Required credentials names 3 | // - jenkins-slack 4 | // - jenkins-something-else 5 | 6 | // Jenkins global env variables (: examples) 7 | // - SLACK_URL : https://att-comdev.slack.com/services/hooks/jenkins-ci/ 8 | // - SLACK_DEFAULT_CHANNEL : #test-jenkins 9 | 10 | 11 | def msg(String message, String channel=SLACK_DEFAULT_CHANNEL){ 12 | // Usage example: notify.msg( "${env.GERRIT_CHANGE_URL} is OK!") 13 | // Custom channel: notify.msg( "${env.GERRIT_CHANGE_URL} is OK!",'#my_channel') 14 | slackSend( 15 | baseUrl: SLACK_URL, 16 | tokenCredentialId: 'jenkins-slack', 17 | channel: channel, 18 | message: "Job <${env.JOB_URL}|${env.JOB_NAME}> said:\n" + message 19 | ) 20 | } 21 | 22 | -------------------------------------------------------------------------------- /vars/pegleg.groovy: -------------------------------------------------------------------------------- 1 | import att.comdev.cicd.config.conf 2 | 3 | /** 4 | * Execution of Pegleg "lint" against a Pegleg 5 | * container. NOTE: Currently setup to ignore 6 | * P001 & P003. See: https://github.com/openstack/airship-pegleg/blob/master/src/bin/pegleg/pegleg/engine/errorcodes.py 7 | * for more information on returned error codes. 8 | * 9 | * @param siteRepo The folder containing your site-level documents (must be at your PWD) 10 | * @param globalRepo The folder containing your global documents (must be at your PWD) 11 | * @param secretsRepo The folder container your security documents (must be at your PWD) 12 | */ 13 | def lint(siteRepo, globalRepo, secretsRepo) { 14 | sh "sudo docker run --rm -i --net=none --workdir='/workspace' -v \$(pwd):/workspace \ 15 | $conf.PEGLEG_IMAGE pegleg -v lint -p ${siteRepo} -a ${globalRepo} -a ${secretsRepo} -x P001 -x P003" 16 | } 17 | 18 | /** 19 | * Execution of Pegleg "lint" within a Pegleg 20 | * container. NOTE: Currently setup to ignore 21 | * P001 & P003. See: https://github.com/openstack/airship-pegleg/blob/master/src/bin/pegleg/pegleg/engine/errorcodes.py 22 | * for more information on returned error codes. 23 | * 24 | * @param siteRepo The folder containing your site-level documents (must be at your PWD) 25 | * @param globalRepo The folder containing your global documents (must be at your PWD) 26 | * @param secretsRepo The folder container your security documents (must be at your PWD) 27 | */ 28 | def lintWithinContainer(siteRepo, globalRepo, secretsRepo) { 29 | sh "pegleg -v lint -p ${siteRepo} -a ${globalRepo} -a ${secretsRepo} -x P001 -x P003" 30 | } 31 | 32 | /** 33 | * Execution of Pegleg "render" against a Pegleg 34 | * container. Redirects the output to a file, so 35 | * it doesn't get written to stdout. 36 | * 37 | * @param siteRepo The folder containing your site-level documents (must be at your PWD) 38 | * @param globalRepo The folder containing your global documents (must be at your PWD) 39 | * @param secretsRepo The folder container your security documents (must be at your PWD) 40 | * @param siteName The name of the site you're looking to render. Must match what's in your site repository's site-definition.yaml 41 | */ 42 | def render(siteRepo, globalRepo, secretsRepo, siteName) { 43 | sh "sudo docker run --rm -i --net=none --workdir='/workspace' -v \$(pwd):/workspace \ 44 | $conf.PEGLEG_IMAGE pegleg site -p ${siteRepo} -a ${globalRepo} -a ${secretsRepo} render ${siteName} -o ${siteName}.yaml" 45 | } 46 | 47 | /** 48 | * Execution of Pegleg "render" within a Pegleg 49 | * container. Redirects the output to a file, so 50 | * it doesn't get written to stdout. 51 | * 52 | * @param siteRepo The folder containing your site-level documents (must be at your PWD) 53 | * @param globalRepo The folder containing your global documents (must be at your PWD) 54 | * @param secretsRepo The folder container your security documents (must be at your PWD) 55 | * @param siteName The name of the site you're looking to render. Must match what's in your site repository's site-definition.yaml 56 | */ 57 | def renderWithinContainer(siteRepo, globalRepo, secretsRepo, siteName) { 58 | sh "pegleg site -p ${siteRepo} -a ${globalRepo} -a ${secretsRepo} render ${siteName} -o ${siteName}.yaml" 59 | } 60 | 61 | /** 62 | * Execution of Pegleg "collect" against a Pegleg 63 | * container. 64 | * 65 | * @param siteRepo The folder containing your site-level documents (must be at your PWD) 66 | * @param globalRepo The folder containing your global documents (must be at your PWD) 67 | * @param secretsRepo The folder container your security documents (must be at your PWD) 68 | * @param siteName The name of the site you're looking to render. Must match what's in your site repository's site-definition.yaml 69 | */ 70 | def collect(siteRepo, globalRepo, secretsRepo, siteName) { 71 | sh "docker run --rm -i --net=none --workdir=/workspace -v \$(pwd):/workspace \ 72 | $conf.PEGLEG_IMAGE pegleg site -p ${siteRepo} -a ${globalRepo} -a ${secretsRepo} collect ${siteName} -s ${siteName}" 73 | } 74 | 75 | /** 76 | * Execution of Pegleg "collect" within a Pegleg 77 | * container. 78 | * 79 | * @param siteRepo The folder containing your site-level documents (must be at your PWD) 80 | * @param globalRepo The folder containing your global documents (must be at your PWD) 81 | * @param secretsRepo The folder container your security documents (must be at your PWD) 82 | * @param siteName The name of the site you're looking to render. Must match what's in your site repository's site-definition.yaml 83 | */ 84 | def collectWithinContainer(siteRepo, globalRepo, secretsRepo, siteName) { 85 | sh "pegleg site -p ${siteRepo} -a ${globalRepo} -a ${secretsRepo} collect ${siteName} -s ${siteName}" 86 | } 87 | -------------------------------------------------------------------------------- /vars/pip.groovy: -------------------------------------------------------------------------------- 1 | /** 2 | * Update hosts and install python and pip 3 | */ 4 | def updateHost() { 5 | sh "sudo apt-get update" 6 | sh "sudo apt install python-minimal -y" 7 | sh "sudo apt install python-pip -y" 8 | } 9 | 10 | /** 11 | * Build pip packages and upload to Artifactory repo 12 | 13 | * @param project project dir to build 14 | */ 15 | def buildPackageAndUpload(String project) { 16 | dir (project) { 17 | sh "python setup.py sdist bdist_wheel upload -r local" 18 | } 19 | 20 | } 21 | 22 | /** 23 | * Create pypirc file which can be used to upload packages to artifactory repo 24 | 25 | * @param credentialsId jenkins SSH credentials ID 26 | */ 27 | def createPypirc(String credentialsId) { 28 | 29 | withCredentials([usernamePassword(credentialsId: credentialsId, 30 | usernameVariable: 'ARTIFACTORY_USER', 31 | passwordVariable: 'ARTIFACTORY_PASSWORD')]) { 32 | sh '''#!/bin/bash 33 | cat << EOF > ~/.pypirc 34 | [distutils] 35 | index-servers = local 36 | [local] 37 | repository: ${ARTF_LOCAL_PYPI_URL} 38 | username: ${ARTIFACTORY_USER} 39 | password: ${ARTIFACTORY_PASSWORD} 40 | EOF 41 | ''' 42 | } 43 | } 44 | 45 | /** 46 | * Install pip packages 47 | * @param packages list of packages to install 48 | */ 49 | def installPackages(List packages) { 50 | sh "pip install ${packages.join(' ')}" 51 | } 52 | -------------------------------------------------------------------------------- /vars/redfish.groovy: -------------------------------------------------------------------------------- 1 | import groovy.json.JsonSlurperClassic 2 | import groovy.json.JsonOutput 3 | 4 | 5 | /** 6 | * Helper for print error from failed request 7 | * @param code Expected response code 8 | * @param res Response object 9 | */ 10 | def _printError(code, res) { 11 | if( !(code instanceof List)) { 12 | code = [code] 13 | } 14 | if ( !(code.contains(res.status))) { 15 | print "See details content: " + res.content 16 | error("Request failed with ${res.status}") 17 | } 18 | } 19 | 20 | def getBasicAuth (String user, String passwd) { 21 | def creds = "${user}:${passwd}" 22 | return creds.bytes.encodeBase64().toString() 23 | } 24 | 25 | def getSystemPath (String ip, String auth) { 26 | def res 27 | retry(5) { 28 | try { 29 | res = httpRequest (url: "https://${ip}/redfish/v1/Systems/", 30 | customHeaders:[[name:'Authorization', value:"Basic ${auth}"]], 31 | httpMode: 'GET', 32 | quiet: true, 33 | ignoreSslErrors: true) 34 | 35 | _printError(200, res) 36 | } catch (err) { 37 | print "Failed to get System Path: ${err}" 38 | sleep 120 39 | error(err.getMessage()) 40 | } 41 | } 42 | 43 | def cont = new JsonSlurperClassic().parseText(res.content) 44 | def systemPath = cont.Members[0]."@odata.id" 45 | 46 | if (systemPath.endsWith("/")) { 47 | systemPath = systemPath[0..-2] 48 | } 49 | 50 | print "System Path: " + systemPath 51 | return systemPath 52 | } 53 | 54 | def getPowerState (String ip, String auth, systemPath=null) { 55 | 56 | systemPath = systemPath ?: getSystemPath(ip, auth) 57 | 58 | def res 59 | retry(5) { 60 | try { 61 | res = httpRequest (url: "https://${ip}${systemPath}/", 62 | customHeaders:[[name:'Authorization', value:"Basic ${auth}"]], 63 | httpMode: 'GET', 64 | quiet: true, 65 | ignoreSslErrors: true) 66 | _printError(200, res) 67 | } catch (err) { 68 | print "Failed to get power state: ${err}" 69 | sleep 120 70 | error(err.getMessage()) 71 | } 72 | } 73 | 74 | def cont = new JsonSlurperClassic().parseText(res.content) 75 | print "PowerState: ${cont.PowerState}" 76 | 77 | return cont.PowerState 78 | } 79 | 80 | def setPowerState (String ip, String auth, String state, systemPath=null) { 81 | 82 | systemPath = systemPath ?: getSystemPath(ip, auth) 83 | 84 | def req = [ 'ResetType': state ] 85 | def jreq = new JsonOutput().toJson(req) 86 | 87 | def res 88 | retry(5) { 89 | try { 90 | res = httpRequest (url: "https://${ip}${systemPath}/Actions/ComputerSystem.Reset/", 91 | customHeaders:[[name:'Authorization', value:"Basic ${auth}"]], 92 | httpMode: 'POST', 93 | ignoreSslErrors: true, 94 | quiet: true, 95 | contentType: 'APPLICATION_JSON', 96 | requestBody: jreq) 97 | 98 | // 204 iDrac, 200 iLO 99 | _printError([200, 204], res) 100 | } catch (err) { 101 | print "Failed to set power state: ${err}" 102 | sleep 120 103 | error(err.getMessage()) 104 | } 105 | } 106 | } 107 | 108 | def powerOn (String ip, String auth, systemPath=null) { 109 | 110 | if (getPowerState(ip, auth, systemPath) == 'On') { 111 | print "Power state already On, skipping" 112 | return 0 113 | } 114 | 115 | print "Setting power state On for node ${ip}" 116 | setPowerState(ip, auth, 'On', systemPath) 117 | 118 | timeout(1) { 119 | def state = getPowerState(ip, auth, systemPath) 120 | while (state != 'On') { 121 | print "Power state is not yet \'On\' (${state}), waiting 10 seconds" 122 | sleep 10 123 | state = getPowerState(ip, auth, systemPath) 124 | } 125 | } 126 | } 127 | 128 | def powerOff (String ip, String auth, systemPath=null) { 129 | 130 | if (getPowerState(ip, auth, systemPath) == 'Off') { 131 | print "Power state already Off, skipping" 132 | return 0 133 | } 134 | 135 | print "Setting power state ForceOff for node ${ip}" 136 | setPowerState(ip, auth, 'ForceOff', systemPath) 137 | 138 | timeout(1) { 139 | def state = getPowerState(ip, auth, systemPath) 140 | while (state != 'Off') { 141 | print "Power state is not yet \'Off\' (${state}), waiting 10 seconds" 142 | sleep 10 143 | state = getPowerState(ip, auth, systemPath) 144 | } 145 | } 146 | } 147 | 148 | def powerReset (String ip, String auth, systemPath=null) { 149 | // ForceRestart is not an option in iDrac 8, changing 150 | // powerReset to 'ForceOff' -> 'On"' 151 | print "Beginning power reset process for node ${ip}" 152 | powerOff(ip, auth, systemPath) 153 | powerOn(ip, auth, systemPath) 154 | } 155 | -------------------------------------------------------------------------------- /vars/ssh.groovy: -------------------------------------------------------------------------------- 1 | 2 | // wrapper for https://jenkins.io/doc/pipeline/steps/ssh-steps/ 3 | // requires 'SSH Steps Plugin' to be installed in Jenkins 4 | 5 | /** 6 | * Helper function to form remote object for the plugin 7 | * https://github.com/jenkinsci/ssh-steps-plugin#remote 8 | * 9 | * @param creds Jenkins credentials ID setup for SSH 10 | * @param ip The IP address or Hostname of the node 11 | */ 12 | def getRemote (String creds, String ip) { 13 | withCredentials([sshUserPrivateKey(credentialsId: creds, 14 | keyFileVariable: 'SSH_KEY', 15 | usernameVariable: 'SSH_USER')]) { 16 | 17 | def remote = [:] 18 | remote.name = 'remote' 19 | remote.host = ip 20 | remote.allowAnyHosts = true 21 | 22 | remote.user = SSH_USER 23 | 24 | def key = readFile SSH_KEY 25 | remote.identity = key 26 | 27 | return remote 28 | } 29 | } 30 | 31 | def cmd (String creds, String ip, String cmd) { 32 | def remote = getRemote(creds, ip) 33 | sshCommand remote: remote, command: cmd 34 | } 35 | 36 | 37 | /** 38 | * Function used to wait and retry execution of a command. 39 | * Good for waiting on the node to boot/restart and become available 40 | */ 41 | def wait (String creds, String ip, String command, attempts = 12, timeout = 60) { 42 | def ie 43 | retry (attempts) { 44 | try { 45 | cmd (creds, ip, command) 46 | } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException | java.lang.InterruptedException err) { 47 | ie = err 48 | echo "Stop retry" 49 | } catch (err) { 50 | sleep timeout 51 | error(err.getMessage()) 52 | } 53 | } 54 | if (ie) { 55 | throw ie 56 | } 57 | } 58 | 59 | def get (String creds, String ip, String src, String dst) { 60 | def remote = getRemote(creds, ip) 61 | sshGet remote: remote, from: src, into: dst, override: true 62 | } 63 | 64 | def put (String creds, String ip, String src, String dst) { 65 | def remote = getRemote(creds, ip) 66 | sshPut remote: remote, from: src, into: dst 67 | } 68 | 69 | def script (String creds, String ip, String script) { 70 | def remote = getRemote(creds, ip) 71 | sshScript remote: remote, script: script 72 | } 73 | 74 | -------------------------------------------------------------------------------- /vars/ssh_pwd.groovy: -------------------------------------------------------------------------------- 1 | 2 | // wrapper for https://jenkins.io/doc/pipeline/steps/ssh-steps/ 3 | // requires 'SSH Steps Plugin' to be installed in Jenkins 4 | 5 | /** 6 | * Helper function to form remote object for the plugin 7 | * https://github.com/jenkinsci/ssh-steps-plugin#remote 8 | * 9 | * @param creds Jenkins credentials ID (Username + Password) setup for SSH 10 | * @param ip The IP address or Hostname of the node 11 | */ 12 | def getRemote (String creds, String ip) { 13 | withCredentials([usernamePassword(credentialsId: creds, 14 | usernameVariable: "SSH_USER", 15 | passwordVariable: "SSH_PASS")]) { 16 | def remote = [:] 17 | remote.name = 'remote' 18 | remote.host = ip 19 | remote.allowAnyHosts = true 20 | remote.user = SSH_USER 21 | remote.password = SSH_PASS 22 | 23 | return remote 24 | } 25 | } 26 | 27 | def cmd (String creds, String ip, String command, failOnError = true) { 28 | def remote = getRemote(creds, ip) 29 | sshCommand remote: remote, failOnError: failOnError, command: command 30 | } 31 | 32 | 33 | /** 34 | * Function used to wait and retry execution of a command. 35 | * Good for waiting on the node to boot/restart and become available 36 | */ 37 | def wait (String creds, String ip, String command, attempts = 12, timeout = 60) { 38 | retry (attempts) { 39 | try { 40 | cmd (creds, ip, command) 41 | } catch (err) { 42 | sleep timeout 43 | error(err.getMessage()) 44 | } 45 | } 46 | } 47 | 48 | def get (String creds, String ip, String src, String dst) { 49 | def remote = getRemote(creds, ip) 50 | sshGet remote: remote, from: src, into: dst, override: true 51 | } 52 | 53 | def put (String creds, String ip, String src, String dst) { 54 | def remote = getRemote(creds, ip) 55 | sshPut remote: remote, from: src, into: dst 56 | } 57 | 58 | def script (String creds, String ip, String script) { 59 | def remote = getRemote(creds, ip) 60 | sshScript remote: remote, script: script 61 | } 62 | 63 | -------------------------------------------------------------------------------- /vars/utils.groovy: -------------------------------------------------------------------------------- 1 | class Params { 2 | static def ABORT_MESSAGE = "Job was aborted." 3 | static def ABORT_ON = ["Aborted by", "Calling Pipeline was cancelled", ABORT_MESSAGE] 4 | } 5 | 6 | /* Method to run downstream jobs to be used in combination with retrier. */ 7 | def runBuild(name, parameters, retries=1) { 8 | retrier (retries) { 9 | job = build( 10 | job: name, 11 | wait: true, 12 | propagate: false, 13 | parameters: parameters 14 | ) 15 | if (job.result == 'SUCCESS') { return job } 16 | else if (job.result == 'ABORTED') { throw new Exception("'${name}': ${Params.ABORT_MESSAGE}") } 17 | else { throw new Exception("'${name}': Job failed.") } 18 | } 19 | } 20 | 21 | 22 | /* Method that allows to retry enclosed body that respects job abort, 23 | including upstream and downstream ones 24 | Usage: 25 | retrier(3) { 26 | ... 27 | } 28 | */ 29 | def retrier(int retries, Closure failCleanup={}, Closure body) { 30 | def lastError 31 | for(int i=0; i> ${HOME}/.ssh/known_hosts" 163 | } 164 | } 165 | --------------------------------------------------------------------------------