├── .gitignore ├── .gitlab-ci.yml ├── koji ├── KojiPipeline └── update_external_repo.sh ├── .github ├── docker │ └── Dockerfile └── workflows │ └── ci.yml ├── release ├── ReleasePipeline ├── publish.sh └── init-env.sh ├── watcher ├── WatcherPipeline └── watcher.sh ├── README.md ├── variables.sh ├── globals.sh ├── logging.sh ├── Makefile └── common.sh /.gitignore: -------------------------------------------------------------------------------- 1 | builder/ 2 | config/ 3 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | check: 2 | script: 3 | - make check 4 | -------------------------------------------------------------------------------- /koji/KojiPipeline: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { label 'clr-builders' } 3 | 4 | stages { 5 | stage('Koji') { 6 | steps { 7 | sh 'koji/update_external_repo.sh' 8 | } 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.github/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM clearlinux:latest 2 | 3 | RUN swupd bundle-add --quiet make network-basic mixer clr-installer sudo 4 | RUN swupd clean --all --quiet 5 | 6 | ARG UID=1000 7 | 8 | RUN useradd -G wheelnopw --uid ${UID} -U -m github 9 | 10 | USER github 11 | 12 | RUN git config --global user.email "github@ci-container.com" 13 | RUN git config --global user.name "Github Actions" 14 | 15 | WORKDIR /mnt 16 | -------------------------------------------------------------------------------- /release/ReleasePipeline: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { label 'clr-builders' } 3 | 4 | environment { 5 | WORK_DIR = "${WORKSPACE}/work" 6 | } 7 | 8 | stages { 9 | stage('Initialize Environment') { 10 | steps { 11 | sh 'release/init-env.sh' 12 | } 13 | } 14 | stage('Publish') { 15 | steps { 16 | sh 'release/publish.sh' 17 | } 18 | } 19 | } 20 | 21 | post { 22 | always { 23 | echo "Release Pipeline OUT - ${env.BUILD_TAG}" 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /release/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (C) 2018 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | set -e 6 | 7 | SCRIPT_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") 8 | 9 | # shellcheck source=common.sh 10 | . "${SCRIPT_DIR}/../common.sh" 11 | . ./config/config.sh 12 | 13 | REMOTE_PATH=${PUBLISHING_HOST}:${PUBLISHING_ROOT}/${NAMESPACE:-${DISTRO_NAME}} 14 | 15 | stage "PUBLISH" 16 | log "From" "${STAGING_DIR}" 17 | log "To" "${REMOTE_PATH}" 18 | 19 | section "Syncing Content" 20 | assert_dir "${STAGING_DIR}" 21 | rsync -vrlHpt --safe-links --delete --exclude '*.src.rpm' -e ssh "${STAGING_DIR}/" "${REMOTE_PATH}" 22 | -------------------------------------------------------------------------------- /watcher/WatcherPipeline: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { label 'clr-builders' } 3 | 4 | stages { 5 | stage('Watcher') { 6 | steps { 7 | script { 8 | def ret = sh script: 'watcher/watcher.sh', returnStatus: true 9 | if (ret == 1) { 10 | currentBuild.result = 'UNSTABLE' 11 | } else if (ret > 1) { 12 | currentBuild.result = 'FAILURE' 13 | } 14 | } 15 | } 16 | } 17 | } 18 | 19 | post { 20 | unstable { 21 | build "${NAMESPACE}" 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /koji/update_external_repo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (C) 2018 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | # shellcheck source=common.sh 6 | 7 | set -e 8 | 9 | SCRIPT_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") 10 | 11 | . "${SCRIPT_DIR}/../common.sh" 12 | 13 | KOJI_TAG=${KOJI_TAG:-"dist-clear"} 14 | repo_name="dist-clear-external-repo" 15 | repo_prefix="${CLR_PUBLIC_DL_URL}/releases/" 16 | repo_suffix="/clear/\$arch/os/" 17 | 18 | current_version=$(koji list-external-repos --name=${repo_name} --quiet) 19 | current_version=${current_version##*releases/} 20 | current_version=${current_version%%${repo_suffix}} 21 | 22 | get_upstream_version 23 | 24 | (( CLR_LATEST <= current_version )) && exit 0 25 | 26 | koji edit-external-repo --url="${repo_prefix}${CLR_LATEST}${repo_suffix}" ${repo_name} 27 | koji regen-repo "${KOJI_TAG}-build" 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](https://github.com/clearlinux/clr-distro-factory/workflows/Continuous%20Integration/badge.svg) 2 | 3 | # Clear Linux* Distro Factory 4 | 'DevOps' tools for maintaining a 'swupd-based' distribution. 5 | 6 | # General Information 7 | Clear Linux Distro Factory hosts the tools necessary to continuously run the 8 | operations required for build and publish releases of a Clear based OS. 9 | 10 | A Downstream Clear is an independent OS that follows the Clear Linux Architecture 11 | and can optionally reuse Clear Linux content as part of its releases, reducing 12 | significantly the cost of running a 'DevOps' operation. 13 | 14 | For more information on how this project was designed to be used, please refer 15 | to our [Documentation](https://github.com/clearlinux/clr-distro-factory/wiki). 16 | 17 | # Coding Style 18 | Please, follow the coding style described [here](https://github.com/clearlinux/clr-distro-factory/wiki/Shell-Code-Style-and-Considerations). 19 | 20 | # Have a Security Issue? 21 | Please go to https://01.org/security to report any security issues or concerns. 22 | 23 | --- 24 | *Other names and brands may be claimed as the property of others. 25 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | environment: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Kernel Info 10 | run: uname -a 11 | - name: Current Dir 12 | run: pwd 13 | - name: Network Interfaces 14 | run: ip a 15 | - name: Partitions Status 16 | run: df -h 17 | - name: Memory 18 | run: free -h 19 | - name: User Info 20 | run: | 21 | whoami 22 | id -u 23 | 24 | static-analysis: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v2.1.0 28 | - name: make check 29 | run: make check 30 | 31 | build: 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v2.1.0 35 | - name: Build Docker Image 36 | run: 37 | docker build --quiet --build-arg UID=$(id -u) --network host --force-rm -t clr-builder .github/docker 38 | - name: Downstream - First Build 39 | run: 40 | docker run --rm -v $(pwd):/mnt clr-builder make build 41 | - name: Downstream - Second Build 42 | run: 43 | docker run --rm -v $(pwd):/mnt clr-builder make build 44 | -------------------------------------------------------------------------------- /variables.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (C) 2018 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | # ================================================================ 6 | # Methodology to easily pass variables among scripts in a pipeline 7 | # ================================================================ 8 | 9 | # Requires: logging.sh 10 | 11 | var_save() { 12 | local VARS_DIR=${VARS_DIR:?"VARS_DIR Cannot be Null/Unset"} 13 | 14 | if (( $# != 1 )); then 15 | error "'var_save' requires a single argument!" 16 | return 1 17 | fi 18 | 19 | # "unsave" 20 | if [[ ! -v ${1} ]]; then 21 | rm -f "${VARS_DIR}/${1}" 22 | return 0 23 | fi 24 | 25 | [[ -d ${VARS_DIR} ]] || mkdir -p "${VARS_DIR}" 26 | 27 | echo "${!1}" > "${VARS_DIR}/${1}" 28 | } 29 | 30 | var_load() { 31 | if (( $# != 1 )); then 32 | error "'var_load' requires a single argument!" 33 | return 1 34 | fi 35 | 36 | # shellcheck disable=2015 37 | [[ -f ${VARS_DIR}/${1} ]] && declare -g "${1}"="$(< "${VARS_DIR}/${1}")" || true 38 | } 39 | 40 | var_load_all() { 41 | [[ ! -d "${VARS_DIR}" ]] && return 42 | 43 | while read -r VAR; do 44 | declare -g "${VAR}"="$(< "${VARS_DIR}/${VAR}")" 45 | done <<< "$(ls "${VARS_DIR}")" 46 | } 47 | -------------------------------------------------------------------------------- /release/init-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (C) 2018 Intel Corporation 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # shellcheck source=common.sh 7 | 8 | set -e 9 | 10 | SCRIPT_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") 11 | 12 | . "${SCRIPT_DIR}/../common.sh" 13 | 14 | # ============================================================================== 15 | # MAIN 16 | # ============================================================================== 17 | stage "Initialize Environment" 18 | 19 | LOG_INDENT=1 fetch_config_repo 20 | . ./config/config.sh 21 | 22 | section "Build Environment" 23 | log "Workflow Repository" "$(git remote get-url origin) ($(git rev-parse --short HEAD))" 24 | log "Workflow Config Repository" "$(git -C config remote get-url origin) ($(git -C config rev-parse --short HEAD))" 25 | 26 | section "Configuration" 27 | log "Distribution" "${DISTRO_NAME}" 28 | log "Distribution Content/Version URL" "${DISTRO_URL}" 29 | 30 | section "Workspace" 31 | log "Namespace" "${NAMESPACE}" 32 | log "Work dir" "${WORK_DIR}" 33 | log "Variables dir" "${VARS_DIR}" 34 | log "Stage dir" "${STAGING_DIR}" 35 | log "Publishing Host" "${PUBLISHING_HOST}" 36 | log "Publishing Root" "${PUBLISHING_ROOT}" 37 | 38 | assert_dep rsync 39 | section "Tools" 40 | log "Clear Linux Version (on Builder)" "$(cat /usr/share/clear/version)" 41 | log "Rsync Version" "$(rsync --version 2>&1 | head -1)" 42 | -------------------------------------------------------------------------------- /globals.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (C) 2019 Intel Corporation 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | NAMESPACE=${NAMESPACE:?"NAMESPACE cannot be Null/Unset"} 7 | WORK_DIR=${WORK_DIR:-"${PWD}/${NAMESPACE}/work"} 8 | 9 | # Distribution 10 | # If this update stream is either a downstream or an upstream 11 | # A Downstream Mix makes reference to an external update stream to reuse content 12 | IS_DOWNSTREAM=${IS_DOWNSTREAM:-true} 13 | # If this build should be a min version 14 | MIN_VERSION=${MIN_VERSION:-false} 15 | # If this build should be a format bump 16 | # Only used by upstream mixes. Downstream mixes can only track upstream mixes 17 | # format bumps, which is automated 18 | FORMAT_BUMP=${FORMAT_BUMP:-false} 19 | 20 | # Servers 21 | CLR_PUBLIC_DL_URL=${CLR_PUBLIC_DL_URL:-"https://download.clearlinux.org"} 22 | 23 | # Mixer 24 | MIX_INCREMENT=${MIX_INCREMENT:-10} 25 | # Global options to apply to all mixer calls 26 | MIXER_OPTS=${MIXER_OPTS:-""} 27 | # Number of builds from the current build to generate deltas 28 | NUM_DELTA_BUILDS=${NUM_DELTA_BUILDS:-10} 29 | # Minimum version of mixer that is supported 30 | MIXER_VER_MIN=${MIXER_VER_MIN:-6.2.3} 31 | 32 | # Width of MCA statistics table 33 | MCA_TABLE_WIDTH=${MCA_TABLE_WIDTH:-120} 34 | 35 | # Workspace 36 | BUNDLES_DIR=${BUNDLES_DIR:-"${WORK_DIR}/bundles"} 37 | LOG_DIR="${WORK_DIR}/logs" 38 | REPO_DIR="${WORK_DIR}/repo" 39 | VARS_DIR="${WORK_DIR}/.vars" 40 | 41 | BUILD_ARCH="${BUILD_ARCH:-x86_64}" 42 | PKGS_DIR_SUFFIX="${BUILD_ARCH}/os/packages" 43 | PKGS_DIR="${REPO_DIR}/${PKGS_DIR_SUFFIX}" 44 | 45 | BUILD_FILE=build-env 46 | BUNDLES_FILE=bundles-def 47 | CONTENT_REPO=content 48 | MCA_FILE=mca-report 49 | PKG_LIST_FILE=packages-nvr 50 | PKG_LICENSES_FILE=packages-license-info 51 | PKG_LIST_TMP=packages_ 52 | RELEASE_NOTES=release-notes 53 | 54 | # List of words to be filtered out from $PKG_LICENSES_FILE as they are not real licenses 55 | # Space-separated 56 | LICENSES_FILTER=${LICENSES_FILTER-"and"} 57 | 58 | # Images 59 | CHKSUM_FILE_SUFFIX=${CHKSUM_FILE_SUFFIX:-"SHA512SUM"} 60 | -------------------------------------------------------------------------------- /watcher/watcher.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (C) 2018 Intel Corporation 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | # shellcheck source=common.sh 7 | 8 | set -e 9 | 10 | # return codes: 11 | # 0 = We are up-to-date. Pipeline Success. 12 | # 1 = A new build is needed. Pipeline Unstable. 13 | # > 1 = Errors. Pipeline Failure. 14 | 15 | SCRIPT_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") 16 | 17 | . "${SCRIPT_DIR}/../common.sh" 18 | 19 | # ============================================================================== 20 | # MAIN 21 | # ============================================================================== 22 | stage "Watcher" 23 | fetch_config_repo 24 | . ./config/config.sh 25 | 26 | log "Upstream Server" "${CLR_PUBLIC_DL_URL}" 27 | log "Distribution Stage" "${STAGING_DIR}" 28 | log_line 29 | 30 | # Check if we are on track with Upstream ClearLinux 31 | get_latest_versions 32 | 33 | log "Clear Linux version" "${CLR_LATEST}" 34 | log "Distribution version" 35 | if [[ -z "${DISTRO_LATEST}" ]]; then 36 | log_line "First Mix! It's Build Time!" 1 37 | exit 1 38 | fi 39 | log_line "${DISTRO_UP_VERSION} ${DISTRO_DOWN_VERSION}" 1 40 | 41 | if (( DISTRO_UP_VERSION < CLR_LATEST )); then 42 | log "Upstream has a new release" "It's Build Time!" 43 | exit 1 44 | fi 45 | 46 | # Check if is there new custom content to be built 47 | ret=0 48 | TMP_PREV_LIST=$(mktemp) 49 | TMP_CURR_LIST=$(mktemp) 50 | PKG_LIST_PATH="${STAGING_DIR}/releases/${DISTRO_LATEST}/${PKG_LIST_FILE}-${DISTRO_LATEST}.txt" 51 | 52 | if ! cat "${PKG_LIST_PATH}" > "${TMP_PREV_LIST}"; then 53 | warn "Failed to fetch Distribution PREVIOUS Package List" "Assuming empty" 54 | fi 55 | 56 | if result=$(koji_cmd list-tagged --latest --quiet "${KOJI_TAG}"); then 57 | echo "${result}" | awk '{print $1}' > "${TMP_CURR_LIST}" 58 | else 59 | warn "Failed to fetch Distribution Package List" "Assuming empty." 60 | fi 61 | 62 | if ! diff "${TMP_CURR_LIST}" "${TMP_PREV_LIST}"; then 63 | log "New custom content" "It's Build Time!" 64 | ret=1 65 | else 66 | log "Nothing to see here." 67 | fi 68 | 69 | rm "${TMP_CURR_LIST}" 70 | rm "${TMP_PREV_LIST}" 71 | exit ${ret} 72 | -------------------------------------------------------------------------------- /logging.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (C) 2019 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | # ================= 6 | # Logging Utilities 7 | # ================= 8 | 9 | # LOG_DOMAIN = Domain name to be printed in the beginning of each log line 10 | # LOG_INDENT = Indentation level to be applied on the log message after 'domain' 11 | # 12 | # To be used with 'run_and_log': 13 | # LOG_DIR = Directory where log files are saved. Default = '${WORK_DIR}/logs' 14 | # LOG_NAME = Prefix used for the log file name. 15 | # LOG_METHOD = One of the following logging alternatives: 16 | # 0 - Log to file (Default) 17 | # 1 - Log to file and std{out,err} 18 | # 2 - Log to std{out,err} 19 | # > 2 - Don't log 20 | 21 | log_line() { 22 | # ${1} - log message 23 | # ${2} - extra indentation 24 | 25 | local indent=${1:+$(( (${LOG_INDENT:-0} + ${2:-0}) * 4 ))} 26 | printf "${LOG_DOMAIN:+"[${LOG_DOMAIN}] "}%${indent}s%s\\n" "" "${1}" 27 | } 28 | 29 | log() { 30 | # ${1} - log title (or message for single line logs) 31 | # ${2} - log message (optional) 32 | # ${3} - extra indentation (optional, requires ${2}) 33 | 34 | if (( $# < 2 )); then 35 | log_line "${1}" 36 | else 37 | log_line "${1}:" "${3}" 38 | log_line "${2}" $((${3:-0} + 1)) 39 | fi 40 | } 41 | 42 | stage() { 43 | log_line "=== ${1^^}" 44 | } 45 | 46 | section() { 47 | log_line 48 | log_line "== ${1} ==" 49 | } 50 | 51 | error() { 52 | log "[ERROR] ${1}" ${2:+"${2}"} "${3}" 53 | } 54 | 55 | info() { 56 | log "[INFO] ${1}" ${2:+"${2}"} "${3}" 57 | } 58 | 59 | warn() { 60 | log "[WARN] ${1}" ${2:+"${2}"} "${3}" 61 | } 62 | 63 | run_and_log() { 64 | if (( ${#} == 0 )); then 65 | error "'run_and_log' requires a command!" 66 | return 1 67 | fi 68 | 69 | mkdir -p "${LOG_DIR}" 70 | 71 | local log_base="${LOG_DIR}/${LOG_NAME:-${1}}" 72 | 73 | # Log to file 74 | if [[ "${LOG_METHOD}" -eq 0 ]]; then 75 | # shellcheck disable=2068 76 | ${@} > "${log_base}.log" 2> "${log_base}_err.log" 77 | # Log to file and stdout/stderr 78 | elif [[ "${LOG_METHOD}" -eq 1 ]]; then 79 | # shellcheck disable=2068 80 | ${@} > >(tee "${log_base}.log") 2> >(tee "${log_base}_err.log") 81 | # Log to stdout/stderr only 82 | elif [[ "${LOG_METHOD}" -eq 2 ]]; then 83 | # shellcheck disable=2068 84 | ${@} 85 | # No output 86 | else 87 | # shellcheck disable=2068 88 | ${@} &> /dev/null 89 | fi 90 | } 91 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | common_CHECKOPTS := --exclude=2034,2164 2 | common_SRC := $(wildcard *.sh) 3 | 4 | pipelines := build koji release watcher 5 | 6 | koji_SRC := $(wildcard $(CURDIR)/koji/*.sh) 7 | koji_STEPS := $(patsubst %.sh,%,$(notdir $(koji_SRC))) 8 | 9 | build_CHECKOPTS := --exclude=2013,2024,2155 10 | build_SRC := $(wildcard $(CURDIR)/build/*.sh) 11 | build_STEPS := init-env local content mixer mca-check images license_info release_notes stage #skipping: koji 12 | 13 | release_SRC := $(wildcard $(CURDIR)/release/*.sh) 14 | release_STEPS := $(patsubst %.sh,%,$(notdir $(release_SRC))) 15 | 16 | watcher_SRC := $(wildcard $(CURDIR)/watcher/*.sh) 17 | watcher_STEPS := $(patsubst %.sh,%,$(notdir $(watcher_SRC))) 18 | 19 | SRC := $(common_SRC) $(build_SRC) $(koji_SRC) $(release_SRC) $(watcher_SRC) 20 | 21 | all: 22 | @echo "Welcome to clr-distro-factory." 23 | @echo "" 24 | @echo "The 'make' targets of this project are for development purpose." 25 | @echo "Please, access the online documentation for information about how" 26 | @echo "to use and deploy this project in a production environment:" 27 | @echo "" 28 | @echo " https://github.com/clearlinux/clr-distro-factory/wiki" 29 | @echo "" 30 | @echo "Usage:" 31 | @echo "" 32 | @echo "'make ' To run all steps of a pipeline'" 33 | @echo "" 34 | @echo "'make /' To run an individual step. Steps may depend on" 35 | @echo " the output of previous steps. It is up for the" 36 | @echo " developer to fullfil its requirements." 37 | @echo "" 38 | @echo "'make serve' To run a webserver hosting updates." 39 | @echo " Requires Python's SimpleHTTPServer." 40 | @echo "" 41 | @echo "pipelines: $(pipelines)" 42 | @echo "" 43 | @echo "build steps: $(build_STEPS)" 44 | @echo "" 45 | @echo "koji steps: $(koji_STEPS)" 46 | @echo "" 47 | @echo "release steps: $(release_STEPS)" 48 | @echo "" 49 | @echo "watcher steps: $(watcher_STEPS)" 50 | 51 | NAMESPACE ?= developer-distro 52 | CONFIG_REPO_HOST ?= $(CURDIR)/builder/ 53 | CLR_BUNDLES ?= "bootloader kernel-native os-core os-core-update" 54 | 55 | HOSTNAME := $(shell hostname -f) 56 | DISTRO_URL ?= http://$(HOSTNAME):8000/ 57 | 58 | MIXER_DIR := $(CURDIR)/builder/mixer 59 | CONFIG_REPO := $(CURDIR)/builder/$(NAMESPACE) 60 | STAGING_DIR := $(CURDIR)/builder/stage 61 | WORK_DIR := $(CURDIR)/builder/work 62 | 63 | $(MIXER_DIR) $(STAGING_DIR) $(WORK_DIR): 64 | @mkdir -p $@ 65 | 66 | $(CONFIG_REPO): 67 | @mkdir -p $@ 68 | @git init $@ 69 | @echo "DISTRO_NAME=$(NAMESPACE)" > $@/config.sh 70 | @echo "DISTRO_URL=$(DISTRO_URL)" >> $@/config.sh 71 | @echo "MIXER_DIR=$(MIXER_DIR)" >> $@/config.sh 72 | @echo "STAGING_DIR=$(STAGING_DIR)" >> $@/config.sh 73 | @git -C $@ add config.sh 74 | @git -C $@ commit -m "Fake config.sh" 75 | 76 | config: $(MIXER_DIR) $(STAGING_DIR) $(CONFIG_REPO) 77 | 78 | .NOTPARALLEL: $(pipelines) 79 | .PHONY: $(pipelines) 80 | 81 | build: $(addprefix build/,$(build_STEPS)) 82 | .PHONY: $(addprefix build/,$(build_STEPS)) 83 | $(addprefix build/,$(build_STEPS)): config $(WORK_DIR) 84 | @NAMESPACE=$(NAMESPACE) \ 85 | CONFIG_REPO_HOST=$(CONFIG_REPO_HOST) \ 86 | CLR_BUNDLES=$(CLR_BUNDLES) \ 87 | WORK_DIR=$(WORK_DIR) \ 88 | $@.sh 89 | 90 | koji: $(addprefix koji/,$(koji_STEPS)) 91 | .PHONY: $(addprefix koji/,$(koji_STEPS)) 92 | $(addprefix koji/,$(koji_STEPS)): 93 | @NAMESPACE=$(NAMESPACE) \ 94 | $@.sh 95 | 96 | watcher: $(addprefix watcher/,$(watcher_STEPS)) 97 | .PHONY: $(addprefix watcher/,$(watcher_STEPS)) 98 | $(addprefix watcher/,$(watcher_STEPS)): config 99 | @NAMESPACE=$(NAMESPACE) \ 100 | CONFIG_REPO_HOST=$(CONFIG_REPO_HOST) \ 101 | $@.sh 102 | 103 | release: $(addprefix release/,$(release_STEPS)) 104 | .PHONY: $(addprefix release/,$(release_STEPS)) 105 | $(addprefix release/,$(release_STEPS)): config $(WORK_DIR) 106 | @NAMESPACE=$(NAMESPACE) \ 107 | CONFIG_REPO_HOST=$(CONFIG_REPO_HOST) \ 108 | CLR_BUNDLES=$(CLR_BUNDLES) \ 109 | WORK_DIR=$(WORK_DIR) \ 110 | $@.sh 111 | 112 | .PHONY: serve 113 | serve: $(STAGING_DIR) 114 | cd $(STAGING_DIR); python3 -m http.server 115 | 116 | .PHONY: clean 117 | clean: 118 | rm -rf $(CURDIR)/builder 119 | 120 | # Static Code Analysis 121 | # ==================== 122 | check_CHECKOPTS := --exclude=1091 123 | check_PIPELINES = check-common $(addprefix check-,$(pipelines)) 124 | $(check_PIPELINES): pipe = $(patsubst check-%,%,$@) 125 | $(check_PIPELINES): 126 | shellcheck -x $(check_CHECKOPTS) $($(pipe)_CHECKOPTS) $($(pipe)_SRC) 127 | 128 | check: check-common $(check_PIPELINES) 129 | -------------------------------------------------------------------------------- /common.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (C) 2018 Intel Corporation 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | # shellcheck source=globals.sh 6 | # shellcheck source=logging.sh 7 | # shellcheck source=variables.sh 8 | 9 | LIB_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") # Do not override SCRIPT_DIR 10 | 11 | . "${LIB_DIR}/globals.sh" 12 | . "${LIB_DIR}/logging.sh" 13 | . "${LIB_DIR}/variables.sh" 14 | 15 | assert_dep () { 16 | command -v "$1" > /dev/null 2>&1 || { error "command '$1' not found"; exit 1; } 17 | } 18 | 19 | assert_dir () { 20 | [[ -d $1 ]] > /dev/null 2>&1 || { error "directory '$1' not found"; exit 1; } 21 | } 22 | 23 | assert_file() { 24 | [[ -f $1 ]] > /dev/null 2>&1 || { error "file '$1' not found"; exit 1; } 25 | } 26 | 27 | function_exists() { 28 | [[ "$(type -t "${1}")" == 'function' ]] 29 | } 30 | 31 | silentkill () { 32 | if [ -n "$2" ]; then 33 | kill "$2" "$1" > /dev/null 2>&1 || true 34 | else 35 | kill -KILL "$1" > /dev/null 2>&1 || true 36 | fi 37 | } 38 | 39 | fetch_git_repo() { 40 | local repo="${1}" 41 | local dir_name="${2}" 42 | 43 | if (( $# != 2 )); then 44 | error "'fetch_git_repo' requires 2 arguments!" 45 | return 1 46 | fi 47 | 48 | if [[ ! -d "${dir_name}" ]]; then 49 | log_line "Cloning..." 1 50 | git clone --quiet "${repo}" "${dir_name}" 51 | log_line "OK!" 2 52 | else 53 | log_line "Updating..." 1 54 | git -C "${dir_name}" fetch --prune -P --quiet origin 55 | git -C "${dir_name}" reset --hard --quiet origin/master 56 | log_line "OK!" 2 57 | fi 58 | 59 | log_line "$(git -C "${dir_name}" remote get-url origin) ($(git -C "${dir_name}" rev-parse --short HEAD))" 1 60 | } 61 | 62 | fetch_config_repo() { 63 | log_line "Config Repository:" 64 | 65 | local REPO_HOST=${CONFIG_REPO_HOST:?"CONFIG_REPO_HOST cannot be Null/Unset"} 66 | local REPO_NAME=${NAMESPACE:?"NAMESPACE cannot be Null/Unset"} 67 | 68 | fetch_git_repo "${REPO_HOST}${REPO_NAME}" "config" 69 | 70 | pushd ./config > /dev/null 71 | log_line "Checking for the required file..." 1 72 | assert_file ./config.sh 73 | log_line "OK!" 2 74 | popd > /dev/null 75 | 76 | log_line "Done!" 1 77 | } 78 | 79 | get_upstream_version() { 80 | CLR_LATEST=${CLR_LATEST:-$(curl "${CLR_PUBLIC_DL_URL}/latest")} || true 81 | if [[ -z "${CLR_LATEST}" ]]; then 82 | error "Failed to fetch Clear Linux latest version." 83 | exit 2 84 | fi 85 | 86 | CLR_FORMAT=$(curl "${CLR_PUBLIC_DL_URL}/update/${CLR_LATEST}/format") || true 87 | if [[ -z "${CLR_FORMAT}" ]]; then 88 | error "Failed to fetch Clear Linux latest format." 89 | exit 2 90 | fi 91 | } 92 | 93 | get_distro_version() { 94 | DISTRO_LATEST=$(cat "${STAGING_DIR}/latest" 2>/dev/null) || true 95 | if [[ -z "${DISTRO_LATEST}" ]]; then 96 | info "Failed to fetch Distribution latest version" "First Mix?" 97 | DISTRO_FORMAT=${CLR_FORMAT:-1} 98 | DISTRO_UP_FORMAT=${CLR_FORMAT} 99 | return 100 | fi 101 | 102 | DISTRO_FORMAT=$(cat "${STAGING_DIR}/update/${DISTRO_LATEST}/format" 2>/dev/null) || true 103 | if [[ -z "${DISTRO_FORMAT}" ]]; then 104 | error "Failed to fetch Distribution latest format." 105 | exit 2 106 | fi 107 | 108 | if "${IS_DOWNSTREAM}"; then 109 | if ((${#DISTRO_LATEST} < 4)); then 110 | error "Distribution version number seems corrupted." 111 | exit 2 112 | fi 113 | 114 | DISTRO_UP_VERSION="${DISTRO_LATEST: : -3}" 115 | DISTRO_DOWN_VERSION="${DISTRO_LATEST: -3}" 116 | 117 | DISTRO_UP_FORMAT=$(curl "${CLR_PUBLIC_DL_URL}/update/${DISTRO_UP_VERSION}/format") || true 118 | if [[ -z "${DISTRO_UP_FORMAT}" ]]; then 119 | error "Failed to fetch Distribution latest base format." 120 | exit 2 121 | fi 122 | fi 123 | } 124 | 125 | get_latest_versions() { 126 | get_upstream_version 127 | get_distro_version 128 | } 129 | 130 | calc_mix_version() { 131 | if "${IS_DOWNSTREAM}"; then 132 | if [[ -z "${DISTRO_LATEST}" || "${CLR_LATEST}" -gt "${DISTRO_UP_VERSION}" ]]; then 133 | MIX_VERSION=$(( CLR_LATEST * 1000 + MIX_INCREMENT )) 134 | MIX_FORMAT=${CLR_FORMAT} 135 | elif [[ "${CLR_LATEST}" -eq "${DISTRO_UP_VERSION}" ]]; then 136 | MIX_VERSION=$(( DISTRO_LATEST + MIX_INCREMENT )) 137 | if [[ "${MIX_VERSION: -3}" -eq 000 ]]; then 138 | error "Invalid Mix Version" \ 139 | "No more Downstream versions available for this Upstream version!" 140 | exit 1 141 | fi 142 | else 143 | error "Invalid Mix version" \ 144 | "Next Upstream Version is less than the Previous Upstream!" 145 | exit 1 146 | fi 147 | 148 | MIX_UP_VERSION="${MIX_VERSION: : -3}" 149 | MIX_DOWN_VERSION="${MIX_VERSION: -3}" 150 | MIX_FORMAT="${DISTRO_FORMAT:-1}" 151 | else 152 | # format bump if not a new mix 153 | if [[ -n "${DISTRO_LATEST}" ]] && "${FORMAT_BUMP}"; then 154 | MIX_VERSION=$(( DISTRO_LATEST + MIX_INCREMENT * 2 )) 155 | MIX_FORMAT=$(( DISTRO_FORMAT + 1 )) 156 | else # new mix or regular mix 157 | MIX_VERSION=$(( DISTRO_LATEST + MIX_INCREMENT )) 158 | MIX_FORMAT="${DISTRO_FORMAT:-1}" 159 | fi 160 | fi 161 | } 162 | 163 | # ================= 164 | # Command Overrides 165 | # ================= 166 | 167 | curl() { 168 | command curl --silent --fail "$@" 169 | } 170 | 171 | koji_cmd() { 172 | # Downloads fail sometime, try harder! 173 | local result="" 174 | local ret=1 175 | for (( i=0; i < 10; i++ )); do 176 | result=$(koji "${@}" 2> /dev/null) \ 177 | || continue 178 | 179 | ret=0 180 | break 181 | done 182 | 183 | [[ -n "${result}" ]] && echo "${result}" 184 | return ${ret} 185 | } 186 | 187 | mixer_cmd() { 188 | # shellcheck disable=SC2086 189 | mixer ${MIXER_OPTS} "${@}" 190 | } 191 | 192 | sudo_mixer_cmd() { 193 | # shellcheck disable=SC2086 194 | sudo -E mixer ${MIXER_OPTS} "${@}" 195 | } 196 | --------------------------------------------------------------------------------