├── .editorconfig ├── .gitignore ├── .gitmodules ├── .travis.yml ├── banner ├── build.sh ├── config.sh ├── issueTemplate ├── readme.md └── update-website.sh /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_size = 2 6 | indent_style = space 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | 10 | [.gitmodules] 11 | indent_style = tab 12 | 13 | [colorEcho/**/*] 14 | indent_size = false 15 | indent_style = false 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build.log 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "colorEcho"] 2 | path = colorEcho 3 | url = https://github.com/PeterDaveHello/ColorEchoForShell.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | dist: xenial 4 | 5 | node_js: 6 | - "8" 7 | 8 | cache: 9 | directories: 10 | - node_modules 11 | 12 | git: 13 | depth: 10 14 | 15 | jobs: 16 | include: 17 | - stage: Static Test 18 | env: 19 | - test: Editor Config check 20 | install: 21 | - npm i -g eclint 22 | script: 23 | - eclint check 24 | -------------------------------------------------------------------------------- /banner: -------------------------------------------------------------------------------- 1 |    ___ ___  _  _    _ ___   _         _ _    _ ___        _      _    2 |   / __|   \| \| |_ | / __| | |__ _  _(_| |__| / __|__ _ _(_)_ __| |_  3 |  | (__| |) | .` | || \__ \ | '_ | || | | / _` \__ / _| '_| | '_ |  _| 4 |   \___|___/|_|\_|\__/|___/ |_.__/\_,_|_|_\__,_|___\__|_| |_| .__/\__| 5 |                                                            |_|        6 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Script for cronjob, will open an issue on cdnjs/cdnjs if build failed. 4 | 5 | StartTimestamp="$(date +%s)" 6 | 7 | pth="$(dirname "$(readlink -f "$0")")" 8 | 9 | cat "$pth/banner" 10 | 11 | . "$pth/config.sh" 12 | 13 | export PATH="$path:$PATH" 14 | 15 | if [ "$(uname)" = "FreeBSD" ] || [ "$(uname)" = "Darwin" ]; then 16 | sed="gsed" 17 | else 18 | sed="sed" 19 | fi 20 | 21 | apiUrl='https://api.github.com/repos/cdnjs/cdnjs/issues' 22 | 23 | IssueTitle="[Build failed] Got error while building meta data/artifact" 24 | IssueAssignee="robocdnjs" 25 | IssueLabels='["Bug - High Priority"]' 26 | IssueContent="$($sed ':a;N;$!ba;s/\n/\\n/g' "$pth/issueTemplate")" 27 | 28 | Issue="{ \"title\": \"$IssueTitle\", \"body\": \"$IssueContent\", \"assignee\": \"$IssueAssignee\", \"labels\": $IssueLabels }" 29 | 30 | flock -E 87 -n build.lock -c "$pth/update-website.sh build" 31 | 32 | error=$? 33 | 34 | if [ $error -eq 87 ]; then 35 | echo -e "\\nPrevious build locked!\\n" 36 | elif [ $error -ne 0 ]; then 37 | curl --silent -H "Authorization: token $githubToken" -d "$Issue" "$apiUrl" > /dev/null 38 | fi 39 | 40 | EndTimestamp="$(date +%s)" 41 | 42 | echo -e "\\nTotal time spent for this build is _$((EndTimestamp - StartTimestamp))_ second(s)\\n" 43 | exit $error 44 | -------------------------------------------------------------------------------- /config.sh: -------------------------------------------------------------------------------- 1 | if [ -z "$BOT_BASE_PATH" ]; then 2 | echo "BOT_BASE_PATH missing" 3 | exit 1 4 | fi 5 | 6 | gitterHook="" 7 | slackHook="" 8 | slackChannel="build-server-logs" 9 | 10 | basePath="$BOT_BASE_PATH" 11 | mainRepo="cdnjs" 12 | webRepo="new-website" 13 | hasLocalRepo=true 14 | forceUpdateMeta=false 15 | forceUpdateRepo=false 16 | 17 | pushMetaOnGitHub=false 18 | 19 | timeout=1200 #maximun execution time for a command, in second 20 | nice=15 21 | retryTimes=3 #retry limit for run_retry() 22 | 23 | logFile='build.log' 24 | logMode='clean' #append|clean 25 | logPath='' #if empty, log will be placed at the root of buildScript 26 | 27 | path="/usr/local/bin:/usr/bin:/bin" 28 | serverOwner="robocdnjs" 29 | -------------------------------------------------------------------------------- /issueTemplate: -------------------------------------------------------------------------------- 1 | Dear all, 2 | 3 | Our build process just got an error while building the meta data/artifact of cdnjs project, our [website and api](https://github.com/cdnjs/new-website) may not be as new as our repository, however, the cdn service will not been affected, you can still use it, and modify the library name, version number and filename in the url can help you access all the content on cdnjs, we will fix the error as soon as possible, sorry for the inconvenience. 4 | 5 | PS: 6 | This issue ticket was created by our [buildScript](https://github.com/cdnjs/buildScript), the automatically meta data/artifact building script of cdnjs, not by the real maintainer, the issuer depends on the API key we are using. 7 | 8 | Best, 9 | Peter 10 | 11 | -- 12 | cc @cdnjs/intern @cdnjs/team-cdnjs 13 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # buildScript 2 | 3 | Script use to rebuild the meta data of [api](https://github.com/cdnjs/cdnjs#api) and [website](https://cdnjs.com/), and then deploy them. 4 | 5 | Most of the codes will be in `update-website.sh`, `build.sh` is the file we need to execute, and `config.sh` is the file contains the configs. 6 | 7 | Take a look at the script, set the path and api key in `config.sh` before use it. 8 | 9 | ## Repositories 10 | 11 | * **cdnjs**, the main repo we are working on 12 | * **cdnjsmaster**, the clean repo be used to build 13 | * **new-website**, the website and api repo 14 | 15 | ## Branches and remotes 16 | 17 | ### Branches 18 | 19 | * **cdnjsmaster** should stay on master, this repo is setup for build process only. 20 | * **new-website** should have two branches 21 | * one is **master**, for the real codes 22 | * another one will be **meta**, **meta** beanch will save additional meta data the we'll use for website and api, but not real codes. 23 | 24 | ### Remotes 25 | 26 | * **cdnjsmaster** should have two remotes, 27 | * **origin** will point to the repo on GitHub 28 | * git@github.com:cdnjs/cdnjs.git 29 | * **local** will points to the local working **cdnjs**, we want most of the objects will be fetched locally. 30 | * For example: `/home/peter/cdnjs/cdnjs_working` 31 | * **new-website** should have 3 remotes, 32 | * **origin** will point to the repo on GitHub 33 | * git@github.com:cdnjs/new-website.git 34 | * the others will be **heroku** and **heroku2**, which will point to the api and website project on heroku. 35 | * git@heroku.com:cdnjs-new-website.git 36 | * git@heroku.com:cdnjsapi.git 37 | -------------------------------------------------------------------------------- /update-website.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | trap "exit 1" EXIT 4 | 5 | function init() { 6 | pth="$(setBasePath)" 7 | . "$pth/config.sh" 8 | 9 | updateMeta="$forceUpdateMeta" 10 | updateRepo="$forceUpdateRepo" 11 | 12 | export PATH="$path:$PATH" 13 | 14 | NVM_DIR=$HOME/.nvm 15 | if [ -s "$NVM_DIR/nvm.sh" ]; then 16 | . "$NVM_DIR/nvm.sh" --no-use 17 | nvm install 10 18 | fi 19 | 20 | if [[ ! $timeout =~ ^[0-9]+$ ]] || [[ $timeout -le 3 ]]; then 21 | timeout=3 22 | fi 23 | 24 | eval logPath="$logPath" 25 | if [ -z "$logPath" ] || [ ! -d "$logPath" ] || [ ! -w "$logPath" ]; then 26 | logPath="$pth" 27 | fi 28 | 29 | [ "$logMode" = "clean" ] && rm -f "$logPath/$logFile" 30 | 31 | . "$pth/colorEcho/dist/ColorEcho.bash" &> /dev/null || { 32 | git -C "$pth" submodule update --init 33 | . "$pth/colorEcho/dist/ColorEcho.bash" 34 | } 35 | } 36 | 37 | function setBasePath() { 38 | dirname "$(realpath "${BASH_SOURCE[0]}")" 39 | } 40 | 41 | function git-checkout-master-if-needed() { 42 | currentBranch="$(git branch | awk '{ if ("*" == $1) print $2}')" 43 | [ "$currentBranch" = "master" ] || run_retry git checkout master 44 | } 45 | 46 | function git-reset-hard-if-needed() { 47 | # the bot already keeps the cdnjs repo up-to-date 48 | echo ignore 49 | # if ! git diff --quiet; then 50 | # output Info "Repo diff found, so reset!" 51 | # run_retry git reset --hard 52 | # else 53 | # output Info "Repo diff not found, so do not reset!" 54 | # fi 55 | } 56 | 57 | function output() { 58 | echo "$(date) [$1] $2" >> "$logPath/$logFile" 59 | case "$1" in 60 | "Warn") 61 | echo.Red "$2" 62 | ;; 63 | "Success") 64 | echo.Green "$2" 65 | ;; 66 | "Info") 67 | echo.Cyan "$2" 68 | ;; 69 | *) 70 | echo "$1" 71 | ;; 72 | esac 73 | if [ -n "$3" ] && [ "$3" = "chat-room" ]; then 74 | curl --silent -d message="[cronjob] $2" "$gitterHook" > /dev/null || output Warn "Error on curl!!! Message may not be posted on our gitter chatroom" 75 | curl --silent -X POST --data-urlencode 'payload={"channel": "#'"$slackChannel"'", "username": "buildScript", "text": "'"$2"'", "icon_emoji": ":building_construction:"}' "$slackHook" > /dev/null || output Warn "Error on curl!!! Message may not be posted on our Slack chatroom" 76 | fi 77 | } 78 | 79 | function error() { 80 | local MSG 81 | if [ "$#" = "0" ]; then 82 | MSG="Error" 83 | else 84 | MSG="$*" 85 | fi 86 | output Warn "$MSG, pwd='$(pwd)'" chat-room 87 | exit 1 88 | } 89 | 90 | function run() { 91 | run_retry_times 1 "$@" 92 | } 93 | 94 | function run_retry() { 95 | run_retry_times "${retryTimes}" "$@" 96 | } 97 | 98 | function run_retry_times() { 99 | local timesLeft="${1}" 100 | local timesLeftOrigin="${1}" 101 | shift 102 | while [ "$timesLeft" -ge 1 ]; do 103 | timesLeft="$((timesLeft - 1))" 104 | if [ $((timesLeftOrigin - timesLeft)) -gt 1 ]; then 105 | sleep 3 106 | echo "$(date) [command (try times: $((timesLeftOrigin - timesLeft))/${timesLeftOrigin})] $*" >> "$logPath/$logFile" 107 | else 108 | echo "$(date) [command] $*" >> "$logPath/$logFile" 109 | fi 110 | 111 | local isBuiltIn=false 112 | local cmd 113 | cmd="$(echo "$1" | awk '{print $1}')" 114 | if type "$cmd" &> /dev/null; then 115 | local temp 116 | temp="$(type "$cmd" | head -n 1)" 117 | [ "$temp" = "$cmd is a shell builtin" ] && isBuiltIn=true 118 | else 119 | error "'$cmd' command not found!" 120 | fi 121 | 122 | if [ "$isBuiltIn" = "false" ]; then 123 | nice -n "$nice" timelimit -q -s 9 -t $((timeout - 2)) -T "$timeout" "$@" || continue 124 | else 125 | "$@" || continue 126 | fi 127 | return 128 | done 129 | error "Got error while running command: '$*'" 130 | } 131 | 132 | function build() { 133 | 134 | [[ -d "$basePath/$mainRepo" ]] || error "Main repo '$basePath/$mainRepo' not found, exit now." 135 | 136 | [[ -d "$basePath/$webRepo" ]] || error "website repo '$basePath/$webRepo' not found, exit now." 137 | 138 | output Info "Start date time: $(date)" 139 | output Info "Start website/api/index building process on $serverOwner's server ..." chat-room 140 | output Info "PATH=$PATH" 141 | output Info "bash path: $(type bash)" 142 | output Info "bash version: $BASH_VERSION" 143 | [[ -z "$NVM_BIN" ]] || output Info "nvm version: $(nvm --version)" 144 | output Info "nodejs path: $(type node)" 145 | output Info "nodejs version: $(node --version)" 146 | output Info "npm path: $(type npm)" 147 | output Info "npm version: $(npm --version)" 148 | output Info "git path: $(type git)" 149 | output Info "git version: $(git --version)" 150 | 151 | output Info "Reset repository to prevent unstaged changes break the build" 152 | run cd "$basePath/$mainRepo" 153 | git-reset-hard-if-needed 154 | 155 | # if [ "$hasLocalRepo" = true ] && [ -d "$basePath" ]; then 156 | # output Success "Exist cdnjs local repo, fetch objects from local branch first" 157 | # run_retry git fetch local 158 | # else 159 | # output Info "Local repo not found, will grab object(s) from GitHub" 160 | # fi 161 | 162 | git-checkout-master-if-needed 163 | 164 | updated=true 165 | # updated=false 166 | # output Info "Pull cdnjs main repo with rebase from origin(GitHub)" 167 | # run_retry git fetch origin master --tags 168 | # if [ "$(run git log --oneline -1 origin/master)" != "$(run git log --oneline -1 master)" ]; then 169 | # run_retry git rebase origin/master master 170 | # updated=true 171 | # fi 172 | 173 | output Info "Current commit: $(run git log --pretty='format:%h - %s - %an %ai' -1)" 174 | if [ "$updated" = false ] && [ "$updateMeta" = false ]; then 175 | msg="Cdnjs main repo is up to date, no need to rebuild" 176 | output Info "$msg" chat-room 177 | else 178 | msg="Cdnjs main repo updates found! Start the rebuild rebuild process" 179 | output Info "$msg" chat-room 180 | msg="Make sure npm package dependencies, do npm install && npm update" 181 | output Info "$msg" 182 | run_retry npm install --no-audit --no-save 183 | run_retry npm update --no-save 184 | msg="Run npm test before building the meta data/artifacts" 185 | output Info "$msg" 186 | run npm test -- --silent 187 | msg="Reset and checkout website repository to meta branch" 188 | output Info "$msg" 189 | (run cd "$basePath/$webRepo" && git-reset-hard-if-needed) 190 | run_retry git -C "$basePath/$webRepo" checkout meta 191 | msg="Rebuild meta data phase 1" 192 | output Info "$msg" chat-room 193 | run_retry node build/packages.json.js 194 | 195 | msg="Rebuild meta data phase 2" 196 | output Info "$msg" chat-room 197 | run cd "$basePath/$webRepo" 198 | msg="Make sure npm package dependencies, do npm install & npm update" 199 | output Info "$msg" chat-room 200 | run_retry npm install --no-audit --no-save 201 | run_retry npm update --no-save 202 | run_retry node update.js 203 | 204 | msg="Upload packages.min.json to Google storage" 205 | output Info "$msg" 206 | run_retry packages set < public/packages.min.json 207 | 208 | msg="Commit meta data update in website repo" 209 | output Info "$msg" chat-room 210 | for file in atom.xml rss.xml sitemap.xml; do 211 | run_retry git add public/$file 212 | done 213 | run_retry git add sri 214 | run_retry git commit --message="meta data" 215 | 216 | updateMeta=true 217 | fi 218 | 219 | output Info "Change directory into website repo" 220 | run cd "$basePath/$webRepo" 221 | 222 | output Info "Reset repository to prevent unstaged changes break the build" 223 | git-reset-hard-if-needed 224 | git-checkout-master-if-needed 225 | 226 | output Info "Pull website repo with rebase from origin(Repo)" 227 | webstatus="$(run_retry git pull --tags --rebase origin master | tail -n 1)" 228 | output Info "Current commit: $(run git log --pretty='format:%h - %s - %an %ai' -1)" 229 | if [ "$webstatus" = "Current branch master is up to date." ] || [ "$webstatus" = "Already up-to-date." ]; then 230 | msg="Cdnjs website repo is up to date" 231 | $updateMeta || msg="$msg too, no need to deploy." 232 | $updateMeta && msg="$msg, but we'll still deploy artifacts since main repo has updates." 233 | output Info "$msg" chat-room 234 | else 235 | msg="Cdnjs website repo updates found!" 236 | output Info "$msg" chat-room 237 | updateRepo=true 238 | fi 239 | 240 | if [ "$updateRepo" = true ]; then 241 | output Info "Update/Initial submodule under website repo" chat-room 242 | run_retry git submodule update --init 243 | 244 | msg="Make sure npm package dependencies, do npm install & npm update" 245 | output Info "$msg" chat-room 246 | run_retry npm install --no-audit --no-save 247 | run_retry npm update --no-save 248 | fi 249 | 250 | msg="Rebase website's meta branch on master" 251 | output Info "$msg" 252 | webstatus="$(run_retry git rebase master meta)" 253 | if [[ "$webstatus" != "Current branch meta is up to date." ]] && [[ "$webstatus" != "Already up-to-date." ]]; then 254 | updateRepo=true 255 | fi 256 | 257 | if [ "$updateMeta" = true ]; then 258 | msg="Now push and deploy website & api" 259 | output Info "$msg" chat-room 260 | for remote in heroku-web heroku-api; do 261 | run_retry git push "$remote" meta:master -f & 262 | sleep 3 263 | done 264 | [ "$pushMetaOnGitHub" = true ] && run_retry git push origin meta -f & 265 | wait 266 | elif [ "$updateRepo" = true ]; then 267 | msg="Now push and deploy website and api" 268 | output Info "$msg" chat-room 269 | for remote in heroku-web heroku-api; do 270 | run_retry git push "$remote" meta:master -f & 271 | sleep 3 272 | done 273 | [ "$pushMetaOnGitHub" = true ] && run_retry git push origin meta -f & 274 | wait 275 | else 276 | msg="Didn't update anything, no need to push or deploy." 277 | output Info "$msg" chat-room 278 | fi 279 | 280 | msg="Update finished." 281 | output Success "$msg" chat-room 282 | output Info "End date time: $(date)" 283 | } 284 | 285 | if [ "$1" = "build" ]; then 286 | set +e 287 | init 288 | build 289 | fi 290 | 291 | trap - EXIT 292 | --------------------------------------------------------------------------------