├── .github ├── issue_template.md ├── pull_request_template.md └── workflows │ └── build.yml ├── .gitignore ├── .gitmodules ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── apt-sync ├── Dockerfile ├── LICENSE ├── sync.py ├── sync.sh └── tunasync │ └── apt-sync.py ├── aptsync ├── Dockerfile ├── LICENSE ├── apt-mirror ├── pre-sync.sh └── sync.sh ├── archvsync ├── .dockerignore ├── .gitignore ├── Dockerfile ├── build ├── pre-sync.sh ├── sync.sh └── update.patch ├── base ├── Dockerfile.alpine ├── Dockerfile.alpine-edge ├── Dockerfile.debian ├── entry.sh ├── pip.conf └── savelog ├── configure.py ├── crates-io-index ├── Dockerfile └── sync.sh ├── curl-helper ├── Dockerfile └── curl-helper.sh ├── debian-cd ├── Dockerfile ├── cd-mirror ├── jigdo-mirror ├── jigdo-mirror.conf.in ├── pre-sync.sh └── sync.sh ├── docker-ce ├── Dockerfile ├── LICENSE ├── sync.sh └── tunasync │ └── sync.py ├── fedora ├── Dockerfile ├── pre-sync.sh └── sync.sh ├── flatpak ├── .gitignore ├── Dockerfile ├── sync.py └── sync.sh ├── freebsd-pkg ├── Dockerfile └── sync.sh ├── freebsd-ports ├── Dockerfile └── sync-ports.sh ├── ghcup ├── Dockerfile ├── README ├── ghcupsync.cabal ├── ghcupsync.hs └── sync.sh ├── github-release ├── Dockerfile ├── LICENSE ├── examples │ └── repos.yaml ├── sync.sh └── tunasync │ └── github-release.py ├── gitsync ├── Dockerfile └── sync.sh ├── google-repo ├── Dockerfile ├── pre-sync.sh └── sync.sh ├── gsutil-rsync ├── Dockerfile ├── pre-sync.sh └── sync.sh ├── hackage ├── Dockerfile └── sync.sh ├── homebrew-bottles ├── Dockerfile ├── bottles-json │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── main.rs │ │ └── structs.rs └── sync.sh ├── julia-storage ├── Dockerfile ├── pre-sync.sh ├── startup.jl └── sync.sh ├── lftpsync ├── Dockerfile └── sync.sh ├── misc ├── Dockerfile └── sync.sh ├── nix-channels ├── Dockerfile ├── nix-channels.py └── sync.sh ├── push.sh ├── pypi ├── Dockerfile ├── pre-sync.sh └── sync.sh ├── rclone ├── Dockerfile └── sync.sh ├── rsync ├── Dockerfile ├── lchmod.c └── sync.sh ├── rubygems ├── Dockerfile ├── pre-sync.sh └── sync.sh ├── shadowmire ├── Dockerfile └── sync.sh ├── stackage ├── Dockerfile ├── README ├── stackage.hs └── sync.sh ├── test ├── Dockerfile └── sync.sh ├── tsumugu ├── Dockerfile └── sync.sh ├── winget-source ├── Dockerfile ├── package-lock.json ├── package.json ├── sync-repo.js ├── sync.sh ├── sysexits.js └── utilities.js ├── yukina ├── Dockerfile └── sync.sh └── yum-sync ├── Dockerfile ├── LICENSE ├── sync.sh └── tunasync └── yum-sync.py /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | ### 预期行为 2 | 3 | 4 | ### 实际行为 5 | 6 | 7 | ### 复现步骤 8 | 9 | * Step 1 10 | * Step 2 11 | 12 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ### Checklist 2 | 3 | - [ ] 更新 README 4 | 5 | (如果方便的话, 可以简述一下您所做的改动) 6 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [ 'master' ] 6 | pull_request: 7 | branches: [ '*' ] 8 | 9 | workflow_dispatch: {} 10 | schedule: 11 | - cron: '30 23 1 * *' 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | with: 20 | submodules: true 21 | fetch-depth: 50 # the same value as Travis CI 22 | 23 | - name: Run build script 24 | run: | 25 | python3 configure.py 26 | make all 27 | env: 28 | COMMIT_FROM: ${{ github.event.before }} 29 | COMMIT_TO: ${{ github.event.after }} 30 | 31 | - name: Test Rsync container 32 | run: | 33 | docker run --rm -t \ 34 | --name=syncing-ubuntu \ 35 | -e RSYNC_HOST=rsync.archive.ubuntu.com \ 36 | -e RSYNC_PATH=ubuntu/ \ 37 | -e RSYNC_EXTRA='--dry-run --no-r --dirs' \ 38 | -e RSYNC_FILTER='- .trace' \ 39 | --tmpfs /data \ 40 | --tmpfs /log \ 41 | ustcmirror/rsync 42 | 43 | - name: Deploy 44 | if: "github.ref == 'refs/heads/master' && github.repository == 'ustclug/ustcmirror-images'" 45 | run: ./push.sh 46 | env: 47 | DOCKER_USER: ${{ secrets.DOCKER_USER }} 48 | DOCKER_PASS: ${{ secrets.DOCKER_PASS }} 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.[oa] 3 | *.py[co] 4 | *~ 5 | *.bac 6 | __pycache__/ 7 | node_modules/ 8 | build/ 9 | Makefile 10 | dist-newstyle/ 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "archvsync/upstream"] 2 | path = archvsync/upstream 3 | url = https://salsa.debian.org/mirror-team/archvsync.git 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | :+1::tada: 非常高兴您愿意花时间来改进这个项目，请接受我最为诚挚的感谢！ :tada::+1: 2 | 3 | 以下是您可能会感兴趣的内容： 4 | 5 | * [如何增加新的同步方式？](#增加新的同步方式) 6 | * [约定](#约定) 7 | * [整个项目是如何构建的？](#整个项目的构建过程) 8 | 9 | ## 增加新的同步方式 10 | 11 | 在项目根目录下新建一个文件夹，命名请参考下面的[约定](#约定)。文件夹中至少应该包含： 12 | 13 | * `Dockerfile` 14 | * `sync.sh`：需要被赋予可执行权限，并添加到根目录 15 | * 其他需要用到的文件 16 | 17 | `sync.sh` 中应该只包含同步的逻辑，如果您需要在同步前或同步后做一些额外的工作的话，可以分别添加 `pre-sync.sh` 或 `post-sync.sh` （均需要可执行权限）到根目录，它们都会以 `root` 的身份执行，而 `sync.sh` 在执行前可能会被降权，取决于 `$OWNER` 的值。可以参考 [pypi](pypi) 文件夹下的内容。 18 | 19 | ### 约定 20 | 21 | * 如果同步方式具有普适性，建议命名为 `xx-sync`，比如 `rsync`，`gitsync` 等。 22 | * 如果同步方式只适用于某一个特定的源，建议命名为那个源的名字，比如 `nodesource`，`stackage` 等。 23 | * 任何同步方式对应的 image 都应该直接或间接地基于 `ustcmirror/base`。 24 | * 如果您构建的镜像需要打上 `latest` 以外的 tag，请创建新的 `Dockerfile` 并把 tag 作为 `Dockerfile` 的后缀名。可以参考 [base](base) 以及 [lftpsync](lftpsync) 文件夹下的内容。 25 | * 如果构建镜像前需要做额外的工作，您可以创建 `$your-sync-method/build` 来实现（需要可执行权限）。可以参考 [aptsync](aptsync)，[archvsync](archvsync) 文件夹下的内容。您的自定义构建程序应该 fail fast，如果是 Bash script 的话，请记得 `set -e`。构建时会以 `cd $your-sync-method/ && ./build $tag` 的方式来调用您的构建程序，如果需要构建多个镜像的话，可以根据第一个参数来决定构建哪一个。 26 | * 同步程序应该读取环境变量作为参数，并且这些参数应该加上合适的前缀以示区分，比如 `RSYNC_HOST`，`GITSYNC_URL` 等。 27 | * 同步程序应该把文件下载到 `$TO` 对应的目录下。 28 | * 如果您的 `sync.sh` 最终只需要调用一个外部程序的话，应该以 `exec program` 的方式调用，方便接收 signal。 29 | * 同步时产生的日志应该都输出到 `stdout` 或 `stderr`。 30 | * 在不会过分麻烦您的前提下，请让构建出来的镜像尽可能小，构建的时间尽可能短。 31 | * 如果 image 添加了对 `BIND_ADDRESS` 的支持，在 `Dockerfile` 中添加 `LABEL bind_support=true`（反之不需要）。 32 | 33 | 如果您对整个项目是如何构建的感兴趣的话，可以继续往下阅读。 34 | 35 | ## 整个项目的构建过程 36 | 37 | 执行 `./configure.py` 后会生成构建需要更新的镜像的 Makefile，接着执行 `make all` 进行构建。 38 | 39 | 生成 Makefile 的大概过程如下： 40 | 41 | 1. 枚举含有 `Dockerfile*` 的文件夹，根据 `Dockerfile` 的名字决定镜像的 tag 42 | 2. 分析 `Dockerfile` 提取依赖信息 43 | 3. 根据以上获取的依赖信息，构建一棵如下格式的 n 叉树： 44 | 45 | ``` 46 | $root$: 47 | base.alpine-3.6: 48 | lftpsync.alpine-3.6 49 | base.alpine: 50 | lftpsync.latest 51 | gitsync.latest: 52 | freebsd-ports.latest 53 | ... 54 | base.debian: 55 | stackage.latest 56 | ``` 57 | 58 | 4. 对该树进行广度优先遍历，如果发现该结点对应的目录下的内容有改变的话（由 `git diff $TRAVIS_COMMIT_RANGE -- dir` 或 `git diff origin/master HEAD -- dir` 决定），就把该结点及其所有的子孙结点视为待构建的 targets，否则把其子结点加入到遍历队列。 59 | 5. 根据 targets 生成 Makefile 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Jian Zeng 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /apt-sync/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="iBug " 3 | LABEL bind_support=true 4 | RUN < "$_LIST" << EOF 14 | set base_path $_BASE 15 | set mirror_path \$base_path/mirror 16 | set skel_path \$base_path/skel 17 | set var_path $LOGDIR 18 | set run_postmirror 0 19 | set nthreads $APTSYNC_NTHREADS 20 | set unlink $APTSYNC_UNLINK 21 | EOF 22 | 23 | chown -R "$OWNER" "$_BASE" 24 | if [[ $APTSYNC_CREATE_DIR == true ]]; then 25 | ln -s "$TO" "$_BASE/mirror/$(get_hostname "$APTSYNC_URL")" 26 | else 27 | _LINK_TARGET="$_BASE/mirror/$(get_hostname_and_path "$APTSYNC_URL")" 28 | mkdir -p $_LINK_TARGET 29 | rmdir $_LINK_TARGET 30 | ln -Ts "$TO" "$_LINK_TARGET" 31 | fi 32 | 33 | IFS=':' read -ra dists <<< "$APTSYNC_DISTS" 34 | for dist in "${dists[@]}"; do 35 | IFS='|' read -ra data <<< "$dist" 36 | # 0: releases 37 | # 1: componenets 38 | # 2: architectures 39 | IFS=' ' read -ra releases <<< "${data[0]}" 40 | for release in "${releases[@]}"; do 41 | for arch in ${data[2]}; do 42 | echo "deb-$arch" "$APTSYNC_URL" "$release" "${data[1]}" 43 | done 44 | done 45 | done | tee -a "$_LIST" 46 | -------------------------------------------------------------------------------- /aptsync/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## EXPORTED IN entry.sh 4 | #TO= 5 | #LOGDIR= 6 | #LOGFILE= 7 | 8 | ## SET IN ENVIRONMENT VARIABLES 9 | #APTSYNC_URL= 10 | #APTSYNC_DISTS= 11 | #BIND_ADDRESS= 12 | 13 | set -e 14 | [[ $DEBUG = true ]] && set -x 15 | 16 | exec apt-mirror 17 | -------------------------------------------------------------------------------- /archvsync/.dockerignore: -------------------------------------------------------------------------------- 1 | upstream/ 2 | update.patch 3 | -------------------------------------------------------------------------------- /archvsync/.gitignore: -------------------------------------------------------------------------------- 1 | ftpsync 2 | common 3 | -------------------------------------------------------------------------------- /archvsync/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Jian Zeng " 3 | LABEL bind_support=true 4 | ENV BASEDIR=/usr/local 5 | RUN apk add --no-cache rsync && mkdir -p "$BASEDIR/etc" 6 | ADD ["common", "ftpsync", "$BASEDIR/bin/"] 7 | ADD ["sync.sh", "pre-sync.sh", "/"] 8 | -------------------------------------------------------------------------------- /archvsync/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd upstream 6 | git apply ../update.patch 7 | cp bin/{common,ftpsync} .. 8 | docker build -t ustcmirror/archvsync .. 9 | -------------------------------------------------------------------------------- /archvsync/pre-sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # override the `killer` func in entry.sh 4 | killer() { 5 | kill -- "$1" 6 | pkill rsync 7 | wait "$1" 8 | } 9 | 10 | touch "$BASEDIR/etc/ftpsync-$REPO.conf" 11 | IGNORE_LOCK="${IGNORE_LOCK:-false}" 12 | if [[ $IGNORE_LOCK = true ]]; then 13 | rm -f "$TO"/Archive-Update-in-Progress-* 14 | fi 15 | -------------------------------------------------------------------------------- /archvsync/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #REPO= 4 | 5 | ## EXPORTED IN entry.sh 6 | #TO= 7 | #LOGDIR= 8 | #LOGFILE= 9 | 10 | ## SET IN ENVIRONMENT VARIABLES 11 | #DEBUG= 12 | #BIND_ADDRESS= 13 | 14 | set -e 15 | [[ $DEBUG = true ]] && set -x 16 | 17 | export LOGROTATE="${LOG_ROTATE_CYCLE:-14}" 18 | export LOG="${LOG:-$LOGFILE}" 19 | if [[ -n $BIND_ADDRESS ]]; then 20 | if [[ $BIND_ADDRESS =~ .*: ]]; then 21 | RSYNC_EXTRA+=" -6 --address $BIND_ADDRESS" 22 | else 23 | RSYNC_EXTRA+=" -4 --address $BIND_ADDRESS" 24 | fi 25 | export RSYNC_EXTRA 26 | fi 27 | 28 | exec ftpsync "sync:archive:$REPO" 29 | -------------------------------------------------------------------------------- /archvsync/update.patch: -------------------------------------------------------------------------------- 1 | diff --git a/bin/common b/bin/common 2 | index 7ac7977..f8e11ba 100644 3 | --- a/bin/common 4 | +++ b/bin/common 5 | @@ -219,7 +219,6 @@ log() { 6 | error () { 7 | log "$@" 8 | LOG_ERROR=1 9 | - mailf -s "[$PROGRAM@$(hostname -s)] ERROR: $*" -b "$*" ${MAILTO} 10 | } 11 | 12 | # log the message using log() but then also send a mail 13 | @@ -229,7 +228,6 @@ error_mailf () { 14 | shift 15 | log "$m" 16 | LOG_ERROR=1 17 | - mailf -s "[$PROGRAM@$(hostname -s)] ERROR: $m" "$@" ${MAILTO} 18 | } 19 | 20 | # run a hook 21 | @@ -284,7 +282,7 @@ savelog() { 22 | # Return rsync version 23 | rsync_protocol() { 24 | RSYNC_VERSION="$(${RSYNC} --version)" 25 | - RSYNC_REGEX="(protocol[ ]+version[ ]+([0-9]+))" 26 | + RSYNC_REGEX="(protocol[ ]+version[ ]+([0-9]+))" 27 | if [[ ${RSYNC_VERSION} =~ ${RSYNC_REGEX} ]]; then 28 | echo ${BASH_REMATCH[2]} 29 | fi 30 | @@ -358,62 +356,3 @@ join_by() { 31 | shift 32 | echo $* 33 | } 34 | - 35 | -# Sends mail 36 | -# mailf [-a attachment] [-b body] [-s subject] to-addr ... 37 | -mailf() { 38 | - local boundary="==--$RANDOM--$RANDOM--$RANDOM--==" 39 | - local attachment=() 40 | - local body=() 41 | - local subject= 42 | - 43 | - OPTIND=1 44 | - while getopts ":a:b:s:" arg; do 45 | - case $arg in 46 | - a) 47 | - attachment+=("$OPTARG") 48 | - ;; 49 | - b) 50 | - body+=("$OPTARG") 51 | - ;; 52 | - s) 53 | - subject="$OPTARG" 54 | - ;; 55 | - esac 56 | - done 57 | - shift $((OPTIND-1)) 58 | - 59 | - ( 60 | - cat < /dev/null 163 | 164 | rm -f "${LOCK}" 165 | 166 | @@ -381,7 +359,6 @@ create_logdir 167 | ######################################################################## 168 | MIRRORNAME=${MIRRORNAME:-$(hostname -f)} 169 | TO=${TO:-"/srv/mirrors/debian/"} 170 | -MAILTO=${MAILTO:-${LOGNAME:?Environment variable LOGNAME unset, please check your system or specify MAILTO}} 171 | HUB=${HUB:-"false"} 172 | 173 | # Connection options 174 | @@ -568,7 +545,7 @@ if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then 175 | fi 176 | 177 | # We want to cleanup always 178 | -trap cleanup EXIT TERM HUP INT QUIT 179 | +trap cleanup EXIT 180 | 181 | # Open log and close stdin 182 | open_log $LOG 183 | @@ -751,39 +728,3 @@ fi 184 | 185 | # Remove the Archive-Update-in-Progress file before we push our downstreams. 186 | rm -f "${LOCK}" 187 | - 188 | -declare -f -F send_mail_new_version > /dev/null && send_mail_new_version || : 189 | - 190 | -if [[ ${HUB} = true ]]; then 191 | - # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop 192 | - if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]] || [[ true = ${SYNCMHOP} ]]; then 193 | - RUNMIRRORARGS="" 194 | - if [[ -n ${ARCHIVE} ]]; then 195 | - # We tell runmirrors about the archive we are running on. 196 | - RUNMIRRORARGS="-a ${ARCHIVE}" 197 | - fi 198 | - # We also tell runmirrors that we are running it from within ftpsync, so it can change 199 | - # the way it works with mhop based on that. 200 | - RUNMIRRORARGS="${RUNMIRRORARGS} -f" 201 | - 202 | - if [[ true = ${SYNCSTAGE1} ]]; then 203 | - # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will 204 | - # not get to this point. 205 | - # So if that happens, tell runmirrors we are doing mhop 206 | - RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" 207 | - elif [[ true = ${SYNCSTAGE2} ]]; then 208 | - RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" 209 | - elif [[ true = ${SYNCALL} ]]; then 210 | - RUNMIRRORARGS="${RUNMIRRORARGS} -k all" 211 | - fi 212 | - log "Trigger slave mirrors using ${RUNMIRRORARGS}" 213 | - ${BINDIR:+${BINDIR}/}runmirrors ${RUNMIRRORARGS} 214 | - log "Trigger slave done" 215 | - 216 | - HOOK=( 217 | - HOOKNR=5 218 | - HOOKSCR="${HOOK5}" 219 | - ) 220 | - hook $HOOK 221 | - fi 222 | -fi 223 | -------------------------------------------------------------------------------- /base/Dockerfile.alpine: -------------------------------------------------------------------------------- 1 | FROM alpine:3.20 2 | LABEL maintainer="Jian Zeng " \ 3 | org.ustcmirror.images=true 4 | RUN < /etc/timezone 10 | dpkg-reconfigure -f noninteractive tzdata 11 | apt-get update && apt-get install -y wget 12 | wget -O /usr/local/bin/su-exec https://ftp.lug.ustc.edu.cn/misc/su-exec 13 | chmod +x /usr/local/bin/su-exec 14 | echo "592f25c51d0e4c90945ece8c4fa35018d20a1091ac109c98b66eb95deef211c7 /usr/local/bin/su-exec" | sha256sum -c - 15 | apt-get purge -y --auto-remove wget 16 | EOF 17 | ADD ["entry.sh", "savelog", "/usr/local/bin/"] 18 | VOLUME ["/data", "/log"] 19 | CMD ["entry.sh"] 20 | -------------------------------------------------------------------------------- /base/entry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | log() { 4 | echo "$@" >&2 5 | } 6 | 7 | killer() { 8 | kill -- "$1" 9 | wait "$1" 10 | } 11 | 12 | rotate_log() { 13 | su-exec "$OWNER" savelog -c "$LOG_ROTATE_CYCLE" "$LOGFILE" 14 | } 15 | 16 | export DEBUG="${DEBUG:-false}" 17 | [[ $DEBUG = true ]] && set -x 18 | set -u 19 | 20 | export SYNC_SCRIPT=${SYNC_SCRIPT:-"/sync.sh"} 21 | export PRE_SYNC_SCRIPT=${PRE_SYNC_SCRIPT:-"/pre-sync.sh"} 22 | export POST_SYNC_SCRIPT=${POST_SYNC_SCRIPT:-"/post-sync.sh"} 23 | export OWNER="${OWNER:-0:0}" 24 | export LOG_ROTATE_CYCLE="${LOG_ROTATE_CYCLE:-0}" 25 | export TO="${TO:-/data/}" 26 | export LOGDIR="${LOGDIR:-/log/}" 27 | export LOGFILE="$LOGDIR/result.log" 28 | declare -i RETRY 29 | export RETRY="${RETRY:-0}" 30 | 31 | main() { 32 | local abort ret 33 | 34 | if [[ ! -x $SYNC_SCRIPT ]]; then 35 | log "$SYNC_SCRIPT not found" 36 | return 1 37 | fi 38 | 39 | chown "$OWNER" "$TO" # not recursive 40 | 41 | [[ -f $PRE_SYNC_SCRIPT ]] && . "$PRE_SYNC_SCRIPT" 42 | 43 | date '+============ SYNC STARTED AT %F %T ============' 44 | 45 | abort=0 46 | while [[ $RETRY -ge 0 ]] && [[ $abort -eq 0 ]]; do 47 | log "*********** 8< ***********" 48 | su-exec "$OWNER" "$SYNC_SCRIPT" & 49 | pid="$!" 50 | trap 'killer $pid; abort=1; log Aborted' INT HUP TERM 51 | wait "$pid" 52 | ret="$?" 53 | [[ $ret -eq 0 ]] && break 54 | RETRY=$((RETRY-1)) 55 | done 56 | 57 | date '+============ SYNC FINISHED AT %F %T ============' 58 | 59 | [[ -f $POST_SYNC_SCRIPT ]] && . "$POST_SYNC_SCRIPT" 60 | 61 | return $ret 62 | } 63 | 64 | if [[ $LOG_ROTATE_CYCLE -ne 0 ]]; then 65 | trap 'rotate_log' EXIT 66 | touch "$LOGFILE" && chown "$OWNER" "$LOGFILE" 67 | else 68 | LOGFILE='/dev/null' 69 | fi 70 | 71 | main &> >(tee -a "$LOGFILE") 72 | -------------------------------------------------------------------------------- /base/pip.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | break-system-packages = true 3 | -------------------------------------------------------------------------------- /base/savelog: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # savelog - save a log file 3 | # Copyright (C) 1987, 1988 Ronald S. Karr and Landon Curt Noll 4 | # Copyright (C) 1992 Ronald S. Karr 5 | # Slight modifications by Ian A. Murdock : 6 | # * uses `gzip' rather than `compress' 7 | # * doesn't use $savedir; keeps saved log files in the same directory 8 | # * reports successful rotation of log files 9 | # * for the sake of consistency, files are rotated even if they are 10 | # empty 11 | # More modifications by Guy Maor : 12 | # * cleanup. 13 | # * -p (preserve) option 14 | # 15 | # usage: savelog [-m mode] [-u user] [-g group] [-t] [-p] [-c cycle] 16 | # [-j] [-C] [-d] [-l] [-r rolldir] [-n] [-q] file... 17 | # -m mode - chmod log files to mode 18 | # -u user - chown log files to user 19 | # -g group - chgrp log files to group 20 | # -c cycle - save cycle versions of the logfile (default: 7) 21 | # -r rolldir- use rolldir instead of . to roll files 22 | # -C - force cleanup of cycled logfiles 23 | # -d - use standard date for rolling 24 | # -D - override date format for -d 25 | # -t - touch file 26 | # -l - don't compress any log files (default: compress) 27 | # -p - preserve mode/user/group of original file 28 | # -j - use bzip2 instead of gzip 29 | # -J - use xz instead of gzip 30 | # -1 .. -9 - compression strength or memory usage (default: 9, except for xz) 31 | # -x script - invoke script with rotated log file in $FILE 32 | # -n - do not rotate empty files 33 | # -q - be quiet 34 | # file - log file names 35 | # 36 | # The savelog command saves and optionally compresses old copies of files. 37 | # Older version of 'file' are named: 38 | # 39 | # 'file'. 40 | # 41 | # where is the version number, 0 being the newest. By default, 42 | # version numbers > 0 are compressed (unless -l prevents it). The 43 | # version number 0 is never compressed on the off chance that a process 44 | # still has 'file' opened for I/O. 45 | # 46 | # if the '-d' option is specified, will be YYMMDDhhmmss 47 | # 48 | # If the 'file' does not exist and -t was given, it will be created. 49 | # 50 | # For files that do exist and have lengths greater than zero, the following 51 | # actions are performed. 52 | # 53 | # 1) Version numered files are cycled. That is version 6 is moved to 54 | # version 7, version is moved to becomes version 6, ... and finally 55 | # version 0 is moved to version 1. Both compressed names and 56 | # uncompressed names are cycled, regardless of -t. Missing version 57 | # files are ignored. 58 | # 59 | # 2) The new file.1 is compressed and is changed subject to 60 | # the -m, -u and -g flags. This step is skipped if the -t flag 61 | # was given. 62 | # 63 | # 3) The main file is moved to file.0. 64 | # 65 | # 4) If the -m, -u, -g, -t, or -p flags are given, then the file is 66 | # touched into existence subject to the given flags. The -p flag 67 | # will preserve the original owner, group, and permissions. 68 | # 69 | # 5) The new file.0 is changed subject to the -m, -u and -g flags. 70 | # 71 | # Note: If no -m, -u, -g, -t, or -p is given, then the primary log file is 72 | # not created. 73 | # 74 | # Note: Since the version numbers start with 0, version number 75 | # is never formed. The count must be at least 2. 76 | # 77 | # Bugs: If a process is still writing to the file.0 and savelog 78 | # moved it to file.1 and compresses it, data could be lost. 79 | # Smail does not have this problem in general because it 80 | # restats files often. 81 | 82 | # common location 83 | export PATH=$PATH:/sbin:/bin:/usr/sbin:/usr/bin 84 | COMPRESS="gzip" 85 | COMPRESS_OPTS="-f" 86 | COMPRESS_STRENGTH_DEF="-9"; 87 | DOT_Z=".gz" 88 | DATUM=`date +%Y%m%d%H%M%S` 89 | 90 | # parse args 91 | exitcode=0 # no problems to far 92 | prog=`basename $0` 93 | mode= 94 | user= 95 | group= 96 | touch= 97 | forceclean= 98 | rolldir= 99 | datum= 100 | preserve= 101 | hookscript= 102 | quiet=0 103 | rotateifempty=yes 104 | count=7 105 | 106 | usage() 107 | { 108 | echo "Usage: $prog [-m mode] [-u user] [-g group] [-t] [-c cycle] [-p]" 109 | echo " [-j] [-C] [-d] [-l] [-r rolldir] [-n] [-q] file ..." 110 | echo " -m mode - chmod log files to mode" 111 | echo " -u user - chown log files to user" 112 | echo " -g group - chgrp log files to group" 113 | echo " -c cycle - save cycle versions of the logfile (default: 7)" 114 | echo " -r rolldir - use rolldir instead of . to roll files" 115 | echo " -C - force cleanup of cycled logfiles" 116 | echo " -d - use standard date for rolling" 117 | echo " -D - override date format for -d" 118 | echo " -t - touch file" 119 | echo " -l - don't compress any log files (default: compress)" 120 | echo " -p - preserve mode/user/group of original file" 121 | echo " -j - use bzip2 instead of gzip" 122 | echo " -J - use xz instead of gzip" 123 | echo " -1 .. -9 - compression strength or memory usage (default: 9, except for xz)" 124 | echo " -x script - invoke script with rotated log file in \$FILE" 125 | echo " -n - do not rotate empty files" 126 | echo " -q - suppress rotation message" 127 | echo " file - log file names" 128 | } 129 | 130 | 131 | fixfile() 132 | { 133 | if [ -n "$user" ]; then 134 | chown -- "$user" "$1" 135 | fi 136 | if [ -n "$group" ]; then 137 | chgrp -- "$group" "$1" 138 | fi 139 | if [ -n "$mode" ]; then 140 | chmod -- "$mode" "$1" 141 | fi 142 | } 143 | 144 | 145 | while getopts m:u:g:c:r:CdD:tlphjJ123456789x:nq opt ; do 146 | case "$opt" in 147 | m) mode="$OPTARG" ;; 148 | u) user="$OPTARG" ;; 149 | g) group="$OPTARG" ;; 150 | c) count="$OPTARG" ;; 151 | r) rolldir="$OPTARG" ;; 152 | C) forceclean=1 ;; 153 | d) datum=1 ;; 154 | D) DATUM=$(date +$OPTARG) ;; 155 | t) touch=1 ;; 156 | j) COMPRESS="bzip2"; COMPRESS_OPTS="-f"; COMPRESS_STRENGTH_DEF="-9"; DOT_Z=".bz2" ;; 157 | J) COMPRESS="xz"; COMPRESS_OPTS="-f"; COMPRESS_STRENGTH_DEF=""; DOT_Z=".xz" ;; 158 | [1-9]) COMPRESS_STRENGTH="-$opt" ;; 159 | x) hookscript="$OPTARG" ;; 160 | l) COMPRESS="" ;; 161 | p) preserve=1 ;; 162 | n) rotateifempty="no" ;; 163 | q) quiet=1 ;; 164 | h) usage; exit 0 ;; 165 | *) usage; exit 1 ;; 166 | esac 167 | done 168 | 169 | shift $(($OPTIND - 1)) 170 | 171 | if [ "$count" -lt 2 ]; then 172 | echo "$prog: count must be at least 2" 1>&2 173 | exit 2 174 | fi 175 | 176 | if [ -n "$COMPRESS" ] && [ -z "`which $COMPRESS`" ]; then 177 | echo "$prog: Compression binary not available, please make sure '$COMPRESS' is installed" 1>&2 178 | exit 2 179 | fi 180 | 181 | if [ -n "$COMPRESS_STRENGTH" ]; then 182 | COMPRESS_OPTS="$COMPRESS_OPTS $COMPRESS_STRENGTH" 183 | else 184 | COMPRESS_OPTS="$COMPRESS_OPTS $COMPRESS_STRENGTH_DEF" 185 | fi 186 | 187 | # cycle thru filenames 188 | while [ $# -gt 0 ]; do 189 | 190 | # get the filename 191 | filename="$1" 192 | shift 193 | 194 | # catch bogus files 195 | if [ -e "$filename" ] && [ ! -f "$filename" ]; then 196 | echo "$prog: $filename is not a regular file" 1>&2 197 | exitcode=3 198 | continue 199 | fi 200 | 201 | # if file does not exist or is empty, and we've been told to not rotate 202 | # empty files, create if requested and skip to the next file. 203 | if [ ! -s "$filename" ] && [ "$rotateifempty" = "no" ]; then 204 | # if -t was given and it does not exist, create it 205 | if test -n "$touch" && [ ! -f "$filename" ]; then 206 | touch -- "$filename" 207 | if [ "$?" -ne 0 ]; then 208 | echo "$prog: could not touch $filename" 1>&2 209 | exitcode=4 210 | continue 211 | fi 212 | fixfile "$filename" 213 | fi 214 | continue 215 | # otherwise if the file does not exist and we've been told to rotate it 216 | # anyway, create an empty file to rotate. 217 | elif [ ! -e "$filename" ]; then 218 | touch -- "$filename" 219 | if [ "$?" -ne 0 ]; then 220 | echo "$prog: could not touch $filename" 1>&2 221 | exitcode=4 222 | continue 223 | fi 224 | fixfile "$filename" 225 | fi 226 | 227 | # be sure that the savedir exists and is writable 228 | # (Debian default: $savedir is . and not ./OLD) 229 | savedir=`dirname -- "$filename"` 230 | if [ -z "$savedir" ]; then 231 | savedir=. 232 | fi 233 | case "$rolldir" in 234 | (/*) 235 | savedir="$rolldir" 236 | ;; 237 | (*) 238 | savedir="$savedir/$rolldir" 239 | ;; 240 | esac 241 | if [ ! -d "$savedir" ]; then 242 | mkdir -p -- "$savedir" 243 | if [ "$?" -ne 0 ]; then 244 | echo "$prog: could not mkdir $savedir" 1>&2 245 | exitcode=5 246 | continue 247 | fi 248 | chmod 0755 -- "$savedir" 249 | fi 250 | if [ ! -w "$savedir" ]; then 251 | echo "$prog: directory $savedir is not writable" 1>&2 252 | exitcode=7 253 | continue 254 | fi 255 | 256 | # determine our uncompressed file names 257 | newname=`basename -- "$filename"` 258 | newname="$savedir/$newname" 259 | 260 | # cycle the old compressed log files 261 | cycle=$(( $count - 1)) 262 | rm -f -- "$newname.$cycle" "$newname.$cycle$DOT_Z" 263 | while [ $cycle -gt 1 ]; do 264 | # --cycle 265 | oldcycle=$cycle 266 | cycle=$(( $cycle - 1 )) 267 | # cycle log 268 | if [ -f "$newname.$cycle$DOT_Z" ]; then 269 | mv -f -- "$newname.$cycle$DOT_Z" \ 270 | "$newname.$oldcycle$DOT_Z" 271 | fi 272 | if [ -f "$newname.$cycle" ]; then 273 | # file was not compressed. move it anyway 274 | mv -f -- "$newname.$cycle" "$newname.$oldcycle" 275 | fi 276 | done 277 | 278 | # compress the old uncompressed log if needed 279 | if [ -f "$newname.0" ]; then 280 | if [ -z "$COMPRESS" ]; then 281 | newfile="$newname.1" 282 | mv -- "$newname.0" "$newfile" 283 | else 284 | newfile="$newname.1$DOT_Z" 285 | # $COMPRESS $COMPRESS_OPTS < $newname.0 > $newfile 286 | # rm -f $newname.0 287 | $COMPRESS $COMPRESS_OPTS "$newname.0" 288 | mv -- "$newname.0$DOT_Z" "$newfile" 289 | fi 290 | fixfile "$newfile" 291 | fi 292 | 293 | # compress the old uncompressed log if needed 294 | if test -n "$datum" && test -n "$COMPRESS"; then 295 | $COMPRESS $COMPRESS_OPTS -- "$newname".[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9] 296 | fi 297 | 298 | # remove old files if so desired 299 | if [ -n "$forceclean" ]; then 300 | cycle=$(( $count - 1)) 301 | if [ -z "$COMPRESS" ]; then 302 | list=$(ls -t -- $newname.[0-9]* 2>/dev/null | sed -e 1,${cycle}d) 303 | if [ -n "$list" ]; then 304 | rm -f -- $list 305 | fi 306 | else 307 | list=$(ls -t -- $newname.[0-9]*$DOT_Z 2>/dev/null | sed -e 1,${cycle}d) 308 | if [ -n "$list" ]; then 309 | rm -f -- $list 310 | fi 311 | fi 312 | fi 313 | 314 | # create new file if needed 315 | if [ -n "$preserve" ]; then 316 | (umask 077 317 | touch -- "$filename.new" 318 | chown --reference="$filename" -- "$filename.new" 319 | chmod --reference="$filename" -- "$filename.new") 320 | filenew=1 321 | elif [ -n "$touch$user$group$mode" ]; then 322 | touch -- "$filename.new" 323 | fixfile "$filename.new" 324 | filenew=1 325 | fi 326 | 327 | newfilename="$newname.0" 328 | # link the file into the file.0 holding place 329 | if [ -f "$filename" ]; then 330 | if [ -n "$filenew" ]; then 331 | if ln -f -- "$filename" "$newfilename"; then 332 | mv -- "$filename.new" "$filename" 333 | else 334 | echo "Error hardlinking $filename to $newfilename" >&2 335 | exitcode=8 336 | continue 337 | fi 338 | else 339 | mv -- "$filename" "$newfilename" 340 | fi 341 | fi 342 | [ ! -f "$newfilename" ] && touch -- "$newfilename" 343 | fixfile "$newfilename" 344 | if [ -n "$datum" ]; then 345 | mv -- "$newfilename" "$newname.$DATUM" 346 | newfilename="$newname.$DATUM" 347 | fi 348 | 349 | if [ -n "$hookscript" ]; then 350 | FILE="$newfilename" $SHELL -c "$hookscript" || \ 351 | { 352 | ret=$? 353 | test "$quiet" -eq 1 || echo "Hook script failed with exit code $ret." 1>&2 354 | } 355 | fi 356 | 357 | # report successful rotation 358 | test "$quiet" -eq 1 || echo "Rotated \`$filename' at `date`." 359 | done 360 | exit $exitcode 361 | -------------------------------------------------------------------------------- /configure.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 -O 2 | 3 | import os 4 | from os import path 5 | import sys 6 | import glob 7 | import subprocess 8 | from datetime import datetime 9 | from collections import defaultdict 10 | 11 | 12 | def invoke(cmd, fail_fast=False): 13 | from subprocess import call, DEVNULL 14 | if fail_fast: 15 | ret = call(cmd, stdout=DEVNULL) 16 | if ret != 0: 17 | sys.exit(ret) 18 | else: 19 | ret = call(cmd, stdout=DEVNULL, stderr=DEVNULL) 20 | return ret 21 | 22 | 23 | class InvalidFrom(Exception): 24 | pass 25 | 26 | 27 | class NoBaseImage(Exception): 28 | pass 29 | 30 | 31 | class Git(): 32 | @staticmethod 33 | def is_invalid_commit(desc): 34 | return invoke(['git', 'cat-file', '-e', desc]) != 0 35 | 36 | @staticmethod 37 | def branch_exists(branch): 38 | return invoke(['git', 'rev-parse', '--verify', branch]) == 0 39 | 40 | @staticmethod 41 | def fetch_branch(branch): 42 | invoke([ 43 | 'git', 'config', 'remote.origin.fetch', 44 | '+refs/heads/*:refs/remotes/origin/*' 45 | ]) 46 | invoke(['git', 'fetch', 'origin', f'{branch}:{branch}'], fail_fast=True) 47 | 48 | @staticmethod 49 | def is_current_branch(branch): 50 | return subprocess.getoutput('git symbolic-ref --short HEAD') == branch 51 | 52 | 53 | class NaryTree(): 54 | """ 55 | A n-ary tree 56 | """ 57 | def __init__(self, name): 58 | self.name = name 59 | self._children = dict() 60 | 61 | def add_child(self, name): 62 | self._children[name] = NaryTree(name) 63 | 64 | def get_child(self, name): 65 | return self._children.get(name, None) 66 | 67 | def find_updated_images(self, check): 68 | q = list(self._children.values()) 69 | while True: 70 | if not q: 71 | break 72 | t = q.pop(0) 73 | encoded = encode_tag(t.name) 74 | img = encoded.split('.')[0] 75 | if img == "base": 76 | # special treatment of base images 77 | # as they are the only images that contain multiple tags 78 | # encoded names look like "base.alpine-edge", "base.debian", etc. 79 | img = f"base/Dockerfile.{encoded.split('.')[1]}" 80 | if not check(img): 81 | q.extend(t._children.values()) 82 | else: 83 | yield (t.name, self.name) 84 | yield from t.enum_all() 85 | 86 | def enum_all(self): 87 | """ 88 | Enumerate all derived images and images based on the derived images 89 | """ 90 | for c in self._children.keys(): 91 | yield (c, self.name) 92 | for c in self._children.values(): 93 | yield from c.enum_all() 94 | 95 | def print(self): 96 | self._print(0) 97 | 98 | def _print(self, lvl): 99 | print(' ' * lvl, end='') 100 | print(self.name) 101 | for v in self._children.values(): 102 | v._print(lvl + 4) 103 | 104 | 105 | class Differ(): 106 | def __init__(self, prev, now): 107 | self._prev = prev 108 | self._now = now 109 | 110 | def changed(self, img): 111 | return invoke( 112 | ['git', 'diff', '--quiet', self._prev, self._now, '--', img]) != 0 113 | 114 | 115 | class Builder(): 116 | def __init__(self): 117 | self._targets = {} 118 | self._now = datetime.today().strftime('%Y%m%d') 119 | self._bases = defaultdict(list) 120 | self._dep_tree = NaryTree('') 121 | 122 | def __enter__(self): 123 | self._fout = open('Makefile', 'w') 124 | self._fout.write('.PHONY: all\r\n') 125 | return self 126 | 127 | def __exit__(self, *args): 128 | self._fout.close() 129 | 130 | def add(self, img, base): 131 | self._bases[base].append(img) 132 | 133 | def finish(self): 134 | root = self._dep_tree 135 | self._build_tree(root) 136 | date_tag = os.environ.get('DATE_TAG', '') != '' 137 | self._generate(force_date_tag=date_tag) 138 | 139 | def _build_tree(self, root): 140 | for derived in self._bases[root.name]: 141 | root.add_child(derived) 142 | if derived in self._bases: 143 | sub = root.get_child(derived) 144 | self._build_tree(sub) 145 | 146 | def _generate(self, *, force_date_tag): 147 | all_targets = set() 148 | event_name = os.environ.get('GITHUB_EVENT_NAME', '') 149 | is_cron = event_name == 'schedule' 150 | 151 | if is_cron or event_name == "workflow_dispatch": 152 | # cron job, or manual "build all" trigger 153 | to_build = self._dep_tree.enum_all() 154 | else: 155 | # GitHub Actions ($COMMIT_FROM & $COMMIT_TO) 156 | commit_from = os.environ.get('COMMIT_FROM', '') 157 | commit_to = os.environ.get('COMMIT_TO', '') 158 | if event_name == "pull_request": 159 | commit_from = os.environ.get('GITHUB_BASE_REF', 'master') 160 | if not Git.branch_exists(commit_from): 161 | print(f'fetching {commit_from} branch...') 162 | Git.fetch_branch(commit_from) 163 | if commit_from and commit_to: 164 | commits_range = "{}...{}".format(commit_from, commit_to) 165 | else: 166 | # git clone --branch on travis 167 | # need to add master branch back 168 | if not Git.branch_exists('master'): 169 | print('fetching master branch...') 170 | Git.fetch_branch('master') 171 | commits_range = 'origin/master...HEAD' 172 | print('COMMITS_RANGE: {}'.format(commits_range)) 173 | prev, current = commits_range.split('...') 174 | if Git.is_invalid_commit(prev): 175 | if Git.is_current_branch('master'): 176 | # fallback 177 | print('invalid commit: {}, fallback to '.format( 178 | prev)) 179 | prev = 'HEAD~5' 180 | else: 181 | prev = 'origin/master' 182 | print('prev: {}'.format(prev)) 183 | print('current: {}'.format(current)) 184 | differ = Differ(prev, current) 185 | to_build = self._dep_tree.find_updated_images(differ.changed) 186 | 187 | for dst, base in to_build: 188 | encoded_dst = encode_tag(dst) 189 | encoded_base = encode_tag(base) 190 | img, tag = encoded_dst.split('.', 1) 191 | 192 | all_targets.add(encoded_dst) 193 | all_targets.add(encoded_base) 194 | 195 | self._print_target(encoded_dst, encoded_base) 196 | 197 | build_script = path.join(img, 'build') 198 | if os.access(build_script, os.X_OK): 199 | self._print_command('cd {} && ./build {}'.format(img, tag)) 200 | elif tag == 'latest': 201 | self._print_command('docker build -t {0} {1}/'.format( 202 | dst, img)) 203 | else: 204 | self._print_command( 205 | 'docker build -t {0} -f {1}/Dockerfile.{2} {1}/'.format( 206 | dst, img, tag)) 207 | 208 | if not is_cron or force_date_tag: 209 | if tag == 'latest': 210 | self._print_command('@docker tag {0} {1}'.format( 211 | dst, dst.replace('latest', self._now))) 212 | else: 213 | self._print_command('@docker tag {0} {0}-{1}'.format( 214 | dst, self._now)) 215 | 216 | self._fout.write('all: {}\r\n'.format(' '.join(all_targets))) 217 | 218 | def _print_target(self, target, dep): 219 | self._fout.write('{}: {}\r\n'.format(target, dep)) 220 | 221 | def _print_command(self, cmd): 222 | self._fout.write('\t' + cmd + '\r\n') 223 | 224 | 225 | def strip_prefix(s, prefix): 226 | if s.startswith(prefix): 227 | return s[len(prefix):] 228 | return s 229 | 230 | 231 | def encode_tag(tag): 232 | return strip_prefix(tag, 'ustcmirror/').replace(':', '.') 233 | 234 | 235 | def get_dest_image(img, f): 236 | n = path.basename(f) 237 | # tag may contain a dot 238 | tag = strip_prefix(n, 'Dockerfile') 239 | if tag: 240 | return 'ustcmirror/{}:{}'.format(img, tag[1:]) 241 | else: 242 | return 'ustcmirror/{}:latest'.format(img) 243 | 244 | 245 | def get_base_image(f): 246 | with open(f) as fin: 247 | # Read Dockerfile in reverse order to get the base image 248 | for l in reversed(fin.readlines()): 249 | l = l.strip() 250 | if not l.startswith('FROM'): 251 | continue 252 | s = l.split() 253 | if len(s) != 2: 254 | raise InvalidFrom(f) 255 | tag = s[1] 256 | if ':' not in tag: 257 | return tag + ':latest' 258 | return tag 259 | raise NoBaseImage(f) 260 | 261 | 262 | def find_all_images(d): 263 | root, dirs, _ = next(os.walk(d)) 264 | imgs = {} 265 | for d in dirs: 266 | rule = path.join(root, d, 'Dockerfile*') 267 | files = glob.glob(rule) 268 | if not files: 269 | continue 270 | for f in files: 271 | dst_img = get_dest_image(d, f) 272 | base_img = get_base_image(f) 273 | imgs[dst_img] = base_img 274 | return imgs 275 | 276 | 277 | def main(): 278 | here = os.getcwd() 279 | 280 | imgs = find_all_images(here) 281 | 282 | with Builder() as b: 283 | for dst, base in imgs.items(): 284 | if base.startswith('ustcmirror'): 285 | b.add(dst, base) 286 | else: 287 | b.add(dst, '') 288 | b.finish() 289 | 290 | return 0 291 | 292 | 293 | if __name__ == '__main__': 294 | sys.exit(main()) 295 | -------------------------------------------------------------------------------- /crates-io-index/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Keyu Tao " 3 | RUN apk add --no-cache git 4 | ADD sync.sh / 5 | -------------------------------------------------------------------------------- /crates-io-index/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Migrated to yuki (docker) by @taoky 4 | # Created by @ksqsf 5 | # Based on the original version written by @knight42 6 | 7 | # ENVS: 8 | # CRATES_PROXY= 9 | # CRATES_GITMSG= 10 | # CRATES_GITMAIL= 11 | # CRATES_GITNAME= 12 | 13 | is_empty() { 14 | [[ -z $(ls -A "$1" 2>/dev/null) ]] 15 | } 16 | 17 | set -e 18 | [[ -n $DEBUG ]] && set -x 19 | 20 | CRATES_PROXY="${CRATES_PROXY:-https://crates-io.proxy.ustclug.org/api/v1/crates}" 21 | CRATES_GITMSG="${CRATES_GITMSG:-Redirect to USTC Mirrors}" 22 | CRATES_GITMAIL="${CRATES_GITMAIL:-lug AT ustc.edu.cn}" 23 | CRATES_GITNAME="${CRATES_GITNAME:-mirror}" 24 | GEOMETRIC_REPACK="${GEOMETRIC_REPACK:-false}" 25 | 26 | ensure_redirect() { 27 | pushd "$TO" 28 | if grep -F -q "$CRATES_PROXY" 'config.json'; then 29 | return 30 | else 31 | cat < 'config.json' 32 | { 33 | "dl": "$CRATES_PROXY", 34 | "api": "https://crates.io/" 35 | } 36 | EOF 37 | git add config.json 38 | git -c user.name="$CRATES_GITNAME" -c user.email="$CRATES_GITMAIL" commit -qm "$CRATES_GITMSG" 39 | fi 40 | popd 41 | } 42 | 43 | # crates.io-index has a custom ensure_redirect logic 44 | # so now we don't use gitsync here. 45 | 46 | if ! is_empty "$TO"; then 47 | cd "$TO" 48 | git fetch origin 49 | git reset --hard origin/master 50 | ensure_redirect 51 | if [[ $GEOMETRIC_REPACK == true ]]; then 52 | git repack --write-midx --write-bitmap-index -d --geometric=2 53 | else 54 | git repack -adb 55 | fi 56 | git gc --auto 57 | git update-server-info 58 | else 59 | git clone 'https://github.com/rust-lang/crates.io-index.git' "$TO" 60 | ensure_redirect 61 | fi 62 | -------------------------------------------------------------------------------- /curl-helper/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Yifan Gao docker@yfgao.com" 3 | LABEL bind_support=true 4 | RUN apk add --no-cache curl coreutils grep findutils bash 5 | ADD curl-helper.sh / 6 | -------------------------------------------------------------------------------- /curl-helper/curl-helper.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | [[ $DEBUG == true ]] && set -x 4 | 5 | urldecode() { 6 | : "${*//+/ }" 7 | echo -e "${_//\%/\\x}" 8 | } 9 | export -f urldecode 10 | 11 | download() { 12 | if [[ $enable_checksum == "true" ]]; then 13 | download_with_checksum 14 | elif [[ $enable_mtime == "true" ]]; then 15 | download_with_mtime 16 | else 17 | download_with_checksum 18 | fi 19 | } 20 | export -f download 21 | 22 | download_with_checksum() { 23 | local local_dir=${local_dir:="$TO"} 24 | local remote_url=${remote_url:=""} 25 | local by_hash=${by_hash:-"$local_dir/.by-hash"} 26 | [[ $DEBUG == true ]] && set -x 27 | curl_init 28 | mkdir -p $by_hash || return 1 29 | while read checksum path alt_path; do 30 | [[ $enable_urldecode == "true" ]] && path=$(urldecode "$path") 31 | if [[ $enable_alternative_path == "true" ]]; then 32 | local p=$local_dir/$alt_path 33 | local url=$path 34 | else 35 | local p=$local_dir/$path 36 | local url=$remote_url/$path 37 | fi 38 | local c=$by_hash/$checksum 39 | if [[ -f $c ]]; then 40 | if [[ ! $c -ef $p ]]; then 41 | mkdir -p $(dirname $p) 42 | ln -f $c $p 43 | fi 44 | else 45 | echo "[INFO] download $url" 46 | $CURL_WRAP -m 600 -sSfRL --create-dirs -o $c.tmp $url 47 | if [[ $? -ne 0 ]]; then 48 | echo "[WARN] download failed $url" 49 | rm -f $c.tmp 50 | continue 51 | fi 52 | 53 | if echo $checksum $c.tmp | sha256sum -c --quiet --status ; then 54 | mv $c.tmp $c 55 | mkdir -p $(dirname $p) 56 | ln -f $c $p 57 | else 58 | echo "[WARN] checksum mismatch $url" 59 | rm -f $c.tmp 60 | fi 61 | fi 62 | done 63 | } 64 | export -f download_with_checksum 65 | 66 | download_with_mtime() { 67 | local local_dir=${local_dir:="$TO"} 68 | local remote_url=${remote_url:=""} 69 | local fail_to_exit=${fail_to_exit:="false"} 70 | curl_init 71 | [[ $DEBUG == true ]] && echo "[DEBUG] fail_to_exit=${fail_to_exit}" 72 | while read path; do 73 | [[ $enable_urldecode == "true" ]] && path=$(urldecode "$path") 74 | local p=$local_dir/$path 75 | local url=$remote_url/$path 76 | if [[ -f $p ]]; then 77 | local remote_mtime=$($CURL_WRAP -sLI $url | grep -oP '(?<=^Last-Modified: ).+$') 78 | local remote_mtime=$(date --date="$remote_mtime" +%s) 79 | local local_mtime=$(stat -c %Y "$p") 80 | if [[ $local_mtime -eq $remote_mtime ]] ; then 81 | echo "[INFO] skip $url" 82 | continue 83 | fi 84 | fi 85 | $CURL_WRAP -m 600 -sSfRL --create-dirs -o $p $url 86 | if [[ $? -ne 0 ]]; then 87 | echo "[WARN] download failed $url" 88 | [[ $fail_to_exit != false ]] && return 1 89 | else 90 | echo "[INFO] downloaded $url" 91 | fi 92 | done 93 | } 94 | export -f download_with_mtime 95 | 96 | 97 | is_ipv6() { 98 | # string contains a colon 99 | [[ $1 =~ .*: ]] 100 | } 101 | export -f is_ipv6 102 | 103 | curl_init() { 104 | [[ -n $CURL_WRAP ]] && return 105 | if [[ -n $BIND_ADDRESS ]]; then 106 | if is_ipv6 "$BIND_ADDRESS"; then 107 | CURL_WRAP="curl -6 --interface $BIND_ADDRESS" 108 | else 109 | CURL_WRAP="curl -4 --interface $BIND_ADDRESS" 110 | fi 111 | else 112 | CURL_WRAP="curl" 113 | fi 114 | export CURL_WRAP 115 | } 116 | export -f curl_init 117 | 118 | 119 | clean_hash_file() { 120 | # purge old hash files 121 | echo "[INFO] clean hash files" 122 | find $by_hash -type f -links 1 -delete 123 | } 124 | export -f clean_hash_file 125 | -------------------------------------------------------------------------------- /debian-cd/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Jian Zeng " 3 | LABEL bind_support=true 4 | VOLUME ["/debian"] 5 | ADD ["sync.sh", "pre-sync.sh", "/"] 6 | ADD ["jigdo-mirror", "cd-mirror", "/usr/local/bin/"] 7 | ADD jigdo-mirror.conf.in /etc/jigdo/ 8 | RUN </dev/null 2>&1 69 | log "Debian-CD mirror sync done." 70 | savelog ${RSYNCLOG} >/dev/null 71 | savelog ${RSYNCELOG} >/dev/null 72 | savelog ${JIGDOLOG} >/dev/null 73 | } 74 | 75 | log() { 76 | echo "$(date +"%b %d %H:%M:%S") ${HOSTNAME} [$$] $@" 77 | } 78 | 79 | checklog() { 80 | if [ `grep -c '^total size is' ${RSYNCLOG} ` -ne $NRSYNCS ]; then 81 | log "Eeek. Debian jigdo rsync broke down..." 82 | exit 1 83 | fi 84 | } 85 | 86 | # Get in the right directory and set the umask to be group writable 87 | # 88 | cd $HOME 89 | umask 022 90 | 91 | # If we are here for the first time, create the 92 | # destination and the trace directory 93 | mkdir -p "${TO}/project/trace" 94 | 95 | touch ${UPDATEREQUIRED} 96 | 97 | # Check to see if another sync is in progress 98 | if lockfile -! -l 43200 -r 0 "$LOCK" >/dev/null 2>&1 ; then 99 | echo "Unable to start mirror sync, lock file $LOCK exists" 100 | exit 101 | fi 102 | 103 | log "Debian-CD mirror sync start" 104 | 105 | # Small pre-work clean up: 106 | [ -f ${RSYNCLOG} ] && rm -f ${RSYNCLOG} 107 | [ -f ${RSYNCELOG} ] && rm -f ${RSYNCELOG} 108 | 109 | trap cleanup EXIT 110 | 111 | PUSHFROM="${SSH_CONNECTION%%\ *}" 112 | if [ -n "${PUSHFROM}" ]; then 113 | log "We got pushed from ${PUSHFROM}" 114 | fi 115 | 116 | log "Acquired main lock" 117 | 118 | while [ -f ${UPDATEREQUIRED} ]; do 119 | 120 | log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" 121 | rm -f ${UPDATEREQUIRED} 122 | 123 | # Deleting the images before the run if DELETEFIRST is set. 124 | 125 | if [ $DELETEFIRST ]; then 126 | log "Removing images not on the other host: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPT_DELFIRST} ${RSYNC_HOST}::${RSYNC_MODULE} ${TO}" 127 | let NRSYNCS++ 128 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPT_DELFIRST} \ 129 | ${RSYNC_HOST}::${RSYNC_MODULE} ${TO} >>${RSYNCLOG} 2>>${RSYNCELOG} 130 | checklog 131 | fi 132 | 133 | log "Perform jigdo rsync stage: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${RSYNC_OPT_NOISO} ${RSYNC_HOST}::${RSYNC_MODULE} ${TO}" 134 | 135 | let NRSYNCS++ 136 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${RSYNC_OPT_NOISO} \ 137 | ${RSYNC_HOST}::${RSYNC_MODULE} ${TO} >>${RSYNCLOG} 2>>${RSYNCELOG} 138 | 139 | checklog 140 | 141 | if [ -n "$masterList" ]; then 142 | log "Receiving master list: ${RSYNC} $RSYNC_OPT_MASTERLIST ${RSYNC_HOST}::${RSYNC_MODULE}" 143 | ${RSYNC} $RSYNC_OPT_MASTERLIST \ 144 | ${RSYNC_HOST}::${RSYNC_MODULE} | awk '/^-/ {print $5}' >$masterList 145 | 146 | if [ -s "$masterList" ]; then 147 | log "Master list generated" 148 | else 149 | log "Master list generation failed" 150 | masterList="" 151 | fi 152 | fi 153 | 154 | log "Now generating images" 155 | 156 | typeset currentVersion=`ls -l ${TO}/current` 157 | currentVersion="${currentVersion##* -> }" 158 | 159 | versionDir="${TO}/${currentVersion}" 160 | 161 | mkdir -p ${LOGDIR}/jigdo/ >/dev/null 2>&1 162 | 163 | for a in ${versionDir}/*/; do 164 | arch=`basename $a` 165 | if [ -f "${TO}/project/build/${currentVersion}/${arch}" ]; then 166 | sets=`cat ${TO}/project/build/${currentVersion}/${arch}` 167 | else 168 | sets="cd dvd" 169 | fi 170 | 171 | for s in $sets; do 172 | typeset jigdoDir=${TO}/${currentVersion}/${arch}/jigdo-${s} 173 | typeset imageDir=${TO}/${currentVersion}/${arch}/iso-${s} 174 | 175 | [ -d $jigdoDir ] || continue 176 | 177 | if [ ! -d $imageDir ]; then 178 | log "Creating $imageDir" 179 | mkdir -p $imageDir 180 | fi 181 | 182 | [ -f $imageDir/MD5SUMS ] || cp $jigdoDir/MD5SUMS $imageDir/MD5SUMS 183 | 184 | echo "jigdoDir=$jigdoDir" > $jigdoConf.$arch.$s 185 | echo "imageDir=$imageDir" >> $jigdoConf.$arch.$s 186 | echo "tmpDir=$tmpDirBase/$arch.$s" >> $jigdoConf.$arch.$s 187 | echo "logfile=${LOGDIR}/jigdo/$arch.$s.log" >> $jigdoConf.$arch.$s 188 | echo "masterList=$masterList" >> $jigdoConf.$arch.$s 189 | cat ${jigdoConf}.in >> $jigdoConf.$arch.$s 190 | log "Start to jigdo $arch-$s" 191 | jigdo-mirror $jigdoConf.$arch.$s >> ${JIGDOLOG} 192 | done 193 | done 194 | 195 | [ -n "$masterList" -a -f "$masterList" ] && rm -f "$masterList" 196 | 197 | ls ${LOGDIR}/jigdo/*.log >/dev/null 2>&1 && savelog ${LOGDIR}/jigdo/*.log >/dev/null 198 | 199 | log "Image generation done" 200 | log "Doing final rsync: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} --size-only ${RSYNC_HOST}::${RSYNC_MODULE}/. ${TO}/." 201 | 202 | let NRSYNCS++ 203 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} \ 204 | --size-only \ 205 | ${RSYNC_HOST}::${RSYNC_MODULE}/. ${TO}/. >>${RSYNCLOG} 2>>${RSYNCELOG} 206 | 207 | checklog 208 | 209 | log "Final rsync with delete done" 210 | done 211 | 212 | rm -f $LOCK > /dev/null 2>&1 213 | 214 | if [ -d "`dirname "${TO}/${TRACE}"`" ]; then 215 | LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" 216 | echo "Used cd-mirror/jigdo script version: ${VERSION}" >> "${TO}/${TRACE}" 217 | echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" 218 | fi 219 | -------------------------------------------------------------------------------- /debian-cd/jigdo-mirror: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ustclug/ustcmirror-images/e154bfc34363410652a93d4098b25f78b8b20f98/debian-cd/jigdo-mirror -------------------------------------------------------------------------------- /debian-cd/jigdo-mirror.conf.in: -------------------------------------------------------------------------------- 1 | debianMirror="file:/debian/" 2 | -------------------------------------------------------------------------------- /debian-cd/pre-sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # override the `killer` func in entry.sh 4 | killer() { 5 | kill -- "$1" 6 | pkill rsync 7 | wait "$1" 8 | } 9 | 10 | IGNORE_LOCK="${IGNORE_LOCK:-false}" 11 | if [[ $IGNORE_LOCK = true ]]; then 12 | rm -f "$TO"/Archive-Update-* 13 | fi 14 | 15 | DELETEFIRST=${DELETEFIRST:-0} 16 | EXCLUDE=${EXCLUDE:-} 17 | 18 | cat > /etc/debian-cd-mirror.conf < create symlink 86 | return 87 | except Exception as e: 88 | print("Panic: get an exception while getting file list") 89 | traceback.print_exc() 90 | # We should exit whole program, because incomplete file list may let script delete existing files! 91 | # sys.exit(1) 92 | os._exit(1) # suicide 93 | return 94 | if not r.ok: 95 | return 96 | 97 | d = pq(r.text) 98 | for link in d('a'): 99 | if link.text.startswith('..'): 100 | continue 101 | href = base_url + link.text 102 | if filter_meta and self.is_metafile_url(href): 103 | self.meta_urls.append(href) 104 | elif link.text.endswith('/'): 105 | yield from self.recursive_get_filelist(href, filter_meta=filter_meta) 106 | else: 107 | yield href 108 | 109 | def relpath(self, url): 110 | assert url.startswith(self.base_url) 111 | return url[len(self.base_url):] 112 | 113 | @property 114 | def files(self): 115 | yield from self.recursive_get_filelist(self.base_url, filter_meta=True) 116 | for url in self.meta_urls: 117 | yield from self.recursive_get_filelist(url, filter_meta=False) 118 | 119 | 120 | def requests_download(remote_url: str, dst_file: Path): 121 | # NOTE the stream=True parameter below 122 | with requests.get(remote_url, timeout=TIMEOUT_OPTION, stream=True) as r: 123 | r.raise_for_status() 124 | remote_ts = parsedate_to_datetime( 125 | r.headers['last-modified']).timestamp() 126 | with open(dst_file, 'wb') as f: 127 | for chunk in r.iter_content(chunk_size=1024**2): 128 | if chunk: # filter out keep-alive new chunks 129 | f.write(chunk) 130 | # f.flush() 131 | os.utime(dst_file, (remote_ts, remote_ts)) 132 | 133 | 134 | def downloading_worker(q): 135 | while True: 136 | item = q.get() 137 | if item is None: 138 | break 139 | 140 | try: 141 | url, dst_file, working_dir = item 142 | if dst_file.is_file(): 143 | print("checking", url, flush=True) 144 | r = requests.head(url, timeout=TIMEOUT_OPTION, allow_redirects=True) 145 | remote_filesize = int(r.headers['content-length']) 146 | remote_date = parsedate_to_datetime(r.headers['last-modified']) 147 | stat = dst_file.stat() 148 | local_filesize = stat.st_size 149 | local_mtime = stat.st_mtime 150 | 151 | if remote_filesize == local_filesize and remote_date.timestamp() == local_mtime: 152 | print("skipping", dst_file.relative_to(working_dir), flush=True) 153 | continue 154 | 155 | dst_file.unlink() 156 | print("downloading", url, flush=True) 157 | requests_download(url, dst_file) 158 | except Exception: 159 | traceback.print_exc() 160 | print("Failed to download", url, flush=True) 161 | if dst_file.is_file(): 162 | dst_file.unlink() 163 | finally: 164 | q.task_done() 165 | 166 | 167 | def create_workers(n): 168 | task_queue = queue.Queue() 169 | for i in range(n): 170 | t = threading.Thread(target=downloading_worker, args=(task_queue, )) 171 | t.start() 172 | return task_queue 173 | 174 | def create_symlink(from_dir: Path, to_dir: Path): 175 | to_dir = to_dir.relative_to(from_dir.parent) 176 | if from_dir.exists(): 177 | if from_dir.is_symlink(): 178 | resolved_symlink = from_dir.resolve().relative_to(from_dir.parent.absolute()) 179 | if resolved_symlink != to_dir: 180 | print(f"WARN: The symlink {from_dir} dest changed from {resolved_symlink} to {to_dir}.") 181 | else: 182 | print(f"WARN: The symlink {from_dir} exists on disk but it is not a symlink.") 183 | else: 184 | if from_dir.is_symlink(): 185 | print(f"WARN: The symlink {from_dir} is probably invalid.") 186 | else: 187 | # create a symlink 188 | from_dir.parent.mkdir(parents=True, exist_ok=True) 189 | from_dir.symlink_to(to_dir) 190 | 191 | def main(): 192 | import argparse 193 | parser = argparse.ArgumentParser() 194 | parser.add_argument("--base-url", default=BASE_URL) 195 | parser.add_argument("--working-dir", default=WORKING_DIR) 196 | parser.add_argument("--workers", default=WORKERS, type=int, 197 | help='number of concurrent downloading jobs') 198 | parser.add_argument("--fast-skip", action='store_true', 199 | help='do not verify size and timestamp of existing package files') 200 | args = parser.parse_args() 201 | 202 | if args.working_dir is None: 203 | raise Exception("Working Directory is None") 204 | 205 | working_dir = Path(args.working_dir) 206 | task_queue = create_workers(args.workers) 207 | 208 | remote_filelist = [] 209 | rs = RemoteSite(args.base_url) 210 | for url in rs.files: 211 | if isinstance(url, tuple): 212 | from_dir, to_dir = url 213 | create_symlink(working_dir / from_dir, working_dir / to_dir) 214 | else: 215 | dst_file = working_dir / rs.relpath(url) 216 | remote_filelist.append(dst_file.relative_to(working_dir)) 217 | 218 | if dst_file.is_file(): 219 | if args.fast_skip and dst_file.suffix in ['.rpm', '.deb', '.tgz', '.zip']: 220 | print("fast skipping", dst_file.relative_to(working_dir), flush=True) 221 | continue 222 | else: 223 | dst_file.parent.mkdir(parents=True, exist_ok=True) 224 | 225 | task_queue.put((url, dst_file, working_dir)) 226 | 227 | # block until all tasks are done 228 | task_queue.join() 229 | # stop workers 230 | for i in range(args.workers): 231 | # TODO: it may hang here when workers threads exit unexpectedly 232 | task_queue.put(None) 233 | 234 | local_filelist = [] 235 | for local_file in working_dir.glob('**/*'): 236 | if local_file.is_file(): 237 | local_filelist.append(local_file.relative_to(working_dir)) 238 | 239 | for old_file in set(local_filelist) - set(remote_filelist): 240 | print("deleting", old_file, flush=True) 241 | old_file = working_dir / old_file 242 | old_file.unlink() 243 | 244 | 245 | if __name__ == "__main__": 246 | main() 247 | 248 | 249 | # vim: ts=4 sw=4 sts=4 expandtab 250 | -------------------------------------------------------------------------------- /fedora/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Shengjing Zhu " 3 | LABEL bind_support=true 4 | RUN < "$_CONF_FILE" << EOF 58 | DESTD=/mirror/ 59 | TIMEFILE=$LOGDIR/timefile 60 | REMOTE=$_REMOTE 61 | MODULES=($MODULE) 62 | RSYNCTIMEOUT=$_RSYNC_TIMEOUT 63 | RSYNCOPTS=(${_RSYNCOPTS[@]}) 64 | CHECKIN_SITE=${CHECKIN_SITE:-''} 65 | CHECKIN_PASSWORD=${CHECKIN_PASSWORD:-''} 66 | CHECKIN_HOST=${CHECKIN_HOST:-''} 67 | VERBOSE=$_VERBOSE 68 | LOGFILE=$LOGFILE 69 | FILTEREXP=${FILTEREXP:-''} 70 | EOF 71 | 72 | cat "$_CONF_FILE" 73 | -------------------------------------------------------------------------------- /fedora/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | [[ $DEBUG = true ]] && set -x 5 | 6 | exec quick-fedora-mirror 7 | -------------------------------------------------------------------------------- /flatpak/.gitignore: -------------------------------------------------------------------------------- 1 | summary.idx* 2 | summaries/ 3 | config 4 | -------------------------------------------------------------------------------- /flatpak/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Keyu Tao " 3 | 4 | RUN < $meta 64 | rm -f $tmpdir/packagesite.yaml 65 | export local_dir=$basedir 66 | enable_checksum=true parallel --line-buffer -j $FBSD_PKG_JOBS --pipepart -a $meta download 67 | 68 | # update meta-data 69 | rsync -a $tmpdir/ $basedir/ 70 | 71 | # purge old packages 72 | local removal_list=$(mktemp) 73 | comm -23 <(find All -type f | sort) <(awk '{print $2}' $meta) | tee $removal_list | xargs rm -f 74 | sed 's/^/[INFO] remove /g' $removal_list 75 | 76 | # clean temp file or dir 77 | rm -f $meta $removal_list 78 | rm -r $tmpdir 79 | 80 | echo "[INFO] sync finished $baseurl" 81 | } 82 | 83 | curl_init 84 | 85 | echo "[INFO] getting version list..." 86 | $CURL_WRAP -sSL $FBSD_PKG_UPSTREAM | grep -oP 'FreeBSD:[0-9]+:[a-z0-9]+' | grep -vP $FBSD_PKG_EXCLUDE | sort -t : -rnk 2 | uniq | tee $FBSD_PLATFORMS 87 | if [[ ${PIPESTATUS[0]} -ne 0 ]]; then 88 | echo "[FATAL] get version list from $FBSD_PKG_UPSTREAM failed." 89 | exit 1 90 | fi 91 | 92 | while read platform; do 93 | echo "[INFO] getting channel list of $platform..." 94 | channels=$($CURL_WRAP -sSL $FBSD_PKG_UPSTREAM/$platform | grep -oP 'latest|quarterly|base_[a-z0-9_]+|kmods_[a-z0-9_]+' | sort -t : -rnk 2 | uniq) 95 | echo $channels 96 | for channel in $channels; do 97 | if $CURL_WRAP -sLIf -o /dev/null $FBSD_PKG_UPSTREAM/$platform/$channel/packagesite.pkg; then 98 | channel_sync $FBSD_PKG_UPSTREAM/$platform/$channel $TO/$platform/$channel 99 | fi 100 | done 101 | done < $FBSD_PLATFORMS 102 | 103 | find $TO -type d -print0 | xargs -0 chmod 755 104 | 105 | rm $FBSD_PLATFORMS 106 | 107 | clean_hash_file 108 | 109 | exit $EXIT_CODE 110 | -------------------------------------------------------------------------------- /freebsd-ports/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/gitsync 2 | LABEL maintainer="Yifan Gao " 3 | LABEL bind_support=true 4 | RUN apk add --no-cache curl coreutils parallel gawk sed grep tar findutils bash 5 | ADD sync-ports.sh / 6 | ENV SYNC_SCRIPT=/sync-ports.sh 7 | -------------------------------------------------------------------------------- /freebsd-ports/sync-ports.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | [[ $DEBUG == true ]] && set -x 4 | 5 | FBSD_PORTS_INDEX_UPSTREAM=${FBSD_PORTS_INDEX_UPSTREAM:-"https://github.com/freebsd/freebsd-ports.git"} 6 | FBSD_PORTS_DISTFILES_UPSTREAM=${FBSD_PORTS_DISTFILES_UPSTREAM:-"http://distcache.freebsd.org/ports-distfiles"} 7 | FBSD_PORTS_JOBS=${FBSD_PORTS_JOBS:-1} 8 | tmpdir=$(mktemp -d) 9 | meta=$(mktemp) 10 | removal_list=$(mktemp) 11 | export BY_HASH=$(realpath $TO/.by-hash) 12 | 13 | is_ipv6() { 14 | # string contains a colon 15 | [[ $1 =~ .*: ]] 16 | } 17 | 18 | if [[ -n $BIND_ADDRESS ]]; then 19 | if is_ipv6 "$BIND_ADDRESS"; then 20 | CURL_WRAP="curl -6 --interface $BIND_ADDRESS" 21 | else 22 | CURL_WRAP="curl -4 --interface $BIND_ADDRESS" 23 | fi 24 | else 25 | CURL_WRAP="curl" 26 | fi 27 | export CURL_WRAP 28 | 29 | download_and_check() { 30 | cd $BY_HASH 31 | while read sum repopath; do 32 | if [[ -f $sum ]]; then 33 | if [[ ! $sum -ef $local_dir/$repopath ]]; then 34 | mkdir -p $(dirname $local_dir/$repopath) 35 | ln -f $sum $local_dir/$repopath 36 | fi 37 | else 38 | echo "[INFO] download $remote_url/$repopath" 39 | $CURL_WRAP -m 600 -sSfRL -o $sum.tmp --globoff $remote_url/$repopath 40 | if [[ $? -ne 0 ]]; then 41 | echo "[WARN] download failed $remote_url/$repopath" 42 | rm -f $sum.tmp 43 | continue 44 | fi 45 | 46 | if echo $sum $sum.tmp | sha256sum -c --quiet --status ; then 47 | mv $sum.tmp $sum 48 | mkdir -p $(dirname $local_dir/$repopath) 49 | ln -f $sum $local_dir/$repopath 50 | else 51 | echo "[WARN] checksum mismatch $remote_url/$repopath" 52 | rm -f $sum.tmp 53 | fi 54 | fi 55 | done 56 | } 57 | 58 | mkdir -p $TO/distfiles 59 | mkdir -p $BY_HASH || return 1 60 | 61 | # update meta 62 | TO=$TO/ports.git GITSYNC_URL=$FBSD_PORTS_INDEX_UPSTREAM GITSYNC_MIRROR=true GITSYNC_BITMAP=true /sync.sh 63 | if [[ $? -ne 0 ]]; then 64 | echo "[FATAL] download meta-data failed." 65 | exit 1 66 | fi 67 | 68 | # prepare meta list 69 | cd $tmpdir 70 | git -C $TO/ports.git archive HEAD | tar -xf - 71 | # remove tailing space each line to make awk correctly work 72 | find . -name distinfo -print0 | parallel -j 8 -0 --xargs cat | sed -E 's/\s+$//g' | awk '-F[() ]' '/^SHA256/ {print $NF,$3}' | sort -k 2 | uniq > $meta 73 | 74 | # clean unfinished downloads 75 | find $TO/distfiles -name '*.tmp' -delete 76 | 77 | # get distfile 78 | export PARALLEL_SHELL=/bin/bash 79 | export -f download_and_check 80 | remote_url=$FBSD_PORTS_DISTFILES_UPSTREAM local_dir=$TO/distfiles parallel --line-buffer -j $FBSD_PORTS_JOBS --pipepart -a $meta download_and_check 81 | 82 | # generate index archive 83 | git -C $TO/ports.git archive --format=tar.gz --prefix=ports/ -o $TO/ports.tar.gz HEAD 84 | 85 | # sanity check before removing old distfiles 86 | awk '{print $2}' $meta > /tmp/check1 87 | awk '{print $2}' $meta | sort > /tmp/check2 88 | if ! cmp -s /tmp/check1 /tmp/check2; then 89 | echo "[FATAL] sanity check failed: metafile is not correctly lexically sorted." 90 | exit 1 91 | fi 92 | 93 | # remove old distfile 94 | cd $TO/distfiles 95 | comm -23 <(find . -type f | sed 's/^\.\///g' | sort) <(awk '{print $2}' $meta) | tee $removal_list | xargs rm -f 96 | sed 's/^/[INFO] remove /g' $removal_list 97 | 98 | # fix dir mode 99 | find $TO -type d -print0 | xargs -0 chmod 755 100 | 101 | # cleam temp file 102 | rm -f $meta $removal_list 103 | rm -r $tmpdir 104 | 105 | # purge old hash files 106 | find $BY_HASH -type f -links 1 -print0 | xargs -0 rm -f 107 | -------------------------------------------------------------------------------- /ghcup/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:debian 2 | LABEL maintainer="Kai Ma " 3 | ADD ["sync.sh", "ghcupsync.hs", "/"] 4 | RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ 5 | --mount=type=cache,sharing=locked,target=/var/lib/apt <= 0.0.6) 5 | 3. 下载这些元数据文件中的链接. 链接 schema://host/path 被转换成 https://mirrors.ustc.edu.cn/ghcup/host/path 6 | 4. 遍历本地文件，删除元数据中没有引用的文件 7 | 5. 将元数据中的链接按上述方式替换成镜像地址 8 | -------------------------------------------------------------------------------- /ghcup/ghcupsync.cabal: -------------------------------------------------------------------------------- 1 | cabal-version: 2.4 2 | name: ghcupsync 3 | version: 0.1.0.0 4 | tested-with: 5 | ghc == 9.0.2 6 | executable ghcupsync 7 | main-is: ghcupsync.hs 8 | build-depends: aeson 9 | , base 10 | , containers 11 | , directory 12 | , filepath 13 | , lens 14 | , lens-aeson 15 | , network-uri 16 | , split 17 | , stm 18 | , text 19 | , typed-process 20 | , unix 21 | , yaml 22 | default-language: Haskell2010 23 | default-extensions: ScopedTypeVariables OverloadedStrings 24 | -------------------------------------------------------------------------------- /ghcup/ghcupsync.hs: -------------------------------------------------------------------------------- 1 | module Main where 2 | 3 | import Control.Applicative 4 | import Control.Concurrent 5 | import Control.Concurrent.STM.TSem 6 | import Control.Exception 7 | import Control.Lens 8 | import Control.Monad 9 | import Control.Monad.STM 10 | import qualified Data.Aeson as Aeson 11 | import Data.Aeson.Lens 12 | import Data.Foldable 13 | import Data.List 14 | import Data.List.Split 15 | import Data.Maybe 16 | import Data.Set (Set) 17 | import qualified Data.Set as Set 18 | import Data.Text (Text) 19 | import qualified Data.Text as Text 20 | import Data.Version 21 | import qualified Data.Yaml.Aeson as Yaml 22 | import GHC.Exts 23 | import Network.URI 24 | import System.Directory 25 | import System.Environment 26 | import System.FilePath 27 | import System.Posix.Directory 28 | import System.Process.Typed 29 | import Text.ParserCombinators.ReadP 30 | import Text.Printf 31 | import Text.Read 32 | import System.Posix.Files (createSymbolicLink) 33 | import Data.Function (on) 34 | 35 | type URL = Text 36 | type SHA256 = Text 37 | 38 | main :: IO () 39 | main = do 40 | basedir <- getEnv "TO" 41 | 42 | -- Set up metadata in a working directory 43 | gitClone ghcupMetadataRepo (mdtmpdir basedir) 44 | 45 | -- Download all artifacts referenced in _supported_ metadata files 46 | files <- listDirectory (mdtmpdir basedir) 47 | let mdfiles = onlySupported $ map parseVersionFromFileName $ filter ("yaml" `isExtensionOf`) files 48 | mdpaths = map (mdtmpdir basedir ) $ map filepath mdfiles 49 | for_ mdpaths $ \mdpath -> 50 | printf "Will sync: %s\n" mdpath 51 | mapM_ (syncByMetadata basedir) mdpaths 52 | 53 | -- Delete unreferenced files (before replacing URLs) 54 | garbageCollect basedir mdpaths 55 | 56 | -- group channels and create symlinks 57 | let 58 | mdByChannel = groupBy sameChannel $ 59 | sortOn channel mdfiles 60 | 61 | linkof :: Channel -> FilePath 62 | linkof channel = mdtmpdir basedir ("ghcup-" ++ channel ++ "latest.yaml") 63 | 64 | latests = maximumBy (compare `on` version) <$> mdByChannel 65 | link'target = do 66 | md <- latests 67 | return (linkof (channel md), filepath md) 68 | 69 | mapM_ link link'target 70 | 71 | -- Replace URLs in tmp metadata, and then copy these files 72 | mapM_ replaceUrls mdpaths 73 | enableMetadata basedir 74 | 75 | where 76 | onlySupported :: [Maybe Metadata] -> [Metadata] 77 | onlySupported xs = do 78 | (Just m) <- xs 79 | guard $ version m >= minimumSupportedVersion 80 | return m 81 | 82 | sameChannel :: Metadata -> Metadata -> Bool 83 | sameChannel m1 m2 = channel m1 == channel m2 84 | 85 | link :: (FilePath, FilePath) -> IO () 86 | link link'target = do 87 | createSymbolicLink (snd link'target) (fst link'target) 88 | createSymbolicLink (snd link'target ++ ".sig") (fst link'target ++ ".sig") 89 | 90 | ghcupMetadataRepo :: URL 91 | ghcupMetadataRepo = "https://github.com/haskell/ghcup-metadata" 92 | 93 | mirroredFileUrl :: IsString str => FilePath -> str 94 | mirroredFileUrl local = fromString $ printf "http://mirrors.ustc.edu.cn/ghcup/%s" local 95 | 96 | minimumSupportedVersion :: Version 97 | minimumSupportedVersion = makeVersion [0, 0, 6] -- Metadata format version 98 | 99 | type Channel = String 100 | 101 | data Metadata = Metadata { 102 | version :: Version, 103 | channel :: Channel, 104 | filepath :: FilePath 105 | } 106 | 107 | parseVersionFromFileName :: FilePath -> Maybe Metadata 108 | parseVersionFromFileName filename = do 109 | let basename = takeBaseName filename 110 | (channel, noPrefix) <- basename `tryChannel` "prereleases-" 111 | <|> basename `tryChannel` "cross-" 112 | <|> basename `tryChannel` "vanilla-" 113 | <|> basename `tryChannel` "" 114 | version <- listToMaybe $ map fst . filter (\(_, rem) -> null rem) $ readP_to_S parseVersion noPrefix 115 | return $ Metadata version channel filename 116 | where 117 | tryChannel basename channel = ((,) channel) <$> stripPrefix ("ghcup-" ++ channel) basename 118 | 119 | -- This version of parseVersionFromFileName may have more forward compatability 120 | -- but poorer performance 121 | -- parseVersionFromFileName filename = do 122 | -- let basename = takeBaseName filename 123 | -- case find isJust . map tryParse $ tails basename of 124 | -- (Just m) -> m -- if found, it will be a double Just, i.e. (Just (Just Version)) 125 | -- Nothing -> Nothing 126 | -- where 127 | -- tryParse :: String -> Maybe Version 128 | -- tryParse = listToMaybe . map fst . reverse . readP_to_S parseVersion 129 | 130 | ------------------------------------------------------------------------ 131 | syncByMetadata :: FilePath -> FilePath -> IO () 132 | syncByMetadata basedir mdpath = do 133 | md <- readMetadata mdpath 134 | let urls = md ^.. deep (key "dlUri" . _String) 135 | sha256s = md ^.. deep (key "dlHash" . _String) 136 | dlUris = Set.toList . Set.fromList $ zip urls sha256s 137 | nthreads = 4 138 | printf "Sync'ing metadata %s... \n" mdpath 139 | mapM_ (downloadConcurrently basedir) (chunksOf nthreads dlUris) 140 | 141 | downloadConcurrently :: FilePath -> [(URL, SHA256)] -> IO () 142 | downloadConcurrently basedir urls = do 143 | barrier <- atomically $ newTSem (1 - fromIntegral (length urls)) 144 | for_ urls $ \url -> forkIO $ do 145 | download basedir url 146 | atomically $ signalTSem barrier 147 | atomically $ waitTSem barrier 148 | 149 | download :: FilePath -> (URL, SHA256) -> IO () 150 | download basedir (url, sha256) = do 151 | let path = basedir urlExtractPath url 152 | exists <- doesFileExist path 153 | if exists 154 | then do printf "%s already exists. Skipped.\n" path 155 | else do printf "Downloading %s to %s ...\n" url path 156 | downloadFile url sha256 path 157 | 158 | where 159 | downloadFile :: URL -> SHA256 -> FilePath -> IO () 160 | downloadFile url sha256 path = do 161 | let dir = takeDirectory path 162 | filename = takeFileName path 163 | args = [Text.unpack url, 164 | "--dir=" ++ dir, 165 | "--out=" ++ filename ++ ".tmp", 166 | "--file-allocation=none", 167 | "--quiet=true", 168 | "--checksum=sha-256=" ++ Text.unpack sha256] 169 | createDirectoryIfMissing True dir 170 | exitCode <- runProcess (proc "aria2c" args) 171 | case exitCode of 172 | ExitSuccess -> do 173 | -- Atomically make the file visible. 174 | renameFile (dir filename ++ ".tmp") (dir filename) 175 | ExitFailure code -> do 176 | -- Probably due to Not Found errors. Just ignore the error 177 | -- so that the signalTSem can be reached, or there will be a 178 | -- deadlock. 179 | printf "! Couldn't download %s (exit code = %d)" url code 180 | removePathForcibly (dir filename ++ ".tmp") 181 | 182 | readMetadata :: FilePath -> IO Aeson.Value 183 | readMetadata path = either throwIO pure =<< Yaml.decodeFileEither path 184 | 185 | urlExtractPath :: Text -> FilePath 186 | urlExtractPath url = fromJust $ do 187 | uri <- parseURI (Text.unpack url) 188 | authority <- uriAuthority uri 189 | pure (uriRegName authority <> uriPath uri) 190 | 191 | ------------------------------------------------------------------------ 192 | mddir :: FilePath -> FilePath 193 | mddir base = base "ghcup-metadata" 194 | 195 | mdtmpdir :: FilePath -> FilePath 196 | mdtmpdir base = base "ghcup-metadata.tmp" 197 | 198 | gitClone :: URL -> FilePath -> IO () 199 | gitClone url path = do 200 | removePathForcibly path 201 | runProcess_ (proc "git" ["clone", "--depth=1", Text.unpack url, path]) 202 | 203 | ------------------------------------------------------------------------ 204 | replaceUrls :: FilePath -> IO () 205 | replaceUrls mdpath = do 206 | md <- readMetadata mdpath 207 | let md' = md & deep (key "dlUri" . _String) %~ localizeUrl 208 | Yaml.encodeFile mdpath md' 209 | where 210 | localizeUrl :: Text -> Text 211 | localizeUrl url = mirroredFileUrl (urlExtractPath url) 212 | 213 | enableMetadata :: FilePath -> IO () 214 | enableMetadata basedir = do 215 | removePathForcibly (mddir basedir) 216 | removePathForcibly (mdtmpdir basedir ".git") 217 | renamePath (mdtmpdir basedir) (mddir basedir) 218 | 219 | ------------------------------------------------------------------------ 220 | garbageCollect :: FilePath -> [FilePath] -> IO () 221 | garbageCollect basedir mdpaths = do 222 | printf "Garbage collecting...\n" 223 | 224 | -- Get all referenced paths 225 | let f path = do 226 | md <- readMetadata path 227 | return . Set.fromList . map urlExtractPath $ (md ^.. deep (key "dlUri" . _String)) 228 | keep <- foldMap f mdpaths 229 | 230 | -- List all local files and remove unused files 231 | let keepAnyway = ["ghcup-metadata.tmp", "ghcup-metadata", "sh"] 232 | files <- listDirectoryRecursive basedir 233 | for_ files $ \file -> 234 | unless (file `Set.member` keep || any (`isPrefixOf` file) keepAnyway) $ do 235 | printf "Delete unreferenced file %s\n" file 236 | removeFile (basedir file) 237 | 238 | where 239 | listDirectoryRecursive :: FilePath -> IO [FilePath] 240 | listDirectoryRecursive = go "" 241 | where 242 | go prefix curpath = do 243 | files <- listDirectory curpath 244 | let f file = do 245 | d <- doesDirectoryExist (curpath file) 246 | if d 247 | then go (prefix file) (curpath file) 248 | else pure [prefix file] 249 | foldMap f files 250 | -------------------------------------------------------------------------------- /ghcup/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | [[ $DEBUG = true ]] && set -x 5 | 6 | /ghcupsync 7 | 8 | mkdir -p "$TO/sh" 9 | pushd "$TO/sh" 10 | curl https://raw.githubusercontent.com/haskell/ghcup-hs/master/scripts/bootstrap/bootstrap-haskell -o bootstrap-haskell 11 | sed -i 's|https://downloads.haskell.org/~ghcup|https://mirrors.ustc.edu.cn/ghcup/downloads.haskell.org/~ghcup|' bootstrap-haskell 12 | sed -i '/edo cabal update/ i \ \ \ \ edo cabal user-config update -a "repository mirrors.ustc.edu.cn" -a " url: https://mirrors.ustc.edu.cn/hackage/" ' bootstrap-haskell 13 | # make ghcup init cabal config before cabal update 14 | curl https://raw.githubusercontent.com/haskell/ghcup-hs/master/scripts/bootstrap/bootstrap-haskell.ps1 -o bootstrap-haskell.ps1 15 | sed -i 's|https://www.haskell.org/ghcup/sh/bootstrap-haskell|https://mirrors.ustc.edu.cn/ghcup/sh/bootstrap-haskell|' bootstrap-haskell.ps1 16 | sed -i 's|https://repo.msys2.org/distrib/|https://mirrors.ustc.edu.cn/msys2/distrib/|' bootstrap-haskell.ps1 17 | popd 18 | -------------------------------------------------------------------------------- /github-release/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Yulong Ming " 3 | LABEL bind_support=true 4 | RUN apk add --no-cache python3 py3-requests py3-yaml 5 | ADD tunasync /usr/local/lib/tunasync 6 | ADD sync.sh / 7 | -------------------------------------------------------------------------------- /github-release/examples/repos.yaml: -------------------------------------------------------------------------------- 1 | - AdoptOpenJDK/openjdk8-binaries 2 | - AdoptOpenJDK/openjdk9-binaries 3 | - AdoptOpenJDK/openjdk10-binaries 4 | - AdoptOpenJDK/openjdk11-binaries 5 | - AdoptOpenJDK/openjdk12-binaries 6 | - AdoptOpenJDK/openjdk13-binaries 7 | - pbatard/rufus 8 | - Homebrew/homebrew-portable-ruby 9 | - a/b 10 | - repo: a/b 11 | tarball: true 12 | - repo: a/b 13 | version: -1 14 | # See also https://github.com/tuna/tunasync-scripts/blob/master/github-release.py#L15 15 | -------------------------------------------------------------------------------- /github-release/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | [ "$DEBUG" = "true" ] && set -x 4 | 5 | exec python3 /usr/local/lib/tunasync/github-release.py --working-dir "$TO" 6 | -------------------------------------------------------------------------------- /github-release/tunasync/github-release.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | import tempfile 6 | import threading 7 | import traceback 8 | import queue 9 | from pathlib import Path 10 | from datetime import datetime 11 | 12 | 13 | # Set BindIP 14 | # https://stackoverflow.com/a/70772914 15 | BINDIP = os.getenv("BIND_ADDRESS", "") 16 | if BINDIP: 17 | import urllib3 18 | real_create_conn = urllib3.util.connection.create_connection 19 | 20 | def set_src_addr(address, timeout, *args, **kw): 21 | source_address = (BINDIP, 0) 22 | return real_create_conn(address, timeout=timeout, source_address=source_address) 23 | 24 | urllib3.util.connection.create_connection = set_src_addr 25 | 26 | import requests 27 | import yaml 28 | 29 | 30 | BASE_URL = os.getenv("UPSTREAM_URL", "https://api.github.com/repos/") 31 | WORKING_DIR = os.getenv("TUNASYNC_WORKING_DIR") 32 | WORKERS = int(os.getenv("WORKERS", "8")) 33 | FAST_SKIP = bool(os.getenv("FAST_SKIP", "")) 34 | 35 | 36 | def get_repos(): 37 | try: 38 | with open('/repos.yaml') as f: 39 | content = f.read() 40 | except FileNotFoundError: 41 | content = os.getenv("REPOS", None) 42 | if content is None: 43 | raise Exception("Loading /repos.yaml file and reading REPOS env both failed") 44 | repos = yaml.safe_load(content) 45 | if isinstance(repos, list): 46 | return repos 47 | else: 48 | repos = repos['repos'] 49 | if not isinstance(repos, list): 50 | raise Exception("Can not inspect repo list from the given file/env") 51 | return repos 52 | 53 | 54 | REPOS = get_repos() 55 | 56 | # connect and read timeout value 57 | TIMEOUT_OPTION = (7, 10) 58 | total_size = 0 59 | 60 | 61 | def sizeof_fmt(num, suffix='iB'): 62 | for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: 63 | if abs(num) < 1024.0: 64 | return "%3.2f%s%s" % (num, unit, suffix) 65 | num /= 1024.0 66 | return "%.2f%s%s" % (num, 'Y', suffix) 67 | 68 | 69 | # wrap around requests.get to use token if available 70 | def github_get(*args, **kwargs): 71 | headers = kwargs.get('headers', {}) 72 | if 'GITHUB_TOKEN' in os.environ: 73 | headers['Authorization'] = 'token {}'.format(os.environ['GITHUB_TOKEN']) 74 | kwargs['headers'] = headers 75 | return requests.get(*args, **kwargs) 76 | 77 | 78 | def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size: int): 79 | # NOTE the stream=True parameter below 80 | with github_get(remote_url, stream=True) as r: 81 | r.raise_for_status() 82 | tmp_dst_file = None 83 | try: 84 | with tempfile.NamedTemporaryFile(prefix="." + dst_file.name + ".", suffix=".tmp", dir=dst_file.parent, delete=False) as f: 85 | tmp_dst_file = Path(f.name) 86 | for chunk in r.iter_content(chunk_size=1024**2): 87 | if chunk: # filter out keep-alive new chunks 88 | f.write(chunk) 89 | # f.flush() 90 | # check for downloaded size 91 | downloaded_size = tmp_dst_file.stat().st_size 92 | if remote_size is not None and downloaded_size != remote_size: 93 | raise ValueError(f'File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes') 94 | os.utime(tmp_dst_file, (remote_ts, remote_ts)) 95 | tmp_dst_file.chmod(0o644) 96 | tmp_dst_file.replace(dst_file) 97 | finally: 98 | if tmp_dst_file is not None: 99 | if tmp_dst_file.is_file(): 100 | tmp_dst_file.unlink() 101 | 102 | 103 | def downloading_worker(q): 104 | while True: 105 | item = q.get() 106 | if item is None: 107 | break 108 | 109 | url, dst_file, working_dir, updated, remote_size = item 110 | 111 | print("downloading", url, "to", 112 | dst_file.relative_to(working_dir), flush=True) 113 | try: 114 | do_download(url, dst_file, updated, remote_size) 115 | except Exception: 116 | print("Failed to download", url, "with this exception:",flush=True) 117 | traceback.print_exc() 118 | 119 | q.task_done() 120 | 121 | 122 | def create_workers(n): 123 | task_queue = queue.Queue() 124 | for i in range(n): 125 | t = threading.Thread(target=downloading_worker, args=(task_queue, )) 126 | t.start() 127 | return task_queue 128 | 129 | 130 | def ensure_safe_name(filename): 131 | filename = filename.replace('\0', ' ') 132 | if filename == '.': 133 | return ' .' 134 | elif filename == '..': 135 | return '. .' 136 | else: 137 | return filename.replace('/', '\\').replace('\\', '_') 138 | 139 | 140 | def main(): 141 | import argparse 142 | parser = argparse.ArgumentParser() 143 | parser.add_argument("--base-url", default=BASE_URL) 144 | parser.add_argument("--working-dir", default=WORKING_DIR) 145 | parser.add_argument("--workers", default=WORKERS, type=int, 146 | help='number of concurrent downloading jobs') 147 | parser.add_argument("--fast-skip", action='store_true', default=FAST_SKIP, 148 | help='do not verify size and timestamp of existing files') 149 | args = parser.parse_args() 150 | 151 | if args.working_dir is None: 152 | raise Exception("Working Directory is None") 153 | 154 | working_dir = Path(args.working_dir) 155 | task_queue = create_workers(args.workers) 156 | remote_filelist = [] 157 | cleaning = False 158 | 159 | def download(release, release_dir, tarball=False): 160 | global total_size 161 | 162 | if tarball: 163 | url = release['tarball_url'] 164 | updated = datetime.strptime( 165 | release['published_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() 166 | dst_file = release_dir / 'repo-snapshot.tar.gz' 167 | remote_filelist.append(dst_file.relative_to(working_dir)) 168 | # no size information, use None to skip size check 169 | remote_size = None 170 | 171 | if dst_file.is_file(): 172 | print("skipping", dst_file.relative_to(working_dir), flush=True) 173 | else: 174 | dst_file.parent.mkdir(parents=True, exist_ok=True) 175 | task_queue.put((url, dst_file, working_dir, updated, remote_size)) 176 | 177 | for asset in release['assets']: 178 | url = asset['browser_download_url'] 179 | updated = datetime.strptime( 180 | asset['updated_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp() 181 | dst_file = release_dir / ensure_safe_name(asset['name']) 182 | remote_filelist.append(dst_file.relative_to(working_dir)) 183 | remote_size = asset['size'] 184 | total_size += remote_size 185 | 186 | if dst_file.is_file(): 187 | if args.fast_skip: 188 | print("fast skipping", dst_file.relative_to( 189 | working_dir), flush=True) 190 | continue 191 | else: 192 | stat = dst_file.stat() 193 | local_filesize = stat.st_size 194 | local_mtime = stat.st_mtime 195 | # print(f"{local_filesize} vs {asset['size']}") 196 | # print(f"{local_mtime} vs {updated}") 197 | if local_mtime > updated or \ 198 | remote_size == local_filesize and local_mtime == updated: 199 | print("skipping", dst_file.relative_to( 200 | working_dir), flush=True) 201 | continue 202 | else: 203 | dst_file.parent.mkdir(parents=True, exist_ok=True) 204 | 205 | task_queue.put((url, dst_file, working_dir, updated, remote_size)) 206 | 207 | def link_latest(name, repo_dir): 208 | try: 209 | os.unlink(repo_dir / "LatestRelease") 210 | except OSError: 211 | pass 212 | try: 213 | os.symlink(name, repo_dir / "LatestRelease") 214 | except OSError: 215 | pass 216 | 217 | success = True 218 | 219 | for cfg in REPOS: 220 | flat = False # build a folder for each release 221 | versions = 1 # keep only one release 222 | tarball = False # do not download the tarball 223 | prerelease = False # filter out pre-releases 224 | if isinstance(cfg, str): 225 | repo = cfg 226 | else: 227 | repo = cfg["repo"] 228 | if "versions" in cfg: 229 | versions = cfg["versions"] 230 | if "flat" in cfg: 231 | flat = cfg["flat"] 232 | if "tarball" in cfg: 233 | tarball = cfg["tarball"] 234 | if "pre_release" in cfg: 235 | prerelease = cfg["pre_release"] 236 | 237 | repo_dir = working_dir / Path(repo) 238 | print(f"syncing {repo} to {repo_dir}") 239 | 240 | try: 241 | r = github_get(f"{args.base_url}{repo}/releases") 242 | r.raise_for_status() 243 | releases = r.json() 244 | except: 245 | traceback.print_exc() 246 | success = False 247 | break 248 | 249 | n_downloaded = 0 250 | for release in releases: 251 | if not release['draft'] and (prerelease or not release['prerelease']): 252 | name = ensure_safe_name(release['name'] or release['tag_name']) 253 | if len(name) == 0: 254 | print("Error: Unnamed release") 255 | continue 256 | download(release, (repo_dir if flat else repo_dir / name), tarball) 257 | if n_downloaded == 0 and not flat: 258 | # create a symbolic link to the latest release folder 259 | link_latest(name, repo_dir) 260 | n_downloaded += 1 261 | if versions > 0 and n_downloaded >= versions: 262 | break 263 | if n_downloaded == 0: 264 | print(f"Error: No release version found for {repo}") 265 | continue 266 | else: 267 | cleaning = True 268 | 269 | # block until all tasks are done 270 | task_queue.join() 271 | # stop workers 272 | for i in range(args.workers): 273 | task_queue.put(None) 274 | 275 | if cleaning: 276 | local_filelist = [] 277 | for local_file in working_dir.glob('**/*'): 278 | if local_file.is_file(): 279 | local_filelist.append(local_file.relative_to(working_dir)) 280 | 281 | for old_file in set(local_filelist) - set(remote_filelist): 282 | print("deleting", old_file, flush=True) 283 | old_file = working_dir / old_file 284 | old_file.unlink() 285 | 286 | for local_dir in working_dir.glob('*/*/*'): 287 | if local_dir.is_dir(): 288 | try: 289 | # remove empty dirs only 290 | local_dir.rmdir() 291 | except: 292 | pass 293 | 294 | print("Total size is", sizeof_fmt(total_size, suffix="")) 295 | 296 | if not success: 297 | sys.exit(1) 298 | 299 | 300 | if __name__ == "__main__": 301 | main() 302 | 303 | 304 | # vim: ts=4 sw=4 sts=4 expandtab 305 | -------------------------------------------------------------------------------- /gitsync/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Jian Zeng " 3 | RUN apk add --no-cache git 4 | ADD sync.sh / 5 | -------------------------------------------------------------------------------- /gitsync/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## EXPORTED IN entry.sh 4 | #TO= 5 | #LOGDIR= 6 | #LOGFILE= 7 | 8 | ## SET IN ENVIRONMENT VARIABLES 9 | #GITSYNC_URL= 10 | #GITSYNC_BRANCH= 11 | #GITSYNC_REMOTE= 12 | #GITSYNC_BITMAP= 13 | #GITSYNC_MIRROR= 14 | #GITSYNC_CHECKOUT= 15 | #GITSYNC_TREELESS= 16 | #GITSYNC_GEOMETRIC= 17 | 18 | is_empty() { 19 | [[ -z $(ls -A "$1" 2>/dev/null) ]] 20 | } 21 | 22 | set -eu 23 | [[ $DEBUG = true ]] && set -x 24 | 25 | GITSYNC_REMOTE="${GITSYNC_REMOTE:-origin}" 26 | GITSYNC_BRANCH="${GITSYNC_BRANCH:-master:master}" 27 | GITSYNC_BITMAP="${GITSYNC_BITMAP:-false}" 28 | GITSYNC_MIRROR="${GITSYNC_MIRROR:-false}" 29 | GITSYNC_CHECKOUT="${GITSYNC_CHECKOUT:-false}" 30 | GITSYNC_TREELESS="${GITSYNC_TREELESS:-false}" 31 | GITSYNC_GEOMETRIC="${GITSYNC_GEOMETRIC:-false}" 32 | 33 | is_empty "$TO" && git clone -v --progress \ 34 | $([ "$GITSYNC_CHECKOUT" = false ] && echo "--bare") \ 35 | $([ "$GITSYNC_CHECKOUT" = true ] && echo "--no-checkout") \ 36 | $([ "$GITSYNC_TREELESS" = true ] && echo "--filter=tree:0") \ 37 | "$GITSYNC_URL" "$TO" 38 | 39 | cd "$TO" || exit 1 40 | if [[ $GITSYNC_MIRROR = true ]]; then 41 | # By default when cloned with --mirror, 42 | # remote.origin.fetch is set to '+refs/*:refs/*' 43 | # But refs/pull will also be fetched, which is not needed. Thus we are not using --mirror option and set $GITSYNC_BRANCH manually 44 | # Tags will be fetched by --tags 45 | # User-provided GITSYNC_BRANCH is ignored here, as all branches (refs/heads/) are mirrored 46 | GITSYNC_BRANCH='+refs/heads/*:refs/heads/*' 47 | fi 48 | 49 | if [[ $GITSYNC_CHECKOUT = true ]]; then 50 | git fetch "$GITSYNC_REMOTE" "$GITSYNC_BRANCH" -u -v --progress 51 | git reset --hard FETCH_HEAD 52 | else 53 | git fetch "$GITSYNC_REMOTE" "$GITSYNC_BRANCH" -v --progress --tags 54 | git update-server-info 55 | fi 56 | 57 | if [[ $GITSYNC_BITMAP = true ]]; then 58 | if [[ $GITSYNC_GEOMETRIC = true ]]; then 59 | git repack --write-midx --write-bitmap-index -d --geometric=2 60 | else 61 | git repack -abd 62 | fi 63 | git gc --auto 64 | fi 65 | -------------------------------------------------------------------------------- /google-repo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Keyu Tao " 3 | RUN < /dev/null 19 | rm index-00.tar.gz || true &> /dev/null 20 | 21 | # snapshot.json contains hashes of 01-index.tar.gz 22 | # Download it first to minimize the chance of race condition 23 | echo "Download snapshot (hashes) ..." 24 | snapshot_json=("snapshot.json" "timestamp.json") 25 | for json in "${snapshot_json[@]}"; do 26 | $CURL_WRAP -sSL -o "$json.bak" "$HACKAGE_BASE_URL/$json" &> /dev/null 27 | done 28 | 29 | echo "Download latest index ..." 30 | $CURL_WRAP -sSL -o index.tar.gz "$HACKAGE_BASE_URL/01-index.tar.gz" &> /dev/null 31 | 32 | echo "Download latest legacy (00-index) index ..." 33 | $CURL_WRAP -sSL -o index-00.tar.gz "$HACKAGE_BASE_URL/00-index.tar.gz" &> /dev/null 34 | 35 | # download extra json files 36 | extra_json=("mirrors.json" "root.json") 37 | for json in "${extra_json[@]}"; do 38 | $CURL_WRAP -sSL -o "$json" "$HACKAGE_BASE_URL/$json" &> /dev/null 39 | done 40 | 41 | mkdir -p package 42 | 43 | # save remote package index to temporary file 44 | echo "Build remote package list ..." 45 | tar -ztf index.tar.gz | awk 'BEGIN{FS="/"}{print($1"-"$2)}' | sort | uniq > $remote_pkgs || true 46 | echo "Remote package list built" 47 | 48 | # save local package index to temporary file 49 | echo "Building local package list ..." 50 | 51 | # check if package/ is empty 52 | local_archives=$(shopt -s nullglob dotglob; echo package/*) 53 | if [[ ${#local_archives[@]} -gt 0 ]]; then 54 | local_archives=(package/*) 55 | local_archives=(${local_archives[@]#package/}) 56 | printf "%s\n" ${local_archives[@]%.tar.gz} | sort > $local_pkgs 57 | else 58 | touch $local_pkgs 59 | fi 60 | 61 | # Files that are unique to remote_pkgs are newer 62 | # download them into local 63 | for pkg in $(comm $remote_pkgs $local_pkgs -23); do 64 | if [[ $pkg = *-preferred-versions ]]; then 65 | continue 66 | fi 67 | 68 | while [[ $(jobs | wc -l) -ge $jobs_max ]]; do 69 | sleep 0.5 70 | done 71 | 72 | download_pkg $pkg & 73 | done 74 | 75 | # wait downloading jobs 76 | wait 77 | 78 | # Files that are unique to local packages are deprecated 79 | # remove them from local 80 | for pkg in $(comm $remote_pkgs $local_pkgs -13); do 81 | if [[ $pkg == "preferred-versions" ]]; then 82 | continue 83 | fi 84 | echo "Removing $pkg.tar.gz ..." 85 | rm "package/$pkg.tar.gz" || true &> /dev/null 86 | done 87 | 88 | cp index.tar.gz 01-index.tar.gz 89 | mv index-00.tar.gz 00-index.tar.gz 90 | mv snapshot.json.bak snapshot.json 91 | mv timestamp.json.bak timestamp.json 92 | } 93 | 94 | function download_pkg () { 95 | pkg=$1 96 | name="$pkg.tar.gz" 97 | echo "Download: $pkg.tar.gz ..." 98 | $CURL_WRAP -sSL -o "package/$name" "$HACKAGE_BASE_URL/package/$pkg/$name" &> /dev/null 99 | echo "Finish: $pkg.tar.gz" 100 | } 101 | 102 | 103 | function rmtemp () { 104 | echo "Remove temporary files" 105 | [[ ! -z $local_pkgs ]] && (rm $local_pkgs $remote_pkgs; true) 106 | } 107 | 108 | trap rmtemp EXIT 109 | pull_hackage 110 | -------------------------------------------------------------------------------- /homebrew-bottles/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:alpine AS builder 2 | WORKDIR /usr/src/bottles-json/ 3 | COPY bottles-json . 4 | RUN apk add --no-cache musl-dev 5 | RUN cargo build --release 6 | 7 | FROM ustcmirror/curl-helper 8 | LABEL maintainer="Yifan Gao docker@yfgao.com" 9 | RUN apk add --no-cache parallel bash sed gzip 10 | ADD sync.sh / 11 | COPY --from=builder /usr/src/bottles-json/target/release/bottles-json /usr/local/bin/ 12 | -------------------------------------------------------------------------------- /homebrew-bottles/bottles-json/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .idea/ 3 | -------------------------------------------------------------------------------- /homebrew-bottles/bottles-json/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "bitflags" 7 | version = "1.3.2" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 10 | 11 | [[package]] 12 | name = "bottles-json" 13 | version = "0.1.0" 14 | dependencies = [ 15 | "clap", 16 | "serde", 17 | "serde_json", 18 | "urlencoding", 19 | ] 20 | 21 | [[package]] 22 | name = "clap" 23 | version = "4.1.4" 24 | source = "registry+https://github.com/rust-lang/crates.io-index" 25 | checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" 26 | dependencies = [ 27 | "bitflags", 28 | "clap_derive", 29 | "clap_lex", 30 | "is-terminal", 31 | "once_cell", 32 | "strsim", 33 | "termcolor", 34 | ] 35 | 36 | [[package]] 37 | name = "clap_derive" 38 | version = "4.1.0" 39 | source = "registry+https://github.com/rust-lang/crates.io-index" 40 | checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" 41 | dependencies = [ 42 | "heck", 43 | "proc-macro-error", 44 | "proc-macro2", 45 | "quote", 46 | "syn", 47 | ] 48 | 49 | [[package]] 50 | name = "clap_lex" 51 | version = "0.3.1" 52 | source = "registry+https://github.com/rust-lang/crates.io-index" 53 | checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" 54 | dependencies = [ 55 | "os_str_bytes", 56 | ] 57 | 58 | [[package]] 59 | name = "errno" 60 | version = "0.3.5" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" 63 | dependencies = [ 64 | "libc", 65 | "windows-sys 0.48.0", 66 | ] 67 | 68 | [[package]] 69 | name = "heck" 70 | version = "0.4.1" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" 73 | 74 | [[package]] 75 | name = "hermit-abi" 76 | version = "0.3.0" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | checksum = "856b5cb0902c2b6d65d5fd97dfa30f9b70c7538e770b98eab5ed52d8db923e01" 79 | 80 | [[package]] 81 | name = "io-lifetimes" 82 | version = "1.0.5" 83 | source = "registry+https://github.com/rust-lang/crates.io-index" 84 | checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" 85 | dependencies = [ 86 | "libc", 87 | "windows-sys 0.45.0", 88 | ] 89 | 90 | [[package]] 91 | name = "is-terminal" 92 | version = "0.4.3" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef" 95 | dependencies = [ 96 | "hermit-abi", 97 | "io-lifetimes", 98 | "rustix", 99 | "windows-sys 0.45.0", 100 | ] 101 | 102 | [[package]] 103 | name = "itoa" 104 | version = "0.4.7" 105 | source = "registry+https://github.com/rust-lang/crates.io-index" 106 | checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" 107 | 108 | [[package]] 109 | name = "libc" 110 | version = "0.2.139" 111 | source = "registry+https://github.com/rust-lang/crates.io-index" 112 | checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" 113 | 114 | [[package]] 115 | name = "linux-raw-sys" 116 | version = "0.1.4" 117 | source = "registry+https://github.com/rust-lang/crates.io-index" 118 | checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" 119 | 120 | [[package]] 121 | name = "once_cell" 122 | version = "1.17.0" 123 | source = "registry+https://github.com/rust-lang/crates.io-index" 124 | checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" 125 | 126 | [[package]] 127 | name = "os_str_bytes" 128 | version = "6.4.1" 129 | source = "registry+https://github.com/rust-lang/crates.io-index" 130 | checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" 131 | 132 | [[package]] 133 | name = "proc-macro-error" 134 | version = "1.0.4" 135 | source = "registry+https://github.com/rust-lang/crates.io-index" 136 | checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" 137 | dependencies = [ 138 | "proc-macro-error-attr", 139 | "proc-macro2", 140 | "quote", 141 | "syn", 142 | "version_check", 143 | ] 144 | 145 | [[package]] 146 | name = "proc-macro-error-attr" 147 | version = "1.0.4" 148 | source = "registry+https://github.com/rust-lang/crates.io-index" 149 | checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" 150 | dependencies = [ 151 | "proc-macro2", 152 | "quote", 153 | "version_check", 154 | ] 155 | 156 | [[package]] 157 | name = "proc-macro2" 158 | version = "1.0.51" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" 161 | dependencies = [ 162 | "unicode-ident", 163 | ] 164 | 165 | [[package]] 166 | name = "quote" 167 | version = "1.0.9" 168 | source = "registry+https://github.com/rust-lang/crates.io-index" 169 | checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" 170 | dependencies = [ 171 | "proc-macro2", 172 | ] 173 | 174 | [[package]] 175 | name = "rustix" 176 | version = "0.36.16" 177 | source = "registry+https://github.com/rust-lang/crates.io-index" 178 | checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" 179 | dependencies = [ 180 | "bitflags", 181 | "errno", 182 | "io-lifetimes", 183 | "libc", 184 | "linux-raw-sys", 185 | "windows-sys 0.45.0", 186 | ] 187 | 188 | [[package]] 189 | name = "ryu" 190 | version = "1.0.5" 191 | source = "registry+https://github.com/rust-lang/crates.io-index" 192 | checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" 193 | 194 | [[package]] 195 | name = "serde" 196 | version = "1.0.125" 197 | source = "registry+https://github.com/rust-lang/crates.io-index" 198 | checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" 199 | dependencies = [ 200 | "serde_derive", 201 | ] 202 | 203 | [[package]] 204 | name = "serde_derive" 205 | version = "1.0.125" 206 | source = "registry+https://github.com/rust-lang/crates.io-index" 207 | checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" 208 | dependencies = [ 209 | "proc-macro2", 210 | "quote", 211 | "syn", 212 | ] 213 | 214 | [[package]] 215 | name = "serde_json" 216 | version = "1.0.64" 217 | source = "registry+https://github.com/rust-lang/crates.io-index" 218 | checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" 219 | dependencies = [ 220 | "itoa", 221 | "ryu", 222 | "serde", 223 | ] 224 | 225 | [[package]] 226 | name = "strsim" 227 | version = "0.10.0" 228 | source = "registry+https://github.com/rust-lang/crates.io-index" 229 | checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 230 | 231 | [[package]] 232 | name = "syn" 233 | version = "1.0.107" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" 236 | dependencies = [ 237 | "proc-macro2", 238 | "quote", 239 | "unicode-ident", 240 | ] 241 | 242 | [[package]] 243 | name = "termcolor" 244 | version = "1.2.0" 245 | source = "registry+https://github.com/rust-lang/crates.io-index" 246 | checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" 247 | dependencies = [ 248 | "winapi-util", 249 | ] 250 | 251 | [[package]] 252 | name = "unicode-ident" 253 | version = "1.0.6" 254 | source = "registry+https://github.com/rust-lang/crates.io-index" 255 | checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" 256 | 257 | [[package]] 258 | name = "urlencoding" 259 | version = "2.1.3" 260 | source = "registry+https://github.com/rust-lang/crates.io-index" 261 | checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" 262 | 263 | [[package]] 264 | name = "version_check" 265 | version = "0.9.4" 266 | source = "registry+https://github.com/rust-lang/crates.io-index" 267 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" 268 | 269 | [[package]] 270 | name = "winapi" 271 | version = "0.3.9" 272 | source = "registry+https://github.com/rust-lang/crates.io-index" 273 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 274 | dependencies = [ 275 | "winapi-i686-pc-windows-gnu", 276 | "winapi-x86_64-pc-windows-gnu", 277 | ] 278 | 279 | [[package]] 280 | name = "winapi-i686-pc-windows-gnu" 281 | version = "0.4.0" 282 | source = "registry+https://github.com/rust-lang/crates.io-index" 283 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 284 | 285 | [[package]] 286 | name = "winapi-util" 287 | version = "0.1.5" 288 | source = "registry+https://github.com/rust-lang/crates.io-index" 289 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 290 | dependencies = [ 291 | "winapi", 292 | ] 293 | 294 | [[package]] 295 | name = "winapi-x86_64-pc-windows-gnu" 296 | version = "0.4.0" 297 | source = "registry+https://github.com/rust-lang/crates.io-index" 298 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 299 | 300 | [[package]] 301 | name = "windows-sys" 302 | version = "0.45.0" 303 | source = "registry+https://github.com/rust-lang/crates.io-index" 304 | checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" 305 | dependencies = [ 306 | "windows-targets 0.42.1", 307 | ] 308 | 309 | [[package]] 310 | name = "windows-sys" 311 | version = "0.48.0" 312 | source = "registry+https://github.com/rust-lang/crates.io-index" 313 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 314 | dependencies = [ 315 | "windows-targets 0.48.5", 316 | ] 317 | 318 | [[package]] 319 | name = "windows-targets" 320 | version = "0.42.1" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" 323 | dependencies = [ 324 | "windows_aarch64_gnullvm 0.42.1", 325 | "windows_aarch64_msvc 0.42.1", 326 | "windows_i686_gnu 0.42.1", 327 | "windows_i686_msvc 0.42.1", 328 | "windows_x86_64_gnu 0.42.1", 329 | "windows_x86_64_gnullvm 0.42.1", 330 | "windows_x86_64_msvc 0.42.1", 331 | ] 332 | 333 | [[package]] 334 | name = "windows-targets" 335 | version = "0.48.5" 336 | source = "registry+https://github.com/rust-lang/crates.io-index" 337 | checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" 338 | dependencies = [ 339 | "windows_aarch64_gnullvm 0.48.5", 340 | "windows_aarch64_msvc 0.48.5", 341 | "windows_i686_gnu 0.48.5", 342 | "windows_i686_msvc 0.48.5", 343 | "windows_x86_64_gnu 0.48.5", 344 | "windows_x86_64_gnullvm 0.48.5", 345 | "windows_x86_64_msvc 0.48.5", 346 | ] 347 | 348 | [[package]] 349 | name = "windows_aarch64_gnullvm" 350 | version = "0.42.1" 351 | source = "registry+https://github.com/rust-lang/crates.io-index" 352 | checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" 353 | 354 | [[package]] 355 | name = "windows_aarch64_gnullvm" 356 | version = "0.48.5" 357 | source = "registry+https://github.com/rust-lang/crates.io-index" 358 | checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" 359 | 360 | [[package]] 361 | name = "windows_aarch64_msvc" 362 | version = "0.42.1" 363 | source = "registry+https://github.com/rust-lang/crates.io-index" 364 | checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" 365 | 366 | [[package]] 367 | name = "windows_aarch64_msvc" 368 | version = "0.48.5" 369 | source = "registry+https://github.com/rust-lang/crates.io-index" 370 | checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" 371 | 372 | [[package]] 373 | name = "windows_i686_gnu" 374 | version = "0.42.1" 375 | source = "registry+https://github.com/rust-lang/crates.io-index" 376 | checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" 377 | 378 | [[package]] 379 | name = "windows_i686_gnu" 380 | version = "0.48.5" 381 | source = "registry+https://github.com/rust-lang/crates.io-index" 382 | checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" 383 | 384 | [[package]] 385 | name = "windows_i686_msvc" 386 | version = "0.42.1" 387 | source = "registry+https://github.com/rust-lang/crates.io-index" 388 | checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" 389 | 390 | [[package]] 391 | name = "windows_i686_msvc" 392 | version = "0.48.5" 393 | source = "registry+https://github.com/rust-lang/crates.io-index" 394 | checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" 395 | 396 | [[package]] 397 | name = "windows_x86_64_gnu" 398 | version = "0.42.1" 399 | source = "registry+https://github.com/rust-lang/crates.io-index" 400 | checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" 401 | 402 | [[package]] 403 | name = "windows_x86_64_gnu" 404 | version = "0.48.5" 405 | source = "registry+https://github.com/rust-lang/crates.io-index" 406 | checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" 407 | 408 | [[package]] 409 | name = "windows_x86_64_gnullvm" 410 | version = "0.42.1" 411 | source = "registry+https://github.com/rust-lang/crates.io-index" 412 | checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" 413 | 414 | [[package]] 415 | name = "windows_x86_64_gnullvm" 416 | version = "0.48.5" 417 | source = "registry+https://github.com/rust-lang/crates.io-index" 418 | checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" 419 | 420 | [[package]] 421 | name = "windows_x86_64_msvc" 422 | version = "0.42.1" 423 | source = "registry+https://github.com/rust-lang/crates.io-index" 424 | checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" 425 | 426 | [[package]] 427 | name = "windows_x86_64_msvc" 428 | version = "0.48.5" 429 | source = "registry+https://github.com/rust-lang/crates.io-index" 430 | checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" 431 | -------------------------------------------------------------------------------- /homebrew-bottles/bottles-json/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bottles-json" 3 | version = "0.1.0" 4 | authors = ["Yifan Gao "] 5 | edition = "2018" 6 | 7 | [profile.release] 8 | strip = "symbols" 9 | 10 | [dependencies] 11 | clap = { version = "4.1.4", features = ["derive"] } 12 | serde = { version = "1.0", features = ["derive"] } 13 | serde_json = "1.0" 14 | urlencoding = "2.1.3" 15 | -------------------------------------------------------------------------------- /homebrew-bottles/bottles-json/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::HashSet, 3 | fs::File, 4 | io::{Read, Write}, 5 | path::{Path, PathBuf}, 6 | }; 7 | 8 | use clap::{Parser, ValueEnum}; 9 | 10 | use serde_json::Value; 11 | 12 | mod structs; 13 | use structs::*; 14 | 15 | #[derive(ValueEnum, Clone)] 16 | enum Mode { 17 | /// List all bottles that need to check and download 18 | /// Output format: sha256 url bottle_file.tar.gz 19 | GetBottlesMetadata, 20 | /// Extract cask/formula json files to folder 21 | ExtractJson, 22 | /// List all cask source files that need to check and download 23 | /// Output format: sha256 url cask_source_file.rb 24 | ListCaskSource, 25 | } 26 | 27 | #[derive(ValueEnum, Clone)] 28 | enum Type { 29 | Formula, 30 | Cask, 31 | } 32 | 33 | #[derive(Parser)] 34 | struct Cli { 35 | /// Parse metadata or extract json to api folder 36 | #[arg(long, value_enum, default_value_t=Mode::GetBottlesMetadata)] 37 | mode: Mode, 38 | 39 | /// The folder to extract json to, required when mode is extract-json 40 | #[arg(long)] 41 | folder: Option, 42 | 43 | /// Formula and cask json has different attribute for its name ("name" or "token") 44 | #[arg(long, value_enum, default_value_t=Type::Formula)] 45 | type_: Type, 46 | } 47 | 48 | fn d(f: &Formula) -> Option<()> { 49 | if f.versions.bottle { 50 | let bs = f.bottle.stable.as_ref()?; 51 | for (platform, v) in bs.files.as_object()?.iter() { 52 | if let Ok(bi) = serde_json::from_value::(v.clone()) { 53 | println!( 54 | "{sha256} {url} {name}-{version}{revision}.{platform}.bottle{rebuild}.tar.gz", 55 | sha256 = bi.sha256, 56 | url = bi.url, 57 | name = f.name, 58 | version = f.versions.stable.as_ref()?, 59 | revision = if f.revision == 0 { 60 | "".to_owned() 61 | } else { 62 | format!("_{}", f.revision) 63 | }, 64 | platform = platform, 65 | rebuild = if bs.rebuild == 0 { 66 | "".to_owned() 67 | } else { 68 | format!(".{}", bs.rebuild) 69 | }, 70 | ); 71 | } 72 | } 73 | } 74 | Some(()) 75 | } 76 | 77 | fn e(f: &Value, name: &str, target: &Path) -> Option<()> { 78 | let tmp_name = format!("{}.json.new", name); 79 | let final_name = format!("{}.json", name); 80 | let contents = serde_json::to_string(f).unwrap(); 81 | 82 | { 83 | // is it the same as existing file? 84 | // this is to avoid unnecessary file metadata modification 85 | let mut existing = String::new(); 86 | if let Ok(mut file) = File::open(target.join(final_name.clone())) { 87 | if let Err(e) = file.read_to_string(&mut existing) { 88 | eprintln!("Failed to read existing file {}: {}", final_name, e); 89 | } else if existing == contents { 90 | return Some(()); 91 | } 92 | } 93 | } 94 | 95 | { 96 | let mut file = File::create(target.join(tmp_name.clone())).unwrap(); 97 | file.write_all(contents.as_bytes()).unwrap(); 98 | } 99 | std::fs::rename(target.join(tmp_name), target.join(final_name)).unwrap(); 100 | Some(()) 101 | } 102 | 103 | fn main() { 104 | let cli = Cli::parse(); 105 | 106 | match cli.mode { 107 | Mode::GetBottlesMetadata => { 108 | let f: Formulae = serde_json::from_reader(std::io::stdin()).unwrap(); 109 | for f in f.0 { 110 | if d(&f).is_none() { 111 | eprintln!("Failed to parse formula: {}", f.name); 112 | } 113 | } 114 | } 115 | Mode::ExtractJson => { 116 | // Handle API json weak-typed here, as it may cause too much trouble to write all types of all fields 117 | let f: Value = serde_json::from_reader(std::io::stdin()).unwrap(); 118 | let target_dir = cli 119 | .folder 120 | .expect("target folder is required as an argument"); 121 | let mut existing_jsons = std::fs::read_dir(&target_dir) 122 | .unwrap() 123 | .filter_map(|e| { 124 | Some( 125 | e.unwrap() 126 | .file_name() 127 | .into_string() 128 | .unwrap() 129 | .strip_suffix(".json")? 130 | .to_owned(), 131 | ) 132 | }) 133 | .collect::>(); 134 | for f in f.as_array().unwrap() { 135 | let fname = match cli.type_ { 136 | Type::Formula => &f["name"], 137 | Type::Cask => &f["token"], 138 | }; 139 | let fname = fname.as_str().unwrap(); 140 | if e(f, fname, &target_dir).is_none() { 141 | eprintln!("Failed to extract json: {}", fname); 142 | } 143 | existing_jsons.remove(fname); 144 | } 145 | // Clean unused jsons 146 | for fname in existing_jsons { 147 | std::fs::remove_file(target_dir.join(format!("{}.json", fname))).unwrap(); 148 | } 149 | } 150 | Mode::ListCaskSource => { 151 | let f: Casks = serde_json::from_reader(std::io::stdin()).unwrap(); 152 | for f in f.0 { 153 | let filename = Path::new(&f.ruby_source_path) 154 | .file_name() 155 | .unwrap() 156 | .to_str() 157 | .unwrap(); 158 | let url = format!("https://formulae.brew.sh/api/cask-source/{}", filename); 159 | let url = urlencoding::encode(&url); 160 | println!( 161 | "{} {} {}", 162 | f.ruby_source_checksum.sha256, url, filename 163 | ); 164 | } 165 | } 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /homebrew-bottles/bottles-json/src/structs.rs: -------------------------------------------------------------------------------- 1 | use serde::Deserialize; 2 | 3 | // Formulae struct types 4 | 5 | /// formula.version 6 | #[derive(Deserialize)] 7 | pub struct Versions { 8 | pub stable: Option, 9 | pub bottle: bool, 10 | } 11 | 12 | /// formula.bottle.stable.files. 13 | #[derive(Deserialize)] 14 | pub struct BottleInfo { 15 | pub url: String, 16 | pub sha256: String, 17 | } 18 | 19 | /// formula.bottle.stable 20 | #[derive(Deserialize)] 21 | pub struct BottleStable { 22 | pub rebuild: u64, 23 | pub files: serde_json::Value, 24 | } 25 | 26 | /// formula.bottle 27 | #[derive(Deserialize)] 28 | pub struct Bottle { 29 | pub stable: Option, 30 | } 31 | 32 | /// formula (one object) 33 | #[derive(Deserialize)] 34 | pub struct Formula { 35 | pub name: String, 36 | pub versions: Versions, 37 | pub bottle: Bottle, 38 | pub revision: u64, 39 | } 40 | 41 | /// formula.json (array of formula) 42 | #[derive(Deserialize)] 43 | pub struct Formulae(pub Vec); 44 | 45 | /// cask.ruby_source_checksum 46 | #[derive(Deserialize)] 47 | pub struct SourceChecksum { 48 | pub sha256: String, 49 | } 50 | 51 | /// cask (one object) 52 | #[derive(Deserialize)] 53 | pub struct Cask { 54 | pub ruby_source_path: String, 55 | pub ruby_source_checksum: SourceChecksum, 56 | } 57 | 58 | /// cask.json (array of cask) 59 | #[derive(Deserialize)] 60 | pub struct Casks(pub Vec); 61 | -------------------------------------------------------------------------------- /homebrew-bottles/sync.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | HOMEBREW_BOTTLES_JOBS=${HOMEBREW_BOTTLES_JOBS:-1} 4 | 5 | source /curl-helper.sh 6 | 7 | mkdir -p "$TO/api" 8 | mkdir -p "$TO/api/formula" 9 | mkdir -p "$TO/api/cask" 10 | mkdir -p "$TO/api/cask-source" 11 | 12 | BOTTLES=$(mktemp) 13 | CASK_SOURCES=$(mktemp) 14 | 15 | BOTTLES_BIND_ADDRESS="$BIND_ADDRESS" 16 | 17 | # Step 1: Download new API jsons and extract 18 | 19 | # 2 special jsons for further processing 20 | FORMULA_JSON="$TO/api/formula.json" 21 | CASK_JSON="$TO/api/cask.json" 22 | 23 | FILES=( 24 | "formula.json" 25 | "cask.json" 26 | "formula.jws.json" 27 | "cask.jws.json" 28 | "formula_tap_migrations.jws.json" 29 | "cask_tap_migrations.jws.json" 30 | ) 31 | URL_BASE="https://formulae.brew.sh/api" 32 | 33 | if [ -n "$BREW_SH_BIND_ADDRESS" ]; then 34 | BIND_ADDRESS="$BREW_SH_BIND_ADDRESS" 35 | fi 36 | 37 | curl_init 38 | 39 | for file in "${FILES[@]}"; do 40 | $CURL_WRAP --compressed -sSL -o "$TO/api/$file".tmp "$URL_BASE/$file" 41 | if [[ $? -ne 0 ]]; then 42 | echo "[FATAL] download $file meta-data failed." 43 | exit 2 44 | fi 45 | mv "$TO/api/$file".tmp "$TO/api/$file" 46 | # create gz files to save bandwidth (with nginx gzip_static) 47 | gzip --keep --force "$TO/api/$file" 48 | done 49 | 50 | bottles-json --mode extract-json --type formula --folder "$TO/api/formula" < "$FORMULA_JSON" 51 | if [[ $? -ne 0 ]]; then 52 | echo "[FATAL] formula API json extracting failed." 53 | exit 5 54 | fi 55 | bottles-json --mode extract-json --type cask --folder "$TO/api/cask" < "$CASK_JSON" 56 | if [[ $? -ne 0 ]]; then 57 | echo "[FATAL] cask API json extracting failed." 58 | exit 6 59 | fi 60 | 61 | # Step 2: Download cask-source 62 | bottles-json --mode list-cask-source > $CASK_SOURCES < "$CASK_JSON" 63 | if [[ $? -ne 0 ]]; then 64 | echo "[FATAL] cask-source list failed." 65 | exit 3 66 | fi 67 | 68 | # Init common envs for parallel downloading 69 | export enable_urldecode=true 70 | export enable_alternative_path=true 71 | export enable_checksum=true 72 | export by_hash_pattern="./.by-hash/*" 73 | 74 | # envs for cask-source 75 | export by_hash=$(realpath $TO/api/cask-source/.by-hash) 76 | 77 | export local_dir="$TO/api/cask-source" 78 | parallel --line-buffer -j $HOMEBREW_BOTTLES_JOBS --pipepart -a $CASK_SOURCES download 79 | 80 | # cleanup 81 | removal_list=$(mktemp) 82 | cd $local_dir 83 | comm -23 <(find . -type f -not -path "$by_hash_pattern" | sed "s|^./||" | sort) <(awk '{print $3}' $CASK_SOURCES | sort) | tee $removal_list | xargs rm -f 84 | sed 's/^/[INFO] remove /g' $removal_list 85 | clean_hash_file 86 | 87 | # chdir back 88 | cd "$TO" 89 | 90 | # Step 3: Download bottles (formula only) 91 | # extract sha256, URL and file name from JSON result 92 | bottles-json < $FORMULA_JSON > $BOTTLES 93 | if [[ $? -ne 0 ]]; then 94 | echo "[FATAL] json parsing failed." 95 | exit 4 96 | fi 97 | 98 | # GitHub Packages auth info 99 | headers_file=$(mktemp) 100 | cat << EOF > $headers_file 101 | Accept: application/vnd.oci.image.index.v1+json 102 | Authorization: Bearer QQ== 103 | EOF 104 | 105 | # Reset BIND_ADDRESS and curl for bottle downloading 106 | BIND_ADDRESS="$BOTTLES_BIND_ADDRESS" 107 | unset CURL_WRAP 108 | curl_init 109 | 110 | # parallel downloading 111 | export by_hash=$(realpath $TO/.by-hash) 112 | export local_dir=$TO 113 | export CURL_WRAP="$CURL_WRAP --header @$headers_file" 114 | parallel --line-buffer -j $HOMEBREW_BOTTLES_JOBS --pipepart -a $BOTTLES download 115 | 116 | # clean up outdated bottles 117 | removal_list=$(mktemp) 118 | cd $local_dir 119 | comm -23 <(find . -type f -not -path "$by_hash_pattern" -not -path "./api/*" | sed "s|^./||" | sort) <(awk '{print $3}' $BOTTLES | sort) | tee $removal_list | xargs rm -f 120 | sed 's/^/[INFO] remove /g' $removal_list 121 | 122 | # clean empty dir. If everything work as expect, this command would do nothing 123 | find . -type d -empty -delete 124 | 125 | clean_hash_file 126 | -------------------------------------------------------------------------------- /julia-storage/Dockerfile: -------------------------------------------------------------------------------- 1 | # Ref: https://github.com/tuna/tunasync-scripts/tree/master/dockerfiles/julia 2 | FROM ustcmirror/base:alpine 3 | 4 | ENV JULIA_DEPOT_PATH="/opt/julia/depot" 5 | RUN < /etc/bandersnatch.conf 4 | [mirror] 5 | directory = $TO 6 | master = $PYPI_MASTER 7 | timeout = $BANDERSNATCH_TIMEOUT 8 | workers = $BANDERSNATCH_WORKERS 9 | hash-index = false 10 | stop-on-error = $BANDERSNATCH_STOP_ON_ERROR 11 | delete-packages = true 12 | EOF 13 | -------------------------------------------------------------------------------- /pypi/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exec bandersnatch mirror 4 | -------------------------------------------------------------------------------- /rclone/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Jian Zeng " 3 | LABEL bind_support=true 4 | 5 | ARG RCLONE_VERSION=v1.50.2 6 | ENV RCLONE_DELETE_AFTER=true \ 7 | RCLONE_DELETE_EXCLUDED=true 8 | 9 | RUN < "$filter_file" 47 | if [ -n "$RSYNC_FILTER" ]; then 48 | echo "$RSYNC_FILTER" >> "$filter_file" 49 | fi 50 | 51 | if [[ -n $RSYNC_RSH ]]; then 52 | RSYNC_URL="$RSYNC_HOST:$RSYNC_PATH" 53 | else 54 | RSYNC_URL="rsync://$RSYNC_HOST/$RSYNC_PATH" 55 | fi 56 | 57 | exec rsync $RSYNC_EXCLUDE --filter="merge $filter_file" --bwlimit "$RSYNC_BW" --max-delete "$RSYNC_MAXDELETE" $opts $RSYNC_EXTRA "$RSYNC_URL" "$TO" 58 | -------------------------------------------------------------------------------- /rubygems/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Yifan Gao " 3 | ENV UPSTREAM=https://rubygems.org 4 | RUN apk add --no-cache ruby ca-certificates && \ 5 | gem install --no-document rubygems-mirror 6 | ADD sync.sh pre-sync.sh / 7 | -------------------------------------------------------------------------------- /rubygems/pre-sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mkdir -p "$HOME/.gem" 4 | chown -R "$OWNER" "$HOME" 5 | -------------------------------------------------------------------------------- /rubygems/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | export HOME=/root 6 | cat << EOF > /root/.gem/.mirrorrc 7 | --- 8 | - from: $UPSTREAM 9 | to: $TO 10 | parallelism: 10 11 | retries: 3 12 | delete: true 13 | skiperror: true 14 | EOF 15 | 16 | # Fetch index 17 | wget -qO "$TO/versions.new" "$UPSTREAM/versions" 18 | md5sum "$TO/versions.new" > "$TO/versions.md5sum.new" 19 | mv -f "$TO/versions.new" "$TO/versions" 20 | mv -f "$TO/versions.md5sum.new" "$TO/versions.md5sum" 21 | 22 | exec gem mirror 23 | -------------------------------------------------------------------------------- /shadowmire/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Keyu Tao " 3 | ARG COMMIT=bbcab99b4d83f9316b7da0ca6adaee53e4dccdc8 4 | 5 | RUN < 2 | -------------------------------------------------------------------------------- /stackage/stackage.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE ImportQualifiedPost #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | {-# LANGUAGE RecordWildCards #-} 4 | 5 | import Control.Exception (try) 6 | import Control.Monad (unless) 7 | import Data.Aeson qualified as A 8 | import Data.ByteString.Lazy qualified as BS 9 | import Data.Either (fromRight) 10 | import Data.List.Split (splitOn) 11 | import Data.Map.Strict qualified as M 12 | import Data.Yaml 13 | import System.Directory (createDirectory, doesDirectoryExist, getFileSize) 14 | import System.Environment (getEnv) 15 | import System.Exit (ExitCode (..)) 16 | import System.FilePath.Posix (()) 17 | import System.Process (callProcess, readProcessWithExitCode) 18 | import Text.Printf (printf) 19 | 20 | type URL = String 21 | 22 | type SHA1 = String 23 | 24 | type SHA256 = String 25 | 26 | type Platform = String 27 | 28 | type Version = String 29 | 30 | type Path = String 31 | 32 | -- 33 | -- following data defs and instances 34 | -- are for stack-setup-2.yaml parsing 35 | -- 36 | 37 | data ResourceInfo = ResourceInfo 38 | { version :: String, 39 | url :: String, 40 | contentLength :: Int, 41 | sha1 :: SHA1, 42 | sha256 :: SHA256 43 | } 44 | deriving (Show) 45 | 46 | newtype GhcJSInfo = GhcJSInfo 47 | { source :: M.Map String ResourceInfo 48 | } 49 | deriving (Show) 50 | 51 | data StackSetup = StackSetup 52 | { stack :: M.Map Platform (M.Map Version ResourceInfo), 53 | sevenzexeInfo :: ResourceInfo, 54 | sevenzdllInfo :: ResourceInfo, 55 | portableGit :: ResourceInfo, 56 | msys2 :: M.Map Platform ResourceInfo, 57 | ghc :: M.Map Platform (M.Map Version ResourceInfo), 58 | ghcjs :: GhcJSInfo 59 | } 60 | deriving (Show) 61 | 62 | data GitHubReleaseAsset = GitHubReleaseAsset 63 | { browser_download_url :: URL, 64 | size :: Int 65 | } 66 | deriving (Show) 67 | 68 | data GitHubReleases = GitHubReleases 69 | { prerelease :: Bool, 70 | assets :: [GitHubReleaseAsset] 71 | } 72 | deriving (Show) 73 | 74 | instance FromJSON GhcJSInfo where 75 | parseJSON = withObject "GhcJSInfo" $ \o -> do 76 | source <- o .: "source" 77 | return GhcJSInfo {..} 78 | 79 | instance ToJSON GhcJSInfo where 80 | toJSON (GhcJSInfo source) = 81 | object 82 | ["source" .= source] 83 | 84 | instance FromJSON ResourceInfo where 85 | parseJSON = withObject "ResourceInfo" $ \o -> do 86 | version <- o .:? "version" .!= "" 87 | url <- o .: "url" 88 | contentLength <- o .:? "content-length" .!= (-1) 89 | sha1 <- o .:? "sha1" .!= "" 90 | sha256 <- o .:? "sha256" .!= "" 91 | return ResourceInfo {..} 92 | 93 | instance ToJSON ResourceInfo where 94 | toJSON (ResourceInfo v u c s1 s256) = 95 | object $ 96 | (["version" .= v | not (null v)]) 97 | ++ ["url" .= u] 98 | ++ (["content-length" .= c | c /= (-1)]) 99 | ++ (["sha1" .= s1 | not (null s1)]) 100 | ++ (["sha256" .= s256 | not (null s256)]) 101 | 102 | instance FromJSON StackSetup where 103 | parseJSON = withObject "StackSetup" $ \o -> do 104 | stack <- o .: "stack" 105 | sevenzexeInfo <- o .: "sevenzexe-info" 106 | sevenzdllInfo <- o .: "sevenzdll-info" 107 | portableGit <- o .: "portable-git" 108 | msys2 <- o .: "msys2" 109 | ghc <- o .: "ghc" 110 | ghcjs <- o .: "ghcjs" 111 | return StackSetup {..} 112 | 113 | instance ToJSON StackSetup where 114 | toJSON (StackSetup stack exe dll pgit msys2 ghc ghcjs) = 115 | object 116 | [ "stack" .= stack, 117 | "sevenzexe-info" .= exe, 118 | "sevenzdll-info" .= dll, 119 | "portable-git" .= pgit, 120 | "msys2" .= msys2, 121 | "ghc" .= ghc, 122 | "ghcjs" .= ghcjs 123 | ] 124 | 125 | instance FromJSON GitHubReleaseAsset where 126 | parseJSON = withObject "GitHubReleaseAsset" $ \o -> do 127 | browser_download_url <- o .: "browser_download_url" 128 | size <- o .: "size" 129 | return GitHubReleaseAsset {..} 130 | 131 | instance FromJSON GitHubReleases where 132 | parseJSON = withObject "GitHubReleases" $ \o -> do 133 | prerelease <- o .: "prerelease" 134 | assets <- o .: "assets" 135 | return GitHubReleases {..} 136 | 137 | redirectToMirror :: Path -> ResourceInfo -> ResourceInfo 138 | redirectToMirror relPath (ResourceInfo ver url conLen s1 s256) = 139 | let redirect = (++) ("https://mirrors.ustc.edu.cn/stackage/" ++ relPath ++ "/") . head . splitOn "?" . last . splitOn "/" 140 | in ResourceInfo ver (redirect url) conLen s1 s256 141 | 142 | -- download a file to given path 143 | -- sha-1 checksum is enabled when sha isn't empty string 144 | download :: URL -> FilePath -> (SHA1, SHA256) -> Int -> Bool -> IO () 145 | download url path sha contentLength force = do 146 | let fileName = head (splitOn "?" (last (splitOn "/" url))) 147 | let filePath = path fileName 148 | putStrLn $ printf "Try to Download %s..." fileName 149 | pathExists <- doesDirectoryExist path 150 | unless pathExists (createDirectory path) 151 | fileSize <- fromRight (-1) <$> (try (getFileSize filePath) :: IO (Either IOError Integer)) 152 | if (fileSize == fromIntegral contentLength) && not force 153 | then putStrLn $ printf "%s already exists. Just skip." filePath 154 | else do 155 | let args_ = 156 | [ url, 157 | "--dir=" ++ path, 158 | "--out=" ++ fileName ++ ".tmp", 159 | "--file-allocation=none", 160 | "--quiet=true" 161 | ] 162 | 163 | -- if sha1 isn't an empty string, append checksum option 164 | let sha1Arg = words (fst sha) >>= \s -> ["--checksum=sha-1=" ++ s] 165 | let sha256Arg = words (snd sha) >>= \s -> ["--checksum=sha-256=" ++ s] 166 | let args = args_ ++ (if null sha256Arg then sha1Arg else sha256Arg) 167 | 168 | (exitCode, _, _) <- readProcessWithExitCode "aria2c" args "" 169 | if exitCode == ExitSuccess 170 | then 171 | callProcess "mv" [filePath ++ ".tmp", filePath] 172 | >> putStrLn (printf "Downloaded %s to %s." fileName filePath) 173 | else putStrLn $ printf "Download failure on %s" fileName 174 | 175 | updateChannels :: FilePath -> IO () 176 | updateChannels basePath = 177 | mapM_ loadChannel ["lts-haskell", "stackage-nightly", "stackage-snapshots", "stackage-content"] 178 | where 179 | loadChannel channel = do 180 | let destPath = basePath channel 181 | exists <- doesDirectoryExist destPath 182 | if exists 183 | then do 184 | putStrLn $ printf "Start to pull latest %s channel" channel 185 | callProcess "git" ["-C", destPath, "pull"] 186 | putStrLn $ printf "Pull %s finish" channel 187 | else do 188 | putStrLn $ printf "%s channel doesn't exist, start first clone" channel 189 | callProcess 190 | "git" 191 | [ "clone", 192 | "--depth", 193 | "1", 194 | "https://github.com/commercialhaskell/" ++ channel ++ ".git", 195 | destPath 196 | ] 197 | putStrLn $ printf "Clone %s finish" channel 198 | 199 | stackSetup :: FilePath -> FilePath -> IO () 200 | stackSetup bp setupPath = do 201 | jr <- decodeFileEither setupPath :: IO (Either ParseException StackSetup) 202 | r <- case jr of 203 | Left err -> do 204 | putStrLn (prettyPrintParseException err) 205 | error "Parse setup yaml failure" 206 | Right obj -> return obj 207 | 208 | let (StackSetup stack exe dll pgit msys2 ghc ghcjs) = r 209 | 210 | -- store stack 211 | -- let filesToDownload = M.toList stack >>= M.toList . snd >>= return . snd 212 | 213 | -- let dlEachStack (ResourceInfo _ u _ s1 s256) = download u (bp "stack") (s1, s256) False 214 | -- in mapM_ dlEachStack filesToDownload 215 | 216 | let newStack = M.map (M.map (redirectToMirror "stack")) stack 217 | 218 | -- store 7z 219 | let dl7z (ResourceInfo _ u l s1 s256) = download u (bp "7z") (s1, s256) l False 220 | in do 221 | dl7z exe 222 | dl7z dll 223 | 224 | let newExe = redirectToMirror "7z" exe 225 | let newDll = redirectToMirror "7z" dll 226 | 227 | -- store portable git 228 | let dlGit (ResourceInfo _ u l s1 s256) = download u (bp "pgit") (s1, s256) l False 229 | in dlGit pgit 230 | 231 | let newPgit = redirectToMirror "pgit" pgit 232 | 233 | -- store ghc 234 | let filesToDownload = map snd $ M.elems ghc >>= M.toList 235 | 236 | let dlGhc (ResourceInfo _ u l s1 s256) = download u (bp "ghc") (s1, s256) l False 237 | in mapM_ dlGhc filesToDownload 238 | 239 | let newGhc = M.map (M.map (redirectToMirror "ghc")) ghc 240 | 241 | -- store msys2 242 | let filesToDownload = snd <$> M.toList msys2 243 | 244 | let dlMsys2 (ResourceInfo _ u l s1 s256) = download u (bp "msys2") (s1, s256) l False 245 | in mapM_ dlMsys2 filesToDownload 246 | 247 | let newMsys2 = M.map (redirectToMirror "msys2") msys2 248 | 249 | encodeFile (bp "stack-setup.yaml") (StackSetup newStack newExe newDll newPgit newMsys2 newGhc ghcjs) 250 | putStrLn $ printf "Stack setup successfully processed" 251 | 252 | syncStack :: FilePath -> IO () 253 | syncStack basePath = do 254 | putStrLn "start to sync stack" 255 | download 256 | "https://api.github.com/repos/commercialhaskell/stack/releases/latest" 257 | "/tmp" 258 | ("", "") 259 | 0 260 | True 261 | let filename = "/tmp/latest" 262 | text <- BS.readFile filename 263 | let latestInfo = case A.decode text of 264 | Just o -> o :: GitHubReleases 265 | _ -> error "decode latest fail!" 266 | let isPreRelease = prerelease latestInfo 267 | let as = assets latestInfo 268 | unless isPreRelease (syncAssets as) 269 | where 270 | syncAssets = mapM_ syncAsset 271 | syncAsset as = do 272 | download 273 | (browser_download_url as) 274 | (basePath "stack") 275 | ("", "") 276 | (size as) 277 | False 278 | 279 | main :: IO () 280 | main = do 281 | -- specify what place to save the mirror TO 282 | basePath <- getEnv "TO" 283 | 284 | -- update channel 285 | updateChannels basePath 286 | 287 | -- load snapshots 288 | download "https://www.stackage.org/download/snapshots.json" basePath ("", "") 0 True 289 | 290 | -- load stack setup 291 | download 292 | "https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml" 293 | "/tmp" 294 | ("", "") 295 | 0 296 | True 297 | 298 | stackSetup basePath "/tmp/stack-setup-2.yaml" 299 | 300 | -- sync stack from github 301 | syncStack basePath 302 | 303 | putStrLn "sync finish" 304 | -------------------------------------------------------------------------------- /stackage/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | [[ $DEBUG = true ]] && set -x 5 | 6 | /stackage 7 | -------------------------------------------------------------------------------- /test/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Jian Zeng " 3 | ADD sync.sh / 4 | -------------------------------------------------------------------------------- /test/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo 'Sleep 6s' 4 | sleep 6 5 | echo 'Sleep 5s' 6 | sleep 5 7 | env 8 | 9 | if [[ -n $HAS_ERROR ]]; then 10 | echo Error occurred 11 | sleep 2 12 | exit 1 13 | fi 14 | 15 | if [[ -n $SLEEP_INFINITY ]]; then 16 | echo Sleep forever 17 | sleep infinity 18 | exit 0 19 | fi 20 | 21 | exit 0 22 | -------------------------------------------------------------------------------- /tsumugu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Keyu Tao " 3 | LABEL bind_support=true 4 | ARG TSUMUGU_VERSION=20250422 5 | 6 | RUN <", 7 | "license": "MIT", 8 | "type": "module", 9 | "dependencies": { 10 | "async": "^3.2.4", 11 | "fetch-retry": "^5.0.4", 12 | "jszip": "^3.10.1", 13 | "node-fetch": "^3.3.1", 14 | "promised-sqlite3": "^2.1.0", 15 | "sqlite3": "^5.1.5", 16 | "winston": "^3.8.2", 17 | "yaml": "^2.4.5" 18 | }, 19 | "scripts": { 20 | "start": "node sync-repo.js" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /winget-source/sync-repo.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import async from 'async' 3 | 4 | import { rm } from 'fs/promises' 5 | import { AsyncDatabase } from 'promised-sqlite3' 6 | import { EX_IOERR, EX_SOFTWARE, EX_TEMPFAIL, EX_UNAVAILABLE } from './sysexits.js' 7 | 8 | import { 9 | buildManifestURIs, 10 | buildManifestURIsFromPackageMetadata, 11 | buildPackageMetadataURIs, 12 | buildPathpartMap, 13 | cacheFileWithURI, 14 | exitWithCode, 15 | extractDatabaseFromBundle, 16 | makeTempDirectory, 17 | setupEnvironment, 18 | syncFile, 19 | } from './utilities.js' 20 | 21 | 22 | const { forceSync, parallelLimit, remote, sqlite3, winston } = setupEnvironment(); 23 | 24 | /** 25 | * Sync with the official WinGet repository index. 26 | * 27 | * @param {number} version WinGet index version to sync. 28 | * @param {(db: AsyncDatabase) => Promise} handler Handler function that reads the index database and syncs necessary files. 29 | * 30 | * @returns {Promise} Fulfills with `undefined` upon success. 31 | */ 32 | async function syncIndex(version, handler) { 33 | const tempDirectory = await makeTempDirectory('winget-repo-'); 34 | const sourceFilename = version > 1 ? `source${version}.msix` : 'source.msix'; 35 | try { 36 | // download index package to buffer 37 | const [indexBuffer, modifiedDate, updated] = await syncFile(sourceFilename, true, false); 38 | if (!updated && !forceSync) { 39 | winston.info(`skip syncing version ${version} from ${remote}`); 40 | return; 41 | } 42 | assert(Buffer.isBuffer(indexBuffer), 'Failed to get the source index buffer!'); 43 | 44 | // unpack, extract and load index database 45 | try { 46 | const databaseFilePath = await extractDatabaseFromBundle(indexBuffer, tempDirectory); 47 | const database = new sqlite3.Database(databaseFilePath, sqlite3.OPEN_READONLY); 48 | try { 49 | // sync files with handler 50 | const asyncDatabase = new AsyncDatabase(database); 51 | await handler(asyncDatabase); 52 | await asyncDatabase.close(); 53 | } catch (error) { 54 | exitWithCode(EX_SOFTWARE, error); 55 | } 56 | } catch (error) { 57 | exitWithCode(EX_IOERR, error); 58 | } 59 | 60 | // update index package 61 | if (updated) { 62 | await cacheFileWithURI(sourceFilename, indexBuffer, modifiedDate); 63 | } 64 | } catch (error) { 65 | try { 66 | await rm(tempDirectory, { recursive: true }); 67 | } finally { 68 | exitWithCode(EX_UNAVAILABLE, error); 69 | } 70 | } 71 | winston.info(`successfully synced version ${version} from ${remote}`); 72 | await rm(tempDirectory, { recursive: true }); 73 | } 74 | 75 | winston.info(`start syncing with ${remote}`); 76 | 77 | await syncIndex(2, async (db) => { 78 | try { 79 | const packageURIs = buildPackageMetadataURIs(await db.all('SELECT id, hash FROM packages')); 80 | try { 81 | // sync latest package metadata and manifests in parallel 82 | await async.eachLimit(packageURIs, parallelLimit, async (uri) => { 83 | const [metadataBuffer, modifiedDate, updated] = await syncFile(uri, forceSync, false); 84 | if (metadataBuffer) { 85 | const manifestURIs = await buildManifestURIsFromPackageMetadata(metadataBuffer); 86 | await async.eachSeries(manifestURIs, async (uri) => await syncFile(uri, forceSync)); 87 | if (updated) { 88 | await cacheFileWithURI(uri, metadataBuffer, modifiedDate); 89 | } 90 | } 91 | }); 92 | } catch (error) { 93 | exitWithCode(EX_TEMPFAIL, error); 94 | } 95 | } catch (error) { 96 | exitWithCode(EX_SOFTWARE, error); 97 | } 98 | }); 99 | 100 | await syncIndex(1, async (db) => { 101 | try { 102 | const pathparts = buildPathpartMap(await db.all('SELECT * FROM pathparts')); 103 | const uris = buildManifestURIs(await db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC'), pathparts); 104 | // sync latest manifests in parallel 105 | try { 106 | await async.eachLimit(uris, parallelLimit, async (uri) => await syncFile(uri, forceSync)); 107 | } catch (error) { 108 | exitWithCode(EX_TEMPFAIL, error); 109 | } 110 | } catch (error) { 111 | exitWithCode(EX_SOFTWARE, error); 112 | } 113 | }); 114 | 115 | winston.info(`successfully synced with ${remote}`); 116 | -------------------------------------------------------------------------------- /winget-source/sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## EXPORTED IN entry.sh 4 | #TO= 5 | 6 | ## SET IN ENVIRONMENT VARIABLES 7 | #BIND_ADDRESS= 8 | #DEBUG= 9 | #WINGET_FORCE_SYNC= 10 | #WINGET_REPO_URL= 11 | #WINGET_REPO_JOBS= 12 | 13 | set -e 14 | 15 | if [[ $DEBUG = true ]]; then 16 | set -x 17 | else 18 | export NODE_ENV=production 19 | fi 20 | 21 | exec node /sync-repo.js 22 | -------------------------------------------------------------------------------- /winget-source/sysexits.js: -------------------------------------------------------------------------------- 1 | // The definitions and documents basically refer to FreeBSD Manual Page for `sysexits`. 2 | // 3 | // https://man.freebsd.org/cgi/man.cgi?query=sysexits 4 | 5 | /** The program runs successfully. */ 6 | export const EX_OK = 0; 7 | 8 | /** The command was used incorrectly, e.g., with the wrong number of arguments, a bad flag, a bad syntax in a parameter, or whatever. */ 9 | export const EX_USAGE = 64; 10 | 11 | /** 12 | * The input data was incorrect in some way. 13 | * 14 | * This should only be used for user's data and not system files. 15 | */ 16 | export const EX_DATAERR = 65; 17 | 18 | /** 19 | * An input file (not a system file) did not exist or was not readable. 20 | * 21 | * This could also include errors like "No message" to a mailer (if it cared to catch it). 22 | */ 23 | export const EX_NOINPUT = 66; 24 | 25 | /** 26 | * The user specified did not exist. 27 | * 28 | * This might be used for mail addresses or remote logins. 29 | */ 30 | export const EX_NOUSER = 67; 31 | 32 | /** 33 | * The host specified did not exist. 34 | * 35 | * This is used in mail addresses or network requests. 36 | */ 37 | export const EX_NOHOST = 68; 38 | 39 | /** 40 | * A service is unavailable. 41 | * 42 | * This can occur if a support program or file does not exist. 43 | * 44 | * This can also be used as a catchall message when something you wanted to do does not work, but you do not know why. 45 | */ 46 | export const EX_UNAVAILABLE = 69; 47 | 48 | /** 49 | * An internal software error has been detected. 50 | * 51 | * This should be limited to non-operating system related errors as possible. 52 | */ 53 | export const EX_SOFTWARE = 70; 54 | 55 | /** 56 | * An operating system error has been detected. 57 | * 58 | * This is intended to be used for such things as "cannot fork", "cannot create pipe", or the like. 59 | * 60 | * It includes things like getuid returning a user that does not exist in the passwd file. 61 | */ 62 | export const EX_OSERR = 71; 63 | 64 | /** Some system file (e.g., `/etc/passwd`, `/var/run/utx.active`, etc.) does not exist, cannot be opened, or has some sort of error (e.g., syntax error). */ 65 | export const EX_OSFILE = 72; 66 | 67 | /** A (user specified) output file cannot be created. */ 68 | export const EX_CANTCREAT = 73; 69 | 70 | /** An error occurred while doing I/O on some file. */ 71 | export const EX_IOERR = 74; 72 | 73 | /** 74 | * Temporary failure, indicating something that is not really an error. 75 | * 76 | * In sendmail, this means that a mailer (e.g.) could not create a connection, and the request should be reattempted later. 77 | */ 78 | export const EX_TEMPFAIL = 75; 79 | 80 | /** The remote system returned something that was "not possible" during a protocol exchange. */ 81 | export const EX_PROTOCOL = 76; 82 | 83 | /** 84 | * You did not have sufficient permission to perform the operation. 85 | * 86 | * This is not intended for file system problems, which should use {@link EX_NOINPUT} or {@link EX_CANTCREAT}, but rather for higher level permissions. 87 | */ 88 | export const EX_NOPERM = 77; 89 | 90 | /** Something was found in an unconfigured or misconfigured state. */ 91 | export const EX_CONFIG = 78; 92 | -------------------------------------------------------------------------------- /winget-source/utilities.js: -------------------------------------------------------------------------------- 1 | import withRetry from 'fetch-retry' 2 | import https from 'https' 3 | import JSZip from 'jszip' 4 | import originalFetch from 'node-fetch' 5 | import os from 'os' 6 | import path from 'path' 7 | import process from 'process' 8 | import sqlite3 from 'sqlite3' 9 | import winston from 'winston' 10 | import YAML from 'yaml' 11 | import Zlib from 'zlib' 12 | 13 | import { existsSync } from 'fs' 14 | import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from 'fs/promises' 15 | import { isIP } from 'net' 16 | import { promisify } from 'util' 17 | 18 | import { EX_IOERR, EX_USAGE } from './sysexits.js' 19 | 20 | 21 | /** 22 | * `fetch` implementation with retry support. 23 | * 24 | * 3 retries with 1000ms delay, on network errors and HTTP code >= 400. 25 | */ 26 | const fetch = withRetry(originalFetch, { 27 | retryOn: (attempt, error, response) => { 28 | if (attempt > 3) return false; 29 | 30 | if (error || response.status >= 400) { 31 | if (response) 32 | winston.warn(`retrying ${response.url} (${attempt})`); 33 | else 34 | winston.warn(`retrying (${attempt}, error: ${error})`); 35 | return true; 36 | } 37 | } 38 | }); 39 | 40 | /** The remote URL of a pre-indexed WinGet source repository. */ 41 | const remote = process.env.WINGET_REPO_URL ?? 'https://cdn.winget.microsoft.com/cache'; 42 | 43 | /** The local path to serve as the root of WinGet source repository. */ 44 | const local = process.env.TO; 45 | 46 | /** Maximum sync jobs to be executed in parallel. Defaults to 8. */ 47 | const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); 48 | 49 | /** Whether the debug mode is enabled. */ 50 | const debugMode = process.env.DEBUG === 'true'; 51 | 52 | /** Whether to perform a forced sync. */ 53 | const forceSync = process.env.WINGET_FORCE_SYNC === 'true'; 54 | 55 | /** Local IP address to be bound to HTTPS requests. */ 56 | const localAddress = process.env.BIND_ADDRESS; 57 | 58 | /** Decompress a deflated stream asynchronously. */ 59 | const inflateRaw = promisify(Zlib.inflateRaw); 60 | 61 | /** 62 | * Get the local sync path of a manifest. 63 | * 64 | * @param {string} uri Manifest URI. 65 | * 66 | * @returns {string} Expected local path of the manifest file. 67 | */ 68 | function getLocalPath(uri) { 69 | return path.join(local, uri); 70 | } 71 | 72 | /** 73 | * Get the remote URL of a manifest. 74 | * 75 | * @param {string} uri Manifest URI. 76 | * 77 | * @returns {URL} Remote URL to get the manifest from. 78 | */ 79 | function getRemoteURL(uri) { 80 | const remoteURL = new URL(remote); 81 | remoteURL.pathname = path.posix.join(remoteURL.pathname, uri); 82 | return remoteURL; 83 | } 84 | 85 | /** 86 | * Decompress a MSZIP-compressed buffer. 87 | * 88 | * @param {Buffer} buffer Compressed buffer using MSZIP. 89 | * 90 | * @returns {Buffer} The decompressed buffer. 91 | */ 92 | async function decompressMSZIP(buffer) { 93 | const magicHeader = Buffer.from([0, 0, 0x43, 0x4b]); 94 | if (!buffer.subarray(26, 30).equals(magicHeader)) { 95 | throw new Error('Invalid MSZIP format'); 96 | } 97 | var chunkIndex = 26; 98 | var decompressed = Buffer.alloc(0); 99 | while ((chunkIndex = buffer.indexOf(magicHeader, chunkIndex)) > -1) { 100 | chunkIndex += magicHeader.byteLength; 101 | const decompressedChunk = await inflateRaw(buffer.subarray(chunkIndex), { 102 | dictionary: decompressed.subarray(-32768) 103 | }); 104 | decompressed = Buffer.concat([decompressed, decompressedChunk]); 105 | } 106 | return decompressed; 107 | } 108 | 109 | /** 110 | * Get last modified date from HTTP response headers. 111 | * 112 | * @param {Response} response The HTTP `fetch` response to parse. 113 | * 114 | * @returns {Date | undefined} Last modified date derived from the response, if exists. 115 | */ 116 | function getLastModifiedDate(response) { 117 | const lastModified = response.headers.get('Last-Modified'); 118 | if (lastModified) { 119 | return new Date(Date.parse(lastModified)); 120 | } else { 121 | return undefined; 122 | } 123 | } 124 | 125 | /** 126 | * Get content length from HTTP response headers. 127 | * 128 | * @param {Response} response The HTTP `fetch` response to parse. 129 | * 130 | * @returns {number} Content length derived from the response, in bytes. 131 | */ 132 | function getContentLength(response) { 133 | const length = response.headers.get('Content-Length'); 134 | if (length) { 135 | return parseInt(length); 136 | } else { 137 | return 0; 138 | } 139 | } 140 | 141 | /** 142 | * Resolve path parts against the local storage. 143 | * 144 | * @param {number} id The ID of the target path part. 145 | * @param {Map} pathparts Path part storage built from the database. 146 | * 147 | * @returns {string} Full URI resolved from the given path part ID. 148 | */ 149 | function resolvePathpart(id, pathparts) { 150 | const pathpart = pathparts.get(id); 151 | if (pathpart === undefined) { 152 | return ''; 153 | } 154 | return path.posix.join(resolvePathpart(pathpart.parent, pathparts), pathpart.pathpart); 155 | } 156 | 157 | /** 158 | * Resolve manifest URIs against package metadata. 159 | * 160 | * Reference: https://github.com/microsoft/winget-cli/blob/master/src/AppInstallerCommonCore/PackageVersionDataManifest.cpp 161 | * 162 | * @param {{ sV: string, vD: { v: string, rP: string | undefined, s256H: string | undefined }[], [key: string]: any }} metadata The parsed package metadata object. 163 | * 164 | * @returns {string[]} URIs resolved from the given metadata. 165 | */ 166 | function resolvePackageManifestURIs(metadata) { 167 | return metadata.vD.map((version) => version.rP).filter(Boolean); 168 | } 169 | 170 | /** 171 | * Set up the default `winston` logger instance. 172 | */ 173 | function setupWinstonLogger() { 174 | const { format, transports } = winston; 175 | winston.configure({ 176 | format: format.errors({ stack: debugMode }), 177 | transports: [ 178 | new transports.Console({ 179 | handleExceptions: true, 180 | level: debugMode ? 'debug' : 'info', 181 | stderrLevels: ['error'], 182 | format: format.combine( 183 | format.timestamp(), 184 | format.printf(({ timestamp, level, message, stack }) => 185 | `[${timestamp}][${level.toUpperCase()}] ${stack ?? message}` 186 | ) 187 | ) 188 | }) 189 | ] 190 | }); 191 | } 192 | 193 | /** 194 | * Build a local storage for path parts from database query. 195 | * 196 | * @param {{ rowid: number, parent: number, pathpart: string }[]} rows Rows returned by the query. 197 | * 198 | * @returns {Map} In-memory path part storage to query against. 199 | */ 200 | export function buildPathpartMap(rows) { 201 | return new Map(rows.map(row => 202 | [row.rowid, { parent: row.parent, pathpart: row.pathpart }] 203 | )); 204 | } 205 | 206 | /** 207 | * Build a list of all manifest URIs from database query. 208 | * 209 | * @param {{ pathpart: string, [key: string]: string }[]} rows Rows returned by the query. 210 | * @param {Map} pathparts Path part storage built from the database. 211 | * 212 | * @returns {string[]} Manifest URIs to sync. 213 | */ 214 | export function buildManifestURIs(rows, pathparts) { 215 | return rows.map(row => resolvePathpart(row.pathpart, pathparts)); 216 | } 217 | 218 | /** 219 | * Build a list of all package metadata URIs from database query. 220 | * 221 | * @param {{ id: string, hash: Buffer, [key: string]: string }[]} rows Rows returned by the query. 222 | * 223 | * @returns {string[]} Package metadata URIs to sync. 224 | */ 225 | export function buildPackageMetadataURIs(rows) { 226 | return rows.map(row => 227 | path.posix.join('packages', row.id, row.hash.toString('hex').slice(0, 8), 'versionData.mszyml') 228 | ); 229 | } 230 | 231 | /** 232 | * Exit with given status with error logging. 233 | * 234 | * @param {number} code Exit code to use. 235 | * @param {Error | string | null | undefined} error Error to log. 236 | * 237 | * @returns {never} Exits the process. 238 | */ 239 | export function exitWithCode(code = 0, error = undefined) { 240 | if (error) { 241 | winston.exitOnError = false; 242 | winston.error(error); 243 | } 244 | process.exit(code); 245 | } 246 | 247 | /** 248 | * Build a list of all manifest URIs from compressed package metadata. 249 | * 250 | * Reference: https://github.com/kyz/libmspack/blob/master/libmspack/mspack/mszipd.c 251 | * 252 | * @param {fs.PathLike | Buffer} mszymlMetadata Path or buffer of the MSZYML metadata file. 253 | * 254 | * @returns {Promise} Manifest URIs to sync. 255 | */ 256 | export async function buildManifestURIsFromPackageMetadata(mszymlMetadata) { 257 | const compressedBuffer = Buffer.isBuffer(mszymlMetadata) ? mszymlMetadata : await readFile(mszymlMetadata); 258 | const buffer = await decompressMSZIP(compressedBuffer); 259 | const metadata = YAML.parse(buffer.toString()); 260 | return resolvePackageManifestURIs(metadata); 261 | } 262 | 263 | /** 264 | * Extract database file from the source bundle. 265 | * 266 | * @param {fs.PathLike | Buffer} msixFile Path or buffer of the MSIX bundle file. 267 | * @param {fs.PathLike} directory Path of directory to save the file. 268 | * 269 | * @returns {Promise} Path of the extracted `index.db` file. 270 | */ 271 | export async function extractDatabaseFromBundle(msixFile, directory) { 272 | const bundle = Buffer.isBuffer(msixFile) ? msixFile : await readFile(msixFile); 273 | const zip = await JSZip.loadAsync(bundle); 274 | const buffer = await zip.file(path.posix.join('Public', 'index.db')).async('Uint8Array'); 275 | const destination = path.join(directory, 'index.db'); 276 | await writeFile(destination, buffer); 277 | return destination; 278 | } 279 | 280 | /** 281 | * Create a unique temporary directory with given prefix. 282 | * 283 | * @param {string} prefix Temporary directory name prefix. Must not contain path separators. 284 | * 285 | * @returns {Promise} Path to the created temporary directory. 286 | */ 287 | export async function makeTempDirectory(prefix) { 288 | try { 289 | return await mkdtemp(path.join(os.tmpdir(), prefix)); 290 | } catch (error) { 291 | exitWithCode(EX_IOERR, error); 292 | } 293 | } 294 | 295 | /** 296 | * Check and set up the environment. 297 | * 298 | * @returns Values and objects to be used in the program. 299 | */ 300 | export function setupEnvironment() { 301 | setupWinstonLogger(); 302 | if (!local) { 303 | exitWithCode(EX_USAGE, "destination path $TO not set!"); 304 | } 305 | if (localAddress) { 306 | https.globalAgent.options.localAddress = localAddress; 307 | https.globalAgent.options.family = isIP(localAddress); 308 | } 309 | return { 310 | debugMode, 311 | forceSync, 312 | local, 313 | parallelLimit, 314 | remote, 315 | sqlite3: debugMode ? sqlite3.verbose() : sqlite3, 316 | winston 317 | }; 318 | } 319 | 320 | /** 321 | * Cache a file with specific modified date. 322 | * 323 | * @param {string} uri File URI to cache. 324 | * @param {Buffer} buffer Whether to save the file to disk. 325 | * @param {Date | null | undefined} modifiedAt Modified date of the file, if applicable. 326 | * 327 | * @returns {Promise} Fulfills with `undefined` upon success. 328 | */ 329 | export async function cacheFileWithURI(uri, buffer, modifiedAt) { 330 | const path = getLocalPath(uri); 331 | await writeFile(path, buffer); 332 | if (modifiedAt) { 333 | await utimes(path, modifiedAt, modifiedAt); 334 | } 335 | } 336 | 337 | /** 338 | * Sync a file with the remote server asynchronously. 339 | * 340 | * @param {string} uri URI to sync. 341 | * @param {boolean} update Whether to allow updating an existing file. 342 | * @param {boolean} save Whether to save the file to disk. 343 | * 344 | * @returns {Promise<[?Buffer, ?Date, boolean]>} File buffer, last modified date and if the file is updated. 345 | */ 346 | export async function syncFile(uri, update = true, save = true) { 347 | const localPath = getLocalPath(uri); 348 | const remoteURL = getRemoteURL(uri); 349 | await mkdir(path.dirname(localPath), { recursive: true }); 350 | if (existsSync(localPath)) { 351 | if (!update) { 352 | winston.debug(`skipped ${uri} because it already exists`); 353 | return [null, null, false]; 354 | } 355 | const response = await fetch(remoteURL, { method: 'HEAD' }); 356 | const lastModified = getLastModifiedDate(response); 357 | const contentLength = getContentLength(response); 358 | if (lastModified) { 359 | const localFile = await stat(localPath); 360 | if (localFile.mtime.getTime() == lastModified.getTime() && localFile.size == contentLength) { 361 | winston.debug(`skipped ${uri} because it's up to date`); 362 | return [await readFile(localPath), lastModified, false]; 363 | } 364 | } 365 | } 366 | winston.info(`downloading from ${remoteURL}`); 367 | const response = await fetch(remoteURL); 368 | const arrayBuffer = await response.arrayBuffer(); 369 | const buffer = Buffer.from(arrayBuffer); 370 | const lastModified = getLastModifiedDate(response); 371 | if (save) { 372 | await cacheFileWithURI(uri, buffer, lastModified); 373 | } 374 | return [buffer, lastModified ?? null, true]; 375 | } 376 | -------------------------------------------------------------------------------- /yukina/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ustcmirror/base:alpine 2 | LABEL maintainer="Keyu Tao " 3 | LABEL bind_support=true 4 | ARG YUKINA_VERSION=20250422 5 | 6 | RUN < 0 else (0, 0) # size can be None 89 | 90 | def check_and_download(url: str, dst_file: Path)->int: 91 | try: 92 | start = time.time() 93 | with requests.get(url, stream=True, timeout=(5, 10)) as r: 94 | r.raise_for_status() 95 | if 'last-modified' in r.headers: 96 | remote_ts = parsedate_to_datetime( 97 | r.headers['last-modified']).timestamp() 98 | else: remote_ts = None 99 | 100 | with dst_file.open('wb') as f: 101 | for chunk in r.iter_content(chunk_size=1024**2): 102 | if time.time() - start > DOWNLOAD_TIMEOUT: 103 | raise TimeoutError("Download timeout") 104 | if not chunk: continue # filter out keep-alive new chunks 105 | 106 | f.write(chunk) 107 | if remote_ts is not None: 108 | os.utime(dst_file, (remote_ts, remote_ts)) 109 | return 0 110 | except BaseException as e: 111 | print(e, flush=True) 112 | if dst_file.is_file(): dst_file.unlink() 113 | return 1 114 | 115 | def download_repodata(url: str, path: Path) -> int: 116 | path = path / "repodata" 117 | path.mkdir(exist_ok=True) 118 | oldfiles = set(path.glob('*.*')) 119 | newfiles = set() 120 | if check_and_download(url + "/repodata/repomd.xml", path / ".repomd.xml") != 0: 121 | print(f"Failed to download the repomd.xml of {url}") 122 | return 1 123 | try: 124 | tree = ET.parse(path / ".repomd.xml") 125 | root = tree.getroot() 126 | assert root.tag.endswith('repomd') 127 | for location in root.findall('./{http://linux.duke.edu/metadata/repo}data/{http://linux.duke.edu/metadata/repo}location'): 128 | href = location.attrib['href'] 129 | assert len(href) > 9 and href[:9] == 'repodata/' 130 | fn = path / href[9:] 131 | newfiles.add(fn) 132 | if check_and_download(url + '/' + href, fn) != 0: 133 | print(f"Failed to download the {href}") 134 | return 1 135 | except BaseException as e: 136 | traceback.print_exc() 137 | return 1 138 | 139 | (path / ".repomd.xml").rename(path / "repomd.xml") # update the repomd.xml 140 | newfiles.add(path / "repomd.xml") 141 | for i in (oldfiles - newfiles): 142 | print(f"Deleting old files: {i}") 143 | i.unlink() 144 | 145 | def check_args(prop: str, lst: List[str]): 146 | for s in lst: 147 | if len(s)==0 or ' ' in s: 148 | raise ValueError(f"Invalid item in {prop}: {repr(s)}") 149 | 150 | def substitute_vars(s: str, vardict: Dict[str, str]) -> str: 151 | for key, val in vardict.items(): 152 | tpl = "@{"+key+"}" 153 | s = s.replace(tpl, val) 154 | return s 155 | 156 | def main(): 157 | parser = argparse.ArgumentParser() 158 | parser.add_argument("base_url", type=str, help="base URL") 159 | parser.add_argument("os_version", type=str, help="e.g. 6-8") 160 | parser.add_argument("component", type=str, help="e.g. mysql56-community,mysql57-community") 161 | parser.add_argument("arch", type=str, help="e.g. x86_64,aarch64") 162 | parser.add_argument("repo_name", type=str, help="e.g. @{comp}-el@{os_ver}, usually the parent name of repodata") 163 | parser.add_argument("working_dir", type=str, help="working directory (with substitution support)") 164 | parser.add_argument("--download-repodata", action='store_true', 165 | help='download repodata files instead of generating them') 166 | parser.add_argument("--pass-arch-to-reposync", action='store_true', 167 | help='''pass --arch to reposync to further filter packages by 'arch' field in metadata (NOT recommended, prone to missing packages in some repositories, e.g. mysql)''') 168 | args = parser.parse_args() 169 | 170 | if '@' == args.os_version[0]: 171 | # current only allow to have one @-prefixed os version 172 | os_list = OS_TEMPLATE[args.os_version[1:]] 173 | elif '-' in args.os_version: 174 | dash = args.os_version.index('-') 175 | os_list = [ str(i) for i in range( 176 | int(args.os_version[:dash]), 177 | 1+int(args.os_version[dash+1:])) ] 178 | elif ',' in args.os_version: 179 | os_list = args.os_version.split(",") 180 | else: 181 | os_list = [args.os_version] 182 | check_args("os_version", os_list) 183 | component_list = args.component.split(',') 184 | check_args("component", component_list) 185 | arch_list = args.arch.split(',') 186 | check_args("arch", arch_list) 187 | 188 | failed = [] 189 | cache_dir = tempfile.mkdtemp() 190 | 191 | def combination_os_comp(arch: str): 192 | for os in os_list: 193 | for comp in component_list: 194 | vardict = { 195 | 'arch': arch, 196 | 'os_ver': os, 197 | 'comp': comp, 198 | } 199 | 200 | name = substitute_vars(args.repo_name, vardict) 201 | url = substitute_vars(args.base_url, vardict) 202 | working_dir = Path(substitute_vars(args.working_dir, vardict)) 203 | try: 204 | probe_url = url + ('' if url.endswith('/') else '/') + "repodata/repomd.xml" 205 | r = requests.head(probe_url, timeout=(7,7)) 206 | if r.status_code < 400 or r.status_code == 403: 207 | yield (name, url, working_dir) 208 | else: 209 | print(probe_url, "->", r.status_code) 210 | except: 211 | traceback.print_exc() 212 | 213 | for arch in arch_list: 214 | # dest_dirs = [] 215 | for name, url, working_dir in combination_os_comp(arch): 216 | working_dir.mkdir(parents=True, exist_ok=True) 217 | conf = tempfile.NamedTemporaryFile("w", suffix=".conf") 218 | conf.write(f''' 219 | [main] 220 | keepcache=0 221 | cachedir={cache_dir} 222 | ''') 223 | conf.write(f''' 224 | [{name}] 225 | name={name} 226 | baseurl={url} 227 | repo_gpgcheck=0 228 | gpgcheck=0 229 | enabled=1 230 | ''') 231 | dst = working_dir.parent.absolute() 232 | # dst.mkdir(parents=True, exist_ok=True) 233 | # dest_dirs.append(dst) 234 | conf.flush() 235 | # sp.run(["cat", conf.name]) 236 | # sp.run(["ls", "-la", cache_dir]) 237 | 238 | # if len(dest_dirs) == 0: 239 | # print("Nothing to sync", flush=True) 240 | # failed.append(('', arch)) 241 | # continue 242 | 243 | cmd_args = ["dnf", "reposync", "-c", conf.name, "--delete", "-p", dst] 244 | if args.pass_arch_to_reposync: 245 | cmd_args += ["--arch", arch] 246 | print(f"Launching reposync with command: {cmd_args}", flush=True) 247 | # print(cmd_args) 248 | ret = sp.run(cmd_args) 249 | if ret.returncode != 0: 250 | failed.append((name, arch)) 251 | continue 252 | 253 | # for path in dest_dirs: 254 | # dst.mkdir(exist_ok=True) 255 | if args.download_repodata: 256 | download_repodata(url, dst) 257 | else: 258 | cmd_args = ["createrepo_c", "--update", "-v", "-c", cache_dir, "-o", str(working_dir), str(working_dir)] 259 | # print(cmd_args) 260 | print(f"Launching createrepo_c with command: {cmd_args}", flush=True) 261 | ret = sp.run(cmd_args) 262 | calc_repo_size(dst) 263 | 264 | conf.close() 265 | 266 | if len(failed) > 0: 267 | print(f"Failed YUM repos: {failed}", flush=True) 268 | exit(len(failed)) 269 | else: 270 | if len(REPO_SIZE_FILE) > 0: 271 | with open(REPO_SIZE_FILE, "a") as fd: 272 | total_size = sum([r[0] for r in REPO_STAT.values()]) 273 | fd.write(f"+{total_size}") 274 | 275 | if __name__ == "__main__": 276 | main() 277 | --------------------------------------------------------------------------------