├── .github └── workflows │ └── main.yml ├── LICENSE ├── README.md ├── action.yml ├── restore.js ├── restore.sh ├── save.js ├── save.sh ├── update_cache_linux.sh ├── update_cache_macos.sh ├── update_cache_ubuntu_16_04.sh ├── update_cache_ubuntu_20_04.sh └── update_cache_windows.sh /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | strategy: 8 | fail-fast: false 9 | matrix: 10 | os: [ubuntu-18.04, ubuntu-20.04, macos-10.15, windows-2019] 11 | 12 | runs-on: ${{ matrix.os }} 13 | 14 | env: 15 | CONAN_USER_HOME: "${{ github.workspace }}/conan/" 16 | 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v2 20 | 21 | - name: Cache Conan modules 22 | id: cache-conan 23 | uses: ./ # Uses an action in the root directory 24 | with: 25 | bot_name: ${{ secrets.BOT_NAME }} 26 | bot_token: ${{ secrets.BOT_TOKEN }} 27 | cache_name: turtlebrowser/conan-center-index-cache 28 | key: test_${{ runner.os }}-${{ matrix.os }}-${{ hashFiles('restore.sh') }}-${{ hashFiles('save.sh') }} 29 | target_os: Test 30 | lfs_limit: 60 31 | 32 | - name: Did we hit the cache? 33 | run: echo "cache-hit was ${{ steps.cache-conan.outputs.cache-hit }}" 34 | 35 | - name: Get Conan 36 | run: | 37 | pip3 install wheel setuptools 38 | pip3 install conan --upgrade 39 | 40 | - name: Fix Path on Ubuntu 41 | if: matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-18.04' || matrix.os == 'ubuntu-16.04' || matrix.os == 'ubuntu-latest' 42 | run: echo "/home/runner/.local/bin" >> $GITHUB_PATH 43 | 44 | - name: Put something in the .conan 45 | if: ${{ steps.cache-conan.outputs.cache-hit == 0 }} 46 | run: conan remote add bincrafters https://api.bintray.com/conan/bincrafters/public-conan 47 | 48 | - name: Print branch 49 | run: | 50 | echo "${{ github.ref }}" 51 | echo "${{ github.head_ref }}" 52 | echo "${{ github.base_ref }}" 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 turtlebrowser 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Conan Cache GitHub Action 2 | 3 | ![CI](https://github.com/turtlebrowser/conan-cache/workflows/CI/badge.svg?branch=master) 4 | 5 | The action uses a GitHub repository as a cache for a .conan directory to speed up very slow builds. It was made specifically to offset the cost of a conan-qt build with QtWebEngine turned on. The cache can be populated in the GitHub action pipeline, or offline on a computer and then pushed to the repo. This second workflow is to help with builds that are too slow for GitHub Actions, where a step cannot take longer than 6 hours, and has limited disk space. See below for how to do an offline filling of the cache. 6 | 7 | The setup process for the action requires the creation of a bot account and a GitHub repo to hold the cache. See below for details. 8 | 9 | **When to use**: The conan-cache action was created to make CI builds possible for projects where building conan modules takes a lot of time, "a lot of time" is here measured in hours. The cache is also useful for local builds. If you have an artifactory then that is probably a better solution than this. Upload a prebuilt artifact there. (That might be added as an option to conan-cache in the future). 10 | 11 | **Works on**: Linux, Windows and MacOS 12 | 13 | This action was inspired by the builtin [GitHub Cache Action](https://help.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows), which might be more than adequate for your needs. See [here](https://github.com/turtlebrowser/conan-cache/blob/master/README.md#example-using-the-builtin-github-cache-instead-for-conan) for how you can use it for conan modules. The builtin GitHub Cache Action has some [limitations](https://help.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy), however, and tends to have a lot of intermittant failures to retrieve the cache, which can be unacceptable when building without takes a long time. 14 | 15 | See also [this trick](https://github.com/turtlebrowser/conan-cache/blob/master/README.md#using-github-cache-action-as-a-pre-cache-since-git-lfs-costs-money) using a combination of GitHub Cache Action and Conan Cache. 16 | 17 | ## Inputs 18 | 19 | ### `bot_name` 20 | **Required** The name of the GitHub bot 21 | 22 | ### `bot_token` 23 | **Required** The personal access token of the GitHub bot 24 | 25 | ### `cache_name` 26 | **Required** The GitHub repo used for this cache 27 | 28 | ### `key` 29 | **Required** An explicit key to store this cache under 30 | 31 | ### `target_os` 32 | **Optional** Target OS if different from host OS 33 | 34 | ### `lfs_limit` 35 | **Optional** In number of MB. Files with a size larger than lfs_limit are added to Git Large File Storage (LFS), defaults to 50MB, GitHub supports max 100MB, and will generate a warning at 50MB files. 36 | 37 | ## Outputs 38 | 39 | ### `cache-hit` 40 | The state of the cache hit: no hit (0), hit on explicit key (1), hit on fallback key (2). 41 | (**An explicit key is represented in the repo as a tag, a fallback key is represented as a branch**) 42 | 43 | ## Example usage 44 | 45 | Assumed environment 46 | ~~~~ 47 | env: 48 | CONAN_USER_HOME: "${{ github.workspace }}/release/" 49 | CONAN_USER_HOME_SHORT: "${{ github.workspace }}/release/short" 50 | ~~~~ 51 | 52 | Action setup 53 | ~~~~ 54 | - name: Cache Conan modules 55 | id: cache-conan 56 | uses: turtlebrowser/conan-cache@master 57 | with: 58 | bot_name: ${{ secrets.BOT_NAME }} 59 | bot_token: ${{ secrets.BOT_TOKEN }} 60 | cache_name: ${{ env.CACHE_GITHUB }}/${{ env.CACHE_GITHUB_REPO }} 61 | key: host-${{ runner.os }}-target-${{ runner.os }}-${{ hashFiles('conanfile.py') }} 62 | target_os: ${{ runner.os }} 63 | lfs_limit: 60 64 | ~~~~ 65 | 66 | After conan has filled the cache, clean it up 67 | ~~~~ 68 | - name: Clean up Conan 69 | run: | 70 | conan remove -f "*" --builds 71 | conan remove -f "*" --src 72 | conan remove -f "*" --system-reqs 73 | ~~~~ 74 | 75 | Use the _cache-hit_ output 76 | ~~~~ 77 | - name: On cache miss add bincrafters remote 78 | if: ${{ steps.cache-conan.outputs.cache-hit == 0 }} 79 | run: conan remote add bincrafters https://api.bintray.com/conan/bincrafters/public-conan 80 | ~~~~ 81 | 82 | ## Using GitHub Cache Action as a pre-cache (since git lfs costs money) 83 | 84 | A trick that can be used is to use the [GitHub Cache Action](https://help.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows) as a first level cache. It has limited space and often fails to fetch the cache for unknown reasons, but many times it will suffice and you can save on git lfs bandwith costs. Here is an example of how that could look. Note that the example doesn't use _restore-keys_, that is because the GitHub Cache Action's _cache-hit_ is a boolean that will only signal if the _key_ was hit (this is **not** how it works in Conan Cache). Also, if the _key_ doesn't hit, then you most likely have a change in your conanfile.py and should fetch from Conan Cache anyway. 85 | 86 | Note: Make sure it has been added to the Conan Cache properly using this action first, otherwise the short paths will be wrong on Windows. They are fixed in save.sh. 87 | 88 | ~~~~ 89 | # Check if GitHub Cache has it, because that's free 90 | - name: Using the builtin GitHub Cache Action for .conan 91 | id: github-cache-conan 92 | uses: actions/cache@v1 93 | env: 94 | cache-name: cache-conan-modules 95 | with: 96 | path: ${{ env.CONAN_USER_HOME }} 97 | key: host-${{ runner.os }}-target-${{ runner.os }}-${{ hashFiles('conanfile.py') }} 98 | 99 | # If GitHub Cache doesn't have it, get from Conan Cache (has git lfs cost) 100 | - name: Cache Conan modules 101 | if: steps.github-cache-conan.outputs.cache-hit != 'true' 102 | id: cache-conan 103 | uses: turtlebrowser/conan-cache@master 104 | with: 105 | bot_name: ${{ secrets.BOT_NAME }} 106 | bot_token: ${{ secrets.BOT_TOKEN }} 107 | cache_name: ${{ env.CACHE_GITHUB }}/${{ env.CACHE_GITHUB_REPO }} 108 | key: host-${{ runner.os }}-target-${{ runner.os }}-${{ hashFiles('conanfile.py') }} 109 | target_os: ${{ runner.os }} 110 | lfs_limit: 60 111 | ~~~~ 112 | 113 | ## Setup 114 | * Create a GitHub bot account that will be used to manage the conan cache 115 | * Create a GitHub repo for your conan cache 116 | * Add your bot as a collaborator on the cache with write access 117 | * Create a personal access token for your bot [https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line] 118 | * Add the personal access token as a secret to your repos workflow [https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets] 119 | * Follow the instructions below to set up an empty cache 120 | 121 | ## Environment in manual steps below 122 | 123 | ### `CACHE_GITHUB` 124 | GitHub account that has the cache repo 125 | ### `CACHE_GITHUB_REPO` 126 | Name of the cache repo 127 | ### `CONAN_USER_HOME` 128 | Conan variable, sets the directory in which .conan will be found 129 | ### `CONAN_USER_HOME_SHORT` 130 | Conan variable, sets the directory which will be used for [Conan short paths](https://docs.conan.io/en/latest/reference/conanfile/attributes.html#short-paths) (Windows only) 131 | ### `LFS_LIMIT` 132 | The file size at which a file should be added to Git LFS rather than to git. GitHub sets a max file size of 100MB. 133 | 134 | ## Make an empty repo 135 | 136 | This is the structure that is expected by conan-cache of an empty cache 137 | 138 | ~~~ 139 | git clone git@github.com:${CACHE_GITHUB}/${CACHE_GITHUB_REPO}.git 140 | cd ${CACHE_GITHUB_REPO} 141 | mkdir .conan && touch .conan/conan-cache.marker 142 | touch .gitattributes 143 | ~~~ 144 | 145 | Fill the .gitattributes with this contents (especially important for Windows) 146 | ~~~ 147 | $ cat .gitattributes 148 | * -text 149 | ~~~ 150 | 151 | ~~~ 152 | mkdir short && touch short/conan-cache.marker 153 | git add -A 154 | git commit -m "Setup cache" 155 | git push 156 | ~~~ 157 | 158 | ## How to use locally 159 | 160 | ~~~ 161 | git clone git@github.com:${CACHE_GITHUB}/${CACHE_GITHUB_REPO}.git 162 | cd ${CACHE_GITHUB_REPO} 163 | git checkout 164 | git lfs pull 165 | export CONAN_USER_HOME="c:/release" 166 | export CONAN_USER_HOME_SHORT=${CONAN_USER_HOME}/short 167 | find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 168 | ~~~ 169 | 170 | ## Populating the cache locally 171 | 172 | ### Prep: Creating the platform branch (first time) 173 | 174 | ~~~ 175 | export CONAN_USER_HOME="c:/release" 176 | export CONAN_USER_HOME_SHORT=${CONAN_USER_HOME}/short 177 | git clone git@github.com:${CACHE_GITHUB}/${CACHE_GITHUB_REPO}.git $CONAN_USER_HOME 178 | cd $CONAN_USER_HOME 179 | git checkout -b 180 | git push -u origin 181 | ~~~ 182 | 183 | ### Prep: Using the platform branch (after the first time) 184 | 185 | ~~~ 186 | export CONAN_USER_HOME="c:/release" 187 | export CONAN_USER_HOME_SHORT=${CONAN_USER_HOME}/short 188 | git clone git@github.com:${CACHE_GITHUB}/${CACHE_GITHUB_REPO}.git $CONAN_USER_HOME 189 | cd $CONAN_USER_HOME 190 | git clean -df 191 | git checkout . 192 | git checkout 193 | git pull 194 | git lfs pull 195 | find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 196 | ~~~ 197 | 198 | The final step is to insert the local CONAN_USER_HOME_SHORT path instead of the placeholder 199 | 200 | ### Populate: Build the target project with CONAN_USER_HOME and CONAN_USER_HOME_SHORT set 201 | 202 | This step assumes it is being run in the same shell as one of the prep steps. 203 | 204 | Build the target project, this will fill the cache. Then remove traces of the build process. Replace hardcoded paths with the placeholder _CONAN_USER_HOME_SHORT_. Then add any files exeeding the LFS limit for the project to _git lfs_. Add/remove everything as is with _git add -A_, commit and push. Start a build on GitHub and the branch will be tested and tagged automatically. 205 | 206 | ~~~ 207 | cd 208 | git pull 209 | rm -rf build 210 | # Make a cmake switch to turn on UPDATE build obsolete: 211 | # mkdir build && cd build && cmake -DUPDATE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. 212 | mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release .. 213 | conan remove -f "*" --builds 214 | conan remove -f "*" --src 215 | conan remove -f "*" --system-reqs 216 | cd $CONAN_USER_HOME 217 | find .conan/ -name .conan_link -exec perl -pi -e 's=$ENV{CONAN_USER_HOME_SHORT}=CONAN_USER_HOME_SHORT/=g' {} + 218 | export LFS_LIMIT=50 219 | find .conan short -type f -size +${LFS_LIMIT}M -exec ls -lh {} \; | awk '{ print $9 ": " $5 }' 220 | find .conan short -type f -size +${LFS_LIMIT}M -execdir git lfs track {} \; 221 | git add -A 222 | git commit -m "Local build" 223 | git push 224 | ~~~ 225 | 226 | ## Example using the builtin GitHub Cache Action instead for .conan 227 | 228 | Assumed environment 229 | ~~~~ 230 | env: 231 | CONAN_USER_HOME: "${{ github.workspace }}/release/" 232 | CONAN_USER_HOME_SHORT: "${{ github.workspace }}/release/short" 233 | ~~~~ 234 | 235 | Action setup 236 | ~~~ 237 | - name: Using the builtin GitHub Cache Action for .conan 238 | if: matrix.os == 'windows-latest' 239 | id: cache-conan 240 | uses: actions/cache@v1 241 | env: 242 | cache-name: cache-conan-modules 243 | with: 244 | path: ${{ env.CONAN_USER_HOME }} 245 | key: ${{ runner.os }}-builder-${{ env.cache-name }}-${{ hashFiles('conanfile.py') }} 246 | restore-keys: ${{ runner.os }}-builder-${{ env.cache-name }}- 247 | ~~~ 248 | -------------------------------------------------------------------------------- /action.yml: -------------------------------------------------------------------------------- 1 | name: 'Conan Cache' 2 | description: 'Cache .conan directories' 3 | author: 'TurtleBrowser' 4 | inputs: 5 | bot_name: 6 | description: 'The name of the GitHub bot' 7 | required: true 8 | bot_token: 9 | description: 'The personal access token of the GitHub bot' 10 | required: true 11 | cache_name: 12 | description: 'The GitHub repo used for this cache' 13 | required: true 14 | key: 15 | description: 'An explicit key' 16 | required: true 17 | target_os: 18 | description: 'Target OS if different from host OS' 19 | required: false 20 | lfs_limit: 21 | description: 'In number of MB. Files with a size larger than lfs_limit are added to Git Large File Storage (LFS), defaults to 50MB, GitHub supports max 100MB' 22 | required: false 23 | outputs: 24 | cache-hit: 25 | description: 'Cache a hit on a key: no hit (0), explicit key (1), fallback key (2)' 26 | runs: 27 | using: 'node12' 28 | main: 'restore.js' 29 | post: 'save.js' 30 | post-if: 'success()' 31 | branding: 32 | icon: 'archive' 33 | color: 'green' 34 | -------------------------------------------------------------------------------- /restore.js: -------------------------------------------------------------------------------- 1 | // From https://github.com/ad-m/github-push-action 2 | // 3 | // MIT License 4 | // 5 | // Copyright (c) 2019 Adam Dobrawy 6 | // 7 | // Permission is hereby granted, free of charge, to any person obtaining a copy 8 | // of this software and associated documentation files (the "Software"), to deal 9 | // in the Software without restriction, including without limitation the rights 10 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | // copies of the Software, and to permit persons to whom the Software is 12 | // furnished to do so, subject to the following conditions: 13 | // 14 | // The above copyright notice and this permission notice shall be included in all 15 | // copies or substantial portions of the Software. 16 | // 17 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | // SOFTWARE. 24 | const spawn = require('child_process').spawn; 25 | const path = require("path"); 26 | 27 | const exec = (cmd, args=[]) => new Promise((resolve, reject) => { 28 | console.log(`Started: ${cmd} ${args.join(" ")}`) 29 | const app = spawn(cmd, args, { stdio: 'inherit' }); 30 | app.on('close', code => { 31 | if(code !== 0){ 32 | err = new Error(`Invalid status code: ${code}`); 33 | err.code = code; 34 | return reject(err); 35 | }; 36 | return resolve(code); 37 | }); 38 | app.on('error', reject); 39 | }); 40 | 41 | const main = async () => { 42 | await exec('bash', [path.join(__dirname, './restore.sh')]); 43 | }; 44 | 45 | main().catch(err => { 46 | console.error(err); 47 | console.error(err.stack); 48 | process.exit(err.code || -1); 49 | }) -------------------------------------------------------------------------------- /restore.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -l 2 | 3 | function is_in_remote() { 4 | local branch=${1} 5 | local existed_in_remote=$(git ls-remote --heads origin ${branch}) 6 | 7 | if [[ -z ${existed_in_remote} ]]; then 8 | echo "-- Conan Cache: ${branch} is NOT in remote" 9 | return 0 10 | else 11 | echo "-- Conan Cache: ${branch} is in remote" 12 | return 1 13 | fi 14 | } 15 | 16 | INPUT_TARGET_OS=${INPUT_TARGET_OS:-$RUNNER_OS} 17 | REPO_BRANCH=main 18 | 19 | echo "-- Conan Cache: $GITHUB_EVENT_NAME : Commit by $GITHUB_ACTOR with SHA $GITHUB_SHA on $GITHUB_REF" 20 | echo "-- Conan Cache: Using cache $INPUT_CACHE_NAME" 21 | 22 | # Make sure path exists and change dir to it 23 | mkdir -p $CONAN_USER_HOME 24 | cd $CONAN_USER_HOME 25 | 26 | # Check out cache 27 | echo "-- Conan Cache: Checking out at CONAN_USER_HOME: $CONAN_USER_HOME" 28 | git clone https://${INPUT_BOT_NAME}:${INPUT_BOT_TOKEN}@github.com/${INPUT_CACHE_NAME}.git ${CONAN_USER_HOME} --branch=main 29 | 30 | echo "-- Conan Cache: Enable long paths" 31 | git config --global core.longpaths true 32 | 33 | hit_cache=0 34 | echo "-- Conan Cache: Setting preliminary cache-hit $hit_cache" 35 | echo "::set-output name=cache-hit::$hit_cache" 36 | 37 | # Check if explicit key exits 38 | echo "-- Conan Cache: Trying explicit key $INPUT_KEY" 39 | if [ $(git tag --list "$INPUT_KEY") ]; then 40 | # If it does - check out explicit and set cache_hit to 1 41 | git checkout ${INPUT_KEY} || exit 1 42 | echo "-- Conan Cache: replace CONAN_USER_HOME_SHORT with ${CONAN_USER_HOME_SHORT}" 43 | find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 44 | hit_cache=1 45 | else 46 | # If it doesn't check if fallback exits 47 | FALLBACK_KEY="host-${RUNNER_OS}-target-${INPUT_TARGET_OS}-${REPO_BRANCH}" 48 | echo "-- Conan Cache: Trying fallback key $FALLBACK_KEY" 49 | 50 | is_in_remote ${FALLBACK_KEY} 51 | 52 | res=$? 53 | 54 | echo "-- Conan Cache: is_in_remote returned $res" 55 | 56 | if [ $res -eq 1 ]; then 57 | # If it does - check out fallback and set cache_hit to 2 58 | echo "-- Conan Cache: Check out fallback key $FALLBACK_KEY" 59 | git checkout ${FALLBACK_KEY} || exit 1 60 | git pull || exit 1 61 | git lfs pull || exit 1 62 | echo "-- Conan Cache: replace CONAN_USER_HOME_SHORT with ${CONAN_USER_HOME_SHORT}" 63 | find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 64 | hit_cache=2 65 | else 66 | # If it doesn't - create the branch and set cache_hit to 0 67 | echo "-- Conan Cache: Creating fallback key $FALLBACK_KEY" 68 | git checkout -b ${FALLBACK_KEY} || exit 1 69 | git push -u origin ${FALLBACK_KEY} || exit 1 70 | hit_cache=0 71 | fi 72 | fi 73 | 74 | echo "-- Conan Cache: Setting cache-hit $hit_cache" 75 | echo "::set-output name=cache-hit::$hit_cache" 76 | -------------------------------------------------------------------------------- /save.js: -------------------------------------------------------------------------------- 1 | // From https://github.com/ad-m/github-push-action 2 | // 3 | // MIT License 4 | // 5 | // Copyright (c) 2019 Adam Dobrawy 6 | // 7 | // Permission is hereby granted, free of charge, to any person obtaining a copy 8 | // of this software and associated documentation files (the "Software"), to deal 9 | // in the Software without restriction, including without limitation the rights 10 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | // copies of the Software, and to permit persons to whom the Software is 12 | // furnished to do so, subject to the following conditions: 13 | // 14 | // The above copyright notice and this permission notice shall be included in all 15 | // copies or substantial portions of the Software. 16 | // 17 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | // SOFTWARE. 24 | const spawn = require('child_process').spawn; 25 | const path = require("path"); 26 | 27 | const exec = (cmd, args=[]) => new Promise((resolve, reject) => { 28 | console.log(`Started: ${cmd} ${args.join(" ")}`) 29 | const app = spawn(cmd, args, { stdio: 'inherit' }); 30 | app.on('close', code => { 31 | if(code !== 0){ 32 | err = new Error(`Invalid status code: ${code}`); 33 | err.code = code; 34 | return reject(err); 35 | }; 36 | return resolve(code); 37 | }); 38 | app.on('error', reject); 39 | }); 40 | 41 | const main = async () => { 42 | await exec('bash', [path.join(__dirname, './save.sh')]); 43 | }; 44 | 45 | main().catch(err => { 46 | console.error(err); 47 | console.error(err.stack); 48 | process.exit(err.code || -1); 49 | }) -------------------------------------------------------------------------------- /save.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -l 2 | 3 | INPUT_LFS_LIMIT=${INPUT_LFS_LIMIT:-50} 4 | 5 | cd $CONAN_USER_HOME 6 | 7 | #FALLBACK_KEY="host-${RUNNER_OS}-target-${INPUT_TARGET_OS}-${REPO_BRANCH}" 8 | echo "-- Conan Cache: Check if on branch" 9 | if [ $(git symbolic-ref --short -q HEAD) ]; then 10 | echo "-- Conan Cache: Currently on fallback key" 11 | 12 | echo "-- Conan Cache: Install Conan" 13 | pip3 install wheel setuptools 14 | pip3 install conan --upgrade 15 | 16 | echo "-- Conan Cache: Clean Conan" 17 | conan remove -f "*" --builds 18 | conan remove -f "*" --src 19 | conan remove -f "*" --system-reqs 20 | 21 | echo "-- Conan Cache: replace ${CONAN_USER_HOME_SHORT} with CONAN_USER_HOME_SHORT" 22 | find .conan/ -name .conan_link -exec perl -pi -e 's|\Q$ENV{CONAN_USER_HOME_SHORT}\E|CONAN_USER_HOME_SHORT|g' {} + 23 | find .conan -name .conan_link.bak -exec rm {} + 24 | 25 | git status 26 | 27 | echo "-- Conan Cache: Configure git" 28 | git config user.email "you@example.com" 29 | git config user.name "${INPUT_BOT_NAME}" 30 | 31 | echo "-- Conan Cache: Find all files bigger than 100MB" 32 | find .conan short -type f -size +100M -exec ls -lh {} \; | awk '{ print $9 ": " $5 }' 33 | 34 | echo "-- Conan Cache: Find all files bigger than 50MB" 35 | find .conan short -type f -size +50M -exec ls -lh {} \; | awk '{ print $9 ": " $5 }' 36 | 37 | echo "-- Conan Cache: Auto LFS track all files bigger than $INPUT_LFS_LIMIT MB" 38 | find .conan short -type f -size +${INPUT_LFS_LIMIT}M -execdir git lfs track {} \; 39 | 40 | echo "-- Conan Cache: Add everything" 41 | git add -A 42 | 43 | echo "-- Conan Cache: Commit locally" 44 | git commit -m "$GITHUB_EVENT_NAME : Commit by $GITHUB_ACTOR with SHA $GITHUB_SHA on $GITHUB_REF" 45 | 46 | echo "-- Conan Cache: Push to GitHub" 47 | git push 48 | 49 | echo "-- Conan Cache: Tag with explicit key : $INPUT_KEY" 50 | git tag $INPUT_KEY 51 | 52 | echo "-- Conan Cache: Push explicit key" 53 | git push origin $INPUT_KEY 54 | else 55 | echo "-- Conan Cache: Got hit on explicit key : $INPUT_KEY" 56 | git status 57 | fi 58 | -------------------------------------------------------------------------------- /update_cache_linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | UBUNTU_VERSION="18.04" 4 | 5 | [ $# -eq 0 ] && { echo "Usage: $0 "; exit 1; } 6 | 7 | [ $# -eq 2 ] && UBUNTU_VERSION="$2" 8 | 9 | confirm() { 10 | read -r -p "${1:-Are you sure? [y/N]} " response 11 | case "$response" in 12 | [yY][eE][sS]|[yY]) 13 | true 14 | ;; 15 | *) 16 | false 17 | ;; 18 | esac 19 | } 20 | 21 | APPLICATION_DIR="$1" 22 | 23 | header() { 24 | echo -e "\e[35m$1\e[0m" 25 | } 26 | 27 | echo "APPLICATION dir: " ${APPLICATION_DIR} 28 | 29 | [ ! -d ${APPLICATION_DIR} ] && { echo "${APPLICATION_DIR} DOES NOT EXIST"; exit 1; } 30 | 31 | export CONAN_USER_HOME="/Code/release/" 32 | 33 | [ ! -d ${CONAN_USER_HOME} ] && { echo "${CONAN_USER_HOME} DOES NOT EXIST"; exit 1; } 34 | 35 | export CONAN_USER_HOME_SHORT=${CONAN_USER_HOME}/short 36 | 37 | [ ! -d ${CONAN_USER_HOME_SHORT} ] && { echo "${CONAN_USER_HOME_SHORT} DOES NOT EXIST"; exit 1; } 38 | 39 | export LFS_LIMIT=50 40 | export CONAN_SYSREQUIRES_MODE=enabled 41 | 42 | header "Prepare Cache for update" 43 | 44 | confirm "Clean cache? [y/N]" && cd ${CONAN_USER_HOME} && git clean -df && git checkout . && git checkout host-Linux-target-Linux-ubuntu-${UBUNTU_VERSION}-main 45 | 46 | confirm "Update cache? [y/N]" && cd ${CONAN_USER_HOME} && git pull && git lfs pull 47 | 48 | confirm "DELETE the whole cache? [y/N]" && cd ${CONAN_USER_HOME} && conan remove -f "*" 49 | 50 | confirm "Prepare short paths [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 51 | 52 | header "Populate the Cache" 53 | 54 | confirm "Update APPLICATION? [y/N]" && cd ${APPLICATION_DIR} && git pull 55 | 56 | confirm "Remove old build directory? [y/N]" && cd ${APPLICATION_DIR} && rm -rf build 57 | 58 | confirm "Create build directory? [y/N]" && cd ${APPLICATION_DIR} && mkdir build 59 | 60 | confirm "Start build? [y/N]" && cd ${APPLICATION_DIR} && cd build && cmake -DUPDATE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. 61 | 62 | header "Clean Conan" 63 | 64 | confirm "Clean Conan builds? [y/N]" && conan remove -f "*" --builds 65 | 66 | confirm "Clean Conan src? [y/N]" && conan remove -f "*" --src 67 | 68 | confirm "Clean Conan system-reqs? [y/N]" && conan remove -f "*" --system-reqs 69 | 70 | header "Prep for cache upload" 71 | 72 | confirm "Fix short paths? [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=$ENV{CONAN_USER_HOME_SHORT}=CONAN_USER_HOME_SHORT=g' {} + 73 | 74 | confirm "Clean up .conan_link.bak? [y/N]" && cd ${CONAN_USER_HOME} && find -name .conan_link.bak -execdir rm {} \; 75 | 76 | confirm "List files over ${LFS_LIMIT}M? [y/N]" && cd ${CONAN_USER_HOME} && find .conan short -type f -size +${LFS_LIMIT}M -exec ls -lh {} \; | awk '{ print $9 ": " $5 }' 77 | 78 | confirm "LFS track files over ${LFS_LIMIT}M? [y/N]" && cd ${CONAN_USER_HOME} && find .conan short -type f -size +${LFS_LIMIT}M -execdir git lfs track {} \; 79 | 80 | header "Upload cache" 81 | 82 | confirm "Add everything to git? [y/N]" && cd ${CONAN_USER_HOME} && git add -A 83 | 84 | confirm "Commit to git? [y/N]" && cd ${CONAN_USER_HOME} && git commit -m "Local build" 85 | 86 | confirm "Push to GitHub? [y/N]" && cd ${CONAN_USER_HOME} && git push 87 | 88 | header "Test cache locally" 89 | 90 | confirm "TEST: Prepare short paths [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 91 | 92 | confirm "TEST: Remove old build directory? [y/N]" && cd ${APPLICATION_DIR} && rm -rf build 93 | 94 | confirm "TEST: Create build directory? [y/N]" && cd ${APPLICATION_DIR} && mkdir build 95 | 96 | confirm "TEST: Start REGULAR build? [y/N]" && cd ${APPLICATION_DIR} && cd build && cmake -DUPDATE_CONAN=OFF -DCMAKE_BUILD_TYPE=Release .. 97 | -------------------------------------------------------------------------------- /update_cache_macos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ $# -eq 0 ] && { echo "Usage: $0 "; exit 1; } 4 | 5 | confirm() { 6 | read -r -p "${1:-Are you sure? [y/N]} " response 7 | case "$response" in 8 | [yY][eE][sS]|[yY]) 9 | true 10 | ;; 11 | *) 12 | false 13 | ;; 14 | esac 15 | } 16 | 17 | APPLICATION_DIR="$1" 18 | 19 | header() { 20 | echo -e "\e[35m$1\e[0m" 21 | } 22 | 23 | echo "APPLICATION dir: " ${APPLICATION_DIR} 24 | 25 | [ ! -d ${APPLICATION_DIR} ] && { echo "${APPLICATION_DIR} DOES NOT EXIST"; exit 1; } 26 | 27 | export CONAN_USER_HOME="/Users/patriciaaas/Code/conan-cache-turtlebrowser" 28 | 29 | [ ! -d ${CONAN_USER_HOME} ] && { echo "${CONAN_USER_HOME} DOES NOT EXIST"; exit 1; } 30 | 31 | export CONAN_USER_HOME_SHORT=${CONAN_USER_HOME}/short 32 | 33 | [ ! -d ${CONAN_USER_HOME_SHORT} ] && { echo "${CONAN_USER_HOME_SHORT} DOES NOT EXIST"; exit 1; } 34 | 35 | export LFS_LIMIT=50 36 | 37 | header "Prepare Cache for update" 38 | 39 | confirm "Clean cache? [y/N]" && cd ${CONAN_USER_HOME} && git clean -df && git checkout . && git checkout host-macOS-target-macOS-macos-11-main 40 | 41 | confirm "Update cache? [y/N]" && cd ${CONAN_USER_HOME} && git pull && git lfs pull 42 | 43 | confirm "DELETE the whole cache? [y/N]" && cd ${CONAN_USER_HOME} && conan remove -f "*" && rm -rf short/* && touch short/conan-cache.marker 44 | 45 | confirm "Prepare short paths [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 46 | 47 | header "Populate the Cache" 48 | 49 | confirm "Update APPLICATION? [y/N]" && cd ${APPLICATION_DIR} && git pull 50 | 51 | confirm "Remove old build directory? [y/N]" && cd ${APPLICATION_DIR} && rm -rf build 52 | 53 | confirm "Create build directory? [y/N]" && cd ${APPLICATION_DIR} && mkdir build 54 | 55 | confirm "Start build? [y/N]" && cd ${APPLICATION_DIR} && cd build && cmake -DUPDATE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. 56 | 57 | header "Clean Conan" 58 | 59 | confirm "Clean Conan builds? [y/N]" && conan remove -f "*" --builds 60 | 61 | confirm "Clean Conan src? [y/N]" && conan remove -f "*" --src 62 | 63 | confirm "Clean Conan system-reqs? [y/N]" && conan remove -f "*" --system-reqs 64 | 65 | header "Prep for cache upload" 66 | 67 | confirm "Fix short paths? [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=$ENV{CONAN_USER_HOME_SHORT}=CONAN_USER_HOME_SHORT=g' {} + 68 | 69 | confirm "Clean up .conan_link.bak? [y/N]" && cd ${CONAN_USER_HOME} && find -name .conan_link.bak -execdir rm {} \; 70 | 71 | confirm "List files over ${LFS_LIMIT}M? [y/N]" && cd ${CONAN_USER_HOME} && find .conan short -type f -size +${LFS_LIMIT}M -exec ls -lh {} \; | awk '{ print $9 ": " $5 }' 72 | 73 | confirm "LFS track files over ${LFS_LIMIT}M? [y/N]" && cd ${CONAN_USER_HOME} && find .conan short -type f -size +${LFS_LIMIT}M -execdir git lfs track {} \; 74 | 75 | header "Upload cache" 76 | 77 | confirm "Add everything to git? [y/N]" && cd ${CONAN_USER_HOME} && git add -A 78 | 79 | confirm "Commit to git? [y/N]" && cd ${CONAN_USER_HOME} && git commit -m "Local build" 80 | 81 | confirm "Push to GitHub? [y/N]" && cd ${CONAN_USER_HOME} && git push 82 | 83 | header "Test cache locally" 84 | 85 | confirm "TEST: Prepare short paths [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 86 | 87 | confirm "TEST: Remove old build directory? [y/N]" && cd ${APPLICATION_DIR} && rm -rf build 88 | 89 | confirm "TEST: Create build directory? [y/N]" && cd ${APPLICATION_DIR} && mkdir build 90 | 91 | confirm "TEST: Start REGULAR build? [y/N]" && cd ${APPLICATION_DIR} && cd build && cmake -DUPDATE_CONAN=OFF -DCMAKE_BUILD_TYPE=Release .. 92 | -------------------------------------------------------------------------------- /update_cache_ubuntu_16_04.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | UBUNTU_VERSION="16.04" 4 | 5 | [ $# -eq 0 ] && { echo "Usage: $0 "; exit 1; } 6 | 7 | ./update_cache_linux.sh "$1" "$UBUNTU_VERSION" -------------------------------------------------------------------------------- /update_cache_ubuntu_20_04.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | UBUNTU_VERSION="20.04" 4 | 5 | [ $# -eq 0 ] && { echo "Usage: $0 "; exit 1; } 6 | 7 | ./update_cache_linux.sh "$1" "$UBUNTU_VERSION" 8 | -------------------------------------------------------------------------------- /update_cache_windows.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ $# -eq 0 ] && { echo "Usage: $0 "; exit 1; } 4 | 5 | confirm() { 6 | read -r -p "${1:-Are you sure? [y/N]} " response 7 | case "$response" in 8 | [yY][eE][sS]|[yY]) 9 | true 10 | ;; 11 | *) 12 | false 13 | ;; 14 | esac 15 | } 16 | 17 | APPLICATION_DIR="$1" 18 | 19 | header() { 20 | echo -e "\e[35m$1\e[0m" 21 | } 22 | 23 | echo "APPLICATION dir: " ${APPLICATION_DIR} 24 | 25 | [ ! -d ${APPLICATION_DIR} ] && { echo "${APPLICATION_DIR} DOES NOT EXIST"; exit 1; } 26 | 27 | export CONAN_USER_HOME="c:/release" 28 | 29 | [ ! -d ${CONAN_USER_HOME} ] && { echo "${CONAN_USER_HOME} DOES NOT EXIST"; exit 1; } 30 | 31 | export CONAN_USER_HOME_SHORT=${CONAN_USER_HOME}/short 32 | 33 | [ ! -d ${CONAN_USER_HOME_SHORT} ] && { echo "${CONAN_USER_HOME_SHORT} DOES NOT EXIST"; exit 1; } 34 | 35 | export LFS_LIMIT=50 36 | 37 | header "Prepare Cache for update" 38 | 39 | confirm "Clean cache? [y/N]" && cd ${CONAN_USER_HOME} && git clean -df && git checkout . && git checkout host-Windows-target-Windows-windows-2019-main 40 | 41 | confirm "Update cache? [y/N]" && cd ${CONAN_USER_HOME} && git pull && git lfs pull 42 | 43 | confirm "DELETE the whole cache? [y/N]" && cd ${CONAN_USER_HOME} && conan remove -f "*" && rm -rf short/* && touch short/conan-cache.marker 44 | 45 | confirm "Prepare short paths [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 46 | 47 | header "Populate the Cache" 48 | 49 | confirm "Update APPLICATION? [y/N]" && cd ${APPLICATION_DIR} && git pull 50 | 51 | confirm "Remove old build directory? [y/N]" && cd ${APPLICATION_DIR} && rm -rf build 52 | 53 | confirm "Create build directory? [y/N]" && cd ${APPLICATION_DIR} && mkdir build 54 | 55 | confirm "Start build? [y/N]" && cd ${APPLICATION_DIR} && cd build && cmake -DUPDATE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. 56 | 57 | header "Clean Conan" 58 | 59 | confirm "Clean Conan builds? [y/N]" && conan remove -f "*" --builds 60 | 61 | confirm "Clean Conan src? [y/N]" && conan remove -f "*" --src 62 | 63 | confirm "Clean Conan system-reqs? [y/N]" && conan remove -f "*" --system-reqs 64 | 65 | header "Prep for cache upload" 66 | 67 | confirm "Fix short paths? [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=$ENV{CONAN_USER_HOME_SHORT}=CONAN_USER_HOME_SHORT=g' {} + 68 | 69 | confirm "Clean up .conan_link.bak? [y/N]" && cd ${CONAN_USER_HOME} && find -name .conan_link.bak -execdir rm {} \; 70 | 71 | confirm "List files over ${LFS_LIMIT}M? [y/N]" && cd ${CONAN_USER_HOME} && find .conan short -type f -size +${LFS_LIMIT}M -exec ls -lh {} \; | awk '{ print $9 ": " $5 }' 72 | 73 | confirm "LFS track files over ${LFS_LIMIT}M? [y/N]" && cd ${CONAN_USER_HOME} && find .conan short -type f -size +${LFS_LIMIT}M -execdir git lfs track {} \; 74 | 75 | header "Upload cache" 76 | 77 | confirm "Add everything to git? [y/N]" && cd ${CONAN_USER_HOME} && git add -A 78 | 79 | confirm "Commit to git? [y/N]" && cd ${CONAN_USER_HOME} && git commit -m "Local build" 80 | 81 | confirm "Push to GitHub? [y/N]" && cd ${CONAN_USER_HOME} && git push 82 | 83 | header "Test cache locally" 84 | 85 | confirm "TEST: Prepare short paths [y/N]" && cd ${CONAN_USER_HOME} && find .conan/ -name .conan_link -exec perl -pi -e 's=CONAN_USER_HOME_SHORT=$ENV{CONAN_USER_HOME_SHORT}=g' {} + 86 | 87 | confirm "TEST: Remove old build directory? [y/N]" && cd ${APPLICATION_DIR} && rm -rf build 88 | 89 | confirm "TEST: Create build directory? [y/N]" && cd ${APPLICATION_DIR} && mkdir build 90 | 91 | confirm "TEST: Start REGULAR build? [y/N]" && cd ${APPLICATION_DIR} && cd build && cmake -DUPDATE_CONAN=OFF -DCMAKE_BUILD_TYPE=Release .. 92 | --------------------------------------------------------------------------------