├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── cache_crate.md │ └── feature_request.md └── workflows │ └── build.yml ├── .gitignore ├── README.md ├── build.py └── requirements.txt /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | max_line_length = 80 8 | indent_size = 4 9 | 10 | [*.yml] 11 | indent_size = 2 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | Do the checklist before filing an issue: 11 | 12 | * [ ] Is this related to the `actions-rs` Actions? If not, use GitHub Community forum to ask questions about Actions in a whole: https://github.community 13 | * [ ] You've read the Contributing section about bugs reporting: https://github.com/actions-rs/.github/blob/master/CONTRIBUTING.md#reporting-bugs 14 | * [ ] Is this something you can debug and fix? Send a pull request! Bug fixes and documentation fixes are welcome. 15 | 16 | ## Description 17 | 18 | A clear and concise description of what the bug is. 19 | 20 | ## Used environment 21 | 22 | ### Workflow code 23 | 24 | ```yaml 25 | Paste that part of your workflow yaml file that causes the bug in here. 26 | 27 | Alternatively you can remove that code block and insert direct link to your workflow file. 28 | Ensure that link points to the specific commit, and not just to the master branch. 29 | ``` 30 | 31 | ### Action output 32 | 33 | ``` 34 | Copy and paste Action output logs in here. 35 | 36 | How else can you help on that step: 37 | 1. Enable debug logs first: https://github.com/actions/toolkit/blob/master/docs/action-debugging.md 38 | 2. Re-run the job 39 | 2. Download logs archive for job run and attach it to this issue 40 | ``` 41 | 42 | ## Expected behavior 43 | 44 | A clear and concise description of what you expected to happen. 45 | 46 | ## Additional context 47 | 48 | Add any other context about the problem here. 49 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/cache_crate.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Request cached crate 3 | about: Request new crate to be cached 4 | title: Add cached crate %CRATE_NAME% 5 | labels: question 6 | assignees: '' 7 | 8 | --- 9 | 10 | Do the checklist before filing an issue: 11 | 12 | * [ ] Is this related to the `actions-rs` Actions? If not, use GitHub Community forum to ask questions about Actions in a whole: https://github.community 13 | * [ ] Does this crate helps with the Rust CI? 14 | * [ ] Is it popular enough to be used by many users? 15 | 16 | ## Crate 17 | 18 | **crates.io**: Add link to the crates.io here 19 | **repository**: Add link to the crate repository 20 | 21 | ## Motivation 22 | 23 | Add any other context about this crate here. 24 | Describe how Rust users can benefit from using this crate in their CI. 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | Do the checklist before filing an issue: 11 | 12 | * [ ] Is this related to the actions-rs Actions? If not, use GitHub Community forum to ask questions about Actions in a whole: https://github.community 13 | * [ ] You've read the Contributing section about feature requests: https://github.com/actions-rs/.github/blob/master/CONTRIBUTING.md#feature-requests 14 | * [ ] Is this something you can debug and fix? Send a pull request! Bug fixes and documentation fixes are welcome. 15 | 16 | ## Motivation 17 | 18 | Describe your idea, motivation, and how Rust community could benefit from this feature. 19 | 20 | ## Additional context 21 | 22 | Add any additional information about this feature; that includes links to the related tools, alternative implementations, blog posts and so on 23 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build tools cache 2 | on: 3 | push: 4 | branches: 5 | - master 6 | 7 | schedule: 8 | # Each day at midnight 9 | - cron: '0 0 * * *' 10 | 11 | jobs: 12 | build: 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | os: 18 | - windows-2019 19 | - ubuntu-18.04 20 | - ubuntu-16.04 21 | - macos-10.15 22 | crate: 23 | - cargo-audit 24 | - cargo-udeps 25 | - cargo-nono 26 | - cargo-geiger 27 | - cargo-tarpaulin 28 | - cargo-cache 29 | - cargo-web 30 | - cargo-tree 31 | - cargo-binutils 32 | - cargo-fuzz 33 | - cargo-deb 34 | 35 | - cross 36 | - sccache 37 | - mdbook 38 | - wasm-pack 39 | - wasm-bindgen-cli 40 | - grcov 41 | 42 | exclude: 43 | # cargo-tarpaulin is available for Linux only 44 | - crate: cargo-tarpaulin 45 | os: windows-2019 46 | - crate: cargo-tarpaulin 47 | os: macos-10.15 48 | # cargo-deb is available for Linux only 49 | - crate: cargo-deb 50 | os: windows-2019 51 | - crate: cargo-deb 52 | os: macos-10.15 53 | steps: 54 | - uses: actions/checkout@v2 55 | - uses: actions/setup-python@v1 56 | with: 57 | python-version: 3.8 58 | - name: Install dependencies 59 | run: python -m pip install -r requirements.txt 60 | 61 | - name: Build ${{ matrix.crate }} 62 | run: python build.py 63 | env: 64 | CRATE: ${{ matrix.crate }} 65 | RUNNER: ${{ matrix.os }} 66 | AWS_S3_REGION: ${{ secrets.AWS_S3_REGION }} 67 | AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} 68 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 69 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 70 | SIGN_CERT: ${{ secrets.SIGN_CERT }} 71 | SIGN_CERT_PASSPHRASE: ${{ secrets.SIGN_CERT_PASSPHRASE }} 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .venv 111 | env/ 112 | venv/ 113 | ENV/ 114 | env.bak/ 115 | venv.bak/ 116 | 117 | # Spyder project settings 118 | .spyderproject 119 | .spyproject 120 | 121 | # Rope project settings 122 | .ropeproject 123 | 124 | # mkdocs documentation 125 | /site 126 | 127 | # mypy 128 | .mypy_cache/ 129 | .dmypy.json 130 | dmypy.json 131 | 132 | # Pyre type checker 133 | .pyre/ 134 | 135 | # pytype static type analyzer 136 | .pytype/ 137 | 138 | # Cython debug symbols 139 | cython_debug/ 140 | 141 | # static files generated from Django application using `collectstatic` 142 | media 143 | static 144 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Binary crates cache 2 | 3 | This repository was responsible for building and persisting binary crates cache, 4 | which were used later by [actions-rs/install](https://github.com/actions-rs/install) Action. 5 | 6 | **Deprecation warning** 7 | 8 | It is archived now and will not be updated anymore. [actions-rs/install](https://github.com/actions-rs/install) Action will be updated eventually to utilize GitHub runners cache to store compiled binaries. 9 | 10 | ## Cached crates 11 | 12 | See [workflow file](https://github.com/actions-rs/tool-cache/blob/master/.github/workflows/build.yml) 13 | for a list of crates, which are stored in the cache storage. 14 | 15 | If you want to suggest a new crate to be added into the crate cache, 16 | check if it was [asked before](https://github.com/actions-rs/tool-cache/issues) already, 17 | and if not - [create a new issue](https://github.com/actions-rs/tool-cache/issues/new?assignees=&labels=question&template=cache_crate.md&title=). 18 | 19 | ## Security considerations 20 | 21 | Binary crates cache is stored at the third party server (AWS S3), 22 | meaning that using this tool cache is potentially less secure 23 | than just calling `cargo install` command. 24 | 25 | Pre-built binaries are executed directly in the virtual environments 26 | and have full access to them, including access to the environment variables, 27 | [secrets](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets), 28 | [access tokens](https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token) 29 | and so on. 30 | 31 | Malicious parties potentially might replace these pre-built binaries, 32 | leading to the security breach. 33 | We try our best to mitigate any potential security problems, 34 | but you must acknowledge that fact before using any @actions-rs Action, 35 | which uses this cache internally and explicitly enabling this functionality. 36 | 37 | ### Security measures 38 | 39 | 1. Crates are compiled [right here at GitHub](https://github.com/actions-rs/tool-cache/actions?query=workflow%3A%22Build+tools+cache%22+event%3Aschedule) 40 | 2. Crates are signed with 4096 bit RSA key 41 | 3. That RSA key is stored in the [GitHub secrets](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets) 42 | 4. Actions at [@actions-rs](https://github.com/actions-rs) are validating 43 | this signature after the file downloading 44 | 5. Compiled crates are stored in the AWS S3 bucket and served via AWS CloudFront 45 | 6. MFA is enabled for AWS root user 46 | 7. Separate AWS user for files uploading has the console access disabled 47 | and only one permission: `PutObject` for this S3 bucket 48 | 8. AWS access key and other confidential details are stored in the 49 | GitHub secrets 50 | 51 | Refer to the [@actions-rs/install](https://github.com/actions-rs/install) 52 | documentation to learn more about files downloading and validating. 53 | 54 | ## Contribute and support 55 | 56 | Any contributions are welcomed! 57 | 58 | If you want to report a bug or have a feature request, 59 | check the [Contributing guide](https://github.com/actions-rs/.github/blob/master/CONTRIBUTING.md). 60 | 61 | You can also support author by funding the ongoing project work, 62 | see [Sponsoring](https://actions-rs.github.io/#sponsoring). 63 | -------------------------------------------------------------------------------- /build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import os.path 5 | import tempfile 6 | import logging 7 | import logging.config 8 | import subprocess 9 | import zipfile 10 | 11 | import boto3 12 | import requests 13 | from dotenv import load_dotenv 14 | 15 | # Mostly for a local testing 16 | load_dotenv() 17 | 18 | S3_OBJECT_URL = 'https://s3.{region}.amazonaws.com/{bucket}/{{object_name}}'.format( 19 | region=os.environ['AWS_S3_REGION'], 20 | bucket=os.environ['AWS_S3_BUCKET'], 21 | ) 22 | S3_OBJECT_NAME = '{crate}/{runner}/{crate}-{version}.zip' 23 | CLOUDFRONT_URL = 'https://d1ad61wkrfbmp3.cloudfront.net/{filename}' 24 | 25 | MAX_VERSIONS_TO_BUILD = 3 26 | 27 | # Set of crates not to be built in any condition. 28 | EXCLUDES = { 29 | # https://github.com/mozilla/grcov/issues/405 30 | ('grcov', '0.5.10'), 31 | } 32 | 33 | 34 | def which(executable): 35 | for path in os.environ['PATH'].split(os.pathsep): 36 | path = path.strip('"') 37 | 38 | fpath = os.path.join(path, executable) 39 | 40 | if os.path.isfile(fpath) and os.access(fpath, os.X_OK): 41 | return fpath 42 | 43 | # checks if os is windows and appends .exe extension to 44 | # `executable`, if not already present, and rechecks. 45 | if os.name == 'nt' and not executable.endswith('.exe'): 46 | return which('{}.exe'.format(executable)) 47 | 48 | 49 | def crate_info(crate): 50 | url = 'https://crates.io/api/v1/crates/{}'.format(crate) 51 | logging.info('Requesting crates.io URL: {}'.format(url)) 52 | 53 | resp = requests.get(url) 54 | resp.raise_for_status() 55 | 56 | def predicate(v): 57 | if v['yanked']: 58 | return False 59 | 60 | if (crate, v['num']) in EXCLUDES: 61 | return False 62 | 63 | return True 64 | 65 | versions = filter(predicate, resp.json()['versions']) 66 | for version in list(versions)[:MAX_VERSIONS_TO_BUILD]: 67 | yield version['num'] 68 | 69 | 70 | def exists(runner, crate, version): 71 | """Check if `crate` with version `version` for `runner` environment 72 | already exists in the S3 bucket.""" 73 | 74 | object_name = S3_OBJECT_NAME.format( 75 | crate=crate, 76 | runner=runner, 77 | version=version, 78 | ) 79 | url = CLOUDFRONT_URL.format(filename=object_name) 80 | logging.info( 81 | 'Check if {crate} == {version} for {runner} exists in S3 bucket at {url}'.format( 82 | crate=crate, 83 | version=version, 84 | runner=runner, 85 | url=url, 86 | )) 87 | resp = requests.head(url, allow_redirects=True) 88 | 89 | if resp.ok: 90 | logging.info( 91 | '{crate} == {version} for {runner} already exists in S3 bucket'.format( 92 | crate=crate, 93 | version=version, 94 | runner=runner, 95 | )) 96 | return True 97 | 98 | else: 99 | logging.warning( 100 | '{crate} == {version} for {runner} does not exists in S3 bucket'.format( 101 | crate=crate, 102 | version=version, 103 | runner=runner, 104 | )) 105 | return False 106 | 107 | 108 | def build(runner, crate, version): 109 | root = os.path.join( 110 | os.getcwd(), 111 | 'build', 112 | '{}-{}-{}'.format(runner, crate, version) 113 | ) 114 | 115 | logging.info('Preparing build root at {}'.format(root)) 116 | os.makedirs(root, exist_ok=True) 117 | 118 | args = [ 119 | 'cargo', 120 | 'install', 121 | '--version', 122 | version, 123 | '--root', 124 | root, 125 | '--no-track', 126 | crate, 127 | ] 128 | subprocess.check_call(args) 129 | 130 | archive_path = '{}.zip'.format(crate) 131 | with zipfile.ZipFile(archive_path, 'w') as archive: 132 | logging.info('Creating archive at {}'.format(archive_path)) 133 | for filename in os.listdir(os.path.join(root, 'bin')): 134 | logging.info('Writing {} into {} archive'.format(filename, archive_path)) 135 | archive.write( 136 | os.path.join(root, 'bin', filename), 137 | filename, 138 | ) 139 | 140 | return archive_path 141 | 142 | 143 | def sign(path): 144 | openssl = which('openssl') 145 | if openssl is None: 146 | raise ValueError('Unable to find OpenSSL!') 147 | 148 | signature_path = '{}.sig'.format(path) 149 | 150 | cert_fd, cert_path = tempfile.mkstemp(prefix='cert_') 151 | os.write(cert_fd, os.environ['SIGN_CERT'].encode()) 152 | os.close(cert_fd) 153 | 154 | args = [ 155 | openssl, 156 | 'dgst', 157 | '-sha256', 158 | '-sign', 159 | cert_path, 160 | '-passin', 161 | 'env:SIGN_CERT_PASSPHRASE', 162 | '-out', 163 | signature_path, 164 | path, 165 | ] 166 | 167 | try: 168 | logging.info('Signing {} at {}'.format(path, signature_path)) 169 | subprocess.check_call(args) 170 | finally: 171 | os.unlink(cert_path) 172 | 173 | if not os.path.exists(signature_path): 174 | raise ValueError('Signature file is missing') 175 | 176 | return signature_path 177 | 178 | 179 | def upload(client, runner, crate, version, path, signature_path): 180 | """Upload prebuilt `crate` with `version` for `runner` environment 181 | located at `path` to the S3 bucket.""" 182 | 183 | object_name = S3_OBJECT_NAME.format( 184 | crate=crate, 185 | runner=runner, 186 | version=version, 187 | ) 188 | object_signature_name = '{}.sig'.format(object_name) 189 | 190 | logging.info('Uploading {path} to {bucket}/{name}'.format( 191 | path=path, 192 | bucket=os.environ['AWS_S3_BUCKET'], 193 | name=object_name, 194 | )) 195 | client.upload_file(path, os.environ['AWS_S3_BUCKET'], object_name) 196 | client.upload_file(signature_path, os.environ['AWS_S3_BUCKET'], object_signature_name) 197 | 198 | 199 | class LogFormatter(logging.Formatter): 200 | def format(self, record): 201 | msg = record.getMessage() 202 | if record.levelno == logging.DEBUG: 203 | return '::debug::{}'.format(msg) 204 | elif record.levelno == logging.INFO: 205 | return msg 206 | elif record.levelno in (logging.WARN, logging.WARNING): 207 | return '::warning::{}'.format(msg) 208 | else: 209 | return '::error::{}'.format(msg) 210 | 211 | 212 | if __name__ == '__main__': 213 | logging.config.dictConfig({ 214 | 'version': 1, 215 | 'disable_existing_loggers': True, 216 | 'formatters': { 217 | 'gha': { 218 | '()': LogFormatter, 219 | }, 220 | }, 221 | 'handlers': { 222 | 'stdout': { 223 | 'class': 'logging.StreamHandler', 224 | 'formatter': 'gha', 225 | }, 226 | }, 227 | 'loggers': { 228 | '': { 229 | 'handlers': ['stdout'], 230 | 'level': 'DEBUG', 231 | } 232 | } 233 | }) 234 | 235 | crate = os.environ['CRATE'] 236 | runner = os.environ['RUNNER'] 237 | 238 | s3_client = boto3.client( 239 | 's3', 240 | region_name=os.environ['AWS_S3_REGION'], 241 | aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], 242 | aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'] 243 | ) 244 | 245 | logging.info('Building {} crate for {} environment'.format(crate, runner)) 246 | for version in crate_info(crate): 247 | if not exists(runner, crate, version): 248 | try: 249 | path = build(runner, crate, version) 250 | except subprocess.CalledProcessError as e: 251 | logging.warning( 252 | 'Unable to build {} == {}: {}'.format(crate, version, e) 253 | ) 254 | else: 255 | logging.info('Built {} at {}'.format(crate, path)) 256 | 257 | signature = sign(path) 258 | upload(s3_client, runner, crate, version, path, signature) 259 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | boto3 3 | python-dotenv 4 | --------------------------------------------------------------------------------