├── .github ├── FUNDING.yml └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSE.md ├── Makefile.toml ├── README.md ├── benches └── benchmarks.rs ├── cliff.toml ├── justfile └── src ├── async_lib.rs ├── content ├── linkto.rs ├── mod.rs ├── path.rs ├── read.rs ├── rm.rs └── write.rs ├── errors.rs ├── get.rs ├── index.rs ├── lib.rs ├── linkto.rs ├── ls.rs ├── put.rs └── rm.rs /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [zkat] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | custom: # Replace with a single custom sponsorship URL 9 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | env: 6 | RUSTFLAGS: -Dwarnings 7 | 8 | jobs: 9 | fmt_and_docs: 10 | name: Check fmt & build docs 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v1 14 | - name: Install Rust 15 | uses: actions-rs/toolchain@v1 16 | with: 17 | profile: minimal 18 | toolchain: stable 19 | components: rustfmt 20 | override: true 21 | - name: rustfmt 22 | run: cargo fmt --all -- --check 23 | - name: docs 24 | run: cargo doc 25 | 26 | build_and_test: 27 | name: Build & Test 28 | runs-on: ${{ matrix.os }} 29 | strategy: 30 | matrix: 31 | rust: [1.70.0, stable] 32 | os: [ubuntu-latest, macOS-latest, windows-latest] 33 | 34 | steps: 35 | - uses: actions/checkout@v1 36 | - name: Install Rust 37 | uses: actions-rs/toolchain@v1 38 | with: 39 | profile: minimal 40 | toolchain: ${{ matrix.rust }} 41 | components: clippy 42 | override: true 43 | - name: Clippy 44 | run: cargo clippy -- -D warnings 45 | - name: Check (async-std) 46 | run: cargo check --features link_to 47 | - name: Run tests (async-std) 48 | run: cargo test --verbose --features link_to 49 | - name: Check (Tokio) 50 | run: cargo check --no-default-features --features tokio-runtime,link_to 51 | - name: Run unit tests (Tokio) 52 | run: cargo test --verbose --no-default-features --features tokio-runtime,link_to --lib 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | /.vscode 4 | Cargo.lock 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # `cacache` Release Changelog 2 | 3 | 4 | ## 13.1.0 (2024-11-26) 5 | 6 | ### Features 7 | 8 | * **link:** add hard_link_hash (#82) ([ab5f1c91](https://github.com/zkat/cacache-rs/commit/ab5f1c91857188f01a93fb60c47b25ee711d545d)) 9 | 10 | ### Bug Fixes 11 | 12 | * **docs:** improved documentation for remove_fully (#78) ([3a712473](https://github.com/zkat/cacache-rs/commit/3a712473645d05b256b430102ae809190e5d4f60)) 13 | * **tokio:** add safe access join handles (#85) ([146a593c](https://github.com/zkat/cacache-rs/commit/146a593c8e3abea8bc4c1888ae6781a3f2e1422e)) 14 | 15 | 16 | ## 13.0.0 (2024-02-15) 17 | 18 | ### Bug Fixes 19 | 20 | * **sync:** Added the feature flags for compilation without the async runtime. Fixes #64. (#65) ([ffa1ab72](https://github.com/zkat/cacache-rs/commit/ffa1ab7254e0b051e6d65fe16b71ab44be04c968)) 21 | * **BREAKING CHANGE**: this bumps the MSRV to 1.70.0 22 | * **remove_fully:** Remove the key content when set `remove_fully` to `true` (#63) ([761d7155](https://github.com/zkat/cacache-rs/commit/761d71558a4b9f06907cb5ff43247fdbd158716b)) 23 | * **BREAKING CHANGE**: this is technically a fix, but maybe people didn't expect this. 24 | 25 | 26 | ## 12.0.0 (2023-10-07) 27 | 28 | ### Features 29 | 30 | * **remove_fully:** add option for remove cache entry (#54) ([34ee6798](https://github.com/zkat/cacache-rs/commit/34ee679816cad95f617fbed11ad93b5f248ab50f)) 31 | * **reflink:** Separate reflink behavior into their own functions (#58) ([cfdf4eed](https://github.com/zkat/cacache-rs/commit/cfdf4eed4d19138ea8937b12333db8280d137b3a)) 32 | * **BREAKING CHANGE**: some signatures for copy have changed, and copy no longer automatically reflinks 33 | 34 | 35 | ## 11.7.1 (2023-09-11) 36 | 37 | ### Bug Fixes 38 | 39 | * **sync-only:** remove futures dep when no async features are being used ([100609ab](https://github.com/zkat/cacache-rs/commit/100609abee0715a12f1bf5688aa6b462b2b8c713)) 40 | * **mmap:** fix unused method warning ([c735d83a](https://github.com/zkat/cacache-rs/commit/c735d83a316ebe2844803132fa81f6dcb9f28b79)) 41 | * **deps:** moved to reflink-copy (#55) ([6624d66c](https://github.com/zkat/cacache-rs/commit/6624d66c16d2aedf6d8369c71c627ff7d817f72a)) 42 | 43 | 44 | ## 11.7.0 (2023-09-11) 45 | 46 | ### Features 47 | 48 | * **sync-only:** add support for removing async runtime dependency altogether (#56) ([60622267](https://github.com/zkat/cacache-rs/commit/606222678931cd809af35c1a38a58933b5fc6d7e)) 49 | 50 | 51 | ## 11.6.0 (2023-05-21) 52 | 53 | ### Features 54 | 55 | * **mmap:** new feature for mmap-based optimization (#49) ([58de0b2d](https://github.com/zkat/cacache-rs/commit/58de0b2d1016c964d6f18f042f10e2a744415b08)) 56 | * **xxhash:** Add xxhash support and some utilities for making it easier to use (#51) ([41392e74](https://github.com/zkat/cacache-rs/commit/41392e7469b7e49b5b35aaacf77eb8a877a1484c)) 57 | 58 | ### Bug Fixes 59 | 60 | * **mmap:** pre-allocate temp file before mmapping (#50) ([1ac1d647](https://github.com/zkat/cacache-rs/commit/1ac1d647e0e25e1230bca376f9f294336dd57943)) 61 | 62 | 63 | ## 11.5.2 (2023-04-01) 64 | 65 | ### Bug Fixes 66 | 67 | * **deps:** move async-attributes over to dev-dependencies ([b7c10eca](https://github.com/zkat/cacache-rs/commit/b7c10ecaaf38893507ac8cba27a8ba45eb309dce)) 68 | 69 | 70 | ## 11.5.1 (2023-04-01) 71 | 72 | ### Bug Fixes 73 | 74 | * **deps:** bump ssri too ([4950998f](https://github.com/zkat/cacache-rs/commit/4950998f77444786ef747cbacfea5214741e350d)) 75 | 76 | 77 | ## 11.5.0 (2023-04-01) 78 | 79 | This release upgrades miette and thiserror in lockstep, bringing in syn2, in 80 | order to avoid [potential issues with the 81 | upgrade](https://github.com/yaahc/eyre/pull/92). 82 | 83 | ### Features 84 | 85 | * **deps:** bump miette and thiserror ([364f9da6](https://github.com/zkat/cacache-rs/commit/364f9da65eccedc51af9e05ab326496fb98d4ed7)) 86 | 87 | 88 | ## 11.4.0 (2023-03-10) 89 | 90 | ### Features 91 | 92 | * **link:** implement hard_link_hash_sync too ([b88217cc](https://github.com/zkat/cacache-rs/commit/b88217cc548a91d0c168f8604a32b78f3366fe46)) 93 | 94 | ### Bug Fixes 95 | 96 | * **security:** bump tempfile version to 3.4.0 (#45) ([02d7f146](https://github.com/zkat/cacache-rs/commit/02d7f146f17c5db3f1c6e3970dc378493254b23d)) 97 | 98 | 99 | ## 11.3.0 (2023-03-05) 100 | 101 | ### Features 102 | 103 | * **links:** add support for hard linking from the cache (#43) ([89eee6c1](https://github.com/zkat/cacache-rs/commit/89eee6c1aebd3b271619a4b55fac7f0afc5f07bb)) 104 | 105 | 106 | ## 11.2.0 (2023-02-28) 107 | 108 | ### Features 109 | 110 | * **link_to:** Add support for linking to existing files from the cache (#41) ([ac56fd9c](https://github.com/zkat/cacache-rs/commit/ac56fd9ce6a453a707df9a0603fc041e4e785d79)) 111 | * **copy:** add support for reflink-copy and unsafe-copy (#42) ([d39e8380](https://github.com/zkat/cacache-rs/commit/d39e83801dc4f1e6479dacd50dcf1372658a598f)) 112 | 113 | 114 | ## 11.1.0 (2023-02-19) 115 | 116 | ### Features 117 | 118 | * **index:** Add support for raw index metadata and expose index functions ([9991f56b](https://github.com/zkat/cacache-rs/commit/9991f56b4c689149257dbe86357cbcab8fa228e9)) 119 | 120 | 121 | ## 11.0.2 (2023-02-19) 122 | 123 | ### Bug Fixes 124 | 125 | * **ls:** make sure `ls` sees entry deletions (#40) ([53c28952](https://github.com/zkat/cacache-rs/commit/53c289523fc0d1bc9708e3e252f84b532e405329)) 126 | 127 | 128 | ## 11.0.1 (2023-02-19) 129 | 130 | ### Features 131 | 132 | * **deps:** bump ssri to 8.0.0 ([f8cbfefc](https://github.com/zkat/cacache-rs/commit/f8cbfefc129861cece61a80dda3e5a4d9d6de053)) 133 | 134 | 135 | ## 11.0.0 (2023-01-29) 136 | 137 | This release includes a number of nice improvements to error reporting, 138 | integrating [`miette`](https://crates.io/crates/miette) to provide more 139 | helpful, contextual error messages, as well as adding 140 | [`tokio`](https://crates.io/crates/tokio) as an optional runtime, instead of 141 | `async-std`. 142 | 143 | It also includes a number of bug fixes and performance improvements. 144 | 145 | Note that this release also bumps the MSRV to 1.67.0, which will hopefully 146 | stay there for a while. Apologies if you're still on an earlier version of 147 | `rustc`: the ecosystem moved forward and ultimately, we needed to move with 148 | it. 149 | 150 | ### Bug Fixes 151 | 152 | * **write:** set tmpfile length in async writer (#35) ([6d84ff0a](https://github.com/zkat/cacache-rs/commit/6d84ff0aed53aea727b32f6b3d16ee0d045aa0e4)) 153 | * **BREAKING CHANGE**: This commit also bumps the MSRV for cacache to 1.66.1. 154 | * **deps:** only include rt-multi-threaded for tokio when running tests ([526386ad](https://github.com/zkat/cacache-rs/commit/526386ada8a6b3ebee3222edb93ef401959e7f6e)) 155 | * **msrv:** be less aggressive about MSRV bump ([dca57e11](https://github.com/zkat/cacache-rs/commit/dca57e11001f8ee06077af77dfffe3bc9df5af2a)) 156 | * **perf:** do the inner fn trick to reduce generic compilation load ([da259ae4](https://github.com/zkat/cacache-rs/commit/da259ae43233e0b0cb23a2195700d3bca2942340)) 157 | 158 | ### Features 159 | 160 | * **async:** Add tokio as an executor option (#36) ([e34dcfdc](https://github.com/zkat/cacache-rs/commit/e34dcfdc250c87010afa07279dc7d6aefce9866b)) 161 | * **errors:** integrate miette and generally improve error reporting (#38) ([c2d5390a](https://github.com/zkat/cacache-rs/commit/c2d5390a84639ddc62b0581c76dd986442cfca7d)) 162 | * **BREAKING CHANGE**: This bumps the MSRV to 1.67.0 and documents it in the README. 163 | 164 | 165 | ## 10.0.1 (2022-02-25) 166 | 167 | ### Bug Fixes 168 | 169 | * **mmap:** mmap needs equal buffer lengths (#33) ([684a3e8a](https://github.com/zkat/cacache-rs/commit/684a3e8a682d7ddc60a18671bc182373f2bc6c28)) 170 | 171 | 172 | ## 10.0.0 (2022-01-23) 173 | 174 | ### Breaking Changes 175 | 176 | * **msrv:** bump MSRV to 1.54, thanks to socket ([323ecacc](https://github.com/zkat/cacache-rs/commit/323ecacc3b1e9b70bfac5f3e0838b099e9c6f8a4)) 177 | 178 | ### Bug Fixes 179 | 180 | * **security:** replaced memmap with memmap2 (#30) ([03d4596b](https://github.com/zkat/cacache-rs/commit/03d4596baa330a057011e5238983095c08039360)) 181 | 182 | 183 | ## 9.0.0 (2021-09-16) 184 | 185 | ### Bug Fixes 186 | 187 | * **clippy:** appease the paperclip ([8d08e452](https://github.com/zkat/cacache-rs/commit/8d08e452984f0bf5a266b1939f3ce2d7ab3aed7d)) 188 | * **cleanup:** general house cleaning, bumping deps, etc ([9163a584](https://github.com/zkat/cacache-rs/commit/9163a58481e28b8ecca62268f1cf32e4e68f47db)) 189 | 190 | ### Features 191 | 192 | * **license:** change license to Apache-2.0 ([97890872](https://github.com/zkat/cacache-rs/commit/97890872d559b0c37aba8414c8f8b1055276c6cb)) 193 | * **BREAKING CHANGE**: This is a significant licensing change. Please review. 194 | 195 | 196 | ## 8.0.0 (2020-07-18) 197 | 198 | #### Breaking Changes 199 | 200 | - **write:** Use mmap for small writes (#26) ([803d0c3e](https://github.com/zkat/cacache-rs/commit/803d0c3ede199c20aec1b514daf21fab9ee68ac2), breaks [#](https://github.com/zkat/cacache-rs/issues/)). This bumps the minimum Rust version from 1.39 to 1.43 due to a dependency's breaking change in a patch version. 201 | 202 | 203 | 204 | ## 7.0.0 (2020-04-30) 205 | 206 | This release is mostly a major overhaul of the external error API, switching out of `anyhow` in favor of more bespoke error handling that works nicer in a library. 207 | 208 | #### Breaking Changes 209 | 210 | - **errors:** remove anyhow and use custom error types (#24) ([bb815f5f](https://github.com/zkat/cacache-rs/commit/bb815f5f22ea932814b8b3e120fd6cac24831d01), breaks [#](https://github.com/zkat/cacache-rs/issues/)) 211 | 212 | #### Bug Fixes 213 | 214 | - **list_sync:** make sure the public interface allows using the Item type (#25) ([88a76189](https://github.com/zkat/cacache-rs/commit/88a76189fce954949ff3026b96158f700f5e2325)) 215 | 216 | 217 | 218 | ## 6.0.0 (2019-11-12) 219 | 220 | #### Breaking Changes 221 | 222 | - **chown:** stop changing owner/group on unix platforms ([d5bb0dff](https://github.com/zkat/cacache-rs/commit/d5bb0dffb623d0a61d7680829ca36ce10ceb2f53)) 223 | - **deps:** upgrade to latest async-std and regular futures crate ([c44b781a](https://github.com/zkat/cacache-rs/commit/c44b781a34bb4f95667ccb784671060ee3c0bcca)) 224 | - **license:** upgrade to Parity 7.0 release ([b54ec598](https://github.com/zkat/cacache-rs/commit/b54ec598cb11272edd685f4db45f6ff8bbeb9747)) 225 | 226 | 227 | 228 | ## 5.0.0 (2019-10-24) 229 | 230 | #### Breaking Changes 231 | 232 | - **api:** rewrite entire API to be like std::fs (#21) ([743476b2](https://github.com/zkat/cacache-rs/commit/743476b274eb07844b7b73137770df856cd7e4c4)) 233 | - **license:** bump Parity license to 7.0.0-pre.3 ([0395b0fb](https://github.com/zkat/cacache-rs/commit/0395b0fbffc65004f2b099aee9075251c8354e06)) 234 | 235 | #### Features 236 | 237 | - **api:** rewrite entire API to be like std::fs (#21) ([743476b2](https://github.com/zkat/cacache-rs/commit/743476b274eb07844b7b73137770df856cd7e4c4)) 238 | - **license:** bump Parity license to 7.0.0-pre.3 ([0395b0fb](https://github.com/zkat/cacache-rs/commit/0395b0fbffc65004f2b099aee9075251c8354e06)) 239 | 240 | 241 | 242 | ## 4.0.0 (2019-10-21) 243 | 244 | #### Bug Fixes 245 | 246 | - **fmt:** cargo fmt --all ([38115599](https://github.com/zkat/cacache-rs/commit/38115599ca9cc9f6426b950d16399f9e03871dd3)) 247 | 248 | #### Breaking Changes 249 | 250 | - **errors:** 251 | - improved errors messaging and context (#20) ([62298cdf](https://github.com/zkat/cacache-rs/commit/62298cdf351d7ed10b54417ae7a702d07b4b4765)) 252 | - Replace failure with anyhow crate (#17) ([ee149a70](https://github.com/zkat/cacache-rs/commit/ee149a70cab9ec37951aef47a21c40a0d6efb234)) 253 | 254 | #### Features 255 | 256 | - **errors:** 257 | - improved errors messaging and context (#20) ([62298cdf](https://github.com/zkat/cacache-rs/commit/62298cdf351d7ed10b54417ae7a702d07b4b4765)) 258 | - Replace failure with anyhow crate (#17) ([ee149a70](https://github.com/zkat/cacache-rs/commit/ee149a70cab9ec37951aef47a21c40a0d6efb234)) 259 | - **license:** Add in Patron license to make proprietary stuff more clear ([fbeb6ec0](https://github.com/zkat/cacache-rs/commit/fbeb6ec0ff77e022d87dc03865d4136bbbd8fbc6)) 260 | - **rm:** Accept AsRef for keys ([64939851](https://github.com/zkat/cacache-rs/commit/649398512f339933605ed70cade3ca16962a6b26)) 261 | 262 | 263 | 264 | ## 3.0.0 (2019-10-19) 265 | 266 | #### Features 267 | 268 | - **api:** get::read -> get::data ([b02f41e0](https://github.com/zkat/cacache-rs/commit/b02f41e07fab0929006e8027395503ff001a6002)) 269 | - **async:** reorganize async APIs to be the primary APIs ([662aea9b](https://github.com/zkat/cacache-rs/commit/662aea9b5a829ca4ca9673f2d82917065d675c62)) 270 | - **get:** get::info -> get::entry ([dafc79f4](https://github.com/zkat/cacache-rs/commit/dafc79f481366f3254c13efaf101c79e018d7e19)) 271 | - **ls:** cacache::ls::all -> ls::all_sync ([c4300167](https://github.com/zkat/cacache-rs/commit/c43001674441e68dd376cf003e17167360ab670e)) 272 | 273 | #### Bug Fixes 274 | 275 | - **check:** {Async}Get::check wasn't working correctly ([d08629cf](https://github.com/zkat/cacache-rs/commit/d08629cf5547f6aad8147f319fee5d30accf89a2)) 276 | - **open:** use actual file paths instead of just cache for open APIs ([03ff1970](https://github.com/zkat/cacache-rs/commit/03ff19709ab13ff4fc61ae8b52ace93db2c9dada)) 277 | 278 | #### Breaking Changes 279 | 280 | - **api:** get::read -> get::data ([b02f41e0](https://github.com/zkat/cacache-rs/commit/b02f41e07fab0929006e8027395503ff001a6002), breaks [#](https://github.com/zkat/cacache-rs/issues/)) 281 | - **async:** reorganize async APIs to be the primary APIs ([662aea9b](https://github.com/zkat/cacache-rs/commit/662aea9b5a829ca4ca9673f2d82917065d675c62), breaks [#](https://github.com/zkat/cacache-rs/issues/)) 282 | - **get:** get::info -> get::entry ([dafc79f4](https://github.com/zkat/cacache-rs/commit/dafc79f481366f3254c13efaf101c79e018d7e19), breaks [#](https://github.com/zkat/cacache-rs/issues/)) 283 | - **ls:** cacache::ls::all -> ls::all_sync ([c4300167](https://github.com/zkat/cacache-rs/commit/c43001674441e68dd376cf003e17167360ab670e), breaks [#](https://github.com/zkat/cacache-rs/issues/)) 284 | 285 | 286 | 287 | ## 2.0.1 (2019-10-15) 288 | 289 | - Just adds some examples of the core API. 290 | 291 | 292 | 293 | ## 2.0.0 (2019-10-15) 294 | 295 | #### Features 296 | 297 | - **async:** add extra async versions of APIs (#6) ([18190bfc](https://github.com/zkat/cacache-rs/commit/18190bfc356fdf871f9f284b54fc48da32e44ead)) 298 | - **license:** 299 | - relicense to Parity+Apache ([4d9404b9](https://github.com/zkat/cacache-rs/commit/4d9404b9a606cfc52fce06999ab5a640bda8fc26)) 300 | 301 | #### Bug Fixes 302 | 303 | - **windows:** add windows support ([97f44573](https://github.com/zkat/cacache-rs/commit/97f44573d55c96172aecf4be553eba064e43d58e)) 304 | 305 | #### Breaking Changes 306 | 307 | - **license:** relicense to Parity+Apache ([4d9404b9](https://github.com/zkat/cacache-rs/commit/4d9404b9a606cfc52fce06999ab5a640bda8fc26)) 308 | 309 | 310 | 311 | ## 1.0.1 (2019-07-01) 312 | 313 | Initial History generation. 314 | 315 | #### Features 316 | 317 | - **api:** AsRef all the things! ([5af622eb](https://github.com/zkat/cacache-rs.git/commit/5af622eb30b9f177117ce2f8ad17690313fba50a)) 318 | - **content:** add baseline read functionality ([e98bfb17](https://github.com/zkat/cacache-rs.git/commit/e98bfb17da0f4b862954e5f7636ea6284cd81367)) 319 | - **error:** 320 | - Add SizeError ([0bbe080a](https://github.com/zkat/cacache-rs.git/commit/0bbe080a6ef636175ce07936ca8a7d26243509fb)) 321 | - add wrapper for atomicwrites ([dbb8c79b](https://github.com/zkat/cacache-rs.git/commit/dbb8c79b00f89e1b6303be179a6389328e1a762c)) 322 | - **errors:** add errors module ([b0464849](https://github.com/zkat/cacache-rs.git/commit/b0464849e6cd32b047bbdfaa000e961dc2d87e86)) 323 | - **exports:** re-export ssri::Algorithm and serde_json::Value ([87adc8cf](https://github.com/zkat/cacache-rs.git/commit/87adc8cf9f63211edc943e72ec28de797de574ea)) 324 | - **get:** 325 | - add get::open() and get::open_hash() ([6e9a2f9f](https://github.com/zkat/cacache-rs.git/commit/6e9a2f9f87ecfb82a7bfd90fb748053a79de4e75)) 326 | - add external cacache::get api ([d91d2141](https://github.com/zkat/cacache-rs.git/commit/d91d2141761abf0e6180dc2ecd8c486637cf9232)) 327 | - **index:** 328 | - make inserter.commit() return integrity ([257fc9b6](https://github.com/zkat/cacache-rs.git/commit/257fc9b6d0cb3f99547059821255b1719dd6be2f)) 329 | - implement delete() ([33a5dbbd](https://github.com/zkat/cacache-rs.git/commit/33a5dbbd51fc8d9ae180e8eac3f0600d8cbe37df)) 330 | - implemented find() ([44eb2acc](https://github.com/zkat/cacache-rs.git/commit/44eb2acc98b242747ff09460e0c276593dfe3840)) 331 | - implemented index::insert() ([322e68ff](https://github.com/zkat/cacache-rs.git/commit/322e68ffaa118ed519e1fe2f395b7cdfa903d91b)) 332 | - port index::insert() ([9ffc090b](https://github.com/zkat/cacache-rs.git/commit/9ffc090b3b2248def2aa9390ca1fd4028fb3663b)) 333 | - **ls:** implemented cacache::ls::all() ([b0f351ea](https://github.com/zkat/cacache-rs.git/commit/b0f351ea269778e2e0be1d1388698d7a4b97ccd0)) 334 | - **path:** ported content_path ([0f768fa5](https://github.com/zkat/cacache-rs.git/commit/0f768fa5c09445cc7dc81bcaea2639cf598f5107)) 335 | - **put:** 336 | - privatize Put and PutOpts fields ([7f1602e2](https://github.com/zkat/cacache-rs.git/commit/7f1602e28fcecc02c47a43867c43dc8b420ca120)) 337 | - make PutOpts Clone ([27ce700b](https://github.com/zkat/cacache-rs.git/commit/27ce700bd69e1b72ab761521b0ba6fe0fc93ece1)) 338 | - Add put::Put and put::PutOpts ([15f017fe](https://github.com/zkat/cacache-rs.git/commit/15f017fe2151ad70dd75fbc90bae4c1cfccc00df)) 339 | - initial implementation of cacache::put ([815d7a3c](https://github.com/zkat/cacache-rs.git/commit/815d7a3c9e880eccd89baf4565e627658c5ac553)) 340 | - **read:** 341 | - added has_content() ([bff95f20](https://github.com/zkat/cacache-rs.git/commit/bff95f20ec3f79a356a30733145f44adc99d2f83)) 342 | - added content read and read_to_string ([70cf52e1](https://github.com/zkat/cacache-rs.git/commit/70cf52e136624bbff415d2641d56331191649f17)) 343 | - **rm:** 344 | - added external rm api ([346cf5fb](https://github.com/zkat/cacache-rs.git/commit/346cf5fb2379b9486186eca6aa14b72106818fc4)) 345 | - added content/rm ([eac29d94](https://github.com/zkat/cacache-rs.git/commit/eac29d941b0e36c143d3262e891fdbf991e316d7)) 346 | - **write:** initial hack for write ([e452fdcd](https://github.com/zkat/cacache-rs.git/commit/e452fdcd16fae12d79602814979312767264a3b7)) 347 | 348 | #### Bug Fixes 349 | 350 | - **api:** use &str keys ([cf0fbe23](https://github.com/zkat/cacache-rs.git/commit/cf0fbe233f721f7ad3637eaf01207e3015f74ecd)) 351 | - **content:** make rm use our own Error ([f3b6abf4](https://github.com/zkat/cacache-rs.git/commit/f3b6abf45c0408228e3bf8a0fe1e744d0b32c0bd)) 352 | - **fmt:** 353 | - cargo fmt ([0349d115](https://github.com/zkat/cacache-rs.git/commit/0349d115f4e8d7aa59c6f7a0455b94be898efd46)) 354 | - cargo fmt ([bc56a1b3](https://github.com/zkat/cacache-rs.git/commit/bc56a1b3fee36f4ec2c3508ab34c3459904e1978)) 355 | - **index:** 356 | - get rid of last compiler warning ([22c4b301](https://github.com/zkat/cacache-rs.git/commit/22c4b3010f9a851dd53073bbe1307ecbf01ef30e)) 357 | - make fields public, too ([65040481](https://github.com/zkat/cacache-rs.git/commit/6504048181415a4818fb6f713c7f9d7be665064a)) 358 | - switch to using new error module ([6f78e00c](https://github.com/zkat/cacache-rs.git/commit/6f78e00c42d59b73c725ebb4105983aee84459ff)) 359 | - make Entry use actual Integrity objects ([7ad0633c](https://github.com/zkat/cacache-rs.git/commit/7ad0633c4363a35a53e832dcac18b4672f462cc8)) 360 | - pass references instead of using .as_path() ([fc067e95](https://github.com/zkat/cacache-rs.git/commit/fc067e95d9c8dbb29ca1732e1e6bbd7b503239cc)) 361 | - remove unneeded integrity() method ([b579be61](https://github.com/zkat/cacache-rs.git/commit/b579be617f32a26ab557fb7944da89754e40c6ea)) 362 | - **lint:** clippy told me to do this ([cba2f0d3](https://github.com/zkat/cacache-rs.git/commit/cba2f0d39afe71293742f97dcfd6c610031e5bfa)) 363 | - **put:** fix warnings ([4a6950ff](https://github.com/zkat/cacache-rs.git/commit/4a6950ff5ddf6d3f110d2cf9bedeb1ef3134d1fa)) 364 | - **write:** use shared Error type for write() ([8bf623b8](https://github.com/zkat/cacache-rs.git/commit/8bf623b8efab138f9a247edc45e477a08ab9213c)) 365 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## When Something Happens 4 | 5 | If you see a Code of Conduct violation, follow these steps: 6 | 7 | 1. Let the person know that what they did is not appropriate and ask them to stop and/or edit their message(s) or commits. 8 | 2. That person should immediately stop the behavior and correct the issue. 9 | 3. If this doesn’t happen, or if you're uncomfortable speaking up, [contact the maintainers](#contacting-maintainers). 10 | 4. As soon as available, a maintainer will look into the issue, and take [further action (see below)](#further-enforcement), starting with a warning, then temporary block, then long-term repo or organization ban. 11 | 12 | When reporting, please include any relevant details, links, screenshots, context, or other information that may be used to better understand and resolve the situation. 13 | 14 | **The maintainer team will prioritize the well-being and comfort of the recipients of the violation over the comfort of the violator.** See [some examples below](#enforcement-examples). 15 | 16 | ## Our Pledge 17 | 18 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers of this project pledge to making participation in our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, technical preferences, nationality, personal appearance, race, religion, or sexual identity and orientation. 19 | 20 | ## Our Standards 21 | 22 | Examples of behavior that contributes to creating a positive environment include: 23 | 24 | * Using welcoming and inclusive language. 25 | * Being respectful of differing viewpoints and experiences. 26 | * Gracefully accepting constructive feedback. 27 | * Focusing on what is best for the community. 28 | * Showing empathy and kindness towards other community members. 29 | * Encouraging and raising up your peers in the project so you can all bask in hacks and glory. 30 | 31 | Examples of unacceptable behavior by participants include: 32 | 33 | * The use of sexualized language or imagery and unwelcome sexual attention or advances, including when simulated online. The only exception to sexual topics is channels/spaces specifically for topics of sexual identity. 34 | * Casual mention of slavery or indentured servitude and/or false comparisons of one's occupation or situation to slavery. Please consider using or asking about alternate terminology when referring to such metaphors in technology. 35 | * Making light of/making mocking comments about trigger warnings and content warnings. 36 | * Trolling, insulting/derogatory comments, and personal or political attacks. 37 | * Public or private harassment, deliberate intimidation, or threats. 38 | * Publishing others' private information, such as a physical or electronic address, without explicit permission. This includes any sort of "outing" of any aspect of someone's identity without their consent. 39 | * Publishing private screenshots or quotes of interactions in the context of this project without all quoted users' *explicit* consent. 40 | * Publishing of private communication that doesn't have to do with reporting harrassment. 41 | * Any of the above even when [presented as "ironic" or "joking"](https://en.wikipedia.org/wiki/Hipster_racism). 42 | * Any attempt to present "reverse-ism" versions of the above as violations. Examples of reverse-isms are "reverse racism", "reverse sexism", "heterophobia", and "cisphobia". 43 | * Unsolicited explanations under the assumption that someone doesn't already know it. Ask before you teach! Don't assume what people's knowledge gaps are. 44 | * [Feigning or exaggerating surprise](https://www.recurse.com/manual#no-feigned-surprise) when someone admits to not knowing something. 45 | * "[Well-actuallies](https://www.recurse.com/manual#no-well-actuallys)" 46 | * Other conduct which could reasonably be considered inappropriate in a professional or community setting. 47 | 48 | ## Scope 49 | 50 | This Code of Conduct applies both within spaces involving this project and in other spaces involving community members. This includes the repository, its Pull Requests and Issue tracker, its Twitter community, private email communications in the context of the project, and any events where members of the project are participating, as well as adjacent communities and venues affecting the project's members. 51 | 52 | Depending on the violation, the maintainers may decide that violations of this code of conduct that have happened outside of the scope of the community may deem an individual unwelcome, and take appropriate action to maintain the comfort and safety of its members. 53 | 54 | ### Other Community Standards 55 | 56 | As a project on GitHub, this project is additionally covered by the [GitHub Community Guidelines](https://help.github.com/articles/github-community-guidelines/). 57 | 58 | Additionally, as a project hosted on npm, is is covered by [npm, Inc's Code of Conduct](https://www.npmjs.com/policies/conduct). 59 | 60 | Enforcement of those guidelines after violations overlapping with the above are the responsibility of the entities, and enforcement may happen in any or all of the services/communities. 61 | 62 | ## Maintainer Enforcement Process 63 | 64 | Once the maintainers get involved, they will follow a documented series of steps and do their best to preserve the well-being of project members. This section covers actual concrete steps. 65 | 66 | ### Contacting Maintainers 67 | 68 | You may get in touch with the maintainer team through any of the following methods: 69 | 70 | * Through email: 71 | * [kzm@sykosomatic.org](mailto:coc@zkat.tech) (Kat Marchán) 72 | 73 | * Through Twitter: 74 | * [@maybekatz](https://twitter.com/maybekatz) (Kat Marchán) 75 | 76 | ### Further Enforcement 77 | 78 | If you've already followed the [initial enforcement steps](#enforcement), these are the steps maintainers will take for further enforcement, as needed: 79 | 80 | 1. Repeat the request to stop. 81 | 2. If the person doubles down, they will have offending messages removed or edited by a maintainers given an official warning. The PR or Issue may be locked. 82 | 3. If the behavior continues or is repeated later, the person will be blocked from participating for 24 hours. 83 | 4. If the behavior continues or is repeated after the temporary block, a long-term (6-12mo) ban will be used. 84 | 85 | On top of this, maintainers may remove any offending messages, images, contributions, etc, as they deem necessary. 86 | 87 | Maintainers reserve full rights to skip any of these steps, at their discretion, if the violation is considered to be a serious and/or immediate threat to the health and well-being of members of the community. These include any threats, serious physical or verbal attacks, and other such behavior that would be completely unacceptable in any social setting that puts our members at risk. 88 | 89 | Members expelled from events or venues with any sort of paid attendance will not be refunded. 90 | 91 | ### Who Watches the Watchers? 92 | 93 | Maintainers and other leaders who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. These may include anything from removal from the maintainer team to a permanent ban from the community. 94 | 95 | Additionally, as a project hosted on both GitHub and npm, [their own Codes of Conducts may be applied against maintainers of this project](#other-community-standards), externally of this project's procedures. 96 | 97 | ### Enforcement Examples 98 | 99 | #### The Best Case 100 | 101 | The vast majority of situations work out like this. This interaction is common, and generally positive. 102 | 103 | > Alex: "Yeah I used X and it was really crazy!" 104 | 105 | > Patt (not a maintainer): "Hey, could you not use that word? What about 'ridiculous' instead?" 106 | 107 | > Alex: "oh sorry, sure." -> edits old comment to say "it was really confusing!" 108 | 109 | #### The Maintainer Case 110 | 111 | Sometimes, though, you need to get maintainers involved. Maintainers will do their best to resolve conflicts, but people who were harmed by something **will take priority**. 112 | 113 | > Patt: "Honestly, sometimes I just really hate using $library and anyone who uses it probably sucks at their job." 114 | 115 | > Alex: "Whoa there, could you dial it back a bit? There's a CoC thing about attacking folks' tech use like that." 116 | 117 | > Patt: "I'm not attacking anyone, what's your problem?" 118 | 119 | > Alex: "@maintainers hey uh. Can someone look at this issue? Patt is getting a bit aggro. I tried to nudge them about it, but nope." 120 | 121 | > KeeperOfCommitBits: (on issue) "Hey Patt, maintainer here. Could you tone it down? This sort of attack is really not okay in this space." 122 | 123 | > Patt: "Leave me alone I haven't said anything bad wtf is wrong with you." 124 | 125 | > KeeperOfCommitBits: (deletes user's comment), "@patt I mean it. Please refer to the CoC over at (URL to this CoC) if you have questions, but you can consider this an actual warning. I'd appreciate it if you reworded your messages in this thread, since they made folks there uncomfortable. Let's try and be kind, yeah?" 126 | 127 | > Patt: "@keeperofbits Okay sorry. I'm just frustrated and I'm kinda burnt out and I guess I got carried away. I'll DM Alex a note apologizing and edit my messages. Sorry for the trouble." 128 | 129 | > KeeperOfCommitBits: "@patt Thanks for that. I hear you on the stress. Burnout sucks :/. Have a good one!" 130 | 131 | #### The Nope Case 132 | 133 | > PepeTheFrog🐸: "Hi, I am a literal actual nazi and I think white supremacists are quite fashionable." 134 | 135 | > Patt: "NOOOOPE. OH NOPE NOPE." 136 | 137 | > Alex: "JFC NO. NOPE. @keeperofbits NOPE NOPE LOOK HERE" 138 | 139 | > KeeperOfCommitBits: "👀 Nope. NOPE NOPE NOPE. 🔥" 140 | 141 | > PepeTheFrog🐸 has been banned from all organization or user repositories belonging to KeeperOfCommitBits. 142 | 143 | ## Attribution 144 | 145 | This Code of Conduct was generated using [WeAllJS Code of Conduct Generator](https://npm.im/weallbehave), which is based on the [WeAllJS Code of 146 | Conduct](https://wealljs.org/code-of-conduct), which is itself based on 147 | [Contributor Covenant](http://contributor-covenant.org), version 1.4, available 148 | at 149 | [http://contributor-covenant.org/version/1/4](http://contributor-covenant.org/version/1/4), 150 | and the LGBTQ in Technology Slack [Code of 151 | Conduct](http://lgbtq.technology/coc.html). 152 | 153 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## How do I... 4 | 5 | * [Use This Guide](#introduction)? 6 | * Ask or Say Something? 🤔🐛😱 7 | * [Request Support](#request-support) 8 | * [Report an Error or Bug](#report-an-error-or-bug) 9 | * [Request a Feature](#request-a-feature) 10 | * Make Something? 🤓👩🏽‍💻📜🍳 11 | * [Project Setup](#project-setup) 12 | * [Contribute Documentation](#contribute-documentation) 13 | * [Contribute Code](#contribute-code) 14 | * Manage Something ✅🙆🏼💃👔 15 | * [Provide Support on Issues](#provide-support-on-issues) 16 | * [Label Issues](#label-issues) 17 | * [Clean Up Issues and PRs](#clean-up-issues-and-prs) 18 | * [Review Pull Requests](#review-pull-requests) 19 | * [Merge Pull Requests](#merge-pull-requests) 20 | * [Tag a Release](#tag-a-release) 21 | * [Join the Project Team](#join-the-project-team) 22 | * Add a Guide Like This One [To My Project](#attribution)? 🤖😻👻 23 | 24 | ## Introduction 25 | 26 | Thank you so much for your interest in contributing!. All types of contributions are encouraged and valued. See the [table of contents](#toc) for different ways to help and details about how this project handles them!📝 27 | 28 | Please make sure to read the relevant section before making your contribution! It will make it a lot easier for us maintainers to make the most of it and smooth out the experience for all involved. 💚 29 | 30 | The [Project Team](#join-the-project-team) looks forward to your contributions. 🙌🏾✨ 31 | 32 | ## Request Support 33 | 34 | If you have a question about this project, how to use it, or just need clarification about something: 35 | 36 | * Open an Issue at https://github.com/zkat/cacache-rs/issues 37 | * Provide as much context as you can about what you're running into. 38 | * Provide project and platform versions, depending on what seems relevant. If not, please be ready to provide that information if maintainers ask for it. 39 | 40 | Once it's filed: 41 | 42 | * The project team will [label the issue](#label-issues). 43 | * Someone will try to have a response soon. 44 | * If you or the maintainers don't respond to an issue for 30 days, the [issue will be closed](#clean-up-issues-and-prs). If you want to come back to it, reply (once, please), and we'll reopen the existing issue. Please avoid filing new issues as extensions of one you already made. 45 | 46 | ## Report an Error or Bug 47 | 48 | If you run into an error or bug with the project: 49 | 50 | * Open an Issue at https://github.com/zkat/cacache-rs/issues 51 | * Include *reproduction steps* that someone else can follow to recreate the bug or error on their own. 52 | * Provide project and platform versions, depending on what seems relevant. If not, please be ready to provide that information if maintainers ask for it. 53 | 54 | Once it's filed: 55 | 56 | * The project team will [label the issue](#label-issues). 57 | * A team member will try to reproduce the issue with your provided steps. If there are no repro steps or no obvious way to reproduce the issue, the team will ask you for those steps and mark the issue as `needs-repro`. Bugs with the `needs-repro` tag will not be addressed until they are reproduced. 58 | * If the team is able to reproduce the issue, it will be marked `needs-fix`, as well as possibly other tags (such as `critical`), and the issue will be left to be [implemented by someone](#contribute-code). 59 | * If you or the maintainers don't respond to an issue for 30 days, the [issue will be closed](#clean-up-issues-and-prs). If you want to come back to it, reply (once, please), and we'll reopen the existing issue. Please avoid filing new issues as extensions of one you already made. 60 | * `critical` issues may be left open, depending on perceived immediacy and severity, even past the 30 day deadline. 61 | 62 | ## Request a Feature 63 | 64 | If the project doesn't do something you need or want it to do: 65 | 66 | * Open an Issue at https://github.com/zkat/cacache-rs/issues 67 | * Provide as much context as you can about what you're running into. 68 | * Please try and be clear about why existing features and alternatives would not work for you. 69 | 70 | Once it's filed: 71 | 72 | * The project team will [label the issue](#label-issues). 73 | * The project team will evaluate the feature request, possibly asking you more questions to understand its purpose and any relevant requirements. If the issue is closed, the team will convey their reasoning and suggest an alternative path forward. 74 | * If the feature request is accepted, it will be marked for implementation with `feature-accepted`, which can then be done by either by a core team member or by anyone in the community who wants to [contribute code](#contribute-code). 75 | 76 | Note: The team is unlikely to be able to accept every single feature request that is filed. Please understand if they need to say no. 77 | 78 | ## Project Setup 79 | 80 | So you wanna contribute some code! That's great! This project uses GitHub Pull Requests to manage contributions, so [read up on how to fork a GitHub project and file a PR](https://guides.github.com/activities/forking) if you've never done it before. 81 | 82 | If this seems like a lot or you aren't able to do all this setup, you might also be able to [edit the files directly](https://help.github.com/articles/editing-files-in-another-user-s-repository/) without having to do any of this setup. Yes, [even code](#contribute-code). 83 | 84 | If you want to go the usual route and run the project locally, though: 85 | 86 | * [Install Rust](https://www.rust-lang.org/learn/get-started) 87 | * [Fork the project](https://guides.github.com/activities/forking/#fork) 88 | 89 | Then in your terminal: 90 | * `cd path/to/your/clone` 91 | * `cargo test` 92 | 93 | And you should be ready to go! 94 | 95 | ## Contribute Documentation 96 | 97 | Documentation is a super important, critical part of this project. Docs are how we keep track of what we're doing, how, and why. It's how we stay on the same page about our policies. And it's how we tell others everything they need in order to be able to use this project -- or contribute to it. So thank you in advance. 98 | 99 | Documentation contributions of any size are welcome! Feel free to file a PR even if you're just rewording a sentence to be more clear, or fixing a spelling mistake! 100 | 101 | To contribute documentation: 102 | 103 | * [Set up the project](#project-setup). 104 | * Edit or add any relevant documentation. 105 | * Make sure your changes are formatted correctly and consistently with the rest of the documentation. 106 | * Re-read what you wrote, and run a spellchecker on it to make sure you didn't miss anything. 107 | * Write clear, concise commit message(s) using [conventional-changelog format](https://github.com/conventional-changelog/conventional-changelog-angular/blob/master/convention.md). Documentation commits should use `docs(): `. 108 | * Go to https://github.com/zkat/cacache-rs/pulls and open a new pull request with your changes. 109 | * If your PR is connected to an open issue, add a line in your PR's description that says `Fixes: #123`, where `#123` is the number of the issue you're fixing. 110 | 111 | Once you've filed the PR: 112 | 113 | * One or more maintainers will use GitHub's review feature to review your PR. 114 | * If the maintainer asks for any changes, edit your changes, push, and ask for another review. 115 | * If the maintainer decides to pass on your PR, they will thank you for the contribution and explain why they won't be accepting the changes. That's ok! We still really appreciate you taking the time to do it, and we don't take that lightly. 💚 116 | * If your PR gets accepted, it will be marked as such, and merged into the `latest` branch soon after. Your contribution will be distributed to the masses next time the maintainers [tag a release](#tag-a-release) 117 | 118 | ## Contribute Code 119 | 120 | We like code commits a lot! They're super handy, and they keep the project going and doing the work it needs to do to be useful to others. 121 | 122 | Code contributions of just about any size are acceptable! 123 | 124 | The main difference between code contributions and documentation contributions is that contributing code requires inclusion of relevant tests for the code being added or changed. Contributions without accompanying tests will be held off until a test is added, unless the maintainers consider the specific tests to be either impossible, or way too much of a burden for such a contribution. 125 | 126 | To contribute code: 127 | 128 | * [Set up the project](#project-setup). 129 | * Make any necessary changes to the source code. 130 | * Include any [additional documentation](#contribute-documentation) the changes might need. 131 | * Write tests that verify that your contribution works as expected. 132 | * Write clear, concise commit message(s) using [conventional-changelog format](https://github.com/conventional-changelog/conventional-changelog-angular/blob/master/convention.md). 133 | * Dependency updates, additions, or removals must be in individual commits, and the message must use the format: `(deps): PKG@VERSION`, where `` is any of the usual `conventional-changelog` prefixes, at your discretion. 134 | * Go to https://github.com/zkat/cacache-rs/pulls and open a new pull request with your changes. 135 | * If your PR is connected to an open issue, add a line in your PR's description that says `Fixes: #123`, where `#123` is the number of the issue you're fixing. 136 | 137 | Once you've filed the PR: 138 | 139 | * Barring special circumstances, maintainers will not review PRs until all checks pass (Travis, AppVeyor, etc). 140 | * One or more maintainers will use GitHub's review feature to review your PR. 141 | * If the maintainer asks for any changes, edit your changes, push, and ask for another review. Additional tags (such as `needs-tests`) will be added depending on the review. 142 | * If the maintainer decides to pass on your PR, they will thank you for the contribution and explain why they won't be accepting the changes. That's ok! We still really appreciate you taking the time to do it, and we don't take that lightly. 💚 143 | * If your PR gets accepted, it will be marked as such, and merged into the `latest` branch soon after. Your contribution will be distributed to the masses next time the maintainers [tag a release](#tag-a-release) 144 | 145 | ## Provide Support on Issues 146 | 147 | [Needs Collaborator](#join-the-project-team): none 148 | 149 | Helping out other users with their questions is a really awesome way of contributing to any community. It's not uncommon for most of the issues on an open source projects being support-related questions by users trying to understand something they ran into, or find their way around a known bug. 150 | 151 | Sometimes, the `support` label will be added to things that turn out to actually be other things, like bugs or feature requests. In that case, suss out the details with the person who filed the original issue, add a comment explaining what the bug is, and change the label from `support` to `bug` or `feature`. If you can't do this yourself, @mention a maintainer so they can do it. 152 | 153 | In order to help other folks out with their questions: 154 | 155 | * Go to the issue tracker and [filter open issues by the `support` label](https://github.com/zkat/cacache-rs/issues?q=is%3Aopen+is%3Aissue+label%3Asupport). 156 | * Read through the list until you find something that you're familiar enough with to give an answer to. 157 | * Respond to the issue with whatever details are needed to clarify the question, or get more details about what's going on. 158 | * Once the discussion wraps up and things are clarified, either close the issue, or ask the original issue filer (or a maintainer) to close it for you. 159 | 160 | Some notes on picking up support issues: 161 | 162 | * Avoid responding to issues you don't know you can answer accurately. 163 | * As much as possible, try to refer to past issues with accepted answers. Link to them from your replies with the `#123` format. 164 | * Be kind and patient with users -- often, folks who have run into confusing things might be upset or impatient. This is ok. Try to understand where they're coming from, and if you're too uncomfortable with the tone, feel free to stay away or withdraw from the issue. (note: if the user is outright hostile or is violating the CoC, [refer to the Code of Conduct](CODE_OF_CONDUCT.md) to resolve the conflict). 165 | 166 | ## Label Issues 167 | 168 | [Needs Collaborator](#join-the-project-team): Issue Tracker 169 | 170 | One of the most important tasks in handling issues is labeling them usefully and accurately. All other tasks involving issues ultimately rely on the issue being classified in such a way that relevant parties looking to do their own tasks can find them quickly and easily. 171 | 172 | In order to label issues, [open up the list of unlabeled issues](https://github.com/zkat/cacache-rs/issues?q=is%3Aopen+is%3Aissue+no%3Alabel) and, **from newest to oldest**, read through each one and apply issue labels according to the table below. If you're unsure about what label to apply, skip the issue and try the next one: don't feel obligated to label each and every issue yourself! 173 | 174 | Label | Apply When | Notes 175 | --- | --- | --- 176 | `bug` | Cases where the code (or documentation) is behaving in a way it wasn't intended to. | If something is happening that surprises the *user* but does not go against the way the code is designed, it should use the `enhancement` label. 177 | `critical` | Added to `bug` issues if the problem described makes the code completely unusable in a common situation. | 178 | `documentation` | Added to issues or pull requests that affect any of the documentation for the project. | Can be combined with other labels, such as `bug` or `enhancement`. 179 | `duplicate` | Added to issues or PRs that refer to the exact same issue as another one that's been previously labeled. | Duplicate issues should be marked and closed right away, with a message referencing the issue it's a duplicate of (with `#123`) 180 | `enhancement` | Added to [feature requests](#request-a-feature), PRs, or documentation issues that are purely additive: the code or docs currently work as expected, but a change is being requested or suggested. | 181 | `help wanted` | Applied by [Committers](#join-the-project-team) to issues and PRs that they would like to get outside help for. Generally, this means it's lower priority for the maintainer team to itself implement, but that the community is encouraged to pick up if they so desire | Never applied on first-pass labeling. 182 | `in-progress` | Applied by [Committers](#join-the-project-team) to PRs that are pending some work before they're ready for review. | The original PR submitter should @mention the team member that applied the label once the PR is complete. 183 | `performance` | This issue or PR is directly related to improving performance. | 184 | `refactor` | Added to issues or PRs that deal with cleaning up or modifying the project for the betterment of it. | 185 | `starter` | Applied by [Committers](#join-the-project-team) to issues that they consider good introductions to the project for people who have not contributed before. These are not necessarily "easy", but rather focused around how much context is necessary in order to understand what needs to be done for this project in particular. | Existing project members are expected to stay away from these unless they increase in priority. 186 | `support` | This issue is either asking a question about how to use the project, clarifying the reason for unexpected behavior, or possibly reporting a `bug` but does not have enough detail yet to determine whether it would count as such. | The label should be switched to `bug` if reliable reproduction steps are provided. Issues primarily with unintended configurations of a user's environment are not considered bugs, even if they cause crashes. 187 | `tests` | This issue or PR either requests or adds primarily tests to the project. | If a PR is pending tests, that will be handled through the [PR review process](#review-pull-requests) 188 | `wontfix` | Labelers may apply this label to issues that clearly have nothing at all to do with the project or are otherwise entirely outside of its scope/sphere of influence. [Committers](#join-the-project-team) may apply this label and close an issue or PR if they decide to pass on an otherwise relevant issue. | The issue or PR should be closed as soon as the label is applied, and a clear explanation provided of why the label was used. Contributors are free to contest the labeling, but the decision ultimately falls on committers as to whether to accept something or not. 189 | 190 | ## Clean Up Issues and PRs 191 | 192 | [Needs Collaborator](#join-the-project-team): Issue Tracker 193 | 194 | Issues and PRs can go stale after a while. Maybe they're abandoned. Maybe the team will just plain not have time to address them any time soon. 195 | 196 | In these cases, they should be closed until they're brought up again or the interaction starts over. 197 | 198 | To clean up issues and PRs: 199 | 200 | * Search the issue tracker for issues or PRs, and add the term `updated:<=YYYY-MM-DD`, where the date is 30 days before today. 201 | * Go through each issue *from oldest to newest*, and close them if **all of the following are true**: 202 | * not opened by a maintainer 203 | * not marked as `critical` 204 | * not marked as `starter` or `help wanted` (these might stick around for a while, in general, as they're intended to be available) 205 | * no explicit messages in the comments asking for it to be left open 206 | * does not belong to a milestone 207 | * Leave a message when closing saying "Cleaning up stale issue. Please reopen or ping us if and when you're ready to resume this. See https://github.com/zkat/cacache-rs/blob/latest/CONTRIBUTING.md#clean-up-issues-and-prs for more details." 208 | 209 | ## Review Pull Requests 210 | 211 | [Needs Collaborator](#join-the-project-team): Issue Tracker 212 | 213 | While anyone can comment on a PR, add feedback, etc, PRs are only *approved* by team members with Issue Tracker or higher permissions. 214 | 215 | PR reviews use [GitHub's own review feature](https://help.github.com/articles/about-pull-request-reviews/), which manages comments, approval, and review iteration. 216 | 217 | Some notes: 218 | 219 | * You may ask for minor changes ("nitpicks"), but consider whether they are really blockers to merging: try to err on the side of "approve, with comments". 220 | * *ALL PULL REQUESTS* should be covered by a test: either by a previously-failing test, an existing test that covers the entire functionality of the submitted code, or new tests to verify any new/changed behavior. All tests must also pass and follow established conventions. Test coverage should not drop, unless the specific case is considered reasonable by maintainers. 221 | * Please make sure you're familiar with the code or documentation being updated, unless it's a minor change (spellchecking, minor formatting, etc). You may @mention another project member who you think is better suited for the review, but still provide a non-approving review of your own. 222 | * Be extra kind: people who submit code/doc contributions are putting themselves in a pretty vulnerable position, and have put time and care into what they've done (even if that's not obvious to you!) -- always respond with respect, be understanding, but don't feel like you need to sacrifice your standards for their sake, either. Just don't be a jerk about it? 223 | 224 | ## Merge Pull Requests 225 | 226 | [Needs Collaborator](#join-the-project-team): Committer 227 | 228 | TBD - need to hash out a bit more of this process. 229 | 230 | ## Tag A Release 231 | 232 | [Needs Collaborator](#join-the-project-team): Committer 233 | 234 | TBD - need to hash out a bit more of this process. The most important bit here is probably that all tests must pass, and tags must use [semver](https://semver.org). 235 | 236 | ## Join the Project Team 237 | 238 | ### Ways to Join 239 | 240 | There are many ways to contribute! Most of them don't require any official status unless otherwise noted. That said, there's a couple of positions that grant special repository abilities, and this section describes how they're granted and what they do. 241 | 242 | All of the below positions are granted based on the project team's needs, as well as their consensus opinion about whether they would like to work with the person and think that they would fit well into that position. The process is relatively informal, and it's likely that people who express interest in participating can just be granted the permissions they'd like. 243 | 244 | You can spot a collaborator on the repo by looking for the `[Collaborator]` or `[Owner]` tags next to their names. 245 | 246 | Permission | Description 247 | --- | --- 248 | Issue Tracker | Granted to contributors who express a strong interest in spending time on the project's issue tracker. These tasks are mainly [labeling issues](#label-issues), [cleaning up old ones](#clean-up-issues-and-prs), and [reviewing pull requests](#review-pull-requests), as well as all the usual things non-team-member contributors can do. Issue handlers should not merge pull requests, tag releases, or directly commit code themselves: that should still be done through the usual pull request process. Becoming an Issue Handler means the project team trusts you to understand enough of the team's process and context to implement it on the issue tracker. 249 | Committer | Granted to contributors who want to handle the actual pull request merges, tagging new versions, etc. Committers should have a good level of familiarity with the codebase, and enough context to understand the implications of various changes, as well as a good sense of the will and expectations of the project team. 250 | Admin/Owner | Granted to people ultimately responsible for the project, its community, etc. 251 | 252 | ## Attribution 253 | 254 | This guide was generated using the WeAllJS `CONTRIBUTING.md` generator. [Make your own](https://npm.im/weallcontribute)! 255 | 256 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cacache" 3 | version = "13.1.0" 4 | authors = ["Kat Marchán "] 5 | edition = "2021" 6 | description = "Content-addressable, key-value, high-performance, on-disk cache." 7 | license = "Apache-2.0" 8 | repository = "https://github.com/zkat/cacache-rs" 9 | homepage = "https://github.com/zkat/cacache-rs" 10 | readme = "README.md" 11 | categories = ["caching", "filesystem"] 12 | 13 | [dependencies] 14 | async-std = { version = "1.10.0", features = ["unstable"], optional = true } 15 | digest = "0.10.6" 16 | either = "1.6.1" 17 | futures = { version = "0.3.17", optional = true } 18 | hex = "0.4.3" 19 | memmap2 = { version = "0.5.8", optional = true } 20 | miette = "5.7.0" 21 | reflink-copy = "0.1.9" 22 | serde = "1.0.130" 23 | serde_derive = "1.0.130" 24 | serde_json = "1.0.68" 25 | sha1 = "0.10.5" 26 | sha2 = "0.10.6" 27 | ssri = "9.0.0" 28 | tempfile = "3.4.0" 29 | thiserror = "1.0.40" 30 | tokio = { version = "1.12.0", features = [ 31 | "fs", 32 | "io-util", 33 | "macros", 34 | "rt", 35 | ], optional = true } 36 | tokio-stream = { version = "0.1.7", features = ["io-util"], optional = true } 37 | walkdir = "2.3.2" 38 | 39 | [target.'cfg(target_os = "linux")'.dependencies] 40 | libc = { version = "0.2.144", optional = true } 41 | 42 | [dev-dependencies] 43 | async-attributes = { version = "1.1.2" } 44 | criterion = "0.4.0" 45 | lazy_static = "1.4.0" 46 | tokio = { version = "1.12.0", features = [ 47 | "fs", 48 | "io-util", 49 | "macros", 50 | "rt", 51 | "rt-multi-thread", 52 | ] } 53 | 54 | [[bench]] 55 | name = "benchmarks" 56 | harness = false 57 | 58 | [features] 59 | default = ["async-std", "mmap"] 60 | mmap = ["memmap2", "libc"] 61 | async-std = ["dep:async-std", "futures"] 62 | link_to = [] 63 | tokio-runtime = ["tokio", "tokio-stream", "futures"] 64 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2019 Cacache Contributors 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /Makefile.toml: -------------------------------------------------------------------------------- 1 | [tasks.changelog] 2 | workspace=false 3 | install_crate="git-cliff" 4 | command = "git-cliff" 5 | args = ["--prepend", "CHANGELOG.md", "-u", "--tag", "${@}"] 6 | 7 | [tasks.release] 8 | workspace=false 9 | install_crate="cargo-release" 10 | command = "cargo" 11 | args = ["release", "--workspace", "${@}"] 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cacache [![CI](https://github.com/zkat/cacache-rs/workflows/CI/badge.svg)](https://github.com/zkat/cacache-rs/actions) [![crates.io](https://img.shields.io/crates/v/cacache.svg)](https://crates.io/crates/cacache) 2 | 3 | A high-performance, concurrent, content-addressable disk cache, optimized for async APIs. 4 | 5 | ## Example 6 | 7 | ```rust 8 | use cacache; 9 | use async_attributes; 10 | 11 | #[async_attributes::main] 12 | async fn main() -> Result<(), cacache::Error> { 13 | let dir = String::from("./my-cache"); 14 | 15 | // Write some data! 16 | cacache::write(&dir, "key", b"my-async-data").await?; 17 | 18 | // Get the data back! 19 | let data = cacache::read(&dir, "key").await?; 20 | assert_eq!(data, b"my-async-data"); 21 | 22 | // Clean up the data! 23 | cacache::rm::all(&dir).await?; 24 | } 25 | ``` 26 | 27 | ## Install 28 | 29 | Using [`cargo-edit`](https://crates.io/crates/cargo-edit) 30 | 31 | `$ cargo add cacache` 32 | 33 | Minimum supported Rust version is `1.43.0`. 34 | 35 | ## Documentation 36 | 37 | - [API Docs](https://docs.rs/cacache) 38 | 39 | ## Features 40 | 41 | - First-class async support, using either 42 | [`async-std`](https://crates.io/crates/async-std) or 43 | [`tokio`](https://crates.io/crates/tokio) as its runtime. Sync APIs are 44 | available but secondary. You can also use sync APIs only and remove the 45 | async runtime dependency. 46 | - `std::fs`-style API 47 | - Extraction by key or by content address (shasum, etc) 48 | - [Subresource Integrity](#integrity) web standard support 49 | - Multi-hash support - safely host sha1, sha512, etc, in a single cache 50 | - Automatic content deduplication 51 | - Atomic content writes even for large data 52 | - Fault tolerance (immune to corruption, partial writes, process races, etc) 53 | - Consistency guarantees on read and write (full data verification) 54 | - Lockless, high-concurrency cache access 55 | - Really helpful, contextual error messages 56 | - Large file support 57 | - Pretty darn fast 58 | - Arbitrary metadata storage 59 | - Cross-platform: Windows and case-(in)sensitive filesystem support 60 | - [`miette`](https://crates.io/crates/miette) integration for detailed, helpful error reporting. 61 | - Punches nazis 62 | 63 | `async-std` is the default async runtime. To use `tokio` instead, turn off 64 | default features and enable the `tokio-runtime` feature, like this: 65 | 66 | ```toml 67 | [dependencies] 68 | cacache = { version = "*", default-features = false, features = ["tokio-runtime", "mmap"] } 69 | ``` 70 | 71 | You can also remove async APIs altogether, including removing async runtime 72 | dependency: 73 | 74 | ```toml 75 | [dependencies] 76 | cacache = { version = "*", default-features = false, features = ["mmap"] } 77 | ``` 78 | 79 | Experimental support for symlinking to existing files is provided via the 80 | "link_to" feature. 81 | 82 | ## Contributing 83 | 84 | The cacache team enthusiastically welcomes contributions and project 85 | participation! There's a bunch of things you can do if you want to contribute! 86 | The [Contributor Guide](CONTRIBUTING.md) has all the information you need for 87 | everything from reporting bugs to contributing entire new features. Please 88 | don't hesitate to jump in if you'd like to, or even ask us questions if 89 | something isn't clear. 90 | 91 | All participants and maintainers in this project are expected to follow [Code 92 | of Conduct](CODE_OF_CONDUCT.md), and just generally be excellent to each 93 | other. 94 | 95 | Happy hacking! 96 | 97 | ## MSRV 98 | 99 | The Minimum Supported Rust Version for cacache is `1.67.0`. Any changes to the 100 | MSRV will be considered breaking changes. 101 | 102 | ## License 103 | 104 | This project is licensed under [the Apache-2.0 License](LICENSE.md). 105 | -------------------------------------------------------------------------------- /benches/benchmarks.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "async-std")] 2 | use async_std::fs as afs; 3 | #[cfg(feature = "link_to")] 4 | use std::path::PathBuf; 5 | #[cfg(all(test, feature = "tokio"))] 6 | use tokio::fs as afs; 7 | 8 | #[cfg(all(test, feature = "async-std"))] 9 | pub use async_std::task::block_on; 10 | 11 | #[cfg(all(test, feature = "tokio"))] 12 | lazy_static::lazy_static! { 13 | static ref TOKIO_RUNTIME: tokio::runtime::Runtime = tokio::runtime::Runtime::new().unwrap(); 14 | } 15 | 16 | #[cfg(all(test, feature = "tokio"))] 17 | #[inline] 18 | pub fn block_on(future: F) -> T 19 | where 20 | F: std::future::Future, 21 | { 22 | TOKIO_RUNTIME.block_on(future) 23 | } 24 | 25 | use std::fs::{self, File}; 26 | use std::io::prelude::*; 27 | 28 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 29 | 30 | const NUM_REPEATS: usize = 10; 31 | 32 | fn baseline_read_sync(c: &mut Criterion) { 33 | let tmp = tempfile::tempdir().unwrap(); 34 | let path = tmp.path().join("test_file"); 35 | let data = b"hello world"; 36 | let mut fd = File::create(&path).unwrap(); 37 | fd.write_all(data).unwrap(); 38 | drop(fd); 39 | c.bench_function("baseline_read_sync", move |b| { 40 | b.iter(|| fs::read(&path).unwrap()) 41 | }); 42 | } 43 | 44 | fn baseline_read_many_sync(c: &mut Criterion) { 45 | let tmp = tempfile::tempdir().unwrap(); 46 | let paths: Vec<_> = (0..) 47 | .take(NUM_REPEATS) 48 | .map(|i| tmp.path().join(format!("test_file_{i}"))) 49 | .collect(); 50 | let data = b"hello world"; 51 | for path in paths.iter() { 52 | let mut fd = File::create(path).unwrap(); 53 | fd.write_all(data).unwrap(); 54 | drop(fd); 55 | } 56 | c.bench_function("baseline_read_many_sync", move |b| { 57 | b.iter(|| { 58 | for path in paths.iter() { 59 | fs::read(black_box(&path)).unwrap(); 60 | } 61 | }) 62 | }); 63 | } 64 | 65 | #[cfg(any(feature = "async-std", feature = "tokio"))] 66 | fn baseline_read_async(c: &mut Criterion) { 67 | let tmp = tempfile::tempdir().unwrap(); 68 | let path = tmp.path().join("test_file"); 69 | let data = b"hello world"; 70 | let mut fd = File::create(&path).unwrap(); 71 | fd.write_all(data).unwrap(); 72 | drop(fd); 73 | c.bench_function("baseline_read_async", move |b| { 74 | b.iter(|| block_on(afs::read(&path))) 75 | }); 76 | } 77 | 78 | #[cfg(any(feature = "async-std", feature = "tokio"))] 79 | fn baseline_read_many_async(c: &mut Criterion) { 80 | let tmp = tempfile::tempdir().unwrap(); 81 | let paths: Vec<_> = (0..) 82 | .take(NUM_REPEATS) 83 | .map(|i| tmp.path().join(format!("test_file_{i}"))) 84 | .collect(); 85 | let data = b"hello world"; 86 | for path in paths.iter() { 87 | let mut fd = File::create(path).unwrap(); 88 | fd.write_all(data).unwrap(); 89 | drop(fd); 90 | } 91 | c.bench_function("baseline_read_many_async", move |b| { 92 | b.iter(|| { 93 | let tasks = paths.iter().map(|path| afs::read(black_box(path))); 94 | block_on(futures::future::join_all(tasks)); 95 | }) 96 | }); 97 | } 98 | 99 | fn read_hash_sync(c: &mut Criterion) { 100 | let tmp = tempfile::tempdir().unwrap(); 101 | let cache = tmp.path().to_owned(); 102 | let data = b"hello world".to_vec(); 103 | let sri = cacache::write_sync(&cache, "hello", data).unwrap(); 104 | c.bench_function("get::data_hash_sync", move |b| { 105 | b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) 106 | }); 107 | } 108 | 109 | fn read_hash_sync_xxh3(c: &mut Criterion) { 110 | let tmp = tempfile::tempdir().unwrap(); 111 | let cache = tmp.path().to_owned(); 112 | let data = b"hello world".to_vec(); 113 | let sri = 114 | cacache::write_sync_with_algo(cacache::Algorithm::Xxh3, &cache, "hello", data).unwrap(); 115 | c.bench_function("get::data_hash_sync::xxh3", move |b| { 116 | b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) 117 | }); 118 | } 119 | 120 | fn read_hash_many_sync(c: &mut Criterion) { 121 | let tmp = tempfile::tempdir().unwrap(); 122 | let cache = tmp.path().to_owned(); 123 | let data: Vec<_> = (0..) 124 | .take(NUM_REPEATS) 125 | .map(|i| format!("test_file_{i}")) 126 | .collect(); 127 | let sris: Vec<_> = data 128 | .iter() 129 | .map(|datum| cacache::write_sync(&cache, "hello", datum).unwrap()) 130 | .collect(); 131 | c.bench_function("get::data_hash_many_sync", move |b| { 132 | b.iter(|| { 133 | for sri in sris.iter() { 134 | cacache::read_hash_sync(black_box(&cache), black_box(sri)).unwrap(); 135 | } 136 | }) 137 | }); 138 | } 139 | 140 | fn read_hash_many_sync_xxh3(c: &mut Criterion) { 141 | let tmp = tempfile::tempdir().unwrap(); 142 | let cache = tmp.path().to_owned(); 143 | let data: Vec<_> = (0..) 144 | .take(NUM_REPEATS) 145 | .map(|i| format!("test_file_{i}")) 146 | .collect(); 147 | let sris: Vec<_> = data 148 | .iter() 149 | .map(|datum| { 150 | cacache::write_sync_with_algo(cacache::Algorithm::Xxh3, &cache, "hello", datum).unwrap() 151 | }) 152 | .collect(); 153 | c.bench_function("get::data_hash_many_sync::xxh3", move |b| { 154 | b.iter(|| { 155 | for sri in sris.iter() { 156 | cacache::read_hash_sync(black_box(&cache), black_box(sri)).unwrap(); 157 | } 158 | }) 159 | }); 160 | } 161 | 162 | fn read_sync(c: &mut Criterion) { 163 | let tmp = tempfile::tempdir().unwrap(); 164 | let cache = tmp.path().to_owned(); 165 | let data = b"hello world".to_vec(); 166 | cacache::write_sync(&cache, "hello", data).unwrap(); 167 | c.bench_function("get::data_sync", move |b| { 168 | b.iter(|| cacache::read_sync(black_box(&cache), black_box(String::from("hello"))).unwrap()) 169 | }); 170 | } 171 | 172 | fn read_hash_sync_big_data(c: &mut Criterion) { 173 | let tmp = tempfile::tempdir().unwrap(); 174 | let cache = tmp.path().to_owned(); 175 | let data = vec![1; 1024 * 1024 * 5]; 176 | let sri = cacache::write_sync(&cache, "hello", data).unwrap(); 177 | c.bench_function("get_hash_big_data", move |b| { 178 | b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) 179 | }); 180 | } 181 | 182 | fn read_hash_sync_big_data_xxh3(c: &mut Criterion) { 183 | let tmp = tempfile::tempdir().unwrap(); 184 | let cache = tmp.path().to_owned(); 185 | let data = vec![1; 1024 * 1024 * 5]; 186 | let sri = 187 | cacache::write_sync_with_algo(cacache::Algorithm::Xxh3, &cache, "hello", data).unwrap(); 188 | c.bench_function("get_hash_big_data::xxh3", move |b| { 189 | b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) 190 | }); 191 | } 192 | 193 | #[cfg(any(feature = "async-std", feature = "tokio"))] 194 | fn read_hash_many_async(c: &mut Criterion) { 195 | let tmp = tempfile::tempdir().unwrap(); 196 | let cache = tmp.path().to_owned(); 197 | let data: Vec<_> = (0..) 198 | .take(NUM_REPEATS) 199 | .map(|i| format!("test_file_{i}")) 200 | .collect(); 201 | let sris: Vec<_> = data 202 | .iter() 203 | .map(|datum| cacache::write_sync(&cache, "hello", datum).unwrap()) 204 | .collect(); 205 | c.bench_function("get::data_hash_many", move |b| { 206 | b.iter(|| { 207 | let tasks = sris 208 | .iter() 209 | .map(|sri| cacache::read_hash(black_box(&cache), black_box(sri))); 210 | block_on(futures::future::join_all(tasks)); 211 | }) 212 | }); 213 | } 214 | 215 | #[cfg(any(feature = "async-std", feature = "tokio"))] 216 | fn read_hash_async(c: &mut Criterion) { 217 | let tmp = tempfile::tempdir().unwrap(); 218 | let cache = tmp.path().to_owned(); 219 | let data = b"hello world".to_vec(); 220 | let sri = cacache::write_sync(&cache, "hello", data).unwrap(); 221 | c.bench_function("get::data_hash", move |b| { 222 | b.iter(|| block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap()) 223 | }); 224 | } 225 | 226 | #[cfg(any(feature = "async-std", feature = "tokio"))] 227 | fn read_async(c: &mut Criterion) { 228 | let tmp = tempfile::tempdir().unwrap(); 229 | let cache = tmp.path().to_owned(); 230 | let data = b"hello world".to_vec(); 231 | cacache::write_sync(&cache, "hello", data).unwrap(); 232 | c.bench_function("get::data", move |b| { 233 | b.iter(|| block_on(cacache::read(black_box(&cache), black_box("hello"))).unwrap()) 234 | }); 235 | } 236 | 237 | #[cfg(any(feature = "async-std", feature = "tokio"))] 238 | fn read_hash_async_big_data(c: &mut Criterion) { 239 | let tmp = tempfile::tempdir().unwrap(); 240 | let cache = tmp.path().to_owned(); 241 | let data = vec![1; 1024 * 1024 * 5]; 242 | let sri = cacache::write_sync(&cache, "hello", data).unwrap(); 243 | c.bench_function("get::data_big_data", move |b| { 244 | b.iter(|| block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap()) 245 | }); 246 | } 247 | 248 | fn write_hash(c: &mut Criterion) { 249 | let tmp = tempfile::tempdir().unwrap(); 250 | let cache = tmp.path().to_owned(); 251 | c.bench_function("put::data::sync", move |b| { 252 | b.iter_custom(|iters| { 253 | let start = std::time::Instant::now(); 254 | for i in 0..iters { 255 | cacache::write_hash_sync(&cache, format!("hello world{i}")).unwrap(); 256 | } 257 | start.elapsed() 258 | }) 259 | }); 260 | } 261 | 262 | fn write_hash_xxh3(c: &mut Criterion) { 263 | let tmp = tempfile::tempdir().unwrap(); 264 | let cache = tmp.path().to_owned(); 265 | c.bench_function("put::data::sync::xxh3", move |b| { 266 | b.iter_custom(|iters| { 267 | let start = std::time::Instant::now(); 268 | for i in 0..iters { 269 | cacache::write_hash_sync_with_algo( 270 | cacache::Algorithm::Xxh3, 271 | &cache, 272 | format!("hello world{i}"), 273 | ) 274 | .unwrap(); 275 | } 276 | start.elapsed() 277 | }) 278 | }); 279 | } 280 | 281 | #[cfg(any(feature = "async-std", feature = "tokio"))] 282 | fn write_hash_async(c: &mut Criterion) { 283 | let tmp = tempfile::tempdir().unwrap(); 284 | let cache = tmp.path().to_owned(); 285 | c.bench_function("put::data", move |b| { 286 | b.iter_custom(|iters| { 287 | let start = std::time::Instant::now(); 288 | for i in 0..iters { 289 | block_on(cacache::write_hash(&cache, format!("hello world{i}"))).unwrap(); 290 | } 291 | start.elapsed() 292 | }) 293 | }); 294 | } 295 | 296 | #[cfg(any(feature = "async-std", feature = "tokio"))] 297 | fn write_hash_async_xxh3(c: &mut Criterion) { 298 | let tmp = tempfile::tempdir().unwrap(); 299 | let cache = tmp.path().to_owned(); 300 | c.bench_function("put::data::xxh3", move |b| { 301 | b.iter_custom(|iters| { 302 | let start = std::time::Instant::now(); 303 | for i in 0..iters { 304 | block_on(cacache::write_hash_with_algo( 305 | cacache::Algorithm::Xxh3, 306 | &cache, 307 | format!("hello world{i}"), 308 | )) 309 | .unwrap(); 310 | } 311 | start.elapsed() 312 | }) 313 | }); 314 | } 315 | 316 | #[cfg(feature = "link_to")] 317 | fn create_tmpfile(tmp: &tempfile::TempDir, buf: &[u8]) -> PathBuf { 318 | let dir = tmp.path().to_owned(); 319 | let target = dir.join("target-file"); 320 | std::fs::create_dir_all(&target.parent().unwrap()).unwrap(); 321 | let mut file = File::create(target.clone()).unwrap(); 322 | file.write_all(buf).unwrap(); 323 | file.flush().unwrap(); 324 | target 325 | } 326 | 327 | #[cfg(feature = "link_to")] 328 | #[cfg(any(feature = "async-std", feature = "tokio"))] 329 | fn link_to_async(c: &mut Criterion) { 330 | let tmp = tempfile::tempdir().unwrap(); 331 | let target = create_tmpfile(&tmp, b"hello world"); 332 | 333 | let tmp = tempfile::tempdir().unwrap(); 334 | let cache = tmp.path().to_owned(); 335 | c.bench_function("link_to::file", move |b| { 336 | b.iter_custom(|iters| { 337 | let start = std::time::Instant::now(); 338 | for i in 0..iters { 339 | block_on(cacache::link_to( 340 | &cache, 341 | format!("key{}", i), 342 | target.clone(), 343 | )) 344 | .unwrap(); 345 | } 346 | start.elapsed() 347 | }) 348 | }); 349 | } 350 | 351 | #[cfg(all(feature = "link_to", any(feature = "async-std", feature = "tokio")))] 352 | fn link_to_hash_async(c: &mut Criterion) { 353 | let tmp = tempfile::tempdir().unwrap(); 354 | let target = create_tmpfile(&tmp, b"hello world"); 355 | 356 | let tmp = tempfile::tempdir().unwrap(); 357 | let cache = tmp.path().to_owned(); 358 | c.bench_function("link_to::file_hash", move |b| { 359 | b.iter(|| block_on(cacache::link_to_hash(&cache, target.clone())).unwrap()) 360 | }); 361 | } 362 | 363 | #[cfg(feature = "link_to")] 364 | fn link_to_sync(c: &mut Criterion) { 365 | let tmp = tempfile::tempdir().unwrap(); 366 | let target = create_tmpfile(&tmp, b"hello world"); 367 | 368 | let tmp = tempfile::tempdir().unwrap(); 369 | let cache = tmp.path().to_owned(); 370 | c.bench_function("link_to::file_sync", move |b| { 371 | b.iter_custom(|iters| { 372 | let start = std::time::Instant::now(); 373 | for i in 0..iters { 374 | cacache::link_to_sync(&cache, format!("key{}", i), target.clone()).unwrap(); 375 | } 376 | start.elapsed() 377 | }) 378 | }); 379 | } 380 | 381 | #[cfg(feature = "link_to")] 382 | fn link_to_hash_sync(c: &mut Criterion) { 383 | let tmp = tempfile::tempdir().unwrap(); 384 | let target = create_tmpfile(&tmp, b"hello world"); 385 | 386 | let tmp = tempfile::tempdir().unwrap(); 387 | let cache = tmp.path().to_owned(); 388 | c.bench_function("link_to::file_hash_sync", move |b| { 389 | b.iter(|| cacache::link_to_hash_sync(&cache, target.clone()).unwrap()) 390 | }); 391 | } 392 | 393 | criterion_group!( 394 | benches, 395 | baseline_read_sync, 396 | baseline_read_many_sync, 397 | write_hash, 398 | write_hash_xxh3, 399 | read_hash_sync, 400 | read_hash_sync_xxh3, 401 | read_hash_many_sync, 402 | read_hash_many_sync_xxh3, 403 | read_sync, 404 | read_hash_sync_big_data, 405 | read_hash_sync_big_data_xxh3, 406 | ); 407 | 408 | #[cfg(any(feature = "async-std", feature = "tokio"))] 409 | criterion_group!( 410 | benches_async, 411 | baseline_read_async, 412 | baseline_read_many_async, 413 | read_hash_async, 414 | read_hash_many_async, 415 | read_async, 416 | write_hash_async, 417 | write_hash_async_xxh3, 418 | read_hash_async_big_data, 419 | ); 420 | 421 | #[cfg(all(feature = "link_to", any(feature = "async-std", feature = "tokio")))] 422 | criterion_group!(link_to_benches_async, link_to_async, link_to_hash_async,); 423 | 424 | #[cfg(feature = "link_to")] 425 | criterion_group!(link_to_benches, link_to_sync, link_to_hash_sync); 426 | 427 | #[cfg(all( 428 | feature = "link_to", 429 | not(any(feature = "async-std", feature = "tokio")) 430 | ))] 431 | criterion_main!(benches, link_to_benches); 432 | #[cfg(all( 433 | not(feature = "link_to"), 434 | any(feature = "async-std", feature = "tokio") 435 | ))] 436 | criterion_main!(benches, benches_async); 437 | #[cfg(all(feature = "link_to", any(feature = "async-std", feature = "tokio")))] 438 | criterion_main!( 439 | benches, 440 | benches_async, 441 | link_to_benches, 442 | link_to_benches_async 443 | ); 444 | #[cfg(all( 445 | not(feature = "link_to"), 446 | not(any(feature = "async-std", feature = "tokio")) 447 | ))] 448 | criterion_main!(benches); 449 | -------------------------------------------------------------------------------- /cliff.toml: -------------------------------------------------------------------------------- 1 | # configuration file for git-cliff (0.1.0) 2 | 3 | [changelog] 4 | # changelog header 5 | header = """ 6 | # `cacache` Release Changelog 7 | 8 | """ 9 | 10 | # template for the changelog body 11 | # https://tera.netlify.app/docs/#introduction 12 | body = """ 13 | {% if version %}\ 14 | 15 | ## {{ version | replace(from="v", to="") }} ({{ timestamp | date(format="%Y-%m-%d") }}) 16 | {% else %}\ 17 | ## Unreleased 18 | {% endif %}\ 19 | {% for group, commits in commits | filter(attribute="scope") | group_by(attribute="group") %} 20 | ### {{ group | upper_first }} 21 | {% for commit in commits %} 22 | {% if commit.scope %}\ 23 | * **{{ commit.scope }}:** {{ commit.message }} ([{{ commit.id | truncate(length=8, end="") }}](https://github.com/zkat/cacache-rs/commit/{{ commit.id }})) 24 | {%- if commit.breaking %} 25 | * **BREAKING CHANGE**: {{ commit.breaking_description }} 26 | {%- endif %}\ 27 | {% endif %}\ 28 | {% endfor %} 29 | {% endfor %} 30 | """ 31 | 32 | # remove the leading and trailing whitespace from the template 33 | trim = false 34 | 35 | # changelog footer 36 | # footer = """ 37 | # 38 | # """ 39 | 40 | [git] 41 | # allow only conventional commits 42 | # https://www.conventionalcommits.org 43 | conventional_commits = true 44 | # regex for parsing and grouping commits 45 | commit_parsers = [ 46 | { message = "^feat*", group = "Features"}, 47 | { message = "^fix*", group = "Bug Fixes"}, 48 | { message = "^doc*", group = "Documentation"}, 49 | { message = "^perf*", group = "Performance"}, 50 | { message = "^refactor*", group = "Refactor"}, 51 | { message = "^style*", group = "Styling"}, 52 | { message = "^test*", group = "Testing"}, 53 | { message = "^chore\\(release\\): prepare for*", skip = true}, 54 | { message = "^chore*", group = "Miscellaneous Tasks"}, 55 | { body = ".*security", group = "Security"}, 56 | ] 57 | # filter out the commits that are not matched by commit parsers 58 | filter_commits = true 59 | # glob pattern for matching git tags 60 | # tag_pattern = "v?[0-9]*" 61 | # regex for skipping tags 62 | # skip_tags = "v0.1.0-beta.1" 63 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | # List available just recipes 2 | @help: 3 | just -l 4 | 5 | # Run tests on both runtimes with cargo nextest 6 | @test: 7 | echo "----------\nasync-std:\n" 8 | cargo nextest run 9 | echo "\n----------\ntokio:\n" 10 | cargo nextest run --no-default-features --features tokio-runtime 11 | 12 | # Run benchmarks with `cargo bench` 13 | @bench: 14 | echo "----------\nasync-std:\n" 15 | cargo bench 16 | echo "\n----------\ntokio:\n" 17 | cargo bench --no-default-features --features tokio-runtime 18 | 19 | # Run benchmarks with `cargo criterion` 20 | @criterion: 21 | echo "----------\nasync-std:\n" 22 | cargo criterion 23 | echo "\n----------\ntokio:\n" 24 | cargo criterion --no-default-features --features tokio-runtime 25 | 26 | # Generate a changelog with git-cliff 27 | changelog TAG: 28 | git-cliff --prepend CHANGELOG.md -u --tag {{TAG}} 29 | 30 | # Prepare a release 31 | release *args: 32 | cargo release --workspace {{args}} 33 | 34 | # Install workspace tools 35 | @install-tools: 36 | cargo install cargo-nextest 37 | cargo install cargo-release 38 | cargo install git-cliff 39 | cargo install cargo-criterion 40 | 41 | # Lint and automatically fix what we can fix 42 | @lint: 43 | cargo clippy --fix --allow-dirty --allow-staged 44 | cargo fmt 45 | -------------------------------------------------------------------------------- /src/async_lib.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "async-std")] 2 | pub use async_std::fs::File; 3 | #[cfg(feature = "tokio")] 4 | pub use tokio::fs::File; 5 | 6 | #[cfg(feature = "async-std")] 7 | pub use futures::io::AsyncRead; 8 | #[cfg(feature = "tokio")] 9 | pub use tokio::io::AsyncRead; 10 | 11 | #[cfg(feature = "async-std")] 12 | pub use futures::io::AsyncReadExt; 13 | #[cfg(feature = "tokio")] 14 | pub use tokio::io::AsyncReadExt; 15 | 16 | #[cfg(feature = "async-std")] 17 | pub use futures::io::AsyncBufReadExt; 18 | #[cfg(feature = "tokio")] 19 | pub use tokio::io::AsyncBufReadExt; 20 | 21 | #[cfg(feature = "async-std")] 22 | pub use futures::io::AsyncWrite; 23 | #[cfg(feature = "tokio")] 24 | pub use tokio::io::AsyncWrite; 25 | 26 | #[cfg(feature = "async-std")] 27 | pub use futures::io::AsyncWriteExt; 28 | #[cfg(feature = "tokio")] 29 | pub use tokio::io::AsyncWriteExt; 30 | 31 | #[cfg(feature = "async-std")] 32 | pub use async_std::fs::read; 33 | #[cfg(feature = "tokio")] 34 | pub use tokio::fs::read; 35 | 36 | #[cfg(feature = "async-std")] 37 | pub use async_std::fs::copy; 38 | #[cfg(feature = "tokio")] 39 | pub use tokio::fs::copy; 40 | 41 | #[cfg(feature = "async-std")] 42 | pub use async_std::fs::metadata; 43 | #[cfg(feature = "tokio")] 44 | pub use tokio::fs::metadata; 45 | 46 | #[cfg(feature = "async-std")] 47 | pub use async_std::fs::remove_file; 48 | #[cfg(feature = "tokio")] 49 | pub use tokio::fs::remove_file; 50 | 51 | #[cfg(feature = "async-std")] 52 | pub use async_std::fs::create_dir_all; 53 | #[cfg(feature = "tokio")] 54 | pub use tokio::fs::create_dir_all; 55 | 56 | #[cfg(feature = "async-std")] 57 | pub use async_std::fs::remove_dir_all; 58 | #[cfg(feature = "tokio")] 59 | pub use tokio::fs::remove_dir_all; 60 | 61 | #[cfg(feature = "async-std")] 62 | pub use async_std::fs::DirBuilder; 63 | #[cfg(feature = "tokio")] 64 | pub use tokio::fs::DirBuilder; 65 | 66 | #[cfg(feature = "async-std")] 67 | pub use async_std::fs::OpenOptions; 68 | #[cfg(feature = "tokio")] 69 | pub use tokio::fs::OpenOptions; 70 | 71 | #[cfg(feature = "async-std")] 72 | pub use async_std::io::BufReader; 73 | #[cfg(feature = "tokio")] 74 | pub use tokio::io::BufReader; 75 | 76 | #[cfg(feature = "async-std")] 77 | #[inline] 78 | pub fn lines_to_stream(lines: futures::io::Lines) -> futures::io::Lines { 79 | lines 80 | } 81 | #[cfg(feature = "tokio")] 82 | #[inline] 83 | pub fn lines_to_stream(lines: tokio::io::Lines) -> tokio_stream::wrappers::LinesStream { 84 | tokio_stream::wrappers::LinesStream::new(lines) 85 | } 86 | 87 | #[cfg(feature = "async-std")] 88 | pub use async_std::task::spawn_blocking; 89 | #[cfg(feature = "tokio")] 90 | pub use tokio::task::spawn_blocking; 91 | 92 | #[cfg(feature = "async-std")] 93 | pub use async_std::task::JoinHandle; 94 | #[cfg(feature = "async-std")] 95 | #[inline] 96 | pub fn unwrap_joinhandle_value(value: T) -> T { 97 | value 98 | } 99 | #[cfg(feature = "tokio")] 100 | pub use tokio::task::JoinHandle; 101 | #[cfg(feature = "tokio")] 102 | #[inline] 103 | pub fn unwrap_joinhandle_value(value: T) -> T { 104 | value 105 | } 106 | 107 | use tempfile::NamedTempFile; 108 | 109 | use crate::errors::IoErrorExt; 110 | 111 | #[cfg(feature = "async-std")] 112 | #[inline] 113 | pub async fn create_named_tempfile( 114 | tmp_path: std::path::PathBuf, 115 | ) -> Option> { 116 | let cloned = tmp_path.clone(); 117 | 118 | Some( 119 | spawn_blocking(|| NamedTempFile::new_in(tmp_path)) 120 | .await 121 | .with_context(|| format!("Failed to create a temp file at {}", cloned.display())), 122 | ) 123 | } 124 | 125 | #[cfg(feature = "tokio")] 126 | #[inline] 127 | pub async fn create_named_tempfile( 128 | tmp_path: std::path::PathBuf, 129 | ) -> Option> { 130 | let cloned = tmp_path.clone(); 131 | match spawn_blocking(|| NamedTempFile::new_in(tmp_path)).await { 132 | Ok(ctx) => Some( 133 | ctx.with_context(|| format!("Failed to create a temp file at {}", cloned.display())), 134 | ), 135 | _ => None, 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/content/linkto.rs: -------------------------------------------------------------------------------- 1 | use ssri::{Algorithm, Integrity, IntegrityOpts}; 2 | use std::fs::DirBuilder; 3 | use std::fs::File; 4 | use std::path::{Path, PathBuf}; 5 | #[cfg(any(feature = "async-std", feature = "tokio"))] 6 | use std::pin::Pin; 7 | #[cfg(any(feature = "async-std", feature = "tokio"))] 8 | use std::task::{Context, Poll}; 9 | 10 | #[cfg(any(feature = "async-std", feature = "tokio"))] 11 | use crate::async_lib::AsyncRead; 12 | use crate::content::path; 13 | use crate::errors::{IoErrorExt, Result}; 14 | 15 | #[cfg(not(any(unix, windows)))] 16 | compile_error!("Symlinking is not supported on this platform."); 17 | 18 | fn symlink_file(src: P, dst: Q) -> std::io::Result<()> 19 | where 20 | P: AsRef, 21 | Q: AsRef, 22 | { 23 | #[cfg(unix)] 24 | { 25 | use std::os::unix::fs::symlink; 26 | symlink(src, dst) 27 | } 28 | #[cfg(windows)] 29 | { 30 | use std::os::windows::fs::symlink_file; 31 | symlink_file(src, dst) 32 | } 33 | } 34 | 35 | fn create_symlink(sri: Integrity, cache: &PathBuf, target: &PathBuf) -> Result { 36 | let cpath = path::content_path(cache.as_ref(), &sri); 37 | DirBuilder::new() 38 | .recursive(true) 39 | // Safe unwrap. cpath always has multiple segments 40 | .create(cpath.parent().unwrap()) 41 | .with_context(|| { 42 | format!( 43 | "Failed to create destination directory for linked cache file, at {}", 44 | cpath.parent().unwrap().display() 45 | ) 46 | })?; 47 | if let Err(e) = symlink_file(target, &cpath) { 48 | // If symlinking fails because there's *already* a file at the desired 49 | // destination, that is ok -- all the cache should care about is that 50 | // there is **some** valid file associated with the computed integrity. 51 | if !cpath.exists() { 52 | return Err(e).with_context(|| { 53 | format!( 54 | "Failed to create cache symlink for {} at {}", 55 | target.display(), 56 | cpath.display() 57 | ) 58 | }); 59 | } 60 | } 61 | Ok(sri) 62 | } 63 | 64 | /// A `Read`-like type that calculates the integrity of a file as it is read. 65 | /// When the linker is committed, a symlink is created from the cache to the 66 | /// target file using the integrity computed from the file's contents. 67 | pub struct ToLinker { 68 | /// The path to the target file that will be symlinked from the cache. 69 | target: PathBuf, 70 | /// The path to the root of the cache directory. 71 | cache: PathBuf, 72 | /// The file descriptor to the target file. 73 | fd: File, 74 | /// The integrity builder for calculating the target file's integrity. 75 | builder: IntegrityOpts, 76 | } 77 | 78 | impl ToLinker { 79 | pub fn new(cache: &Path, algo: Algorithm, target: &Path) -> Result { 80 | let file = File::open(target) 81 | .with_context(|| format!("Failed to open reader to {}", target.display()))?; 82 | Ok(Self { 83 | target: target.to_path_buf(), 84 | cache: cache.to_path_buf(), 85 | fd: file, 86 | builder: IntegrityOpts::new().algorithm(algo), 87 | }) 88 | } 89 | 90 | /// Add the symlink to the target file from the cache. 91 | pub fn commit(self) -> Result { 92 | create_symlink(self.builder.result(), &self.cache, &self.target) 93 | } 94 | } 95 | 96 | impl std::io::Read for ToLinker { 97 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 98 | let amt = self.fd.read(buf)?; 99 | if amt > 0 { 100 | self.builder.input(&buf[..amt]); 101 | } 102 | Ok(amt) 103 | } 104 | } 105 | 106 | /// An `AsyncRead`-like type that calculates the integrity of a file as it is 107 | /// read. When the linker is committed, a symlink is created from the cache to 108 | /// the target file using the integrity computed from the file's contents. 109 | #[cfg(any(feature = "async-std", feature = "tokio"))] 110 | pub struct AsyncToLinker { 111 | /// The path to the target file that will be symlinked from the cache. 112 | target: PathBuf, 113 | /// The path to the root of the cache directory. 114 | cache: PathBuf, 115 | /// The async-enabled file descriptor to the target file. 116 | fd: crate::async_lib::File, 117 | /// The integrity builder for calculating the target file's integrity. 118 | builder: IntegrityOpts, 119 | } 120 | 121 | #[cfg(any(feature = "async-std", feature = "tokio"))] 122 | impl AsyncRead for AsyncToLinker { 123 | #[cfg(feature = "async-std")] 124 | fn poll_read( 125 | mut self: Pin<&mut Self>, 126 | cx: &mut Context<'_>, 127 | buf: &mut [u8], 128 | ) -> Poll> { 129 | let amt = futures::ready!(Pin::new(&mut self.fd).poll_read(cx, buf))?; 130 | if amt > 0 { 131 | self.builder.input(&buf[..amt]); 132 | } 133 | Poll::Ready(Ok(amt)) 134 | } 135 | 136 | #[cfg(feature = "tokio")] 137 | fn poll_read( 138 | mut self: Pin<&mut Self>, 139 | cx: &mut Context<'_>, 140 | buf: &mut tokio::io::ReadBuf<'_>, 141 | ) -> Poll> { 142 | let pre_len = buf.filled().len(); 143 | futures::ready!(Pin::new(&mut self.fd).poll_read(cx, buf))?; 144 | if buf.filled().len() > pre_len { 145 | self.builder.input(&buf.filled()[pre_len..]); 146 | } 147 | Poll::Ready(Ok(())) 148 | } 149 | } 150 | 151 | #[cfg(any(feature = "async-std", feature = "tokio"))] 152 | impl AsyncToLinker { 153 | pub async fn new(cache: &Path, algo: Algorithm, target: &Path) -> Result { 154 | let file = crate::async_lib::File::open(target) 155 | .await 156 | .with_context(|| format!("Failed to open reader to {}", target.display()))?; 157 | Ok(Self { 158 | target: target.to_path_buf(), 159 | cache: cache.to_path_buf(), 160 | fd: file, 161 | builder: IntegrityOpts::new().algorithm(algo), 162 | }) 163 | } 164 | 165 | /// Add the symlink to the target file from the cache. 166 | pub async fn commit(self) -> Result { 167 | create_symlink(self.builder.result(), &self.cache, &self.target) 168 | } 169 | } 170 | 171 | #[cfg(test)] 172 | mod tests { 173 | use std::io::{Read, Write}; 174 | 175 | use super::*; 176 | 177 | #[cfg(feature = "async-std")] 178 | use async_attributes::test as async_test; 179 | #[cfg(feature = "tokio")] 180 | use tokio::test as async_test; 181 | 182 | #[cfg(feature = "async-std")] 183 | use futures::io::AsyncReadExt; 184 | #[cfg(feature = "tokio")] 185 | use tokio::io::AsyncReadExt; 186 | 187 | fn create_tmpfile(tmp: &tempfile::TempDir, buf: &[u8]) -> PathBuf { 188 | let dir = tmp.path().to_owned(); 189 | let target = dir.join("target-file"); 190 | std::fs::create_dir_all(&target.parent().unwrap()).unwrap(); 191 | let mut file = File::create(&target).unwrap(); 192 | file.write_all(buf).unwrap(); 193 | file.flush().unwrap(); 194 | target 195 | } 196 | 197 | #[test] 198 | fn basic_link() { 199 | let tmp = tempfile::tempdir().unwrap(); 200 | let target = create_tmpfile(&tmp, b"hello world"); 201 | 202 | let tmp = tempfile::tempdir().unwrap(); 203 | let dir = tmp.path().to_owned(); 204 | let mut linker = ToLinker::new(&dir, Algorithm::Sha256, &target).unwrap(); 205 | 206 | // read all of the data from the linker, which will calculate the integrity 207 | // hash. 208 | let mut buf = Vec::new(); 209 | linker.read_to_end(&mut buf).unwrap(); 210 | assert_eq!(buf, b"hello world"); 211 | 212 | // commit the linker, creating a symlink in the cache and an integrity 213 | // hash. 214 | let sri = linker.commit().unwrap(); 215 | assert_eq!(sri.to_string(), Integrity::from(b"hello world").to_string()); 216 | 217 | let cpath = path::content_path(&dir, &sri); 218 | assert!(cpath.exists()); 219 | let metadata = std::fs::symlink_metadata(&cpath).unwrap(); 220 | let file_type = metadata.file_type(); 221 | assert!(file_type.is_symlink()); 222 | assert_eq!(std::fs::read(cpath).unwrap(), b"hello world"); 223 | } 224 | 225 | #[cfg(any(feature = "async-std", feature = "tokio"))] 226 | #[async_test] 227 | async fn basic_async_link() { 228 | let tmp = tempfile::tempdir().unwrap(); 229 | let target = create_tmpfile(&tmp, b"hello world"); 230 | 231 | let tmp = tempfile::tempdir().unwrap(); 232 | let dir = tmp.path().to_owned(); 233 | let mut linker = AsyncToLinker::new(&dir, Algorithm::Sha256, &target) 234 | .await 235 | .unwrap(); 236 | 237 | // read all of the data from the linker, which will calculate the integrity 238 | // hash. 239 | let mut buf: Vec = Vec::new(); 240 | AsyncReadExt::read_to_end(&mut linker, &mut buf) 241 | .await 242 | .unwrap(); 243 | assert_eq!(buf, b"hello world"); 244 | 245 | // commit the linker, creating a symlink in the cache and an integrity 246 | // hash. 247 | let sri = linker.commit().await.unwrap(); 248 | assert_eq!(sri.to_string(), Integrity::from(b"hello world").to_string()); 249 | 250 | let cpath = path::content_path(&dir, &sri); 251 | assert!(cpath.exists()); 252 | let metadata = std::fs::symlink_metadata(&cpath).unwrap(); 253 | let file_type = metadata.file_type(); 254 | assert!(file_type.is_symlink()); 255 | assert_eq!(std::fs::read(cpath).unwrap(), b"hello world"); 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /src/content/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod path; 2 | pub mod read; 3 | pub mod rm; 4 | pub mod write; 5 | 6 | #[cfg(feature = "link_to")] 7 | pub mod linkto; 8 | -------------------------------------------------------------------------------- /src/content/path.rs: -------------------------------------------------------------------------------- 1 | use ssri::Integrity; 2 | use std::path::{Path, PathBuf}; 3 | 4 | const CONTENT_VERSION: &str = "2"; 5 | 6 | // Current format of content file path: 7 | // 8 | // sha512-BaSE64Hex= -> 9 | // ~/.my-cache/content-v2/sha512/ba/da/55deadbeefc0ffee 10 | // 11 | pub fn content_path(cache: &Path, sri: &Integrity) -> PathBuf { 12 | let mut path = PathBuf::new(); 13 | let (algo, hex) = sri.to_hex(); 14 | path.push(cache); 15 | path.push(format!("content-v{CONTENT_VERSION}")); 16 | path.push(algo.to_string()); 17 | path.push(&hex[0..2]); 18 | path.push(&hex[2..4]); 19 | path.push(&hex[4..]); 20 | path 21 | } 22 | 23 | #[cfg(test)] 24 | mod tests { 25 | use super::*; 26 | use ssri::Integrity; 27 | use std::path::Path; 28 | 29 | #[test] 30 | fn basic_test() { 31 | let sri = Integrity::from(b"hello world"); 32 | let cpath = content_path(Path::new("~/.my-cache"), &sri); 33 | let mut wanted = PathBuf::new(); 34 | wanted.push("~/.my-cache"); 35 | wanted.push(format!("content-v{CONTENT_VERSION}")); 36 | wanted.push("sha256"); 37 | wanted.push("b9"); 38 | wanted.push("4d"); 39 | wanted.push("27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"); 40 | assert_eq!(cpath.to_str().unwrap(), wanted.to_str().unwrap()); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/content/read.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{self, File}; 2 | use std::io::Read; 3 | use std::path::Path; 4 | #[cfg(any(feature = "async-std", feature = "tokio"))] 5 | use std::pin::Pin; 6 | #[cfg(any(feature = "async-std", feature = "tokio"))] 7 | use std::task::{Context, Poll}; 8 | 9 | #[cfg(any(feature = "async-std", feature = "tokio"))] 10 | use crate::async_lib::AsyncReadExt; 11 | 12 | use ssri::{Algorithm, Integrity, IntegrityChecker}; 13 | 14 | #[cfg(any(feature = "async-std", feature = "tokio"))] 15 | use crate::async_lib::AsyncRead; 16 | use crate::content::path; 17 | use crate::errors::{IoErrorExt, Result}; 18 | 19 | pub struct Reader { 20 | fd: File, 21 | checker: IntegrityChecker, 22 | } 23 | 24 | impl std::io::Read for Reader { 25 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 26 | let amt = self.fd.read(buf)?; 27 | self.checker.input(&buf[..amt]); 28 | Ok(amt) 29 | } 30 | } 31 | 32 | impl Reader { 33 | pub fn check(self) -> Result { 34 | Ok(self.checker.result()?) 35 | } 36 | } 37 | 38 | #[cfg(any(feature = "async-std", feature = "tokio"))] 39 | pub struct AsyncReader { 40 | fd: crate::async_lib::File, 41 | checker: IntegrityChecker, 42 | } 43 | 44 | #[cfg(any(feature = "async-std", feature = "tokio"))] 45 | impl AsyncRead for AsyncReader { 46 | #[cfg(feature = "async-std")] 47 | fn poll_read( 48 | mut self: Pin<&mut Self>, 49 | cx: &mut Context<'_>, 50 | buf: &mut [u8], 51 | ) -> Poll> { 52 | let amt = futures::ready!(Pin::new(&mut self.fd).poll_read(cx, buf))?; 53 | self.checker.input(&buf[..amt]); 54 | Poll::Ready(Ok(amt)) 55 | } 56 | 57 | #[cfg(feature = "tokio")] 58 | fn poll_read( 59 | mut self: Pin<&mut Self>, 60 | cx: &mut Context<'_>, 61 | buf: &mut tokio::io::ReadBuf<'_>, 62 | ) -> Poll> { 63 | let pre_len = buf.filled().len(); 64 | futures::ready!(Pin::new(&mut self.fd).poll_read(cx, buf))?; 65 | let post_len = buf.filled().len(); 66 | if post_len - pre_len == 0 { 67 | return Poll::Ready(Ok(())); 68 | } 69 | self.checker.input(&buf.filled()[pre_len..]); 70 | Poll::Ready(Ok(())) 71 | } 72 | } 73 | 74 | #[cfg(any(feature = "async-std", feature = "tokio"))] 75 | impl AsyncReader { 76 | pub fn check(self) -> Result { 77 | Ok(self.checker.result()?) 78 | } 79 | } 80 | 81 | pub fn open(cache: &Path, sri: Integrity) -> Result { 82 | let cpath = path::content_path(cache, &sri); 83 | Ok(Reader { 84 | fd: File::open(cpath).with_context(|| { 85 | format!( 86 | "Failed to open reader to {}", 87 | path::content_path(cache, &sri).display() 88 | ) 89 | })?, 90 | checker: IntegrityChecker::new(sri), 91 | }) 92 | } 93 | 94 | #[cfg(any(feature = "async-std", feature = "tokio"))] 95 | pub async fn open_async(cache: &Path, sri: Integrity) -> Result { 96 | let cpath = path::content_path(cache, &sri); 97 | Ok(AsyncReader { 98 | fd: crate::async_lib::File::open(cpath).await.with_context(|| { 99 | format!( 100 | "Failed to open reader to {}", 101 | path::content_path(cache, &sri).display() 102 | ) 103 | })?, 104 | checker: IntegrityChecker::new(sri), 105 | }) 106 | } 107 | 108 | pub fn read(cache: &Path, sri: &Integrity) -> Result> { 109 | let cpath = path::content_path(cache, sri); 110 | let ret = fs::read(cpath).with_context(|| { 111 | format!( 112 | "Failed to read contents for file at {}", 113 | path::content_path(cache, sri).display() 114 | ) 115 | })?; 116 | sri.check(&ret)?; 117 | Ok(ret) 118 | } 119 | 120 | #[cfg(any(feature = "async-std", feature = "tokio"))] 121 | pub async fn read_async<'a>(cache: &'a Path, sri: &'a Integrity) -> Result> { 122 | let cpath = path::content_path(cache, sri); 123 | let ret = crate::async_lib::read(&cpath).await.with_context(|| { 124 | format!( 125 | "Failed to read contents for file at {}", 126 | path::content_path(cache, sri).display() 127 | ) 128 | })?; 129 | sri.check(&ret)?; 130 | Ok(ret) 131 | } 132 | 133 | pub fn reflink_unchecked(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { 134 | let cpath = path::content_path(cache, sri); 135 | reflink_copy::reflink(cpath, to).with_context(|| { 136 | format!( 137 | "Failed to reflink cache contents from {} to {}", 138 | path::content_path(cache, sri).display(), 139 | to.display() 140 | ) 141 | })?; 142 | Ok(()) 143 | } 144 | 145 | pub fn reflink(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { 146 | let mut reader = open(cache, sri.clone())?; 147 | let mut buf: [u8; 1024] = [0; 1024]; 148 | loop { 149 | let read = reader.read(&mut buf).with_context(|| { 150 | format!( 151 | "Failed to read cache contents while verifying integrity for {}", 152 | path::content_path(cache, sri).display() 153 | ) 154 | })?; 155 | if read == 0 { 156 | break; 157 | } 158 | } 159 | reader.check()?; 160 | reflink_unchecked(cache, sri, to) 161 | } 162 | 163 | #[cfg(any(feature = "async-std", feature = "tokio"))] 164 | pub async fn reflink_async(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { 165 | let mut reader = open_async(cache, sri.clone()).await?; 166 | let mut buf = [0u8; 1024 * 8]; 167 | loop { 168 | let read = AsyncReadExt::read(&mut reader, &mut buf) 169 | .await 170 | .with_context(|| { 171 | format!( 172 | "Failed to read cache contents while verifying integrity for {}", 173 | path::content_path(cache, sri).display() 174 | ) 175 | })?; 176 | if read == 0 { 177 | break; 178 | } 179 | } 180 | reader.check()?; 181 | reflink_unchecked(cache, sri, to) 182 | } 183 | 184 | pub fn copy_unchecked(cache: &Path, sri: &Integrity, to: &Path) -> Result { 185 | let cpath = path::content_path(cache, sri); 186 | std::fs::copy(cpath, to).with_context(|| { 187 | format!( 188 | "Failed to copy cache contents from {} to {}", 189 | path::content_path(cache, sri).display(), 190 | to.display() 191 | ) 192 | }) 193 | } 194 | 195 | pub fn copy(cache: &Path, sri: &Integrity, to: &Path) -> Result { 196 | let mut reader = open(cache, sri.clone())?; 197 | let mut buf: [u8; 1024] = [0; 1024]; 198 | let mut size = 0; 199 | loop { 200 | let read = reader.read(&mut buf).with_context(|| { 201 | format!( 202 | "Failed to read cache contents while verifying integrity for {}", 203 | path::content_path(cache, sri).display() 204 | ) 205 | })?; 206 | size += read; 207 | if read == 0 { 208 | break; 209 | } 210 | } 211 | reader.check()?; 212 | copy_unchecked(cache, sri, to)?; 213 | 214 | Ok(size as u64) 215 | } 216 | 217 | #[cfg(any(feature = "async-std", feature = "tokio"))] 218 | pub async fn copy_unchecked_async<'a>( 219 | cache: &'a Path, 220 | sri: &'a Integrity, 221 | to: &'a Path, 222 | ) -> Result { 223 | let cpath = path::content_path(cache, sri); 224 | crate::async_lib::copy(&cpath, to).await.with_context(|| { 225 | format!( 226 | "Failed to copy cache contents from {} to {}", 227 | path::content_path(cache, sri).display(), 228 | to.display() 229 | ) 230 | }) 231 | } 232 | 233 | #[cfg(any(feature = "async-std", feature = "tokio"))] 234 | pub async fn copy_async<'a>(cache: &'a Path, sri: &'a Integrity, to: &'a Path) -> Result { 235 | let mut reader = open_async(cache, sri.clone()).await?; 236 | let mut buf: [u8; 1024] = [0; 1024]; 237 | let mut size = 0; 238 | loop { 239 | let read = AsyncReadExt::read(&mut reader, &mut buf) 240 | .await 241 | .with_context(|| { 242 | format!( 243 | "Failed to read cache contents while verifying integrity for {}", 244 | path::content_path(cache, sri).display() 245 | ) 246 | })?; 247 | size += read; 248 | if read == 0 { 249 | break; 250 | } 251 | } 252 | reader.check()?; 253 | copy_unchecked_async(cache, sri, to).await?; 254 | Ok(size as u64) 255 | } 256 | 257 | pub fn hard_link_unchecked(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { 258 | let cpath = path::content_path(cache, sri); 259 | std::fs::hard_link(cpath, to).with_context(|| { 260 | format!( 261 | "Failed to link cache contents from {} to {}", 262 | path::content_path(cache, sri).display(), 263 | to.display() 264 | ) 265 | })?; 266 | Ok(()) 267 | } 268 | 269 | pub fn hard_link(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { 270 | hard_link_unchecked(cache, sri, to)?; 271 | let mut reader = open(cache, sri.clone())?; 272 | let mut buf = [0u8; 1024 * 8]; 273 | loop { 274 | let read = reader.read(&mut buf).with_context(|| { 275 | format!( 276 | "Failed to read cache contents while verifying integrity for {}", 277 | path::content_path(cache, sri).display() 278 | ) 279 | })?; 280 | if read == 0 { 281 | break; 282 | } 283 | } 284 | reader.check()?; 285 | Ok(()) 286 | } 287 | 288 | #[cfg(any(feature = "async-std", feature = "tokio"))] 289 | pub async fn hard_link_async(cache: &Path, sri: &Integrity, to: &Path) -> Result<()> { 290 | let mut reader = open_async(cache, sri.clone()).await?; 291 | let mut buf = [0u8; 1024 * 8]; 292 | loop { 293 | let read = AsyncReadExt::read(&mut reader, &mut buf) 294 | .await 295 | .with_context(|| { 296 | format!( 297 | "Failed to read cache contents while verifying integrity for {}", 298 | path::content_path(cache, sri).display() 299 | ) 300 | })?; 301 | if read == 0 { 302 | break; 303 | } 304 | } 305 | reader.check()?; 306 | hard_link_unchecked(cache, sri, to)?; 307 | Ok(()) 308 | } 309 | 310 | pub fn has_content(cache: &Path, sri: &Integrity) -> Option { 311 | if path::content_path(cache, sri).exists() { 312 | Some(sri.clone()) 313 | } else { 314 | None 315 | } 316 | } 317 | 318 | #[cfg(any(feature = "async-std", feature = "tokio"))] 319 | pub async fn has_content_async(cache: &Path, sri: &Integrity) -> Option { 320 | if crate::async_lib::metadata(path::content_path(cache, sri)) 321 | .await 322 | .is_ok() 323 | { 324 | Some(sri.clone()) 325 | } else { 326 | None 327 | } 328 | } 329 | -------------------------------------------------------------------------------- /src/content/rm.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::path::Path; 3 | 4 | use ssri::Integrity; 5 | 6 | use crate::content::path; 7 | use crate::errors::{IoErrorExt, Result}; 8 | 9 | pub fn rm(cache: &Path, sri: &Integrity) -> Result<()> { 10 | fs::remove_file(path::content_path(cache, sri)).with_context(|| { 11 | format!( 12 | "Failed to remove cache file {}", 13 | path::content_path(cache, sri).display() 14 | ) 15 | })?; 16 | Ok(()) 17 | } 18 | 19 | #[cfg(any(feature = "async-std", feature = "tokio"))] 20 | pub async fn rm_async(cache: &Path, sri: &Integrity) -> Result<()> { 21 | crate::async_lib::remove_file(path::content_path(cache, sri)) 22 | .await 23 | .with_context(|| { 24 | format!( 25 | "Failed to remove cache file {}", 26 | path::content_path(cache, sri).display() 27 | ) 28 | })?; 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /src/content/write.rs: -------------------------------------------------------------------------------- 1 | use std::fs::DirBuilder; 2 | use std::io::prelude::*; 3 | use std::path::{Path, PathBuf}; 4 | #[cfg(any(feature = "async-std", feature = "tokio"))] 5 | use std::pin::Pin; 6 | #[cfg(any(feature = "async-std", feature = "tokio"))] 7 | use std::sync::Mutex; 8 | #[cfg(any(feature = "async-std", feature = "tokio"))] 9 | use std::task::{Context, Poll}; 10 | 11 | #[cfg(any(feature = "async-std", feature = "tokio"))] 12 | use futures::prelude::*; 13 | #[cfg(feature = "mmap")] 14 | use memmap2::MmapMut; 15 | use ssri::{Algorithm, Integrity, IntegrityOpts}; 16 | use tempfile::NamedTempFile; 17 | 18 | #[cfg(any(feature = "async-std", feature = "tokio"))] 19 | use crate::async_lib::{AsyncWrite, JoinHandle}; 20 | use crate::content::path; 21 | use crate::errors::{IoErrorExt, Result}; 22 | use crate::Error; 23 | 24 | #[cfg(feature = "mmap")] 25 | pub const MAX_MMAP_SIZE: usize = 1024 * 1024; 26 | 27 | #[cfg(not(feature = "mmap"))] 28 | struct MmapMut; 29 | 30 | #[cfg(not(feature = "mmap"))] 31 | impl MmapMut { 32 | #[allow(dead_code)] 33 | fn flush_async(&self) -> std::io::Result<()> { 34 | panic!() 35 | } 36 | 37 | fn copy_from_slice(&self, _: &[u8]) { 38 | panic!() 39 | } 40 | } 41 | 42 | pub struct Writer { 43 | cache: PathBuf, 44 | builder: IntegrityOpts, 45 | mmap: Option, 46 | tmpfile: NamedTempFile, 47 | } 48 | 49 | impl Writer { 50 | pub fn new(cache: &Path, algo: Algorithm, size: Option) -> Result { 51 | let cache_path = cache.to_path_buf(); 52 | let mut tmp_path = cache_path.clone(); 53 | tmp_path.push("tmp"); 54 | DirBuilder::new() 55 | .recursive(true) 56 | .create(&tmp_path) 57 | .with_context(|| { 58 | format!( 59 | "Failed to create cache directory for temporary files, at {}", 60 | tmp_path.display() 61 | ) 62 | })?; 63 | let tmp_path_clone = tmp_path.clone(); 64 | let mut tmpfile = NamedTempFile::new_in(tmp_path).with_context(|| { 65 | format!( 66 | "Failed to create temp file while initializing a writer, inside {}", 67 | tmp_path_clone.display() 68 | ) 69 | })?; 70 | let mmap = make_mmap(&mut tmpfile, size)?; 71 | Ok(Writer { 72 | cache: cache_path, 73 | builder: IntegrityOpts::new().algorithm(algo), 74 | tmpfile, 75 | mmap, 76 | }) 77 | } 78 | 79 | pub fn close(self) -> Result { 80 | let sri = self.builder.result(); 81 | let cpath = path::content_path(&self.cache, &sri); 82 | DirBuilder::new() 83 | .recursive(true) 84 | // Safe unwrap. cpath always has multiple segments 85 | .create(cpath.parent().unwrap()) 86 | .with_context(|| { 87 | format!( 88 | "Failed to create destination directory for cache contents, at {}", 89 | path::content_path(&self.cache, &sri) 90 | .parent() 91 | .unwrap() 92 | .display() 93 | ) 94 | })?; 95 | let res = self.tmpfile.persist(&cpath); 96 | match res { 97 | Ok(_) => {} 98 | Err(e) => { 99 | // We might run into conflicts sometimes when persisting files. 100 | // This is ok. We can deal. Let's just make sure the destination 101 | // file actually exists, and we can move on. 102 | if !cpath.exists() { 103 | return Err(e.error).with_context(|| { 104 | format!( 105 | "Failed to persist cache contents while closing writer, at {}", 106 | path::content_path(&self.cache, &sri).display() 107 | ) 108 | })?; 109 | } 110 | } 111 | } 112 | Ok(sri) 113 | } 114 | } 115 | 116 | impl Write for Writer { 117 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 118 | self.builder.input(buf); 119 | if let Some(mmap) = &mut self.mmap { 120 | mmap.copy_from_slice(buf); 121 | Ok(buf.len()) 122 | } else { 123 | self.tmpfile.write(buf) 124 | } 125 | } 126 | 127 | fn flush(&mut self) -> std::io::Result<()> { 128 | self.tmpfile.flush() 129 | } 130 | } 131 | 132 | #[cfg(any(feature = "async-std", feature = "tokio"))] 133 | pub struct AsyncWriter(Mutex); 134 | 135 | #[cfg(any(feature = "async-std", feature = "tokio"))] 136 | enum State { 137 | Idle(Option), 138 | Busy(JoinHandle), 139 | } 140 | 141 | #[cfg(any(feature = "async-std", feature = "tokio"))] 142 | struct Inner { 143 | cache: PathBuf, 144 | builder: IntegrityOpts, 145 | tmpfile: NamedTempFile, 146 | mmap: Option, 147 | buf: Vec, 148 | last_op: Option, 149 | } 150 | 151 | #[cfg(any(feature = "async-std", feature = "tokio"))] 152 | enum Operation { 153 | Write(std::io::Result), 154 | Flush(std::io::Result<()>), 155 | } 156 | 157 | #[cfg(any(feature = "async-std", feature = "tokio"))] 158 | impl AsyncWriter { 159 | #[allow(clippy::new_ret_no_self)] 160 | #[allow(clippy::needless_lifetimes)] 161 | pub async fn new(cache: &Path, algo: Algorithm, size: Option) -> Result { 162 | let cache_path = cache.to_path_buf(); 163 | let mut tmp_path = cache_path.clone(); 164 | tmp_path.push("tmp"); 165 | crate::async_lib::DirBuilder::new() 166 | .recursive(true) 167 | .create(&tmp_path) 168 | .await 169 | .with_context(|| { 170 | format!( 171 | "Failed to create cache directory for temporary files, at {}", 172 | tmp_path.display() 173 | ) 174 | })?; 175 | 176 | match crate::async_lib::create_named_tempfile(tmp_path).await { 177 | Some(tmpfile) => { 178 | let mut tmpfile = tmpfile?; 179 | let mmap = make_mmap(&mut tmpfile, size)?; 180 | Ok(AsyncWriter(Mutex::new(State::Idle(Some(Inner { 181 | cache: cache_path, 182 | builder: IntegrityOpts::new().algorithm(algo), 183 | mmap, 184 | tmpfile, 185 | buf: vec![], 186 | last_op: None, 187 | }))))) 188 | } 189 | _ => Err(Error::IoError( 190 | std::io::Error::new(std::io::ErrorKind::Other, "temp file create error"), 191 | "Possible memory issues for file handle".into(), 192 | )), 193 | } 194 | } 195 | 196 | pub async fn close(self) -> Result { 197 | // NOTE: How do I even get access to `inner` safely??? 198 | // let inner = ???; 199 | // Blocking, but should be a very fast op. 200 | futures::future::poll_fn(|cx| { 201 | let state = &mut *self.0.lock().unwrap(); 202 | 203 | loop { 204 | match state { 205 | State::Idle(opt) => match opt.take() { 206 | None => return Poll::Ready(None), 207 | Some(inner) => { 208 | let (s, r) = futures::channel::oneshot::channel(); 209 | let tmpfile = inner.tmpfile; 210 | let sri = inner.builder.result(); 211 | let cpath = path::content_path(&inner.cache, &sri); 212 | 213 | // Start the operation asynchronously. 214 | *state = State::Busy(crate::async_lib::spawn_blocking(|| { 215 | let res = std::fs::DirBuilder::new() 216 | .recursive(true) 217 | // Safe unwrap. cpath always has multiple segments 218 | .create(cpath.parent().unwrap()) 219 | .with_context(|| { 220 | format!( 221 | "building directory {} failed", 222 | cpath.parent().unwrap().display() 223 | ) 224 | }); 225 | if res.is_err() { 226 | let _ = s.send(res.map(|_| sri)); 227 | } else { 228 | let res = tmpfile 229 | .persist(&cpath) 230 | .map_err(|e| e.error) 231 | .with_context(|| { 232 | format!("persisting file {} failed", cpath.display()) 233 | }); 234 | if res.is_err() { 235 | // We might run into conflicts 236 | // sometimes when persisting files. 237 | // This is ok. We can deal. Let's just 238 | // make sure the destination file 239 | // actually exists, and we can move 240 | // on. 241 | let _ = s.send( 242 | std::fs::metadata(cpath) 243 | .with_context(|| { 244 | String::from("File still doesn't exist") 245 | }) 246 | .map(|_| sri), 247 | ); 248 | } else { 249 | let _ = s.send(res.map(|_| sri)); 250 | } 251 | } 252 | State::Idle(None) 253 | })); 254 | 255 | return Poll::Ready(Some(r)); 256 | } 257 | }, 258 | // Poll the asynchronous operation the file is currently blocked on. 259 | State::Busy(task) => { 260 | let next_state = crate::async_lib::unwrap_joinhandle_value( 261 | futures::ready!(Pin::new(task).poll(cx)), 262 | ); 263 | 264 | update_state(state, next_state); 265 | } 266 | } 267 | } 268 | }) 269 | .map(|opt| opt.ok_or_else(|| crate::errors::io_error("file closed"))) 270 | .await 271 | .with_context(|| "Error while closing cache contents".to_string())? 272 | .await 273 | .map_err(|_| crate::errors::io_error("Operation cancelled")) 274 | .with_context(|| "Error while closing cache contents".to_string())? 275 | } 276 | } 277 | 278 | #[cfg(any(feature = "async-std", feature = "tokio"))] 279 | impl AsyncWrite for AsyncWriter { 280 | fn poll_write( 281 | self: Pin<&mut Self>, 282 | cx: &mut Context<'_>, 283 | buf: &[u8], 284 | ) -> Poll> { 285 | match self.0.lock() { 286 | Ok(mut state) => { 287 | let state = &mut *state; 288 | 289 | loop { 290 | match state { 291 | State::Idle(opt) => { 292 | // Grab a reference to the inner representation of the file or return an error 293 | // if the file is closed. 294 | let inner = opt 295 | .as_mut() 296 | .ok_or_else(|| crate::errors::io_error("file closed"))?; 297 | 298 | // Check if the operation has completed. 299 | if let Some(Operation::Write(res)) = inner.last_op.take() { 300 | let n = res?; 301 | 302 | // If more data was written than is available in the buffer, let's retry 303 | // the write operation. 304 | if n <= buf.len() { 305 | return Poll::Ready(Ok(n)); 306 | } 307 | } else { 308 | let mut inner = opt.take().unwrap(); 309 | 310 | // Set the length of the inner buffer to the length of the provided buffer. 311 | if inner.buf.len() < buf.len() { 312 | inner.buf.reserve(buf.len() - inner.buf.len()); 313 | } 314 | unsafe { 315 | inner.buf.set_len(buf.len()); 316 | } 317 | 318 | // Copy the data to write into the inner buffer. 319 | inner.buf[..buf.len()].copy_from_slice(buf); 320 | 321 | // Start the operation asynchronously. 322 | *state = State::Busy(crate::async_lib::spawn_blocking(|| { 323 | inner.builder.input(&inner.buf); 324 | if let Some(mmap) = &mut inner.mmap { 325 | mmap.copy_from_slice(&inner.buf); 326 | inner.last_op = Some(Operation::Write(Ok(inner.buf.len()))); 327 | State::Idle(Some(inner)) 328 | } else { 329 | let res = inner.tmpfile.write(&inner.buf); 330 | inner.last_op = Some(Operation::Write(res)); 331 | State::Idle(Some(inner)) 332 | } 333 | })); 334 | } 335 | } 336 | // Poll the asynchronous operation the file is currently blocked on. 337 | State::Busy(task) => { 338 | let next_state = crate::async_lib::unwrap_joinhandle_value( 339 | futures::ready!(Pin::new(task).poll(cx)), 340 | ); 341 | 342 | update_state(state, next_state); 343 | } 344 | } 345 | } 346 | } 347 | _ => Poll::Pending, 348 | } 349 | } 350 | 351 | fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 352 | match self.0.lock() { 353 | Ok(mut state) => { 354 | let state = &mut *state; 355 | loop { 356 | match state { 357 | State::Idle(opt) => { 358 | // Grab a reference to the inner representation of the file or return if the 359 | // file is closed. 360 | let inner = match opt.as_mut() { 361 | None => return Poll::Ready(Ok(())), 362 | Some(s) => s, 363 | }; 364 | 365 | // Check if the operation has completed. 366 | if let Some(Operation::Flush(res)) = inner.last_op.take() { 367 | return Poll::Ready(res); 368 | } else { 369 | let mut inner = opt.take().unwrap(); 370 | 371 | if let Some(mmap) = &inner.mmap { 372 | match mmap.flush_async() { 373 | Ok(_) => (), 374 | Err(e) => return Poll::Ready(Err(e)), 375 | }; 376 | } 377 | 378 | // Start the operation asynchronously. 379 | *state = State::Busy(crate::async_lib::spawn_blocking(|| { 380 | let res = inner.tmpfile.flush(); 381 | inner.last_op = Some(Operation::Flush(res)); 382 | State::Idle(Some(inner)) 383 | })); 384 | } 385 | } 386 | // Poll the asynchronous operation the file is currently blocked on. 387 | State::Busy(task) => { 388 | let next_state = crate::async_lib::unwrap_joinhandle_value( 389 | futures::ready!(Pin::new(task).poll(cx)), 390 | ); 391 | 392 | update_state(state, next_state); 393 | } 394 | } 395 | } 396 | } 397 | _ => Poll::Pending, 398 | } 399 | } 400 | 401 | #[cfg(feature = "async-std")] 402 | fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 403 | self.poll_close_impl(cx) 404 | } 405 | 406 | #[cfg(feature = "tokio")] 407 | fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 408 | self.poll_close_impl(cx) 409 | } 410 | } 411 | 412 | #[cfg(feature = "tokio")] 413 | /// Update the state. 414 | fn update_state( 415 | current_state: &mut State, 416 | next_state: std::result::Result, 417 | ) { 418 | match next_state { 419 | Ok(next) => { 420 | *current_state = next; 421 | } 422 | _ => { 423 | *current_state = State::Idle(None); 424 | } 425 | } 426 | } 427 | 428 | #[cfg(not(feature = "tokio"))] 429 | /// Update the state. 430 | fn update_state(current_state: &mut State, next_state: State) { 431 | *current_state = next_state; 432 | } 433 | 434 | #[cfg(any(feature = "async-std", feature = "tokio"))] 435 | impl AsyncWriter { 436 | #[inline] 437 | fn poll_close_impl( 438 | self: Pin<&mut Self>, 439 | cx: &mut std::task::Context<'_>, 440 | ) -> Poll> { 441 | match self.0.lock() { 442 | Ok(mut state) => { 443 | let state = &mut *state; 444 | loop { 445 | match state { 446 | State::Idle(opt) => { 447 | // Grab a reference to the inner representation of the file or return if the 448 | // file is closed. 449 | let inner = match opt.take() { 450 | None => return Poll::Ready(Ok(())), 451 | Some(s) => s, 452 | }; 453 | 454 | // Start the operation asynchronously. 455 | *state = State::Busy(crate::async_lib::spawn_blocking(|| { 456 | drop(inner); 457 | State::Idle(None) 458 | })); 459 | } 460 | // Poll the asynchronous operation the file is currently blocked on. 461 | State::Busy(task) => { 462 | let next_state = crate::async_lib::unwrap_joinhandle_value( 463 | futures::ready!(Pin::new(task).poll(cx)), 464 | ); 465 | 466 | update_state(state, next_state); 467 | } 468 | } 469 | } 470 | } 471 | _ => Poll::Pending, 472 | } 473 | } 474 | } 475 | 476 | #[cfg(feature = "mmap")] 477 | fn make_mmap(tmpfile: &mut NamedTempFile, size: Option) -> Result> { 478 | if let Some(size @ 0..=MAX_MMAP_SIZE) = size { 479 | allocate_file(tmpfile.as_file(), size).with_context(|| { 480 | format!( 481 | "Failed to configure file length for temp file at {}", 482 | tmpfile.path().display() 483 | ) 484 | })?; 485 | Ok(unsafe { MmapMut::map_mut(tmpfile.as_file()).ok() }) 486 | } else { 487 | Ok(None) 488 | } 489 | } 490 | 491 | #[cfg(feature = "mmap")] 492 | #[cfg(target_os = "linux")] 493 | fn allocate_file(file: &std::fs::File, size: usize) -> std::io::Result<()> { 494 | use std::io::{Error, ErrorKind}; 495 | use std::os::fd::AsRawFd; 496 | 497 | let fd = file.as_raw_fd(); 498 | match unsafe { libc::posix_fallocate64(fd, 0, size as i64) } { 499 | 0 => Ok(()), 500 | libc::ENOSPC => Err(Error::new( 501 | ErrorKind::Other, // ErrorKind::StorageFull is unstable 502 | "cannot allocate file: no space left on device", 503 | )), 504 | err => Err(Error::new( 505 | ErrorKind::Other, 506 | format!("posix_fallocate64 failed with code {err}"), 507 | )), 508 | } 509 | } 510 | 511 | #[cfg(feature = "mmap")] 512 | #[cfg(not(target_os = "linux"))] 513 | fn allocate_file(file: &std::fs::File, size: usize) -> std::io::Result<()> { 514 | file.set_len(size as u64) 515 | } 516 | 517 | #[cfg(not(feature = "mmap"))] 518 | fn make_mmap(_: &mut NamedTempFile, _: Option) -> Result> { 519 | Ok(None) 520 | } 521 | 522 | #[cfg(test)] 523 | mod tests { 524 | use super::*; 525 | #[cfg(any(feature = "async-std", feature = "tokio"))] 526 | use crate::async_lib::AsyncWriteExt; 527 | use tempfile; 528 | 529 | #[cfg(feature = "async-std")] 530 | use async_attributes::test as async_test; 531 | #[cfg(feature = "tokio")] 532 | use tokio::test as async_test; 533 | 534 | #[test] 535 | fn basic_write() { 536 | let tmp = tempfile::tempdir().unwrap(); 537 | let dir = tmp.path().to_owned(); 538 | let mut writer = Writer::new(&dir, Algorithm::Sha256, None).unwrap(); 539 | writer.write_all(b"hello world").unwrap(); 540 | let sri = writer.close().unwrap(); 541 | assert_eq!(sri.to_string(), Integrity::from(b"hello world").to_string()); 542 | assert_eq!( 543 | std::fs::read(path::content_path(&dir, &sri)).unwrap(), 544 | b"hello world" 545 | ); 546 | } 547 | 548 | #[cfg(any(feature = "async-std", feature = "tokio"))] 549 | #[async_test] 550 | async fn basic_async_write() { 551 | let tmp = tempfile::tempdir().unwrap(); 552 | let dir = tmp.path().to_owned(); 553 | let mut writer = AsyncWriter::new(&dir, Algorithm::Sha256, None) 554 | .await 555 | .unwrap(); 556 | writer.write_all(b"hello world").await.unwrap(); 557 | let sri = writer.close().await.unwrap(); 558 | assert_eq!(sri.to_string(), Integrity::from(b"hello world").to_string()); 559 | assert_eq!( 560 | std::fs::read(path::content_path(&dir, &sri)).unwrap(), 561 | b"hello world" 562 | ); 563 | } 564 | } 565 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use miette::Diagnostic; 4 | use thiserror::Error; 5 | 6 | /// Error type returned by all API calls. 7 | #[derive(Error, Debug, Diagnostic)] 8 | pub enum Error { 9 | /// Returned when an index entry could not be found during 10 | /// lookup. 11 | #[error("Entry not found for key {1:?} in cache {0:?}")] 12 | #[diagnostic(code(cacache::entry_not_found), url(docsrs))] 13 | EntryNotFound(PathBuf, String), 14 | 15 | /// Returned when a size check has failed. 16 | #[error("Size check failed.\n\tWanted: {0}\n\tActual: {1}")] 17 | #[diagnostic(code(cacache::size_mismatch), url(docsrs))] 18 | SizeMismatch(usize, usize), 19 | 20 | /// Returned when a general IO error has occurred. 21 | #[error("{1}")] 22 | #[diagnostic(code(cacache::io_error), url(docsrs))] 23 | IoError(#[source] std::io::Error, String), 24 | 25 | /// Returned when a general serde error has occurred. 26 | #[error("{1}")] 27 | #[diagnostic(code(cacache::serde_error), url(docsrs))] 28 | SerdeError(#[source] serde_json::Error, String), 29 | 30 | /// Returned when an integrity check has failed. 31 | #[error(transparent)] 32 | #[diagnostic(code(cacache::integrity_error), url(docsrs))] 33 | IntegrityError(#[from] ssri::Error), 34 | } 35 | 36 | /// The result type returned by calls to this library 37 | pub type Result = std::result::Result; 38 | 39 | pub trait IoErrorExt { 40 | fn with_context String>(self, f: F) -> Result; 41 | } 42 | 43 | impl IoErrorExt for std::result::Result { 44 | fn with_context String>(self, f: F) -> Result { 45 | match self { 46 | Ok(t) => Ok(t), 47 | Err(e) => Err(Error::IoError(e, f())), 48 | } 49 | } 50 | } 51 | 52 | impl IoErrorExt for std::result::Result { 53 | fn with_context String>(self, f: F) -> Result { 54 | match self { 55 | Ok(t) => Ok(t), 56 | Err(e) => Err(Error::SerdeError(e, f())), 57 | } 58 | } 59 | } 60 | 61 | pub fn io_error(err: impl Into>) -> std::io::Error { 62 | std::io::Error::new(std::io::ErrorKind::Other, err) 63 | } 64 | -------------------------------------------------------------------------------- /src/index.rs: -------------------------------------------------------------------------------- 1 | //! Raw access to the cache index. Use with caution! 2 | 3 | use std::collections::HashSet; 4 | use std::fs::{self, OpenOptions}; 5 | use std::hash::{Hash, Hasher}; 6 | use std::io::{ErrorKind, Write}; 7 | use std::path::{Path, PathBuf}; 8 | use std::time::{SystemTime, UNIX_EPOCH}; 9 | 10 | use digest::Digest; 11 | use either::{Left, Right}; 12 | #[cfg(any(feature = "async-std", feature = "tokio"))] 13 | use futures::stream::StreamExt; 14 | use serde_derive::{Deserialize, Serialize}; 15 | use serde_json::Value; 16 | use sha1::Sha1; 17 | use sha2::Sha256; 18 | use ssri::Integrity; 19 | use walkdir::WalkDir; 20 | 21 | #[cfg(any(feature = "async-std", feature = "tokio"))] 22 | use crate::async_lib::{AsyncBufReadExt, AsyncWriteExt}; 23 | use crate::content::path::content_path; 24 | use crate::errors::{IoErrorExt, Result}; 25 | use crate::put::WriteOpts; 26 | 27 | const INDEX_VERSION: &str = "5"; 28 | 29 | /// Represents a cache index entry, which points to content. 30 | #[derive(PartialEq, Debug)] 31 | pub struct Metadata { 32 | /// Key this entry is stored under. 33 | pub key: String, 34 | /// Integrity hash for the stored data. Acts as a key into {cache}/content. 35 | pub integrity: Integrity, 36 | /// Timestamp in unix milliseconds when this entry was written. 37 | pub time: u128, 38 | /// Size of data associated with this entry. 39 | pub size: usize, 40 | /// Arbitrary JSON associated with this entry. 41 | pub metadata: Value, 42 | /// Raw metadata in binary form. Can be different from JSON metadata. 43 | pub raw_metadata: Option>, 44 | } 45 | 46 | #[derive(Deserialize, Serialize, Debug)] 47 | struct SerializableMetadata { 48 | key: String, 49 | integrity: Option, 50 | time: u128, 51 | size: usize, 52 | metadata: Value, 53 | raw_metadata: Option>, 54 | } 55 | 56 | impl PartialEq for SerializableMetadata { 57 | fn eq(&self, other: &Self) -> bool { 58 | self.key == other.key 59 | } 60 | } 61 | 62 | impl Eq for SerializableMetadata {} 63 | 64 | impl Hash for SerializableMetadata { 65 | fn hash(&self, state: &mut H) { 66 | self.key.hash(state); 67 | } 68 | } 69 | 70 | /// Raw insertion into the cache index. 71 | pub fn insert(cache: &Path, key: &str, opts: WriteOpts) -> Result { 72 | let bucket = bucket_path(cache, key); 73 | fs::create_dir_all(bucket.parent().unwrap()).with_context(|| { 74 | format!( 75 | "Failed to create index bucket directory: {:?}", 76 | bucket.parent().unwrap() 77 | ) 78 | })?; 79 | let stringified = serde_json::to_string(&SerializableMetadata { 80 | key: key.to_owned(), 81 | integrity: opts.sri.clone().map(|x| x.to_string()), 82 | time: opts.time.unwrap_or_else(now), 83 | size: opts.size.unwrap_or(0), 84 | metadata: opts.metadata.unwrap_or(serde_json::Value::Null), 85 | raw_metadata: opts.raw_metadata, 86 | }) 87 | .with_context(|| format!("Failed to serialize entry with key `{key}`"))?; 88 | 89 | let mut buck = OpenOptions::new() 90 | .create(true) 91 | .append(true) 92 | .open(&bucket) 93 | .with_context(|| format!("Failed to create or open index bucket at {bucket:?}"))?; 94 | 95 | let out = format!("\n{}\t{}", hash_entry(&stringified), stringified); 96 | buck.write_all(out.as_bytes()) 97 | .with_context(|| format!("Failed to write to index bucket at {bucket:?}"))?; 98 | buck.flush() 99 | .with_context(|| format!("Failed to flush bucket at {bucket:?}"))?; 100 | Ok(opts 101 | .sri 102 | .or_else(|| "sha1-deadbeef".parse::().ok()) 103 | .unwrap()) 104 | } 105 | 106 | #[cfg(any(feature = "async-std", feature = "tokio"))] 107 | /// Asynchronous raw insertion into the cache index. 108 | pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: WriteOpts) -> Result { 109 | let bucket = bucket_path(cache, key); 110 | crate::async_lib::create_dir_all(bucket.parent().unwrap()) 111 | .await 112 | .with_context(|| { 113 | format!( 114 | "Failed to create index bucket directory: {:?}", 115 | bucket.parent().unwrap() 116 | ) 117 | })?; 118 | let stringified = serde_json::to_string(&SerializableMetadata { 119 | key: key.to_owned(), 120 | integrity: opts.sri.clone().map(|x| x.to_string()), 121 | time: opts.time.unwrap_or_else(now), 122 | size: opts.size.unwrap_or(0), 123 | metadata: opts.metadata.unwrap_or(serde_json::Value::Null), 124 | raw_metadata: opts.raw_metadata, 125 | }) 126 | .with_context(|| format!("Failed to serialize entry with key `{key}`"))?; 127 | 128 | let mut buck = crate::async_lib::OpenOptions::new() 129 | .create(true) 130 | .append(true) 131 | .open(&bucket) 132 | .await 133 | .with_context(|| format!("Failed to create or open index bucket at {bucket:?}"))?; 134 | 135 | let out = format!("\n{}\t{}", hash_entry(&stringified), stringified); 136 | buck.write_all(out.as_bytes()) 137 | .await 138 | .with_context(|| format!("Failed to write to index bucket at {bucket:?}"))?; 139 | buck.flush() 140 | .await 141 | .with_context(|| format!("Failed to flush bucket at {bucket:?}"))?; 142 | Ok(opts 143 | .sri 144 | .or_else(|| "sha1-deadbeef".parse::().ok()) 145 | .unwrap()) 146 | } 147 | 148 | /// Raw index Metadata access. 149 | pub fn find(cache: &Path, key: &str) -> Result> { 150 | let bucket = bucket_path(cache, key); 151 | Ok(bucket_entries(&bucket) 152 | .with_context(|| format!("Failed to read index bucket entries from {bucket:?}"))? 153 | .into_iter() 154 | .fold(None, |acc, entry| { 155 | if entry.key == key { 156 | if let Some(integrity) = entry.integrity { 157 | let integrity: Integrity = match integrity.parse() { 158 | Ok(sri) => sri, 159 | _ => return acc, 160 | }; 161 | Some(Metadata { 162 | key: entry.key, 163 | integrity, 164 | size: entry.size, 165 | time: entry.time, 166 | metadata: entry.metadata, 167 | raw_metadata: entry.raw_metadata, 168 | }) 169 | } else { 170 | None 171 | } 172 | } else { 173 | acc 174 | } 175 | })) 176 | } 177 | 178 | #[cfg(any(feature = "async-std", feature = "tokio"))] 179 | /// Asynchronous raw index Metadata access. 180 | pub async fn find_async(cache: &Path, key: &str) -> Result> { 181 | let bucket = bucket_path(cache, key); 182 | Ok(bucket_entries_async(&bucket) 183 | .await 184 | .with_context(|| format!("Failed to read index bucket entries from {bucket:?}"))? 185 | .into_iter() 186 | .fold(None, |acc, entry| { 187 | if entry.key == key { 188 | if let Some(integrity) = entry.integrity { 189 | let integrity: Integrity = match integrity.parse() { 190 | Ok(sri) => sri, 191 | _ => return acc, 192 | }; 193 | Some(Metadata { 194 | key: entry.key, 195 | integrity, 196 | size: entry.size, 197 | time: entry.time, 198 | metadata: entry.metadata, 199 | raw_metadata: entry.raw_metadata, 200 | }) 201 | } else { 202 | None 203 | } 204 | } else { 205 | acc 206 | } 207 | })) 208 | } 209 | 210 | /// Deletes an index entry, without deleting the actual cache data entry. 211 | pub fn delete(cache: &Path, key: &str) -> Result<()> { 212 | insert( 213 | cache, 214 | key, 215 | WriteOpts { 216 | algorithm: None, 217 | size: None, 218 | sri: None, 219 | time: None, 220 | metadata: None, 221 | raw_metadata: None, 222 | }, 223 | ) 224 | .map(|_| ()) 225 | } 226 | 227 | #[cfg(any(feature = "async-std", feature = "tokio"))] 228 | /// Asynchronously deletes an index entry, without deleting the actual cache 229 | /// data entry. 230 | pub async fn delete_async(cache: &Path, key: &str) -> Result<()> { 231 | insert( 232 | cache, 233 | key, 234 | WriteOpts { 235 | algorithm: None, 236 | size: None, 237 | sri: None, 238 | time: None, 239 | metadata: None, 240 | raw_metadata: None, 241 | }, 242 | ) 243 | .map(|_| ()) 244 | } 245 | 246 | /// Lists raw index Metadata entries. 247 | pub fn ls(cache: &Path) -> impl Iterator> { 248 | let cache_path = cache.join(format!("index-v{INDEX_VERSION}")); 249 | let cloned = cache_path.clone(); 250 | WalkDir::new(&cache_path) 251 | .into_iter() 252 | .map(move |bucket| { 253 | let bucket = bucket 254 | .map_err(|e| match e.io_error() { 255 | Some(io_err) => std::io::Error::new(io_err.kind(), io_err.kind().to_string()), 256 | None => crate::errors::io_error("Unexpected error"), 257 | }) 258 | .with_context(|| { 259 | format!( 260 | "Error while walking cache index directory at {}", 261 | cloned.display() 262 | ) 263 | })?; 264 | 265 | if bucket.file_type().is_dir() { 266 | return Ok(Vec::new()); 267 | } 268 | 269 | let owned_path = bucket.path().to_owned(); 270 | Ok(bucket_entries(bucket.path()) 271 | .with_context(|| { 272 | format!("Error getting bucket entries from {}", owned_path.display()) 273 | })? 274 | .into_iter() 275 | .rev() 276 | .collect::>() 277 | .into_iter() 278 | .filter_map(|se| { 279 | if let Some(i) = se.integrity { 280 | Some(Metadata { 281 | key: se.key, 282 | integrity: i.parse().unwrap(), 283 | time: se.time, 284 | size: se.size, 285 | metadata: se.metadata, 286 | raw_metadata: se.raw_metadata, 287 | }) 288 | } else { 289 | None 290 | } 291 | }) 292 | .collect()) 293 | }) 294 | .flat_map(|res| match res { 295 | Ok(it) => Left(it.into_iter().map(Ok)), 296 | Err(err) => Right(std::iter::once(Err(err))), 297 | }) 298 | } 299 | 300 | fn bucket_path(cache: &Path, key: &str) -> PathBuf { 301 | let hashed = hash_key(key); 302 | cache 303 | .join(format!("index-v{INDEX_VERSION}")) 304 | .join(&hashed[0..2]) 305 | .join(&hashed[2..4]) 306 | .join(&hashed[4..]) 307 | } 308 | 309 | fn hash_key(key: &str) -> String { 310 | let mut hasher = Sha1::new(); 311 | hasher.update(key); 312 | hex::encode(hasher.finalize()) 313 | } 314 | 315 | fn hash_entry(key: &str) -> String { 316 | let mut hasher = Sha256::new(); 317 | hasher.update(key); 318 | hex::encode(hasher.finalize()) 319 | } 320 | 321 | fn now() -> u128 { 322 | SystemTime::now() 323 | .duration_since(UNIX_EPOCH) 324 | .unwrap() 325 | .as_millis() 326 | } 327 | 328 | fn bucket_entries(bucket: &Path) -> std::io::Result> { 329 | use std::io::{BufRead, BufReader}; 330 | fs::File::open(bucket) 331 | .map(|file| { 332 | BufReader::new(file) 333 | .lines() 334 | .map_while(std::result::Result::ok) 335 | .filter_map(|entry| { 336 | let entry_str = match entry.split('\t').collect::>()[..] { 337 | [hash, entry_str] if hash_entry(entry_str) == hash => entry_str, 338 | // Something's wrong with the entry. Abort. 339 | _ => return None, 340 | }; 341 | serde_json::from_str::(entry_str).ok() 342 | }) 343 | .collect() 344 | }) 345 | .or_else(|err| { 346 | if err.kind() == ErrorKind::NotFound { 347 | Ok(Vec::new()) 348 | } else { 349 | Err(err)? 350 | } 351 | }) 352 | } 353 | 354 | #[cfg(any(feature = "async-std", feature = "tokio"))] 355 | async fn bucket_entries_async(bucket: &Path) -> std::io::Result> { 356 | let file_result = crate::async_lib::File::open(bucket).await; 357 | let file = if let Err(err) = file_result { 358 | if err.kind() == ErrorKind::NotFound { 359 | return Ok(Vec::new()); 360 | } 361 | return Err(err)?; 362 | } else { 363 | file_result.unwrap() 364 | }; 365 | let mut vec = Vec::new(); 366 | let mut lines = 367 | crate::async_lib::lines_to_stream(crate::async_lib::BufReader::new(file).lines()); 368 | while let Some(line) = lines.next().await { 369 | if let Ok(entry) = line { 370 | let entry_str = match entry.split('\t').collect::>()[..] { 371 | [hash, entry_str] if hash_entry(entry_str) == hash => entry_str, 372 | // Something's wrong with the entry. Abort. 373 | _ => continue, 374 | }; 375 | if let Ok(serialized) = serde_json::from_str::(entry_str) { 376 | vec.push(serialized); 377 | } 378 | } 379 | } 380 | Ok(vec) 381 | } 382 | 383 | /// Builder for options and flags for remove cache entry. 384 | #[derive(Clone, Default)] 385 | pub struct RemoveOpts { 386 | pub(crate) remove_fully: bool, 387 | } 388 | 389 | impl RemoveOpts { 390 | /// Creates cache remove options. 391 | pub fn new() -> Self { 392 | Default::default() 393 | } 394 | 395 | /// Set the remove fully option 396 | /// If remove_fully is set to true then the index and content file itself will be physically deleted rather than appending a null. 397 | pub fn remove_fully(mut self, remove_fully: bool) -> Self { 398 | self.remove_fully = remove_fully; 399 | self 400 | } 401 | 402 | /// Removes an individual index metadata entry. 403 | /// If remove_fully is set to false (default), the associated content will be left in the cache. 404 | /// If remove_fully is true, both the index entry and the contents will be physically removed from the disk 405 | pub fn remove_sync(self, cache: P, key: K) -> Result<()> 406 | where 407 | P: AsRef, 408 | K: AsRef, 409 | { 410 | if !self.remove_fully { 411 | delete(cache.as_ref(), key.as_ref()) 412 | } else { 413 | if let Some(meta) = crate::metadata_sync(cache.as_ref(), key.as_ref())? { 414 | let content = content_path(cache.as_ref(), &meta.integrity); 415 | fs::remove_file(&content) 416 | .with_context(|| format!("Failed to remove content at {content:?}"))?; 417 | } 418 | let bucket = bucket_path(cache.as_ref(), key.as_ref()); 419 | fs::remove_file(&bucket) 420 | .with_context(|| format!("Failed to remove bucket at {bucket:?}")) 421 | } 422 | } 423 | 424 | /// Removes an individual index metadata entry. 425 | /// If remove_fully is set to false (default), the associated content will be left in the cache. 426 | /// If remove_fully is true, both the index entry and the contents will be physically removed from the disk 427 | #[cfg(any(feature = "async-std", feature = "tokio"))] 428 | pub async fn remove(self, cache: P, key: K) -> Result<()> 429 | where 430 | P: AsRef, 431 | K: AsRef, 432 | { 433 | if !self.remove_fully { 434 | delete_async(cache.as_ref(), key.as_ref()).await 435 | } else { 436 | if let Some(meta) = crate::metadata(cache.as_ref(), key.as_ref()).await? { 437 | let content = content_path(cache.as_ref(), &meta.integrity); 438 | crate::async_lib::remove_file(&content) 439 | .await 440 | .with_context(|| format!("Failed to remove content at {content:?}"))?; 441 | } 442 | let bucket = bucket_path(cache.as_ref(), key.as_ref()); 443 | crate::async_lib::remove_file(&bucket) 444 | .await 445 | .with_context(|| format!("Failed to remove bucket at {bucket:?}")) 446 | } 447 | } 448 | } 449 | 450 | #[cfg(test)] 451 | mod tests { 452 | use super::*; 453 | use serde_json::json; 454 | 455 | #[cfg(feature = "async-std")] 456 | use async_attributes::test as async_test; 457 | #[cfg(feature = "tokio")] 458 | use tokio::test as async_test; 459 | 460 | const MOCK_ENTRY: &str = "\n9cbbfe2553e7c7e1773f53f0f643fdd72008faa38da53ebcb055e5e20321ae47\t{\"key\":\"hello\",\"integrity\":\"sha1-deadbeef\",\"time\":1234567,\"size\":0,\"metadata\":null,\"raw_metadata\":null}"; 461 | 462 | fn ls_entries(dir: &Path) -> Vec { 463 | let mut entries = ls(dir) 464 | .map(|x| Ok(x?.key)) 465 | .collect::>>() 466 | .unwrap(); 467 | entries.sort(); 468 | entries 469 | } 470 | 471 | #[test] 472 | fn insert_basic() { 473 | let tmp = tempfile::tempdir().unwrap(); 474 | let dir = tmp.path().to_owned(); 475 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 476 | let time = 1_234_567; 477 | let opts = WriteOpts::new().integrity(sri).time(time); 478 | insert(&dir, "hello", opts).unwrap(); 479 | let entry = std::fs::read_to_string(bucket_path(&dir, "hello")).unwrap(); 480 | assert_eq!(entry, MOCK_ENTRY); 481 | } 482 | 483 | #[cfg(any(feature = "async-std", feature = "tokio"))] 484 | #[async_test] 485 | async fn insert_async_basic() { 486 | let tmp = tempfile::tempdir().unwrap(); 487 | let dir = tmp.path().to_owned(); 488 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 489 | let time = 1_234_567; 490 | let opts = WriteOpts::new().integrity(sri).time(time); 491 | futures::executor::block_on(async { 492 | insert_async(&dir, "hello", opts).await.unwrap(); 493 | }); 494 | let entry = std::fs::read_to_string(bucket_path(&dir, "hello")).unwrap(); 495 | assert_eq!(entry, MOCK_ENTRY); 496 | } 497 | 498 | #[test] 499 | fn find_basic() { 500 | let tmp = tempfile::tempdir().unwrap(); 501 | let dir = tmp.path().to_owned(); 502 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 503 | let time = 1_234_567; 504 | let bucket = bucket_path(&dir, "hello"); 505 | fs::create_dir_all(bucket.parent().unwrap()).unwrap(); 506 | fs::write(bucket, MOCK_ENTRY).unwrap(); 507 | let entry = find(&dir, "hello").unwrap().unwrap(); 508 | assert_eq!( 509 | entry, 510 | Metadata { 511 | key: String::from("hello"), 512 | integrity: sri, 513 | time, 514 | size: 0, 515 | metadata: json!(null), 516 | raw_metadata: None, 517 | } 518 | ); 519 | } 520 | 521 | #[test] 522 | fn find_none() { 523 | let tmp = tempfile::tempdir().unwrap(); 524 | let dir = tmp.path().to_owned(); 525 | assert_eq!(find(&dir, "hello").unwrap(), None); 526 | } 527 | 528 | #[test] 529 | fn delete_basic() { 530 | let tmp = tempfile::tempdir().unwrap(); 531 | let dir = tmp.path().to_owned(); 532 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 533 | let time = 1_234_567; 534 | let opts = WriteOpts::new().integrity(sri).time(time); 535 | insert(&dir, "hello", opts).unwrap(); 536 | delete(&dir, "hello").unwrap(); 537 | assert_eq!(find(&dir, "hello").unwrap(), None); 538 | } 539 | 540 | #[cfg(any(feature = "async-std", feature = "tokio"))] 541 | #[async_test] 542 | async fn delete_async_basic() { 543 | let tmp = tempfile::tempdir().unwrap(); 544 | let dir = tmp.path().to_owned(); 545 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 546 | let time = 1_234_567; 547 | let opts = WriteOpts::new().integrity(sri).time(time); 548 | insert(&dir, "hello", opts).unwrap(); 549 | futures::executor::block_on(async { 550 | delete_async(&dir, "hello").await.unwrap(); 551 | }); 552 | assert_eq!(find(&dir, "hello").unwrap(), None); 553 | } 554 | 555 | #[test] 556 | fn delete_fully() { 557 | let tmp = tempfile::tempdir().unwrap(); 558 | let dir = tmp.path().to_owned(); 559 | let content = content_path(&dir, &"sha1-deadbeef".parse().unwrap()); 560 | fs::create_dir_all(content.parent().unwrap()).unwrap(); 561 | fs::write(content.as_path(), "hello").unwrap(); 562 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 563 | let time = 1_234_567; 564 | insert(&dir, "hello", WriteOpts::new().integrity(sri).time(time)).unwrap(); 565 | RemoveOpts::new() 566 | .remove_fully(true) 567 | .remove_sync(&dir, "hello") 568 | .unwrap(); 569 | assert_eq!(find(&dir, "hello").unwrap(), None); 570 | assert!(!content.exists()); 571 | } 572 | 573 | #[cfg(any(feature = "async-std", feature = "tokio"))] 574 | #[async_test] 575 | async fn delete_fully_async() { 576 | let tmp = tempfile::tempdir().unwrap(); 577 | let dir = tmp.path().to_owned(); 578 | let content = content_path(&dir, &"sha1-deadbeef".parse().unwrap()); 579 | fs::create_dir_all(content.parent().unwrap()).unwrap(); 580 | fs::write(content.as_path(), "hello").unwrap(); 581 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 582 | let time = 1_234_567; 583 | insert(&dir, "hello", WriteOpts::new().integrity(sri).time(time)).unwrap(); 584 | RemoveOpts::new() 585 | .remove_fully(true) 586 | .remove(&dir, "hello") 587 | .await 588 | .unwrap(); 589 | assert_eq!(find(&dir, "hello").unwrap(), None); 590 | assert!(!content.exists()); 591 | } 592 | 593 | #[test] 594 | fn round_trip() { 595 | let tmp = tempfile::tempdir().unwrap(); 596 | let dir = tmp.path().to_owned(); 597 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 598 | let time = 1_234_567; 599 | let opts = WriteOpts::new().integrity(sri.clone()).time(time); 600 | insert(&dir, "hello", opts).unwrap(); 601 | let entry = find(&dir, "hello").unwrap().unwrap(); 602 | assert_eq!( 603 | entry, 604 | Metadata { 605 | key: String::from("hello"), 606 | integrity: sri, 607 | time, 608 | size: 0, 609 | metadata: json!(null), 610 | raw_metadata: None, 611 | } 612 | ); 613 | } 614 | 615 | #[cfg(any(feature = "async-std", feature = "tokio"))] 616 | #[async_test] 617 | async fn round_trip_async() { 618 | let tmp = tempfile::tempdir().unwrap(); 619 | let dir = tmp.path().to_owned(); 620 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 621 | let time = 1_234_567; 622 | let opts = WriteOpts::new().integrity(sri.clone()).time(time); 623 | futures::executor::block_on(async { 624 | insert_async(&dir, "hello", opts).await.unwrap(); 625 | }); 626 | let entry = futures::executor::block_on(async { 627 | find_async(&dir, "hello").await.unwrap().unwrap() 628 | }); 629 | assert_eq!( 630 | entry, 631 | Metadata { 632 | key: String::from("hello"), 633 | integrity: sri, 634 | time, 635 | size: 0, 636 | metadata: json!(null), 637 | raw_metadata: None, 638 | } 639 | ); 640 | } 641 | 642 | #[test] 643 | fn ls_basic() { 644 | let tmp = tempfile::tempdir().unwrap(); 645 | let dir = tmp.path().to_owned(); 646 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 647 | let time = 1_234_567; 648 | let opts = WriteOpts::new().integrity(sri.clone()).time(time); 649 | insert(&dir, "hello", opts).unwrap(); 650 | let opts = WriteOpts::new().integrity(sri).time(time); 651 | insert(&dir, "world", opts).unwrap(); 652 | 653 | let entries = ls_entries(&dir); 654 | assert_eq!(entries, vec![String::from("hello"), String::from("world")]) 655 | } 656 | 657 | #[test] 658 | fn ls_basic_with_delete() { 659 | let tmp = tempfile::tempdir().unwrap(); 660 | let dir = tmp.path().to_owned(); 661 | let sri: Integrity = "sha1-deadbeef".parse().unwrap(); 662 | let time = 1_234_567; 663 | let opts = WriteOpts::new().integrity(sri.clone()).time(time); 664 | insert(&dir, "hello", opts).unwrap(); 665 | let opts = WriteOpts::new().integrity(sri).time(time); 666 | insert(&dir, "world", opts).unwrap(); 667 | 668 | let entries = ls_entries(&dir); 669 | assert_eq!(entries, vec![String::from("hello"), String::from("world")]); 670 | 671 | delete(&dir, "hello").unwrap(); 672 | let entries = ls_entries(&dir); 673 | assert_eq!(entries, vec![String::from("world")]) 674 | } 675 | } 676 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! cacache is a Rust library for managing local key and content address 2 | //! caches. It's really fast, really good at concurrency, and it will never 3 | //! give you corrupted data, even if cache files get corrupted or manipulated. 4 | //! 5 | //! ## API Layout 6 | //! 7 | //! The cacache API is organized roughly similar to `std::fs`; most of the 8 | //! toplevel functionality is available as free functions directly in the 9 | //! `cacache` module, with some additional functionality available through 10 | //! returned objects, as well as `WriteOpts`, which is analogous to 11 | //! `OpenOpts`, but is only able to write. 12 | //! 13 | //! One major difference is that the default APIs are all async functions, as 14 | //! opposed to `std::fs`, where they're all synchronous. Synchronous APIs in 15 | //! cacache are accessible through the `_sync` suffix. 16 | //! 17 | //! ### Suffixes 18 | //! 19 | //! You may notice various suffixes associated with otherwise familiar 20 | //! functions: 21 | //! 22 | //! * `_sync` - Most cacache APIs are asynchronous by default. Anything using 23 | //! the `_sync` suffix behaves just like its unprefixed counterpart, except 24 | //! the operation is synchronous. 25 | //! * `_hash` - Since cacache is a content-addressable cache, the `_hash` 26 | //! suffix means you're interacting directly with content data, skipping the 27 | //! index and its metadata. These functions use an `Integrity` to look up 28 | //! data, instead of a string key. 29 | //! 30 | //! ## Examples 31 | //! 32 | //! Un-suffixed APIs are all async, using 33 | //! [`async-std`](https://crates.io/crates/async-std). They let you put data 34 | //! in and get it back out -- asynchronously! 35 | //! 36 | //! ```no_run 37 | //! use async_attributes; 38 | //! 39 | //! #[async_attributes::main] 40 | //! async fn main() -> cacache::Result<()> { 41 | //! // Data goes in... 42 | //! cacache::write("./my-cache", "key", b"hello").await?; 43 | //! 44 | //! // ...data comes out! 45 | //! let data = cacache::read("./my-cache", "key").await?; 46 | //! assert_eq!(data, b"hello"); 47 | //! 48 | //! Ok(()) 49 | //! } 50 | //! ``` 51 | //! 52 | //! ### Lookup by hash 53 | //! 54 | //! What makes `cacache` content addressable, though, is its ability to fetch 55 | //! data by its "content address", which in our case is a ["subresource 56 | //! integrity" hash](https://crates.io/crates/ssri), which `cacache::put` 57 | //! conveniently returns for us. Fetching data by hash is significantly faster 58 | //! than doing key lookups: 59 | //! 60 | //! ```no_run 61 | //! use async_attributes; 62 | //! 63 | //! #[async_attributes::main] 64 | //! async fn main() -> cacache::Result<()> { 65 | //! // Data goes in... 66 | //! let sri = cacache::write("./my-cache", "key", b"hello").await?; 67 | //! 68 | //! // ...data gets looked up by `sri` ("Subresource Integrity"). 69 | //! let data = cacache::read_hash("./my-cache", &sri).await?; 70 | //! assert_eq!(data, b"hello"); 71 | //! 72 | //! Ok(()) 73 | //! } 74 | //! ``` 75 | //! 76 | //! ### Large file support 77 | //! 78 | //! `cacache` supports large file reads, in both async and sync mode, through 79 | //! an API reminiscent of `std::fs::OpenOptions`: 80 | //! 81 | //! ```no_run 82 | //! use async_attributes; 83 | //! use async_std::prelude::*; 84 | //! 85 | //! #[async_attributes::main] 86 | //! async fn main() -> cacache::Result<()> { 87 | //! let mut fd = cacache::Writer::create("./my-cache", "key").await?; 88 | //! for _ in 0..10 { 89 | //! fd.write_all(b"very large data").await.expect("Failed to write to cache"); 90 | //! } 91 | //! // Data is only committed to the cache after you do `fd.commit()`! 92 | //! let sri = fd.commit().await?; 93 | //! println!("integrity: {}", &sri); 94 | //! 95 | //! let mut fd = cacache::Reader::open("./my-cache", "key").await?; 96 | //! let mut buf = String::new(); 97 | //! fd.read_to_string(&mut buf).await.expect("Failed to read to string"); 98 | //! 99 | //! // Make sure to call `.check()` when you're done! It makes sure that what 100 | //! // you just read is actually valid. `cacache` always verifies the data 101 | //! // you get out is what it's supposed to be. The check is very cheap! 102 | //! fd.check()?; 103 | //! 104 | //! Ok(()) 105 | //! } 106 | //! ``` 107 | //! 108 | //! ### Sync API 109 | //! 110 | //! There are also sync APIs available if you don't want to use async/await. 111 | //! The synchronous APIs are generally faster for linear operations -- that 112 | //! is, doing one thing after another, as opposed to doing many things at 113 | //! once. If you're only reading and writing one thing at a time across your 114 | //! application, you probably want to use these instead. 115 | //! 116 | //! If you wish to _only_ use sync APIs and not pull in an async runtime, you 117 | //! can disable default features: 118 | //! 119 | //! ```toml 120 | //! # Cargo.toml 121 | //! [dependencies] 122 | //! cacache = { version = "X.Y.Z", default-features = false, features = ["mmap"] } 123 | //! ``` 124 | //! 125 | //! ```no_run 126 | //! fn main() -> cacache::Result<()> { 127 | //! cacache::write_sync("./my-cache", "key", b"my-data").unwrap(); 128 | //! let data = cacache::read_sync("./my-cache", "key").unwrap(); 129 | //! assert_eq!(data, b"my-data"); 130 | //! Ok(()) 131 | //! } 132 | //! ``` 133 | //! 134 | //! ### Linking to existing files 135 | //! 136 | //! The `link_to` feature enables an additional set of APIs for adding 137 | //! existing files into the cache via symlinks, without having to duplicate 138 | //! their data. Once the cache links to them, these files can be accessed by 139 | //! key just like other cached data, with the same integrity checking. 140 | //! 141 | //! The `link_to` methods are available in both async and sync variants, using 142 | //! the same suffixes as the other APIs. 143 | //! 144 | //! ```no_run 145 | //! #[async_attributes::main] 146 | //! async fn main() -> cacache::Result<()> { 147 | //! #[cfg(feature = "link_to")] 148 | //! cacache::link_to("./my-cache", "key", "/path/to/my-other-file.txt").await?; 149 | //! let data = cacache::read("./my-cache", "key").await?; 150 | //! assert_eq!(data, b"my-data"); 151 | //! Ok(()) 152 | //! } 153 | //! ``` 154 | #![warn(missing_docs)] 155 | 156 | #[cfg(all(feature = "async-std", feature = "tokio-runtime"))] 157 | compile_error!("Only either feature \"async-std\" or \"tokio-runtime\" must be enabled for this crate, not both."); 158 | 159 | pub use serde_json::Value; 160 | pub use ssri::{Algorithm, Integrity}; 161 | 162 | #[cfg(any(feature = "async-std", feature = "tokio"))] 163 | mod async_lib; 164 | 165 | mod content; 166 | mod errors; 167 | pub mod index; 168 | 169 | mod get; 170 | #[cfg(feature = "link_to")] 171 | mod linkto; 172 | mod ls; 173 | mod put; 174 | mod rm; 175 | 176 | pub use errors::{Error, Result}; 177 | pub use index::{Metadata, RemoveOpts}; 178 | 179 | pub use get::*; 180 | #[cfg(feature = "link_to")] 181 | pub use linkto::*; 182 | pub use ls::*; 183 | pub use put::*; 184 | pub use rm::*; 185 | -------------------------------------------------------------------------------- /src/linkto.rs: -------------------------------------------------------------------------------- 1 | #[cfg(any(feature = "async-std", feature = "tokio"))] 2 | use crate::async_lib::AsyncRead; 3 | #[cfg(any(feature = "async-std", feature = "tokio"))] 4 | use crate::async_lib::AsyncReadExt; 5 | use crate::content::linkto; 6 | use crate::errors::{Error, IoErrorExt, Result}; 7 | use crate::{index, WriteOpts}; 8 | use ssri::{Algorithm, Integrity}; 9 | use std::io::Read; 10 | use std::path::{Path, PathBuf}; 11 | #[cfg(any(feature = "async-std", feature = "tokio"))] 12 | use std::pin::Pin; 13 | #[cfg(any(feature = "async-std", feature = "tokio"))] 14 | use std::task::{Context as TaskContext, Poll}; 15 | 16 | const BUF_SIZE: usize = 16 * 1024; 17 | const PROBE_SIZE: usize = 8; 18 | 19 | /// Asynchronously adds `target` to the `cache` with a symlink, indexing it 20 | /// under `key`. 21 | /// 22 | /// ## Example 23 | /// ```no_run 24 | /// use async_attributes; 25 | /// use std::path::Path; 26 | /// 27 | /// #[async_attributes::main] 28 | /// async fn main() -> cacache::Result<()> { 29 | /// cacache::link_to("./my-cache", "my-key", "../my-other-files/my-file.tgz").await?; 30 | /// Ok(()) 31 | /// } 32 | /// ``` 33 | #[cfg(any(feature = "async-std", feature = "tokio"))] 34 | pub async fn link_to(cache: P, key: K, target: T) -> Result 35 | where 36 | P: AsRef, 37 | K: AsRef, 38 | T: AsRef, 39 | { 40 | ToLinker::open(cache, key, target).await?.commit().await 41 | } 42 | 43 | /// Asynchrounously adds `target` to the `cache` with a symlink, skipping 44 | /// associating an index key with it. 45 | /// 46 | /// ## Example 47 | /// ```no_run 48 | /// use async_attributes; 49 | /// use std::path::Path; 50 | /// 51 | /// #[async_attributes::main] 52 | /// async fn main() -> cacache::Result<()> { 53 | /// cacache::link_to_hash("./my-cache", "../my-other-files/my-file.tgz").await?; 54 | /// Ok(()) 55 | /// } 56 | /// ``` 57 | #[cfg(any(feature = "async-std", feature = "tokio"))] 58 | pub async fn link_to_hash(cache: P, target: T) -> Result 59 | where 60 | P: AsRef, 61 | T: AsRef, 62 | { 63 | ToLinker::open_hash(cache, target).await?.commit().await 64 | } 65 | 66 | /// Synchronously creates a symlink in the `cache` to the `target`, indexing it 67 | /// under `key`. 68 | /// 69 | /// ## Example 70 | /// ```no_run 71 | /// use std::io::Read; 72 | /// use std::path::Path; 73 | /// 74 | /// fn main() -> cacache::Result<()> { 75 | /// cacache::link_to_sync("./my-cache", "my-key", "../my-other-files/my-file.tgz")?; 76 | /// Ok(()) 77 | /// } 78 | /// ``` 79 | pub fn link_to_sync(cache: P, key: K, target: T) -> Result 80 | where 81 | P: AsRef, 82 | K: AsRef, 83 | T: AsRef, 84 | { 85 | SyncToLinker::open(cache, key, target)?.commit() 86 | } 87 | 88 | /// Synchronously creates a symlink in the `cache` to the `target`, skipping 89 | /// associating an index key with it. 90 | /// 91 | /// ## Example 92 | /// ```no_run 93 | /// use std::io::Read; 94 | /// use std::path::Path; 95 | /// 96 | /// fn main() -> cacache::Result<()> { 97 | /// cacache::link_to_hash_sync("./my-cache", "../foo/bar.tgz")?; 98 | /// Ok(()) 99 | /// } 100 | /// ``` 101 | pub fn link_to_hash_sync(cache: P, target: T) -> Result 102 | where 103 | P: AsRef, 104 | T: AsRef, 105 | { 106 | SyncToLinker::open_hash(cache, target)?.commit() 107 | } 108 | 109 | /// Extend the `WriteOpts` struct with factories for creating `ToLinker` and 110 | /// `SyncToLinker` instances. 111 | impl WriteOpts { 112 | /// Opens the target file handle for reading, returning a ToLinker instance. 113 | #[cfg(any(feature = "async-std", feature = "tokio"))] 114 | pub async fn link_to(self, cache: P, key: K, target: T) -> Result 115 | where 116 | P: AsRef, 117 | K: AsRef, 118 | T: AsRef, 119 | { 120 | async fn inner( 121 | opts: WriteOpts, 122 | cache: &Path, 123 | key: &str, 124 | target: &Path, 125 | ) -> Result { 126 | Ok(ToLinker { 127 | cache: cache.to_path_buf(), 128 | key: Some(String::from(key)), 129 | read: 0, 130 | linker: linkto::AsyncToLinker::new( 131 | cache, 132 | opts.algorithm.unwrap_or(Algorithm::Sha256), 133 | target, 134 | ) 135 | .await?, 136 | opts, 137 | }) 138 | } 139 | inner(self, cache.as_ref(), key.as_ref(), target.as_ref()).await 140 | } 141 | 142 | /// Opens the target file handle for reading, without a key, returning a 143 | /// ToLinker instance. 144 | #[cfg(any(feature = "async-std", feature = "tokio"))] 145 | pub async fn link_to_hash(self, cache: P, target: T) -> Result 146 | where 147 | P: AsRef, 148 | T: AsRef, 149 | { 150 | async fn inner(opts: WriteOpts, cache: &Path, target: &Path) -> Result { 151 | Ok(ToLinker { 152 | cache: cache.to_path_buf(), 153 | key: None, 154 | read: 0, 155 | linker: linkto::AsyncToLinker::new( 156 | cache, 157 | opts.algorithm.unwrap_or(Algorithm::Sha256), 158 | target, 159 | ) 160 | .await?, 161 | opts, 162 | }) 163 | } 164 | inner(self, cache.as_ref(), target.as_ref()).await 165 | } 166 | 167 | /// Opens the target file handle for reading synchronously, returning a 168 | /// SyncToLinker instance. 169 | pub fn link_to_sync(self, cache: P, key: K, target: T) -> Result 170 | where 171 | P: AsRef, 172 | K: AsRef, 173 | T: AsRef, 174 | { 175 | fn inner(opts: WriteOpts, cache: &Path, key: &str, target: &Path) -> Result { 176 | Ok(SyncToLinker { 177 | cache: cache.to_path_buf(), 178 | key: Some(String::from(key)), 179 | read: 0, 180 | linker: linkto::ToLinker::new( 181 | cache, 182 | opts.algorithm.unwrap_or(Algorithm::Sha256), 183 | target, 184 | )?, 185 | opts, 186 | }) 187 | } 188 | inner(self, cache.as_ref(), key.as_ref(), target.as_ref()) 189 | } 190 | 191 | /// Opens the target file handle for reading synchronously, without a key, 192 | /// returning a SyncToLinker instance. 193 | pub fn link_to_hash_sync(self, cache: P, target: T) -> Result 194 | where 195 | P: AsRef, 196 | T: AsRef, 197 | { 198 | fn inner(opts: WriteOpts, cache: &Path, target: &Path) -> Result { 199 | Ok(SyncToLinker { 200 | cache: cache.to_path_buf(), 201 | key: None, 202 | read: 0, 203 | linker: linkto::ToLinker::new( 204 | cache, 205 | opts.algorithm.unwrap_or(Algorithm::Sha256), 206 | target, 207 | )?, 208 | opts, 209 | }) 210 | } 211 | inner(self, cache.as_ref(), target.as_ref()) 212 | } 213 | } 214 | 215 | /// A file handle for asynchronously reading in data from a file to be added to 216 | /// the cache via a symlink to the target file. 217 | /// 218 | /// Make sure to call `.commit()` when done reading to actually add the file to 219 | /// the cache. 220 | #[cfg(any(feature = "async-std", feature = "tokio"))] 221 | pub struct ToLinker { 222 | cache: PathBuf, 223 | key: Option, 224 | read: usize, 225 | pub(crate) linker: linkto::AsyncToLinker, 226 | opts: WriteOpts, 227 | } 228 | 229 | #[cfg(any(feature = "async-std", feature = "tokio"))] 230 | impl AsyncRead for ToLinker { 231 | #[cfg(feature = "async-std")] 232 | fn poll_read( 233 | mut self: Pin<&mut Self>, 234 | cx: &mut TaskContext<'_>, 235 | buf: &mut [u8], 236 | ) -> Poll> { 237 | let amt = futures::ready!(Pin::new(&mut self.linker).poll_read(cx, buf))?; 238 | self.read += amt; 239 | Poll::Ready(Ok(amt)) 240 | } 241 | 242 | #[cfg(feature = "tokio")] 243 | fn poll_read( 244 | mut self: Pin<&mut Self>, 245 | cx: &mut TaskContext<'_>, 246 | buf: &mut tokio::io::ReadBuf<'_>, 247 | ) -> Poll> { 248 | let pre_len = buf.filled().len(); 249 | futures::ready!(Pin::new(&mut self.linker).poll_read(cx, buf))?; 250 | self.read += buf.filled().len() - pre_len; 251 | Poll::Ready(Ok(())) 252 | } 253 | } 254 | 255 | fn filesize(target: &Path) -> Result { 256 | Ok(target 257 | .metadata() 258 | .with_context(|| format!("Failed to get metadata of {}", target.display()))? 259 | .len() as usize) 260 | } 261 | 262 | #[cfg(any(feature = "async-std", feature = "tokio"))] 263 | impl ToLinker { 264 | /// Creates a new asynchronous readable file handle into the cache. 265 | pub async fn open(cache: P, key: K, target: T) -> Result 266 | where 267 | P: AsRef, 268 | K: AsRef, 269 | T: AsRef, 270 | { 271 | async fn inner(cache: &Path, key: &str, target: &Path) -> Result { 272 | let size = filesize(target)?; 273 | WriteOpts::new() 274 | .algorithm(Algorithm::Sha256) 275 | .size(size) 276 | .link_to(cache, key, target) 277 | .await 278 | } 279 | inner(cache.as_ref(), key.as_ref(), target.as_ref()).await 280 | } 281 | 282 | /// Creates a new asynchronous readable file handle into the cache. 283 | pub async fn open_hash(cache: P, target: T) -> Result 284 | where 285 | P: AsRef, 286 | T: AsRef, 287 | { 288 | async fn inner(cache: &Path, target: &Path) -> Result { 289 | let size = filesize(target)?; 290 | WriteOpts::new() 291 | .algorithm(Algorithm::Sha256) 292 | .size(size) 293 | .link_to_hash(cache, target) 294 | .await 295 | } 296 | inner(cache.as_ref(), target.as_ref()).await 297 | } 298 | 299 | /// Consumes the rest of the file handle, creates an symlink into 300 | /// the cache, and creates index entries for the linked file. Also verifies 301 | /// data against `size` and `integrity` options, if provided. Must be called 302 | /// manually in order to complete the writing process, otherwise everything 303 | /// will be thrown out. 304 | pub async fn commit(mut self) -> Result { 305 | self.consume().await?; 306 | let linker_sri = self.linker.commit().await?; 307 | if let Some(sri) = &self.opts.sri { 308 | if sri.matches(&linker_sri).is_none() { 309 | return Err(ssri::Error::IntegrityCheckError(sri.clone(), linker_sri).into()); 310 | } 311 | } else { 312 | self.opts.sri = Some(linker_sri.clone()); 313 | } 314 | if let Some(size) = self.opts.size { 315 | if size != self.read { 316 | return Err(Error::SizeMismatch(size, self.read)); 317 | } 318 | } 319 | if let Some(key) = self.key { 320 | index::insert(&self.cache, &key, self.opts) 321 | } else { 322 | Ok(linker_sri) 323 | } 324 | } 325 | 326 | // "Consume" the remainder of the reader, so that the integrity is properly 327 | // calculated. 328 | async fn consume(&mut self) -> Result<()> { 329 | // Do a small 'test' read to avoid allocating a larger buffer if it 330 | // isn't necessary. 331 | let mut probe = [0; PROBE_SIZE]; 332 | if self.context_read(&mut probe).await? > 0 { 333 | // Make sure all the bytes are read so that the integrity is 334 | // properly calculated. 335 | let mut buf = [0; BUF_SIZE]; 336 | while self.context_read(&mut buf).await? > 0 {} 337 | } 338 | Ok(()) 339 | } 340 | 341 | async fn context_read(&mut self, buf: &mut [u8]) -> Result { 342 | AsyncReadExt::read(self, buf).await.with_context(|| { 343 | "Failed to read target file contents while calculating integrity".into() 344 | }) 345 | } 346 | } 347 | 348 | /// A file handle for synchronously reading data from a file to be added to the 349 | /// cache via a symlink. 350 | /// 351 | /// Make sure to call `.commit()` when done reading to actually add the file 352 | /// to the cache. 353 | pub struct SyncToLinker { 354 | cache: PathBuf, 355 | key: Option, 356 | read: usize, 357 | pub(crate) linker: linkto::ToLinker, 358 | opts: WriteOpts, 359 | } 360 | 361 | impl std::io::Read for SyncToLinker { 362 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 363 | let amt = self.linker.read(buf)?; 364 | self.read += amt; 365 | Ok(amt) 366 | } 367 | } 368 | 369 | impl SyncToLinker { 370 | /// Creates a new readable file handle to a file the cache will link to, 371 | /// indexed at the provided key, on commit. 372 | /// 373 | /// It is not necessary to read any of the file before calling `.commit()`. 374 | /// 375 | /// ## Example 376 | /// ```no_run 377 | /// use std::io::prelude::*; 378 | /// 379 | /// fn main() -> cacache::Result<()> { 380 | /// let path = "../my-other-files/my-file.tgz"; 381 | /// let mut fd = cacache::SyncToLinker::open("./my-cache", "my-key", path)?; 382 | /// let mut str = String::new(); 383 | /// fd.read_to_string(&mut str).expect("Failed to read to string"); 384 | /// // The file is not linked into the cache until you commit it. 385 | /// fd.commit()?; 386 | /// Ok(()) 387 | /// } 388 | /// ``` 389 | pub fn open(cache: P, key: K, target: T) -> Result 390 | where 391 | P: AsRef, 392 | K: AsRef, 393 | T: AsRef, 394 | { 395 | fn inner(cache: &Path, key: &str, target: &Path) -> Result { 396 | let size = filesize(target)?; 397 | WriteOpts::new() 398 | .algorithm(Algorithm::Sha256) 399 | .size(size) 400 | .link_to_sync(cache, key, target) 401 | } 402 | inner(cache.as_ref(), key.as_ref(), target.as_ref()) 403 | } 404 | 405 | /// Creates a new readable file handle to a file that the cache will link 406 | /// to, without an indexe key, on commit. 407 | /// 408 | /// It is not necessary to read any of the file before calling `.commit()`. 409 | /// 410 | /// ## Example 411 | /// ```no_run 412 | /// use std::io::prelude::*; 413 | /// 414 | /// fn main() -> cacache::Result<()> { 415 | /// let path = "../my-other-files/my-file.tgz"; 416 | /// let mut fd = cacache::SyncToLinker::open_hash("./my-cache", path)?; 417 | /// let mut str = String::new(); 418 | /// fd.read_to_string(&mut str).expect("Failed to read to string"); 419 | /// // The file is not linked into the cache until you commit it. 420 | /// fd.commit()?; 421 | /// Ok(()) 422 | /// } 423 | /// ``` 424 | pub fn open_hash(cache: P, target: T) -> Result 425 | where 426 | P: AsRef, 427 | T: AsRef, 428 | { 429 | fn inner(cache: &Path, target: &Path) -> Result { 430 | let size = filesize(target)?; 431 | WriteOpts::new() 432 | .algorithm(Algorithm::Sha256) 433 | .size(size) 434 | .link_to_hash_sync(cache, target) 435 | } 436 | inner(cache.as_ref(), target.as_ref()) 437 | } 438 | 439 | /// Consumes the rest of the file handle, creates a symlink to the file, and 440 | /// creates index entries for the linked file. Also verifies data against 441 | /// `size` and `integrity` options, if provided. Must be called manually in 442 | /// order to complete the writing process, otherwise everything will be 443 | /// thrown out. 444 | pub fn commit(mut self) -> Result { 445 | self.consume()?; 446 | let cache = self.cache; 447 | let linker_sri = self.linker.commit()?; 448 | if let Some(sri) = &self.opts.sri { 449 | if sri.matches(&linker_sri).is_none() { 450 | return Err(ssri::Error::IntegrityCheckError(sri.clone(), linker_sri).into()); 451 | } 452 | } else { 453 | self.opts.sri = Some(linker_sri.clone()); 454 | } 455 | if let Some(size) = self.opts.size { 456 | if size != self.read { 457 | return Err(Error::SizeMismatch(size, self.read)); 458 | } 459 | } 460 | if let Some(key) = self.key { 461 | index::insert(&cache, &key, self.opts) 462 | } else { 463 | Ok(linker_sri) 464 | } 465 | } 466 | 467 | fn consume(&mut self) -> Result<()> { 468 | // Do a small 'test' read to avoid allocating a larger buffer if it 469 | // isn't necessary. 470 | let mut probe = [0; PROBE_SIZE]; 471 | if self.context_read(&mut probe)? > 0 { 472 | // Make sure all the bytes are read so that the integrity is 473 | // properly calculated. 474 | let mut buf = [0; BUF_SIZE]; 475 | while self.context_read(&mut buf)? > 0 {} 476 | } 477 | Ok(()) 478 | } 479 | 480 | fn context_read(&mut self, buf: &mut [u8]) -> Result { 481 | self.read(buf).with_context(|| { 482 | "Failed to read target file contents while calculating integrity".into() 483 | }) 484 | } 485 | } 486 | 487 | #[cfg(test)] 488 | mod tests { 489 | use std::fs::File; 490 | use std::io::Write; 491 | 492 | use super::*; 493 | 494 | #[cfg(feature = "async-std")] 495 | use async_attributes::test as async_test; 496 | #[cfg(feature = "tokio")] 497 | use tokio::test as async_test; 498 | 499 | fn create_tmpfile(tmp: &tempfile::TempDir, buf: &[u8]) -> PathBuf { 500 | let dir = tmp.path().to_owned(); 501 | let target = dir.join("target-file"); 502 | std::fs::create_dir_all(target.parent().unwrap()).unwrap(); 503 | let mut file = File::create(&target).unwrap(); 504 | file.write_all(buf).unwrap(); 505 | file.flush().unwrap(); 506 | target 507 | } 508 | 509 | #[cfg(any(feature = "async-std", feature = "tokio"))] 510 | #[async_test] 511 | async fn test_link() { 512 | let tmp = tempfile::tempdir().unwrap(); 513 | let target = create_tmpfile(&tmp, b"hello world"); 514 | 515 | let tmp = tempfile::tempdir().unwrap(); 516 | let dir = tmp.path().to_owned(); 517 | crate::link_to(&dir, "my-key", target).await.unwrap(); 518 | 519 | let buf = crate::read(&dir, "my-key").await.unwrap(); 520 | assert_eq!(buf, b"hello world"); 521 | } 522 | 523 | #[cfg(any(feature = "async-std", feature = "tokio"))] 524 | #[async_test] 525 | async fn test_link_to_hash() { 526 | let tmp = tempfile::tempdir().unwrap(); 527 | let target = create_tmpfile(&tmp, b"hello world"); 528 | 529 | let tmp = tempfile::tempdir().unwrap(); 530 | let dir = tmp.path().to_owned(); 531 | let sri = crate::link_to_hash(&dir, target).await.unwrap(); 532 | 533 | let buf = crate::read_hash(&dir, &sri).await.unwrap(); 534 | assert_eq!(buf, b"hello world"); 535 | } 536 | 537 | #[test] 538 | fn test_link_to_sync() { 539 | let tmp = tempfile::tempdir().unwrap(); 540 | let target = create_tmpfile(&tmp, b"hello world"); 541 | 542 | let tmp = tempfile::tempdir().unwrap(); 543 | let dir = tmp.path().to_owned(); 544 | crate::link_to_sync(&dir, "my-key", target).unwrap(); 545 | 546 | let buf = crate::read_sync(&dir, "my-key").unwrap(); 547 | assert_eq!(buf, b"hello world"); 548 | } 549 | 550 | #[test] 551 | fn test_link_to_hash_sync() { 552 | let tmp = tempfile::tempdir().unwrap(); 553 | let target = create_tmpfile(&tmp, b"hello world"); 554 | 555 | let tmp = tempfile::tempdir().unwrap(); 556 | let dir = tmp.path().to_owned(); 557 | let sri = crate::link_to_hash_sync(&dir, target).unwrap(); 558 | 559 | let buf = crate::read_hash_sync(&dir, &sri).unwrap(); 560 | assert_eq!(buf, b"hello world"); 561 | } 562 | 563 | #[cfg(any(feature = "async-std", feature = "tokio"))] 564 | #[async_test] 565 | async fn test_open() { 566 | let tmp = tempfile::tempdir().unwrap(); 567 | let target = create_tmpfile(&tmp, b"hello world"); 568 | 569 | let tmp = tempfile::tempdir().unwrap(); 570 | let dir = tmp.path().to_owned(); 571 | let mut handle = crate::ToLinker::open(&dir, "my-key", target).await.unwrap(); 572 | 573 | let mut buf = Vec::new(); 574 | handle.read_to_end(&mut buf).await.unwrap(); 575 | handle.commit().await.unwrap(); 576 | assert_eq!(buf, b"hello world"); 577 | 578 | let buf = crate::read_sync(&dir, "my-key").unwrap(); 579 | assert_eq!(buf, b"hello world"); 580 | } 581 | 582 | #[cfg(any(feature = "async-std", feature = "tokio"))] 583 | #[async_test] 584 | async fn test_open_hash() { 585 | let tmp = tempfile::tempdir().unwrap(); 586 | let target = create_tmpfile(&tmp, b"hello world"); 587 | 588 | let tmp = tempfile::tempdir().unwrap(); 589 | let dir = tmp.path().to_owned(); 590 | let mut handle = crate::ToLinker::open_hash(&dir, target).await.unwrap(); 591 | 592 | let mut buf = Vec::new(); 593 | handle.read_to_end(&mut buf).await.unwrap(); 594 | let sri = handle.commit().await.unwrap(); 595 | assert_eq!(buf, b"hello world"); 596 | 597 | let buf = crate::read_hash_sync(&dir, &sri).unwrap(); 598 | assert_eq!(buf, b"hello world"); 599 | } 600 | 601 | #[test] 602 | fn test_open_sync() { 603 | let tmp = tempfile::tempdir().unwrap(); 604 | let target = create_tmpfile(&tmp, b"hello world"); 605 | 606 | let tmp = tempfile::tempdir().unwrap(); 607 | let dir = tmp.path().to_owned(); 608 | let mut handle = crate::SyncToLinker::open(&dir, "my-key", target).unwrap(); 609 | 610 | let mut buf = Vec::new(); 611 | handle.read_to_end(&mut buf).unwrap(); 612 | handle.commit().unwrap(); 613 | assert_eq!(buf, b"hello world"); 614 | 615 | let buf = crate::read_sync(&dir, "my-key").unwrap(); 616 | assert_eq!(buf, b"hello world"); 617 | } 618 | 619 | #[test] 620 | fn test_open_hash_sync() { 621 | let tmp = tempfile::tempdir().unwrap(); 622 | let target = create_tmpfile(&tmp, b"hello world"); 623 | 624 | let tmp = tempfile::tempdir().unwrap(); 625 | let dir = tmp.path().to_owned(); 626 | let mut handle = crate::SyncToLinker::open_hash(&dir, target).unwrap(); 627 | 628 | let mut buf = Vec::new(); 629 | handle.read_to_end(&mut buf).unwrap(); 630 | let sri = handle.commit().unwrap(); 631 | assert_eq!(buf, b"hello world"); 632 | 633 | let buf = crate::read_hash_sync(&dir, &sri).unwrap(); 634 | assert_eq!(buf, b"hello world"); 635 | } 636 | } 637 | -------------------------------------------------------------------------------- /src/ls.rs: -------------------------------------------------------------------------------- 1 | //! Functions for iterating over the cache. 2 | use std::path::Path; 3 | 4 | use crate::errors::Result; 5 | use crate::index; 6 | 7 | /// Returns a synchronous iterator that lists all cache index entries. 8 | pub fn list_sync>(cache: P) -> impl Iterator> { 9 | index::ls(cache.as_ref()) 10 | } 11 | 12 | #[cfg(test)] 13 | mod tests { 14 | use super::*; 15 | 16 | #[test] 17 | fn test_list_sync() { 18 | // check that the public interface to list elements can actually use the 19 | // Iterator::Item 20 | let tmp = tempfile::tempdir().unwrap(); 21 | let dir = tmp.path().to_owned(); 22 | 23 | assert!(list_sync(dir) 24 | .map(|x| Ok(x?.key)) 25 | .collect::>>() 26 | .is_err()) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/put.rs: -------------------------------------------------------------------------------- 1 | //! Functions for writing to cache. 2 | use std::io::prelude::*; 3 | use std::path::{Path, PathBuf}; 4 | #[cfg(any(feature = "async-std", feature = "tokio"))] 5 | use std::pin::Pin; 6 | 7 | use serde_json::Value; 8 | use ssri::{Algorithm, Integrity}; 9 | 10 | #[cfg(any(feature = "async-std", feature = "tokio"))] 11 | use crate::async_lib::{AsyncWrite, AsyncWriteExt}; 12 | use crate::content::write; 13 | use crate::errors::{Error, IoErrorExt, Result}; 14 | use crate::index; 15 | 16 | #[cfg(any(feature = "async-std", feature = "tokio"))] 17 | use std::task::{Context as TaskContext, Poll}; 18 | 19 | /// Writes `data` to the `cache`, indexing it under `key`. 20 | /// 21 | /// ## Example 22 | /// ```no_run 23 | /// use async_attributes; 24 | /// 25 | /// #[async_attributes::main] 26 | /// async fn main() -> cacache::Result<()> { 27 | /// cacache::write("./my-cache", "my-key", b"hello").await?; 28 | /// Ok(()) 29 | /// } 30 | /// ``` 31 | #[cfg(any(feature = "async-std", feature = "tokio"))] 32 | pub async fn write(cache: P, key: K, data: D) -> Result 33 | where 34 | P: AsRef, 35 | D: AsRef<[u8]>, 36 | K: AsRef, 37 | { 38 | write_with_algo(Algorithm::Sha256, cache, key, data).await 39 | } 40 | 41 | /// Writes `data` to the `cache`, indexing it under `key`. Use this function 42 | /// to customize the hashing algorithm. 43 | /// 44 | /// ## Example 45 | /// ```no_run 46 | /// use async_attributes; 47 | /// 48 | /// #[async_attributes::main] 49 | /// async fn main() -> cacache::Result<()> { 50 | /// cacache::write_with_algo(cacache::Algorithm::Xxh3, "./my-cache", "my-key", b"hello").await?; 51 | /// Ok(()) 52 | /// } 53 | /// ``` 54 | #[cfg(any(feature = "async-std", feature = "tokio"))] 55 | pub async fn write_with_algo( 56 | algo: Algorithm, 57 | cache: P, 58 | key: K, 59 | data: D, 60 | ) -> Result 61 | where 62 | P: AsRef, 63 | D: AsRef<[u8]>, 64 | K: AsRef, 65 | { 66 | async fn inner(algo: Algorithm, cache: &Path, key: &str, data: &[u8]) -> Result { 67 | let mut writer = WriteOpts::new() 68 | .algorithm(algo) 69 | .size(data.len()) 70 | .open(cache, key) 71 | .await?; 72 | writer.write_all(data).await.with_context(|| { 73 | format!("Failed to write to cache data for key {key} for cache at {cache:?}") 74 | })?; 75 | writer.commit().await 76 | } 77 | inner(algo, cache.as_ref(), key.as_ref(), data.as_ref()).await 78 | } 79 | 80 | /// Writes `data` to the `cache`, skipping associating an index key with it. 81 | /// 82 | /// ## Example 83 | /// ```no_run 84 | /// use async_attributes; 85 | /// 86 | /// #[async_attributes::main] 87 | /// async fn main() -> cacache::Result<()> { 88 | /// cacache::write_hash("./my-cache", b"hello").await?; 89 | /// Ok(()) 90 | /// } 91 | /// ``` 92 | #[cfg(any(feature = "async-std", feature = "tokio"))] 93 | pub async fn write_hash(cache: P, data: D) -> Result 94 | where 95 | P: AsRef, 96 | D: AsRef<[u8]>, 97 | { 98 | write_hash_with_algo(Algorithm::Sha256, cache, data).await 99 | } 100 | 101 | /// Writes `data` to the `cache`, skipping associating an index key with it. 102 | /// Use this to customize the hashing algorithm. 103 | /// 104 | /// ## Example 105 | /// ```no_run 106 | /// use async_attributes; 107 | /// 108 | /// #[async_attributes::main] 109 | /// async fn main() -> cacache::Result<()> { 110 | /// cacache::write_hash_with_algo(cacache::Algorithm::Xxh3, "./my-cache", b"hello").await?; 111 | /// Ok(()) 112 | /// } 113 | /// ``` 114 | #[cfg(any(feature = "async-std", feature = "tokio"))] 115 | pub async fn write_hash_with_algo(algo: Algorithm, cache: P, data: D) -> Result 116 | where 117 | P: AsRef, 118 | D: AsRef<[u8]>, 119 | { 120 | async fn inner(algo: Algorithm, cache: &Path, data: &[u8]) -> Result { 121 | let mut writer = WriteOpts::new() 122 | .algorithm(algo) 123 | .size(data.len()) 124 | .open_hash(cache) 125 | .await?; 126 | writer 127 | .write_all(data) 128 | .await 129 | .with_context(|| format!("Failed to write to cache data for cache at {cache:?}"))?; 130 | writer.commit().await 131 | } 132 | inner(algo, cache.as_ref(), data.as_ref()).await 133 | } 134 | /// A reference to an open file writing to the cache. 135 | #[cfg(any(feature = "async-std", feature = "tokio"))] 136 | pub struct Writer { 137 | cache: PathBuf, 138 | key: Option, 139 | written: usize, 140 | pub(crate) writer: write::AsyncWriter, 141 | opts: WriteOpts, 142 | } 143 | 144 | #[cfg(any(feature = "async-std", feature = "tokio"))] 145 | impl AsyncWrite for Writer { 146 | fn poll_write( 147 | mut self: Pin<&mut Self>, 148 | cx: &mut TaskContext<'_>, 149 | buf: &[u8], 150 | ) -> Poll> { 151 | let amt = futures::ready!(Pin::new(&mut self.writer).poll_write(cx, buf))?; 152 | self.written += amt; 153 | Poll::Ready(Ok(amt)) 154 | } 155 | 156 | fn poll_flush(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { 157 | Pin::new(&mut self.writer).poll_flush(cx) 158 | } 159 | 160 | #[cfg(feature = "async-std")] 161 | fn poll_close(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { 162 | Pin::new(&mut self.writer).poll_close(cx) 163 | } 164 | 165 | #[cfg(feature = "tokio")] 166 | fn poll_shutdown( 167 | mut self: Pin<&mut Self>, 168 | cx: &mut TaskContext<'_>, 169 | ) -> Poll> { 170 | Pin::new(&mut self.writer).poll_shutdown(cx) 171 | } 172 | } 173 | 174 | #[cfg(any(feature = "async-std", feature = "tokio"))] 175 | impl Writer { 176 | /// Creates a new writable file handle into the cache. 177 | /// 178 | /// ## Example 179 | /// ```no_run 180 | /// use async_attributes; 181 | /// use async_std::prelude::*; 182 | /// 183 | /// #[async_attributes::main] 184 | /// async fn main() -> cacache::Result<()> { 185 | /// let mut fd = cacache::Writer::create("./my-cache", "my-key").await?; 186 | /// fd.write_all(b"hello world").await.expect("Failed to write to cache"); 187 | /// // Data is not saved into the cache until you commit it. 188 | /// fd.commit().await?; 189 | /// Ok(()) 190 | /// } 191 | /// ``` 192 | pub async fn create(cache: P, key: K) -> Result 193 | where 194 | P: AsRef, 195 | K: AsRef, 196 | { 197 | Self::create_with_algo(Algorithm::Sha256, cache, key).await 198 | } 199 | 200 | /// Creates a new writable file handle into the cache. Use this to 201 | /// customize the algorithm used for hashing. 202 | /// 203 | /// ## Example 204 | /// ```no_run 205 | /// use async_attributes; 206 | /// use async_std::prelude::*; 207 | /// 208 | /// #[async_attributes::main] 209 | /// async fn main() -> cacache::Result<()> { 210 | /// let mut fd = cacache::Writer::create_with_algo(cacache::Algorithm::Xxh3, "./my-cache", "my-key").await?; 211 | /// fd.write_all(b"hello world").await.expect("Failed to write to cache"); 212 | /// // Data is not saved into the cache until you commit it. 213 | /// fd.commit().await?; 214 | /// Ok(()) 215 | /// } 216 | /// ``` 217 | pub async fn create_with_algo(algo: Algorithm, cache: P, key: K) -> Result 218 | where 219 | P: AsRef, 220 | K: AsRef, 221 | { 222 | async fn inner(algo: Algorithm, cache: &Path, key: &str) -> Result { 223 | WriteOpts::new().algorithm(algo).open(cache, key).await 224 | } 225 | inner(algo, cache.as_ref(), key.as_ref()).await 226 | } 227 | 228 | /// Closes the Writer handle and writes content and index entries. Also 229 | /// verifies data against `size` and `integrity` options, if provided. 230 | /// Must be called manually in order to complete the writing process, 231 | /// otherwise everything will be thrown out. 232 | pub async fn commit(mut self) -> Result { 233 | let cache = self.cache; 234 | let writer_sri = self.writer.close().await?; 235 | if let Some(sri) = &self.opts.sri { 236 | if sri.matches(&writer_sri).is_none() { 237 | return Err(ssri::Error::IntegrityCheckError(sri.clone(), writer_sri).into()); 238 | } 239 | } else { 240 | self.opts.sri = Some(writer_sri.clone()); 241 | } 242 | if let Some(size) = self.opts.size { 243 | if size != self.written { 244 | return Err(Error::SizeMismatch(size, self.written)); 245 | } 246 | } 247 | if let Some(key) = self.key { 248 | index::insert_async(&cache, &key, self.opts).await 249 | } else { 250 | Ok(writer_sri) 251 | } 252 | } 253 | } 254 | 255 | /// Writes `data` to the `cache` synchronously, indexing it under `key`. 256 | /// 257 | /// ## Example 258 | /// ```no_run 259 | /// use std::io::Read; 260 | /// 261 | /// fn main() -> cacache::Result<()> { 262 | /// let data = cacache::write_sync("./my-cache", "my-key", b"hello")?; 263 | /// Ok(()) 264 | /// } 265 | /// ``` 266 | pub fn write_sync(cache: P, key: K, data: D) -> Result 267 | where 268 | P: AsRef, 269 | D: AsRef<[u8]>, 270 | K: AsRef, 271 | { 272 | write_sync_with_algo(Algorithm::Sha256, cache, key, data) 273 | } 274 | 275 | /// Writes `data` to the `cache` synchronously, indexing it under `key`. Use 276 | /// this to customize the hashing algorithm. 277 | /// 278 | /// ## Example 279 | /// ```no_run 280 | /// use std::io::Read; 281 | /// 282 | /// fn main() -> cacache::Result<()> { 283 | /// let data = cacache::write_sync_with_algo(cacache::Algorithm::Xxh3, "./my-cache", "my-key", b"hello")?; 284 | /// Ok(()) 285 | /// } 286 | /// ``` 287 | pub fn write_sync_with_algo( 288 | algo: Algorithm, 289 | cache: P, 290 | key: K, 291 | data: D, 292 | ) -> Result 293 | where 294 | P: AsRef, 295 | D: AsRef<[u8]>, 296 | K: AsRef, 297 | { 298 | fn inner(algo: Algorithm, cache: &Path, key: &str, data: &[u8]) -> Result { 299 | let mut writer = SyncWriter::create_with_algo(algo, cache, key)?; 300 | writer.write_all(data).with_context(|| { 301 | format!("Failed to write to cache data for key {key} for cache at {cache:?}") 302 | })?; 303 | writer.written = data.as_ref().len(); 304 | writer.commit() 305 | } 306 | inner(algo, cache.as_ref(), key.as_ref(), data.as_ref()) 307 | } 308 | 309 | /// Writes `data` to the `cache` synchronously, skipping associating a key with it. 310 | /// 311 | /// ## Example 312 | /// ```no_run 313 | /// use std::io::Read; 314 | /// 315 | /// fn main() -> cacache::Result<()> { 316 | /// let data = cacache::write_hash_sync("./my-cache", b"hello")?; 317 | /// Ok(()) 318 | /// } 319 | /// ``` 320 | pub fn write_hash_sync(cache: P, data: D) -> Result 321 | where 322 | P: AsRef, 323 | D: AsRef<[u8]>, 324 | { 325 | write_hash_sync_with_algo(Algorithm::Sha256, cache, data) 326 | } 327 | 328 | /// Writes `data` to the `cache` synchronously, skipping associating a key with it. 329 | /// 330 | /// ## Example 331 | /// ```no_run 332 | /// use std::io::Read; 333 | /// 334 | /// fn main() -> cacache::Result<()> { 335 | /// let data = cacache::write_hash_sync_with_algo(cacache::Algorithm::Xxh3, "./my-cache", b"hello")?; 336 | /// Ok(()) 337 | /// } 338 | /// ``` 339 | pub fn write_hash_sync_with_algo(algo: Algorithm, cache: P, data: D) -> Result 340 | where 341 | P: AsRef, 342 | D: AsRef<[u8]>, 343 | { 344 | fn inner(algo: Algorithm, cache: &Path, data: &[u8]) -> Result { 345 | let mut writer = WriteOpts::new() 346 | .algorithm(algo) 347 | .size(data.len()) 348 | .open_hash_sync(cache)?; 349 | writer 350 | .write_all(data) 351 | .with_context(|| format!("Failed to write to cache data for cache at {cache:?}"))?; 352 | writer.written = data.len(); 353 | writer.commit() 354 | } 355 | inner(algo, cache.as_ref(), data.as_ref()) 356 | } 357 | /// Builder for options and flags for opening a new cache file to write data into. 358 | #[derive(Clone, Default)] 359 | pub struct WriteOpts { 360 | pub(crate) algorithm: Option, 361 | pub(crate) sri: Option, 362 | pub(crate) size: Option, 363 | pub(crate) time: Option, 364 | pub(crate) metadata: Option, 365 | pub(crate) raw_metadata: Option>, 366 | } 367 | 368 | impl WriteOpts { 369 | /// Creates a blank set of cache writing options. 370 | pub fn new() -> WriteOpts { 371 | Default::default() 372 | } 373 | 374 | /// Opens the file handle for writing, returning an Writer instance. 375 | #[cfg(any(feature = "async-std", feature = "tokio"))] 376 | pub async fn open(self, cache: P, key: K) -> Result 377 | where 378 | P: AsRef, 379 | K: AsRef, 380 | { 381 | async fn inner(me: WriteOpts, cache: &Path, key: &str) -> Result { 382 | Ok(Writer { 383 | cache: cache.to_path_buf(), 384 | key: Some(String::from(key)), 385 | written: 0, 386 | writer: write::AsyncWriter::new( 387 | cache, 388 | me.algorithm.unwrap_or(Algorithm::Sha256), 389 | None, 390 | ) 391 | .await?, 392 | opts: me, 393 | }) 394 | } 395 | inner(self, cache.as_ref(), key.as_ref()).await 396 | } 397 | 398 | /// Opens the file handle for writing, without a key returning an Writer instance. 399 | #[cfg(any(feature = "async-std", feature = "tokio"))] 400 | pub async fn open_hash

(self, cache: P) -> Result 401 | where 402 | P: AsRef, 403 | { 404 | async fn inner(me: WriteOpts, cache: &Path) -> Result { 405 | Ok(Writer { 406 | cache: cache.to_path_buf(), 407 | key: None, 408 | written: 0, 409 | writer: write::AsyncWriter::new( 410 | cache, 411 | me.algorithm.unwrap_or(Algorithm::Sha256), 412 | me.size, 413 | ) 414 | .await?, 415 | opts: me, 416 | }) 417 | } 418 | inner(self, cache.as_ref()).await 419 | } 420 | 421 | /// Opens the file handle for writing synchronously, returning a SyncWriter instance. 422 | pub fn open_sync(self, cache: P, key: K) -> Result 423 | where 424 | P: AsRef, 425 | K: AsRef, 426 | { 427 | fn inner(me: WriteOpts, cache: &Path, key: &str) -> Result { 428 | Ok(SyncWriter { 429 | cache: cache.to_path_buf(), 430 | key: Some(String::from(key)), 431 | written: 0, 432 | writer: write::Writer::new( 433 | cache, 434 | me.algorithm.unwrap_or(Algorithm::Sha256), 435 | me.size, 436 | )?, 437 | opts: me, 438 | }) 439 | } 440 | inner(self, cache.as_ref(), key.as_ref()) 441 | } 442 | 443 | /// Opens the file handle for writing, without a key returning an SyncWriter instance. 444 | pub fn open_hash_sync

(self, cache: P) -> Result 445 | where 446 | P: AsRef, 447 | { 448 | fn inner(me: WriteOpts, cache: &Path) -> Result { 449 | Ok(SyncWriter { 450 | cache: cache.to_path_buf(), 451 | key: None, 452 | written: 0, 453 | writer: write::Writer::new( 454 | cache, 455 | me.algorithm.unwrap_or(Algorithm::Sha256), 456 | me.size, 457 | )?, 458 | opts: me, 459 | }) 460 | } 461 | inner(self, cache.as_ref()) 462 | } 463 | 464 | /// Configures the algorithm to write data under. 465 | pub fn algorithm(mut self, algo: Algorithm) -> Self { 466 | self.algorithm = Some(algo); 467 | self 468 | } 469 | 470 | /// Sets the expected size of the data to write. If there's a date size 471 | /// mismatch, `put.commit()` will return an error. 472 | pub fn size(mut self, size: usize) -> Self { 473 | self.size = Some(size); 474 | self 475 | } 476 | 477 | /// Sets arbitrary additional metadata to associate with the index entry. 478 | pub fn metadata(mut self, metadata: Value) -> Self { 479 | self.metadata = Some(metadata); 480 | self 481 | } 482 | 483 | /// Sets arbitrary additional binary metadata to associate with the index entry. 484 | pub fn raw_metadata(mut self, metadata: Vec) -> Self { 485 | self.raw_metadata = Some(metadata); 486 | self 487 | } 488 | 489 | /// Sets the specific time in unix milliseconds to associate with this 490 | /// entry. This is usually automatically set to the write time, but can be 491 | /// useful to change for tests and such. 492 | pub fn time(mut self, time: u128) -> Self { 493 | self.time = Some(time); 494 | self 495 | } 496 | 497 | /// Sets the expected integrity hash of the written data. If there's a 498 | /// mismatch between this Integrity and the one calculated by the write, 499 | /// `put.commit()` will error. 500 | pub fn integrity(mut self, sri: Integrity) -> Self { 501 | self.sri = Some(sri); 502 | self 503 | } 504 | } 505 | 506 | /// A reference to an open file writing to the cache. 507 | pub struct SyncWriter { 508 | cache: PathBuf, 509 | key: Option, 510 | written: usize, 511 | pub(crate) writer: write::Writer, 512 | opts: WriteOpts, 513 | } 514 | 515 | impl Write for SyncWriter { 516 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 517 | let written = self.writer.write(buf)?; 518 | self.written += written; 519 | Ok(written) 520 | } 521 | fn flush(&mut self) -> std::io::Result<()> { 522 | self.writer.flush() 523 | } 524 | } 525 | 526 | impl SyncWriter { 527 | /// Creates a new writable file handle into the cache. 528 | /// 529 | /// ## Example 530 | /// ```no_run 531 | /// use std::io::prelude::*; 532 | /// 533 | /// fn main() -> cacache::Result<()> { 534 | /// let mut fd = cacache::SyncWriter::create("./my-cache", "my-key")?; 535 | /// fd.write_all(b"hello world").expect("Failed to write to cache"); 536 | /// // Data is not saved into the cache until you commit it. 537 | /// fd.commit()?; 538 | /// Ok(()) 539 | /// } 540 | /// ``` 541 | pub fn create(cache: P, key: K) -> Result 542 | where 543 | P: AsRef, 544 | K: AsRef, 545 | { 546 | fn inner(cache: &Path, key: &str) -> Result { 547 | WriteOpts::new() 548 | .algorithm(Algorithm::Sha256) 549 | .open_sync(cache, key) 550 | } 551 | inner(cache.as_ref(), key.as_ref()) 552 | } 553 | 554 | /// Creates a new writable file handle into the cache. Use this to 555 | /// customize the hashing algorithm. 556 | /// 557 | /// ## Example 558 | /// ```no_run 559 | /// use std::io::prelude::*; 560 | /// 561 | /// fn main() -> cacache::Result<()> { 562 | /// let mut fd = cacache::SyncWriter::create_with_algo(cacache::Algorithm::Xxh3, "./my-cache", "my-key")?; 563 | /// fd.write_all(b"hello world").expect("Failed to write to cache"); 564 | /// // Data is not saved into the cache until you commit it. 565 | /// fd.commit()?; 566 | /// Ok(()) 567 | /// } 568 | /// ``` 569 | pub fn create_with_algo(algo: Algorithm, cache: P, key: K) -> Result 570 | where 571 | P: AsRef, 572 | K: AsRef, 573 | { 574 | fn inner(algo: Algorithm, cache: &Path, key: &str) -> Result { 575 | WriteOpts::new().algorithm(algo).open_sync(cache, key) 576 | } 577 | inner(algo, cache.as_ref(), key.as_ref()) 578 | } 579 | /// Closes the Writer handle and writes content and index entries. Also 580 | /// verifies data against `size` and `integrity` options, if provided. 581 | /// Must be called manually in order to complete the writing process, 582 | /// otherwise everything will be thrown out. 583 | pub fn commit(mut self) -> Result { 584 | let cache = self.cache; 585 | let writer_sri = self.writer.close()?; 586 | if let Some(sri) = &self.opts.sri { 587 | if sri.matches(&writer_sri).is_none() { 588 | return Err(ssri::Error::IntegrityCheckError(sri.clone(), writer_sri).into()); 589 | } 590 | } else { 591 | self.opts.sri = Some(writer_sri.clone()); 592 | } 593 | if let Some(size) = self.opts.size { 594 | if size != self.written { 595 | return Err(Error::SizeMismatch(size, self.written)); 596 | } 597 | } 598 | if let Some(key) = self.key { 599 | index::insert(&cache, &key, self.opts) 600 | } else { 601 | Ok(writer_sri) 602 | } 603 | } 604 | } 605 | 606 | #[cfg(test)] 607 | mod tests { 608 | #[cfg(feature = "async-std")] 609 | use async_attributes::test as async_test; 610 | #[cfg(feature = "tokio")] 611 | use tokio::test as async_test; 612 | 613 | #[cfg(any(feature = "async-std", feature = "tokio"))] 614 | #[async_test] 615 | async fn round_trip() { 616 | let tmp = tempfile::tempdir().unwrap(); 617 | let dir = tmp.path().to_owned(); 618 | crate::write(&dir, "hello", b"hello").await.unwrap(); 619 | let data = crate::read(&dir, "hello").await.unwrap(); 620 | assert_eq!(data, b"hello"); 621 | } 622 | 623 | #[test] 624 | fn round_trip_sync() { 625 | let tmp = tempfile::tempdir().unwrap(); 626 | let dir = tmp.path().to_owned(); 627 | crate::write_sync(&dir, "hello", b"hello").unwrap(); 628 | let data = crate::read_sync(&dir, "hello").unwrap(); 629 | assert_eq!(data, b"hello"); 630 | } 631 | 632 | #[test] 633 | fn hash_write_sync() { 634 | let tmp = tempfile::tempdir().unwrap(); 635 | let dir = tmp.path().to_owned(); 636 | let original = format!("hello world{}", 5); 637 | let integrity = crate::write_hash_sync(&dir, &original) 638 | .expect("should be able to write a hash synchronously"); 639 | let bytes = crate::read_hash_sync(&dir, &integrity) 640 | .expect("should be able to read the data we just wrote"); 641 | let result = 642 | String::from_utf8(bytes).expect("we wrote valid utf8 but did not read valid utf8 back"); 643 | assert_eq!(result, original, "we did not read back what we wrote"); 644 | } 645 | 646 | #[cfg(any(feature = "async-std", feature = "tokio"))] 647 | #[async_test] 648 | async fn hash_write_async() { 649 | let tmp = tempfile::tempdir().unwrap(); 650 | let dir = tmp.path().to_owned(); 651 | let original = format!("hello world{}", 12); 652 | let integrity = crate::write_hash(&dir, &original) 653 | .await 654 | .expect("should be able to write a hash asynchronously"); 655 | let bytes = crate::read_hash(&dir, &integrity) 656 | .await 657 | .expect("should be able to read back what we wrote"); 658 | let result = 659 | String::from_utf8(bytes).expect("we wrote valid utf8 but did not read valid utf8 back"); 660 | assert_eq!(result, original, "we did not read back what we wrote"); 661 | } 662 | } 663 | -------------------------------------------------------------------------------- /src/rm.rs: -------------------------------------------------------------------------------- 1 | //! Functions for removing things from the cache. 2 | use std::fs; 3 | use std::path::Path; 4 | 5 | use ssri::Integrity; 6 | 7 | use crate::content::rm; 8 | use crate::errors::{IoErrorExt, Result}; 9 | use crate::index; 10 | 11 | /// Removes an individual index metadata entry. The associated content will be 12 | /// left in the cache. 13 | /// 14 | /// ## Example 15 | /// ```no_run 16 | /// use async_std::prelude::*; 17 | /// use async_attributes; 18 | /// 19 | /// #[async_attributes::main] 20 | /// async fn main() -> cacache::Result<()> { 21 | /// let sri = cacache::write("./my-cache", "my-key", b"hello").await?; 22 | /// 23 | /// cacache::remove("./my-cache", "my-key").await?; 24 | /// 25 | /// // This fails: 26 | /// cacache::read("./my-cache", "my-key").await?; 27 | /// 28 | /// // But this succeeds: 29 | /// cacache::read_hash("./my-cache", &sri).await?; 30 | /// 31 | /// Ok(()) 32 | /// } 33 | /// ``` 34 | #[cfg(any(feature = "async-std", feature = "tokio"))] 35 | pub async fn remove(cache: P, key: K) -> Result<()> 36 | where 37 | P: AsRef, 38 | K: AsRef, 39 | { 40 | index::delete_async(cache.as_ref(), key.as_ref()).await 41 | } 42 | 43 | /// Removes an individual content entry. Any index entries pointing to this 44 | /// content will become invalidated. 45 | /// 46 | /// ## Example 47 | /// ```no_run 48 | /// use async_std::prelude::*; 49 | /// use async_attributes; 50 | /// 51 | /// #[async_attributes::main] 52 | /// async fn main() -> cacache::Result<()> { 53 | /// let sri = cacache::write("./my-cache", "my-key", b"hello").await?; 54 | /// 55 | /// cacache::remove_hash("./my-cache", &sri).await?; 56 | /// 57 | /// // These fail: 58 | /// cacache::read("./my-cache", "my-key").await?; 59 | /// cacache::read_hash("./my-cache", &sri).await?; 60 | /// 61 | /// // But this succeeds: 62 | /// cacache::metadata("./my-cache", "my-key").await?; 63 | /// 64 | /// Ok(()) 65 | /// } 66 | /// ``` 67 | #[cfg(any(feature = "async-std", feature = "tokio"))] 68 | pub async fn remove_hash>(cache: P, sri: &Integrity) -> Result<()> { 69 | rm::rm_async(cache.as_ref(), sri).await 70 | } 71 | 72 | /// Removes entire contents of the cache, including temporary files, the entry 73 | /// index, and all content data. 74 | /// 75 | /// ## Example 76 | /// ```no_run 77 | /// use async_std::prelude::*; 78 | /// use async_attributes; 79 | /// 80 | /// #[async_attributes::main] 81 | /// async fn main() -> cacache::Result<()> { 82 | /// let sri = cacache::write("./my-cache", "my-key", b"hello").await?; 83 | /// 84 | /// cacache::clear("./my-cache").await?; 85 | /// 86 | /// // These all fail: 87 | /// cacache::read("./my-cache", "my-key").await?; 88 | /// cacache::metadata("./my-cache", "my-key").await?; 89 | /// cacache::read_hash("./my-cache", &sri).await?; 90 | /// 91 | /// Ok(()) 92 | /// } 93 | /// ``` 94 | #[cfg(any(feature = "async-std", feature = "tokio"))] 95 | pub async fn clear>(cache: P) -> Result<()> { 96 | async fn inner(cache: &Path) -> Result<()> { 97 | for entry in cache 98 | .read_dir() 99 | .with_context(|| { 100 | format!( 101 | "Failed to read directory contents while clearing cache, at {}", 102 | cache.display() 103 | ) 104 | })? 105 | .flatten() 106 | { 107 | crate::async_lib::remove_dir_all(entry.path()) 108 | .await 109 | .with_context(|| format!("Failed to clear cache at {}", cache.display()))?; 110 | } 111 | Ok(()) 112 | } 113 | inner(cache.as_ref()).await 114 | } 115 | 116 | /// Removes an individual index entry synchronously. The associated content 117 | /// will be left in the cache. 118 | /// 119 | /// ## Example 120 | /// ```no_run 121 | /// use std::io::Read; 122 | /// 123 | /// fn main() -> cacache::Result<()> { 124 | /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; 125 | /// 126 | /// cacache::remove_sync("./my-cache", "my-key")?; 127 | /// 128 | /// // This fails: 129 | /// cacache::read_sync("./my-cache", "my-key")?; 130 | /// 131 | /// // But this succeeds: 132 | /// cacache::read_hash_sync("./my-cache", &sri)?; 133 | /// 134 | /// Ok(()) 135 | /// } 136 | /// ``` 137 | pub fn remove_sync(cache: P, key: K) -> Result<()> 138 | where 139 | P: AsRef, 140 | K: AsRef, 141 | { 142 | index::delete(cache.as_ref(), key.as_ref()) 143 | } 144 | 145 | /// Removes an individual content entry synchronously. Any index entries 146 | /// pointing to this content will become invalidated. 147 | /// 148 | /// ## Example 149 | /// ```no_run 150 | /// use std::io::Read; 151 | /// 152 | /// fn main() -> cacache::Result<()> { 153 | /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; 154 | /// 155 | /// cacache::remove_hash_sync("./my-cache", &sri)?; 156 | /// 157 | /// // These fail: 158 | /// cacache::read_sync("./my-cache", "my-key")?; 159 | /// cacache::read_hash_sync("./my-cache", &sri)?; 160 | /// 161 | /// // But this succeeds: 162 | /// cacache::metadata_sync("./my-cache", "my-key")?; 163 | /// 164 | /// Ok(()) 165 | /// } 166 | /// ``` 167 | pub fn remove_hash_sync>(cache: P, sri: &Integrity) -> Result<()> { 168 | rm::rm(cache.as_ref(), sri) 169 | } 170 | 171 | /// Removes entire contents of the cache synchronously, including temporary 172 | /// files, the entry index, and all content data. 173 | /// 174 | /// ## Example 175 | /// ```no_run 176 | /// use std::io::Read; 177 | /// 178 | /// fn main() -> cacache::Result<()> { 179 | /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; 180 | /// 181 | /// cacache::clear_sync("./my-cache")?; 182 | /// 183 | /// // These all fail: 184 | /// cacache::read_sync("./my-cache", "my-key")?; 185 | /// cacache::read_hash_sync("./my-cache", &sri)?; 186 | /// cacache::metadata_sync("./my-cache", "my-key")?; 187 | /// 188 | /// Ok(()) 189 | /// } 190 | /// ``` 191 | pub fn clear_sync>(cache: P) -> Result<()> { 192 | fn inner(cache: &Path) -> Result<()> { 193 | for entry in cache 194 | .read_dir() 195 | .with_context(|| { 196 | format!( 197 | "Failed to read directory contents while clearing cache, at {}", 198 | cache.display() 199 | ) 200 | })? 201 | .flatten() 202 | { 203 | fs::remove_dir_all(entry.path()) 204 | .with_context(|| format!("Failed to clear cache at {}", cache.display()))?; 205 | } 206 | Ok(()) 207 | } 208 | inner(cache.as_ref()) 209 | } 210 | 211 | #[cfg(test)] 212 | mod tests { 213 | 214 | #[cfg(feature = "async-std")] 215 | use async_attributes::test as async_test; 216 | #[cfg(feature = "tokio")] 217 | use tokio::test as async_test; 218 | 219 | #[cfg(any(feature = "async-std", feature = "tokio"))] 220 | #[async_test] 221 | async fn test_remove() { 222 | futures::executor::block_on(async { 223 | let tmp = tempfile::tempdir().unwrap(); 224 | let dir = tmp.path().to_owned(); 225 | let sri = crate::write(&dir, "key", b"my-data").await.unwrap(); 226 | 227 | crate::remove(&dir, "key").await.unwrap(); 228 | 229 | let entry = crate::metadata(&dir, "key").await.unwrap(); 230 | assert_eq!(entry, None); 231 | 232 | let data_exists = crate::exists(&dir, &sri).await; 233 | assert!(data_exists); 234 | }); 235 | } 236 | 237 | #[cfg(any(feature = "async-std", feature = "tokio"))] 238 | #[async_test] 239 | async fn test_remove_data() { 240 | futures::executor::block_on(async { 241 | let tmp = tempfile::tempdir().unwrap(); 242 | let dir = tmp.path().to_owned(); 243 | let sri = crate::write(&dir, "key", b"my-data").await.unwrap(); 244 | 245 | crate::remove_hash(&dir, &sri).await.unwrap(); 246 | 247 | let entry = crate::metadata(&dir, "key").await.unwrap(); 248 | assert!(entry.is_some()); 249 | 250 | let data_exists = crate::exists(&dir, &sri).await; 251 | assert!(!data_exists); 252 | }); 253 | } 254 | 255 | #[cfg(any(feature = "async-std", feature = "tokio"))] 256 | #[async_test] 257 | async fn test_clear() { 258 | futures::executor::block_on(async { 259 | let tmp = tempfile::tempdir().unwrap(); 260 | let dir = tmp.path().to_owned(); 261 | let sri = crate::write(&dir, "key", b"my-data").await.unwrap(); 262 | 263 | crate::clear(&dir).await.unwrap(); 264 | 265 | let entry = crate::metadata(&dir, "key").await.unwrap(); 266 | assert!(entry.is_none()); 267 | 268 | let data_exists = crate::exists(&dir, &sri).await; 269 | assert!(!data_exists); 270 | }); 271 | } 272 | 273 | #[test] 274 | fn test_remove_sync() { 275 | let tmp = tempfile::tempdir().unwrap(); 276 | let dir = tmp.path().to_owned(); 277 | let sri = crate::write_sync(&dir, "key", b"my-data").unwrap(); 278 | 279 | crate::remove_sync(&dir, "key").unwrap(); 280 | 281 | let new_entry = crate::metadata_sync(&dir, "key").unwrap(); 282 | assert!(new_entry.is_none()); 283 | 284 | let data_exists = crate::exists_sync(&dir, &sri); 285 | assert!(data_exists); 286 | } 287 | 288 | #[test] 289 | fn test_remove_data_sync() { 290 | let tmp = tempfile::tempdir().unwrap(); 291 | let dir = tmp.path().to_owned(); 292 | let sri = crate::write_sync(&dir, "key", b"my-data").unwrap(); 293 | 294 | crate::remove_hash_sync(&dir, &sri).unwrap(); 295 | 296 | let entry = crate::metadata_sync(&dir, "key").unwrap(); 297 | assert!(entry.is_some()); 298 | 299 | let data_exists = crate::exists_sync(&dir, &sri); 300 | assert!(!data_exists); 301 | } 302 | 303 | #[test] 304 | fn test_clear_sync() { 305 | let tmp = tempfile::tempdir().unwrap(); 306 | let dir = tmp.path().to_owned(); 307 | let sri = crate::write_sync(&dir, "key", b"my-data").unwrap(); 308 | 309 | crate::clear_sync(&dir).unwrap(); 310 | 311 | let entry = crate::metadata_sync(&dir, "key").unwrap(); 312 | assert_eq!(entry, None); 313 | 314 | let data_exists = crate::exists_sync(&dir, &sri); 315 | assert!(!data_exists); 316 | } 317 | } 318 | --------------------------------------------------------------------------------