├── .claude └── CLAUDE.md ├── .gitignore ├── .bootc-dev-infra-commit.txt ├── renovate.json ├── .gemini └── config.yaml ├── Cargo.toml ├── AGENTS.md ├── LICENSE-MIT ├── README.md ├── .github └── workflows │ └── ci.yml ├── examples ├── zstd.rs ├── custom_compressor.rs └── ocidir-import-tar.rs ├── LICENSE-APACHE └── src └── lib.rs /.claude/CLAUDE.md: -------------------------------------------------------------------------------- 1 | ../AGENTS.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock -------------------------------------------------------------------------------- /.bootc-dev-infra-commit.txt: -------------------------------------------------------------------------------- 1 | 3e0c644d172f697e20e5bb4450d407dd293ea14a 2 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "github>platform-engineering-org/.github" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /.gemini/config.yaml: -------------------------------------------------------------------------------- 1 | # NOTE: This file is canonically maintained in 2 | # 3 | # DO NOT EDIT 4 | # 5 | # This config mainly overrides `summary: false` by default 6 | # as it's really noisy. 7 | have_fun: true 8 | code_review: 9 | disable: false 10 | # Even medium level can be quite noisy, I don't think 11 | # we need LOW. Anyone who wants that type of stuff should 12 | # be able to get it locally or before review. 13 | comment_severity_threshold: MEDIUM 14 | max_review_comments: -1 15 | pull_request_opened: 16 | help: false 17 | summary: false # turned off by default 18 | code_review: true 19 | ignore_patterns: [] 20 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ocidir" 3 | description = "A Rust library for reading and writing OCI (opencontainers) layout directories" 4 | version = "0.6.0" 5 | edition = "2024" 6 | license = "MIT OR Apache-2.0" 7 | repository = "https://github.com/containers/ocidir-rs" 8 | keywords = ["oci", "opencontainers", "docker", "podman", "containers"] 9 | 10 | [dependencies] 11 | camino = "1.0.4" 12 | chrono = "0.4.19" 13 | cap-std-ext = "4.0" 14 | flate2 = { features = ["zlib"], default-features = false, version = "1.0.20" } 15 | hex = "0.4.3" 16 | openssl = "0.10.33" 17 | serde = { features = ["derive"], version = "1.0.125" } 18 | serde_json = "1.0.64" 19 | tar = "0.4.38" 20 | thiserror = "2" 21 | # See https://github.com/youki-dev/oci-spec-rs/pull/288 22 | oci-spec = "0.8.2" 23 | zstd = { version = "0.13.2", optional = true } 24 | canon-json = "0.2.0" 25 | 26 | [dev-dependencies] 27 | anyhow = "1.0.89" 28 | cap-tempfile = "3.3.0" 29 | 30 | [features] 31 | zstd = ["dep:zstd"] 32 | zstdmt = ["zstd", "zstd/zstdmt"] 33 | -------------------------------------------------------------------------------- /AGENTS.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Instructions for AI agents 4 | 5 | ## CRITICAL instructions for generating commits 6 | 7 | ### Signed-off-by 8 | 9 | Human review is required for all code that is generated 10 | or assisted by a large language model. If you 11 | are a LLM, you MUST NOT include a `Signed-off-by` 12 | on any automatically generated git commits. Only explicit 13 | human action or request should include a Signed-off-by. 14 | If for example you automatically create a pull request 15 | and the DCO check fails, tell the human to review 16 | the code and give them instructions on how to add 17 | a signoff. 18 | 19 | ### Attribution 20 | 21 | When generating substantial amounts of code, you SHOULD 22 | include an `Assisted-by: TOOLNAME (MODELNAME)`. For example, 23 | `Assisted-by: Goose (Sonnet 4.5)`. 24 | 25 | ## Follow other guidelines 26 | 27 | Look at the project README.md and look for guidelines 28 | related to contribution, such as a CONTRIBUTING.md 29 | and follow those. 30 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 The openat Developers 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ocidir 2 | 3 | [![Crates.io][crates-badge]][crates-url] 4 | 5 | [crates-badge]: https://img.shields.io/crates/v/ocidir.svg 6 | [crates-url]: https://crates.io/crates/ocidir 7 | [![docs.rs](https://docs.rs/ocidir/badge.svg)](https://docs.rs/ocidir) 8 | 9 | # Read and write to OCI image layout directories 10 | 11 | This library contains medium and low-level APIs for working with 12 | [OCI images], which are basically a directory with blobs and JSON files 13 | for metadata. 14 | 15 | ## Dependency on cap-std 16 | 17 | This library makes use of [cap-std] to operate in a capability-oriented 18 | fashion. In practice, the code in this project is well tested and would 19 | not traverse outside its own path root. However, using capabilities 20 | is a generally good idea when operating in the container ecosystem, 21 | in particular when actively processing tar streams. 22 | 23 | ## Examples 24 | 25 | To access an existing OCI directory: 26 | 27 | ```rust,no_run 28 | # use ocidir::cap_std; 29 | # use anyhow::{anyhow, Result}; 30 | # fn main() -> anyhow::Result<()> { 31 | let d = cap_std::fs::Dir::open_ambient_dir("/path/to/ocidir", cap_std::ambient_authority())?; 32 | let d = ocidir::OciDir::open(d)?; 33 | println!("{:?}", d.read_index()?); 34 | # Ok(()) 35 | # } 36 | ``` 37 | 38 | [cap-std]: https://docs.rs/cap-std/ 39 | [OCI images]: https://github.com/opencontainers/image-spec 40 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # Maintained in https://github.com/coreos/repo-templates 2 | # Do not edit downstream. 3 | 4 | name: Rust 5 | on: 6 | push: 7 | branches: [main] 8 | pull_request: 9 | branches: [main] 10 | permissions: 11 | contents: read 12 | 13 | # don't waste job slots on superseded code 14 | concurrency: 15 | group: ${{ github.workflow }}-${{ github.ref }} 16 | cancel-in-progress: true 17 | 18 | env: 19 | CARGO_TERM_COLOR: always 20 | # Pinned toolchain for linting 21 | ACTIONS_LINTS_TOOLCHAIN: 1.86.0 22 | 23 | jobs: 24 | tests-stable: 25 | name: Tests, stable toolchain 26 | runs-on: ubuntu-latest 27 | steps: 28 | - name: Check out repository 29 | uses: actions/checkout@v4 30 | - name: Install toolchain 31 | uses: dtolnay/rust-toolchain@v1 32 | with: 33 | toolchain: stable 34 | - name: Cache build artifacts 35 | uses: Swatinem/rust-cache@v2 36 | - name: cargo build 37 | run: cargo build --all-targets --all-features 38 | - name: cargo test 39 | run: cargo test --all-targets --all-features 40 | # https://github.com/rust-lang/cargo/issues/6669 41 | - name: cargo test --doc 42 | run: cargo test --doc --all-features 43 | linting: 44 | name: Lints, pinned toolchain 45 | runs-on: ubuntu-latest 46 | steps: 47 | - name: Check out repository 48 | uses: actions/checkout@v4 49 | - name: Install toolchain 50 | uses: dtolnay/rust-toolchain@v1 51 | with: 52 | toolchain: ${{ env.ACTIONS_LINTS_TOOLCHAIN }} 53 | components: rustfmt, clippy 54 | - name: Cache build artifacts 55 | uses: Swatinem/rust-cache@v2 56 | - name: cargo fmt (check) 57 | run: cargo fmt -- --check -l 58 | - name: cargo clippy (warnings) 59 | run: cargo clippy --all-targets --all-features -- -D warnings 60 | -------------------------------------------------------------------------------- /examples/zstd.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "zstdmt")] 2 | fn main() { 3 | use std::{env, path::PathBuf}; 4 | 5 | use cap_tempfile::TempDir; 6 | use oci_spec::image::Platform; 7 | use ocidir::OciDir; 8 | let dir = TempDir::new(ocidir::cap_std::ambient_authority()).unwrap(); 9 | let oci_dir = OciDir::ensure(dir.try_clone().unwrap()).unwrap(); 10 | 11 | let mut manifest = oci_dir.new_empty_manifest().unwrap().build().unwrap(); 12 | let mut config = ocidir::oci_spec::image::ImageConfigurationBuilder::default() 13 | .build() 14 | .unwrap(); 15 | 16 | // Add the src as a layer 17 | let writer = oci_dir.create_layer_zstd(Some(0)).unwrap(); 18 | let mut builder = tar::Builder::new(writer); 19 | builder.follow_symlinks(false); 20 | 21 | builder 22 | .append_dir_all(".", PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src")) 23 | .unwrap(); 24 | 25 | let layer = builder.into_inner().unwrap().complete().unwrap(); 26 | oci_dir.push_layer(&mut manifest, &mut config, layer, "src", None); 27 | 28 | // Add the examples as a layer, using multithreaded compression 29 | let writer = oci_dir.create_layer_zstd_multithread(Some(0), 4).unwrap(); 30 | let mut builder = tar::Builder::new(writer); 31 | builder.follow_symlinks(false); 32 | builder 33 | .append_dir_all( 34 | ".", 35 | PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples"), 36 | ) 37 | .unwrap(); 38 | let layer = builder.into_inner().unwrap().complete().unwrap(); 39 | oci_dir.push_layer(&mut manifest, &mut config, layer, "examples", None); 40 | 41 | println!( 42 | "Created image with manifest: {}", 43 | manifest.to_string_pretty().unwrap() 44 | ); 45 | 46 | // Add the image manifest 47 | let _descriptor = oci_dir 48 | .insert_manifest_and_config(manifest.clone(), config, None, Platform::default()) 49 | .unwrap(); 50 | } 51 | 52 | #[cfg(not(feature = "zstdmt"))] 53 | fn main() { 54 | println!("Run this example with `cargo run --example zstd --features zstdmt`"); 55 | } 56 | -------------------------------------------------------------------------------- /examples/custom_compressor.rs: -------------------------------------------------------------------------------- 1 | /// Example that shows how to use a custom compression and media type for image layers. 2 | /// The example below does no compression. 3 | use std::{env, io, path::PathBuf}; 4 | 5 | use oci_spec::image::Platform; 6 | use ocidir::{BlobWriter, OciDir, WriteComplete, cap_std::fs::Dir}; 7 | 8 | struct NoCompression<'a>(BlobWriter<'a>); 9 | 10 | impl io::Write for NoCompression<'_> { 11 | fn write(&mut self, buf: &[u8]) -> io::Result { 12 | self.0.write(buf) 13 | } 14 | 15 | fn flush(&mut self) -> io::Result<()> { 16 | self.0.flush() 17 | } 18 | } 19 | 20 | impl<'a> WriteComplete> for NoCompression<'a> { 21 | fn complete(self) -> io::Result> { 22 | Ok(self.0) 23 | } 24 | } 25 | 26 | fn main() { 27 | let dir = Dir::open_ambient_dir(env::temp_dir(), ocidir::cap_std::ambient_authority()).unwrap(); 28 | let oci_dir = OciDir::ensure(dir).unwrap(); 29 | 30 | let mut manifest = oci_dir.new_empty_manifest().unwrap().build().unwrap(); 31 | let mut config = ocidir::oci_spec::image::ImageConfigurationBuilder::default() 32 | .build() 33 | .unwrap(); 34 | 35 | // Add the src as a layer 36 | let writer = oci_dir 37 | .create_custom_layer( 38 | |bw| Ok(NoCompression(bw)), 39 | oci_spec::image::MediaType::ImageLayer, 40 | ) 41 | .unwrap(); 42 | let mut builder = tar::Builder::new(writer); 43 | builder.follow_symlinks(false); 44 | 45 | builder 46 | .append_dir_all(".", PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src")) 47 | .unwrap(); 48 | 49 | let layer = builder.into_inner().unwrap().complete().unwrap(); 50 | oci_dir.push_layer(&mut manifest, &mut config, layer, "src", None); 51 | 52 | println!( 53 | "Created image with manifest: {}", 54 | manifest.to_string_pretty().unwrap() 55 | ); 56 | 57 | // Add the image manifest 58 | let _descriptor = oci_dir 59 | .insert_manifest_and_config(manifest.clone(), config, None, Platform::default()) 60 | .unwrap(); 61 | } 62 | -------------------------------------------------------------------------------- /examples/ocidir-import-tar.rs: -------------------------------------------------------------------------------- 1 | //! # Import a pre-generated tarball, wrapping with OCI metadata into an OCI directory. 2 | //! 3 | //! This little exmaple shows a bit of how to use the ocidir API. But it has a use case 4 | //! for low-level testing of OCI runtimes by injecting arbitrary tarballs. 5 | 6 | use std::io::BufReader; 7 | use std::{fs::File, path::Path}; 8 | 9 | use anyhow::Context; 10 | use cap_tempfile::cap_std; 11 | use chrono::Utc; 12 | use oci_spec::image::{MediaType, Platform}; 13 | use ocidir::OciDir; 14 | 15 | fn import(oci_dir: &OciDir, name: &str, src: File) -> anyhow::Result<()> { 16 | let mtime = src.metadata()?.modified()?; 17 | let mut input_tar = BufReader::new(src); 18 | let created = chrono::DateTime::::from(mtime); 19 | 20 | let mut manifest = oci_dir.new_empty_manifest().unwrap().build().unwrap(); 21 | let mut config = ocidir::oci_spec::image::ImageConfigurationBuilder::default() 22 | .build() 23 | .unwrap(); 24 | 25 | // Add the src as a layer 26 | let mut writer = oci_dir.create_blob().unwrap(); 27 | std::io::copy(&mut input_tar, &mut writer)?; 28 | 29 | let blob = writer.complete()?; 30 | let descriptor = blob 31 | .descriptor() 32 | .media_type(MediaType::ImageLayer) 33 | .build() 34 | .unwrap(); 35 | let blob_digest = descriptor.digest().to_string(); 36 | manifest.layers_mut().push(descriptor); 37 | let mut rootfs = config.rootfs().clone(); 38 | rootfs.diff_ids_mut().push(blob_digest); 39 | config.set_rootfs(rootfs); 40 | let h = oci_spec::image::HistoryBuilder::default() 41 | .created(created.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) 42 | .created_by(name.to_string()) 43 | .build() 44 | .unwrap(); 45 | config.history_mut().get_or_insert_default().push(h); 46 | 47 | println!( 48 | "Created image with manifest: {}", 49 | manifest.to_string_pretty().unwrap() 50 | ); 51 | 52 | // Add the image manifest 53 | let _descriptor = oci_dir 54 | .insert_manifest_and_config( 55 | manifest.clone(), 56 | config, 57 | Some("latest"), 58 | Platform::default(), 59 | ) 60 | .unwrap(); 61 | 62 | Ok(()) 63 | } 64 | 65 | fn main() -> anyhow::Result<()> { 66 | let args = std::env::args().collect::>(); 67 | let ocidir = args[1].as_str(); 68 | let path = Path::new(args[2].as_str()); 69 | let Some(name) = path.file_stem().and_then(|v| v.to_str()) else { 70 | anyhow::bail!("Invalid path: {path:?}"); 71 | }; 72 | let f = File::open(path).with_context(|| format!("Opening {path:?}"))?; 73 | 74 | let dir = &cap_std::fs::Dir::open_ambient_dir(ocidir, cap_std::ambient_authority()) 75 | .with_context(|| format!("Opening {ocidir}"))?; 76 | let oci_dir = OciDir::ensure(dir.try_clone()?)?; 77 | 78 | import(&oci_dir, name, f)?; 79 | Ok(()) 80 | } 81 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | #![deny(missing_docs)] 3 | 4 | use canon_json::CanonicalFormatter; 5 | use cap_std::fs::{Dir, DirBuilderExt}; 6 | use cap_std_ext::cap_tempfile; 7 | use cap_std_ext::dirext::CapStdExtDirExt; 8 | use flate2::write::GzEncoder; 9 | use oci_image::MediaType; 10 | use oci_spec::image::{ 11 | self as oci_image, Descriptor, Digest, ImageConfiguration, ImageIndex, ImageManifest, 12 | Sha256Digest, 13 | }; 14 | use openssl::hash::{Hasher, MessageDigest}; 15 | use serde::{Deserialize, Serialize}; 16 | use std::collections::{HashMap, HashSet}; 17 | use std::fmt::Debug; 18 | use std::fs::File; 19 | use std::io::{BufReader, BufWriter, prelude::*}; 20 | use std::marker::PhantomData; 21 | use std::path::{Path, PathBuf}; 22 | use std::str::FromStr; 23 | use thiserror::Error; 24 | 25 | // Re-export our dependencies that are used as part of the public API. 26 | pub use cap_std_ext::cap_std; 27 | pub use oci_spec; 28 | 29 | /// Path inside an OCI directory to the blobs 30 | const BLOBDIR: &str = "blobs/sha256"; 31 | 32 | const OCI_TAG_ANNOTATION: &str = "org.opencontainers.image.ref.name"; 33 | 34 | /// Errors returned by this crate. 35 | #[derive(Error, Debug)] 36 | #[non_exhaustive] 37 | pub enum Error { 38 | #[error("i/o error")] 39 | /// An input/output error 40 | Io(#[from] std::io::Error), 41 | #[error("serialization error")] 42 | /// Returned when serialization or deserialization fails 43 | SerDe(#[from] serde_json::Error), 44 | #[error("parsing OCI value")] 45 | /// Returned when an OCI spec error occurs 46 | OciSpecError(#[from] oci_spec::OciSpecError), 47 | #[error("unexpected cryptographic routine error")] 48 | /// Returned when a cryptographic routine encounters an unexpected problem 49 | CryptographicError(Box), 50 | #[error("Expected digest {expected} but found {found}")] 51 | /// Returned when a digest does not match 52 | DigestMismatch { 53 | /// Expected digest value 54 | expected: Box, 55 | /// Found digest value 56 | found: Box, 57 | }, 58 | #[error("Expected size {expected} but found {found}")] 59 | /// Returned when a descriptor digest does not match what was expected 60 | SizeMismatch { 61 | /// Expected size value 62 | expected: u64, 63 | /// Found size value 64 | found: u64, 65 | }, 66 | #[error("Expected digest algorithm sha256 but found {found}")] 67 | /// Returned when a digest algorithm is not supported 68 | UnsupportedDigestAlgorithm { 69 | /// The unsupported digest algorithm that was found 70 | found: Box, 71 | }, 72 | #[error("Cannot find the Image Index (index.json)")] 73 | /// Returned when the OCI Image Index (index.json) is missing 74 | MissingImageIndex, 75 | #[error("Unexpected media type {media_type}")] 76 | /// Returned when there's an unexpected media type 77 | UnexpectedMediaType { 78 | /// The unexpected media type that was encountered 79 | media_type: MediaType, 80 | }, 81 | #[error("error")] 82 | /// An unknown other error 83 | Other(Box), 84 | } 85 | 86 | /// The error type returned from this crate. 87 | pub type Result = std::result::Result; 88 | 89 | impl From for Error { 90 | fn from(value: openssl::error::Error) -> Self { 91 | Self::CryptographicError(value.to_string().into()) 92 | } 93 | } 94 | 95 | impl From for Error { 96 | fn from(value: openssl::error::ErrorStack) -> Self { 97 | Self::CryptographicError(value.to_string().into()) 98 | } 99 | } 100 | 101 | // This is intentionally an empty struct 102 | // See https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor 103 | #[derive(Serialize, Deserialize)] 104 | struct EmptyDescriptor {} 105 | 106 | /// Completed blob metadata 107 | #[derive(Debug)] 108 | pub struct Blob { 109 | /// SHA-256 digest 110 | sha256: oci_image::Sha256Digest, 111 | /// Size 112 | size: u64, 113 | } 114 | 115 | impl Blob { 116 | /// The SHA-256 digest for this blob 117 | pub fn sha256(&self) -> &oci_image::Sha256Digest { 118 | &self.sha256 119 | } 120 | 121 | /// Descriptor 122 | pub fn descriptor(&self) -> oci_image::DescriptorBuilder { 123 | oci_image::DescriptorBuilder::default() 124 | .digest(self.sha256.clone()) 125 | .size(self.size) 126 | } 127 | 128 | /// Return the size of this blob 129 | pub fn size(&self) -> u64 { 130 | self.size 131 | } 132 | } 133 | 134 | /// Completed layer metadata 135 | #[derive(Debug)] 136 | pub struct Layer { 137 | /// The underlying blob (usually compressed) 138 | pub blob: Blob, 139 | /// The uncompressed digest, which will be used for "diffid"s 140 | pub uncompressed_sha256: Sha256Digest, 141 | /// The media type of the layer 142 | pub media_type: MediaType, 143 | } 144 | 145 | impl Layer { 146 | /// Return the descriptor for this layer 147 | pub fn descriptor(&self) -> oci_image::DescriptorBuilder { 148 | self.blob.descriptor().media_type(self.media_type.clone()) 149 | } 150 | 151 | /// Return a Digest instance for the uncompressed SHA-256. 152 | pub fn uncompressed_sha256_as_digest(&self) -> Digest { 153 | self.uncompressed_sha256.clone().into() 154 | } 155 | } 156 | 157 | /// Create an OCI blob. 158 | pub struct BlobWriter<'a> { 159 | /// Compute checksum 160 | hash: Hasher, 161 | /// Target file 162 | target: Option>>, 163 | size: u64, 164 | } 165 | 166 | impl Debug for BlobWriter<'_> { 167 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 168 | f.debug_struct("BlobWriter") 169 | .field("target", &self.target) 170 | .field("size", &self.size) 171 | .finish() 172 | } 173 | } 174 | 175 | #[derive(Debug)] 176 | /// An opened OCI directory. 177 | pub struct OciDir { 178 | /// The underlying directory. 179 | dir: Dir, 180 | blobs_dir: Dir, 181 | } 182 | 183 | fn sha256_of_descriptor(desc: &Descriptor) -> Result<&str> { 184 | desc.as_digest_sha256() 185 | .ok_or_else(|| Error::UnsupportedDigestAlgorithm { 186 | found: desc.digest().to_string().into(), 187 | }) 188 | } 189 | 190 | impl OciDir { 191 | /// Create an empty config descriptor. 192 | /// See https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor 193 | /// Our API right now always mutates a manifest, which means we need 194 | /// a "valid" manifest, which requires a "valid" config descriptor. 195 | fn empty_config_descriptor(&self) -> Result { 196 | let empty_descriptor = oci_image::DescriptorBuilder::default() 197 | .media_type(MediaType::EmptyJSON) 198 | .size(2_u32) 199 | .digest(Sha256Digest::from_str( 200 | "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", 201 | )?) 202 | .data("e30=") 203 | .build()?; 204 | 205 | if !self 206 | .dir 207 | .exists(OciDir::parse_descriptor_to_path(&empty_descriptor)?) 208 | { 209 | let mut blob = self.create_blob()?; 210 | serde_json::to_writer(&mut blob, &EmptyDescriptor {})?; 211 | blob.complete_verified_as(&empty_descriptor)?; 212 | } 213 | 214 | Ok(empty_descriptor) 215 | } 216 | 217 | /// Generate a valid empty manifest. See above. 218 | pub fn new_empty_manifest(&self) -> Result { 219 | Ok(oci_image::ImageManifestBuilder::default() 220 | .schema_version(oci_image::SCHEMA_VERSION) 221 | .config(self.empty_config_descriptor()?) 222 | .layers(Vec::new())) 223 | } 224 | 225 | /// Open the OCI directory at the target path; if it does not already 226 | /// have the standard OCI metadata, it is created. 227 | pub fn ensure(dir: Dir) -> Result { 228 | let mut db = cap_std::fs::DirBuilder::new(); 229 | db.recursive(true).mode(0o755); 230 | dir.ensure_dir_with(BLOBDIR, &db)?; 231 | if !dir.try_exists("oci-layout")? { 232 | dir.atomic_write("oci-layout", r#"{"imageLayoutVersion":"1.0.0"}"#)?; 233 | } 234 | Self::open(dir) 235 | } 236 | 237 | /// Clone an OCI directory, using reflinks for blobs. 238 | pub fn clone_to(&self, destdir: &Dir, p: impl AsRef) -> Result { 239 | let p = p.as_ref(); 240 | destdir.create_dir(p)?; 241 | let cloned = Self::ensure(destdir.open_dir(p)?)?; 242 | for blob in self.blobs_dir.entries()? { 243 | let blob = blob?; 244 | let path = Path::new(BLOBDIR).join(blob.file_name()); 245 | let mut src = self.dir.open(&path).map(BufReader::new)?; 246 | self.dir 247 | .atomic_replace_with(&path, |w| std::io::copy(&mut src, w))?; 248 | } 249 | Ok(cloned) 250 | } 251 | 252 | /// Open an existing OCI directory. 253 | pub fn open(dir: Dir) -> Result { 254 | let blobs_dir = dir.open_dir(BLOBDIR)?; 255 | Self::open_with_external_blobs(dir, blobs_dir) 256 | } 257 | 258 | /// Open an existing OCI directory with a separate cap_std::Dir for blobs/sha256 259 | /// This is useful when `blobs/sha256` might contain symlinks pointing outside the oci 260 | /// directory, e.g. when sharing blobs across OCI repositories. The LXC OCI template uses this 261 | /// feature. 262 | pub fn open_with_external_blobs(dir: Dir, blobs_dir: Dir) -> Result { 263 | Ok(Self { dir, blobs_dir }) 264 | } 265 | 266 | /// Return the underlying directory. 267 | pub fn dir(&self) -> &Dir { 268 | &self.dir 269 | } 270 | 271 | /// Return the underlying directory for blobs. 272 | pub fn blobs_dir(&self) -> &Dir { 273 | &self.blobs_dir 274 | } 275 | 276 | /// Write a serializable data (JSON) as an OCI blob 277 | pub fn write_json_blob( 278 | &self, 279 | v: &S, 280 | media_type: oci_image::MediaType, 281 | ) -> Result { 282 | let mut w = BlobWriter::new(&self.dir)?; 283 | let mut ser = serde_json::Serializer::with_formatter(&mut w, CanonicalFormatter::new()); 284 | v.serialize(&mut ser)?; 285 | let blob = w.complete()?; 286 | Ok(blob.descriptor().media_type(media_type)) 287 | } 288 | 289 | /// Create a blob (can be anything). 290 | pub fn create_blob(&self) -> Result> { 291 | BlobWriter::new(&self.dir) 292 | } 293 | 294 | /// Create a layer writer with a custom encoder and 295 | /// media type 296 | pub fn create_custom_layer<'a, W: WriteComplete>>( 297 | &'a self, 298 | create: impl FnOnce(BlobWriter<'a>) -> std::io::Result, 299 | media_type: MediaType, 300 | ) -> Result> { 301 | let bw = BlobWriter::new(&self.dir)?; 302 | Ok(LayerWriter::new(create(bw)?, media_type)) 303 | } 304 | 305 | /// Create a writer for a new gzip+tar blob; the contents 306 | /// are not parsed, but are expected to be a tarball. 307 | pub fn create_gzip_layer<'a>( 308 | &'a self, 309 | c: Option, 310 | ) -> Result>>> { 311 | let creator = |bw: BlobWriter<'a>| Ok(GzEncoder::new(bw, c.unwrap_or_default())); 312 | self.create_custom_layer(creator, MediaType::ImageLayerGzip) 313 | } 314 | 315 | /// Create a tar output stream, backed by a blob 316 | pub fn create_layer( 317 | &'_ self, 318 | c: Option, 319 | ) -> Result>>>> { 320 | Ok(tar::Builder::new(self.create_gzip_layer(c)?)) 321 | } 322 | 323 | #[cfg(feature = "zstd")] 324 | /// Create a writer for a new zstd+tar blob; the contents 325 | /// are not parsed, but are expected to be a tarball. 326 | /// 327 | /// This method is only available when the `zstd` feature is enabled. 328 | pub fn create_layer_zstd<'a>( 329 | &'a self, 330 | compression_level: Option, 331 | ) -> Result>>> { 332 | let creator = |bw: BlobWriter<'a>| zstd::Encoder::new(bw, compression_level.unwrap_or(0)); 333 | self.create_custom_layer(creator, MediaType::ImageLayerZstd) 334 | } 335 | 336 | #[cfg(feature = "zstdmt")] 337 | /// Create a writer for a new zstd+tar blob; the contents 338 | /// are not parsed, but are expected to be a tarball. 339 | /// The compression is multithreaded. 340 | /// 341 | /// The `n_workers` parameter specifies the number of threads to use for compression, per 342 | /// [zstd::Encoder::multithread]] 343 | /// 344 | /// This method is only available when the `zstdmt` feature is enabled. 345 | pub fn create_layer_zstd_multithread<'a>( 346 | &'a self, 347 | compression_level: Option, 348 | n_workers: u32, 349 | ) -> Result>>> { 350 | let creator = |bw: BlobWriter<'a>| { 351 | let mut encoder = zstd::Encoder::new(bw, compression_level.unwrap_or(0))?; 352 | encoder.multithread(n_workers)?; 353 | Ok(encoder) 354 | }; 355 | self.create_custom_layer(creator, MediaType::ImageLayerZstd) 356 | } 357 | 358 | /// Add a layer to the top of the image stack. The firsh pushed layer becomes the root. 359 | pub fn push_layer( 360 | &self, 361 | manifest: &mut oci_image::ImageManifest, 362 | config: &mut oci_image::ImageConfiguration, 363 | layer: Layer, 364 | description: &str, 365 | annotations: Option>, 366 | ) { 367 | self.push_layer_annotated(manifest, config, layer, annotations, description); 368 | } 369 | 370 | /// Add a layer to the top of the image stack with optional annotations. 371 | /// 372 | /// This is otherwise equivalent to [`Self::push_layer`]. 373 | pub fn push_layer_annotated( 374 | &self, 375 | manifest: &mut oci_image::ImageManifest, 376 | config: &mut oci_image::ImageConfiguration, 377 | layer: Layer, 378 | annotations: Option>>, 379 | description: &str, 380 | ) { 381 | let created = chrono::offset::Utc::now(); 382 | self.push_layer_full(manifest, config, layer, annotations, description, created) 383 | } 384 | 385 | /// Add a layer to the top of the image stack with optional annotations and desired timestamp. 386 | /// 387 | /// This is otherwise equivalent to [`Self::push_layer_annotated`]. 388 | pub fn push_layer_full( 389 | &self, 390 | manifest: &mut oci_image::ImageManifest, 391 | config: &mut oci_image::ImageConfiguration, 392 | layer: Layer, 393 | annotations: Option>>, 394 | description: &str, 395 | created: chrono::DateTime, 396 | ) { 397 | let history = oci_image::HistoryBuilder::default() 398 | .created(created.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) 399 | .created_by(description.to_string()) 400 | .build() 401 | .unwrap(); 402 | self.push_layer_with_history_annotated(manifest, config, layer, annotations, Some(history)); 403 | } 404 | 405 | /// Add a layer to the top of the image stack with optional annotations and desired history entry. 406 | /// 407 | /// This is otherwise equivalent to [`Self::push_layer_annotated`]. 408 | pub fn push_layer_with_history_annotated( 409 | &self, 410 | manifest: &mut oci_image::ImageManifest, 411 | config: &mut oci_image::ImageConfiguration, 412 | layer: Layer, 413 | annotations: Option>>, 414 | history: Option, 415 | ) { 416 | let mut builder = layer.descriptor(); 417 | if let Some(annotations) = annotations { 418 | builder = builder.annotations(annotations); 419 | } 420 | let blobdesc = builder.build().unwrap(); 421 | manifest.layers_mut().push(blobdesc); 422 | let mut rootfs = config.rootfs().clone(); 423 | rootfs 424 | .diff_ids_mut() 425 | .push(layer.uncompressed_sha256_as_digest().to_string()); 426 | config.set_rootfs(rootfs); 427 | let history = if let Some(history) = history { 428 | history 429 | } else { 430 | oci_image::HistoryBuilder::default().build().unwrap() 431 | }; 432 | config.history_mut().get_or_insert_default().push(history); 433 | } 434 | 435 | /// Add a layer to the top of the image stack with desired history entry. 436 | /// 437 | /// This is otherwise equivalent to [`Self::push_layer`]. 438 | pub fn push_layer_with_history( 439 | &self, 440 | manifest: &mut oci_image::ImageManifest, 441 | config: &mut oci_image::ImageConfiguration, 442 | layer: Layer, 443 | history: Option, 444 | ) { 445 | let annotations: Option> = None; 446 | self.push_layer_with_history_annotated(manifest, config, layer, annotations, history); 447 | } 448 | 449 | fn parse_descriptor_to_path(desc: &oci_spec::image::Descriptor) -> Result { 450 | let digest = sha256_of_descriptor(desc)?; 451 | Ok(PathBuf::from(digest)) 452 | } 453 | 454 | /// Open a blob; its size is validated as a sanity check. 455 | pub fn read_blob(&self, desc: &oci_spec::image::Descriptor) -> Result { 456 | let path = Self::parse_descriptor_to_path(desc)?; 457 | let f = self.blobs_dir.open(path).map(|f| f.into_std())?; 458 | let expected: u64 = desc.size(); 459 | let found = f.metadata()?.len(); 460 | if expected != found { 461 | return Err(Error::SizeMismatch { expected, found }); 462 | } 463 | Ok(f) 464 | } 465 | 466 | /// Returns `true` if the blob with this digest is already present. 467 | pub fn has_blob(&self, desc: &oci_spec::image::Descriptor) -> Result { 468 | let path = Self::parse_descriptor_to_path(desc)?; 469 | self.blobs_dir.try_exists(path).map_err(Into::into) 470 | } 471 | 472 | /// Returns `true` if the manifest is already present. 473 | pub fn has_manifest(&self, desc: &oci_spec::image::Descriptor) -> Result { 474 | let index = self.read_index()?; 475 | Ok(index 476 | .manifests() 477 | .iter() 478 | .any(|m| m.digest() == desc.digest())) 479 | } 480 | 481 | /// Read a JSON blob. 482 | pub fn read_json_blob( 483 | &self, 484 | desc: &oci_spec::image::Descriptor, 485 | ) -> Result { 486 | let blob = BufReader::new(self.read_blob(desc)?); 487 | serde_json::from_reader(blob).map_err(Into::into) 488 | } 489 | 490 | /// Write a configuration blob. 491 | pub fn write_config( 492 | &self, 493 | config: oci_image::ImageConfiguration, 494 | ) -> Result { 495 | Ok(self 496 | .write_json_blob(&config, MediaType::ImageConfig)? 497 | .build() 498 | .unwrap()) 499 | } 500 | 501 | /// Read the image index. 502 | pub fn read_index(&self) -> Result { 503 | let r = if let Some(index) = self.dir.open_optional("index.json")?.map(BufReader::new) { 504 | oci_image::ImageIndex::from_reader(index)? 505 | } else { 506 | return Err(Error::MissingImageIndex); 507 | }; 508 | Ok(r) 509 | } 510 | 511 | /// Write a manifest as a blob, and replace the index with a reference to it. 512 | pub fn insert_manifest( 513 | &self, 514 | manifest: oci_image::ImageManifest, 515 | tag: Option<&str>, 516 | platform: oci_image::Platform, 517 | ) -> Result { 518 | let mut manifest = self 519 | .write_json_blob(&manifest, MediaType::ImageManifest)? 520 | .platform(platform) 521 | .build() 522 | .unwrap(); 523 | if let Some(tag) = tag { 524 | let annotations: HashMap<_, _> = [(OCI_TAG_ANNOTATION.to_string(), tag.to_string())] 525 | .into_iter() 526 | .collect(); 527 | manifest.set_annotations(Some(annotations)); 528 | } 529 | 530 | let index = match self.read_index() { 531 | Ok(mut index) => { 532 | let mut manifests = index.manifests().clone(); 533 | if let Some(tag) = tag { 534 | manifests.retain(|d| !Self::descriptor_is_tagged(d, tag)); 535 | } 536 | manifests.push(manifest.clone()); 537 | index.set_manifests(manifests); 538 | index 539 | } 540 | Err(Error::MissingImageIndex) => oci_image::ImageIndexBuilder::default() 541 | .schema_version(oci_image::SCHEMA_VERSION) 542 | .manifests(vec![manifest.clone()]) 543 | .build()?, 544 | Err(e) => { 545 | return Err(e); 546 | } 547 | }; 548 | 549 | self.dir 550 | .atomic_replace_with("index.json", |mut w| -> Result<()> { 551 | let mut ser = 552 | serde_json::Serializer::with_formatter(&mut w, CanonicalFormatter::new()); 553 | index.serialize(&mut ser)?; 554 | Ok(()) 555 | })?; 556 | Ok(manifest) 557 | } 558 | 559 | /// Convenience helper to write the provided config, update the manifest to use it, then call [`insert_manifest`]. 560 | pub fn insert_manifest_and_config( 561 | &self, 562 | mut manifest: oci_image::ImageManifest, 563 | config: oci_image::ImageConfiguration, 564 | tag: Option<&str>, 565 | platform: oci_image::Platform, 566 | ) -> Result { 567 | let config = self.write_config(config)?; 568 | manifest.set_config(config); 569 | self.insert_manifest(manifest, tag, platform) 570 | } 571 | 572 | /// Write a manifest as a blob, and replace the index with a reference to it. 573 | pub fn replace_with_single_manifest( 574 | &self, 575 | manifest: oci_image::ImageManifest, 576 | platform: oci_image::Platform, 577 | ) -> Result<()> { 578 | let manifest = self 579 | .write_json_blob(&manifest, MediaType::ImageManifest)? 580 | .platform(platform) 581 | .build() 582 | .unwrap(); 583 | 584 | let index_data = oci_image::ImageIndexBuilder::default() 585 | .schema_version(oci_image::SCHEMA_VERSION) 586 | .manifests(vec![manifest]) 587 | .build() 588 | .unwrap(); 589 | self.dir 590 | .atomic_replace_with("index.json", |mut w| -> Result<()> { 591 | let mut ser = 592 | serde_json::Serializer::with_formatter(&mut w, CanonicalFormatter::new()); 593 | index_data.serialize(&mut ser)?; 594 | Ok(()) 595 | })?; 596 | Ok(()) 597 | } 598 | 599 | fn descriptor_is_tagged(d: &Descriptor, tag: &str) -> bool { 600 | d.annotations() 601 | .as_ref() 602 | .and_then(|annos| annos.get(OCI_TAG_ANNOTATION)) 603 | .filter(|tagval| tagval.as_str() == tag) 604 | .is_some() 605 | } 606 | 607 | /// Find the manifest with the provided tag 608 | pub fn find_manifest_with_tag(&self, tag: &str) -> Result> { 609 | let desc = self.find_manifest_descriptor_with_tag(tag)?; 610 | desc.map(|img| self.read_json_blob(&img)).transpose() 611 | } 612 | 613 | /// Find the manifest descriptor with the provided tag 614 | pub fn find_manifest_descriptor_with_tag( 615 | &self, 616 | tag: &str, 617 | ) -> Result> { 618 | let idx = self.read_index()?; 619 | Ok(idx 620 | .manifests() 621 | .iter() 622 | .find(|desc| Self::descriptor_is_tagged(desc, tag)) 623 | .cloned()) 624 | } 625 | 626 | /// Verify a single manifest and all of its referenced objects. 627 | /// Skips already validated blobs referenced by digest in `validated`, 628 | /// and updates that set with ones we did validate. 629 | fn fsck_one_manifest( 630 | &self, 631 | manifest: &ImageManifest, 632 | validated: &mut HashSet>, 633 | ) -> Result<()> { 634 | let config_digest = sha256_of_descriptor(manifest.config())?; 635 | match manifest.config().media_type() { 636 | MediaType::ImageConfig => { 637 | let _: ImageConfiguration = self.read_json_blob(manifest.config())?; 638 | } 639 | MediaType::EmptyJSON => { 640 | let _: EmptyDescriptor = self.read_json_blob(manifest.config())?; 641 | } 642 | media_type => { 643 | return Err(Error::UnexpectedMediaType { 644 | media_type: media_type.clone(), 645 | }); 646 | } 647 | } 648 | validated.insert(config_digest.into()); 649 | for layer in manifest.layers() { 650 | let expected = sha256_of_descriptor(layer)?; 651 | if validated.contains(expected) { 652 | continue; 653 | } 654 | let mut f = self.read_blob(layer)?; 655 | let mut digest = Hasher::new(MessageDigest::sha256())?; 656 | std::io::copy(&mut f, &mut digest)?; 657 | let found = hex::encode( 658 | digest 659 | .finish() 660 | .map_err(|e| Error::Other(e.to_string().into()))?, 661 | ); 662 | if expected != found { 663 | return Err(Error::DigestMismatch { 664 | expected: expected.into(), 665 | found: found.into(), 666 | }); 667 | } 668 | validated.insert(expected.into()); 669 | } 670 | Ok(()) 671 | } 672 | 673 | /// Verify consistency of the index, its manifests, the config and blobs (all the latter) 674 | /// by verifying their descriptor. 675 | pub fn fsck(&self) -> Result { 676 | let index = self.read_index()?; 677 | let mut validated_blobs = HashSet::new(); 678 | for manifest_descriptor in index.manifests() { 679 | let expected_sha256 = sha256_of_descriptor(manifest_descriptor)?; 680 | let manifest: ImageManifest = self.read_json_blob(manifest_descriptor)?; 681 | validated_blobs.insert(expected_sha256.into()); 682 | self.fsck_one_manifest(&manifest, &mut validated_blobs)?; 683 | } 684 | Ok(validated_blobs.len().try_into().unwrap()) 685 | } 686 | } 687 | 688 | impl<'a> BlobWriter<'a> { 689 | fn new(ocidir: &'a Dir) -> Result { 690 | Ok(Self { 691 | hash: Hasher::new(MessageDigest::sha256())?, 692 | // FIXME add ability to choose filename after completion 693 | target: Some(BufWriter::new(cap_tempfile::TempFile::new(ocidir)?)), 694 | size: 0, 695 | }) 696 | } 697 | 698 | /// Finish writing this blob, verifying its digest and size against the expected descriptor. 699 | pub fn complete_verified_as(mut self, descriptor: &Descriptor) -> Result { 700 | let expected_digest = sha256_of_descriptor(descriptor)?; 701 | let found_digest = hex::encode(self.hash.finish()?); 702 | if found_digest.as_str() != expected_digest { 703 | return Err(Error::DigestMismatch { 704 | expected: expected_digest.into(), 705 | found: found_digest.into(), 706 | }); 707 | } 708 | let descriptor_size: u64 = descriptor.size(); 709 | if self.size != descriptor_size { 710 | return Err(Error::SizeMismatch { 711 | expected: descriptor_size, 712 | found: self.size, 713 | }); 714 | } 715 | self.complete_as(&found_digest) 716 | } 717 | 718 | /// Finish writing this blob object with the supplied name 719 | fn complete_as(mut self, sha256_digest: &str) -> Result { 720 | let destname = &format!("{}/{}", BLOBDIR, sha256_digest); 721 | let target = self.target.take().unwrap(); 722 | target.into_inner().unwrap().replace(destname)?; 723 | Ok(Blob { 724 | sha256: Sha256Digest::from_str(sha256_digest).unwrap(), 725 | size: self.size, 726 | }) 727 | } 728 | 729 | /// Finish writing this blob object. 730 | pub fn complete(mut self) -> Result { 731 | let sha256 = hex::encode(self.hash.finish()?); 732 | self.complete_as(&sha256) 733 | } 734 | } 735 | 736 | impl std::io::Write for BlobWriter<'_> { 737 | fn write(&mut self, srcbuf: &[u8]) -> std::io::Result { 738 | let written = self.target.as_mut().unwrap().write(srcbuf)?; 739 | self.hash.update(&srcbuf[..written])?; 740 | self.size += written as u64; 741 | Ok(written) 742 | } 743 | 744 | fn flush(&mut self) -> std::io::Result<()> { 745 | Ok(()) 746 | } 747 | } 748 | 749 | /// A writer that can be finalized to return an inner writer. 750 | pub trait WriteComplete: Write { 751 | /// Complete the write operation and return the inner writer 752 | fn complete(self) -> std::io::Result; 753 | } 754 | 755 | impl WriteComplete for GzEncoder 756 | where 757 | W: Write, 758 | { 759 | fn complete(self) -> std::io::Result { 760 | self.finish() 761 | } 762 | } 763 | 764 | #[cfg(feature = "zstd")] 765 | impl WriteComplete for zstd::Encoder<'_, W> 766 | where 767 | W: Write, 768 | { 769 | fn complete(self) -> std::io::Result { 770 | self.finish() 771 | } 772 | } 773 | 774 | /// A writer for a layer. 775 | pub struct LayerWriter<'a, W> 776 | where 777 | W: WriteComplete>, 778 | { 779 | inner: Sha256Writer, 780 | media_type: MediaType, 781 | marker: PhantomData<&'a ()>, 782 | } 783 | 784 | impl<'a, W> LayerWriter<'a, W> 785 | where 786 | W: WriteComplete>, 787 | { 788 | /// Create a new LayerWriter with the given inner writer and media type 789 | pub fn new(inner: W, media_type: oci_image::MediaType) -> Self { 790 | Self { 791 | inner: Sha256Writer::new(inner), 792 | media_type, 793 | marker: PhantomData, 794 | } 795 | } 796 | 797 | /// Complete the layer writing and return the layer descriptor 798 | pub fn complete(self) -> Result { 799 | let (uncompressed_sha256, enc) = self.inner.finish(); 800 | let blob = enc.complete()?.complete()?; 801 | Ok(Layer { 802 | blob, 803 | uncompressed_sha256, 804 | media_type: self.media_type, 805 | }) 806 | } 807 | } 808 | 809 | impl<'a, W> std::io::Write for LayerWriter<'a, W> 810 | where 811 | W: WriteComplete>, 812 | { 813 | fn write(&mut self, data: &[u8]) -> std::io::Result { 814 | self.inner.write(data) 815 | } 816 | 817 | fn flush(&mut self) -> std::io::Result<()> { 818 | self.inner.flush() 819 | } 820 | } 821 | 822 | /// Wraps a writer and calculates the sha256 digest of data written to the inner writer 823 | struct Sha256Writer { 824 | inner: W, 825 | sha: openssl::sha::Sha256, 826 | } 827 | 828 | impl Sha256Writer { 829 | pub(crate) fn new(inner: W) -> Self { 830 | Self { 831 | inner, 832 | sha: openssl::sha::Sha256::new(), 833 | } 834 | } 835 | 836 | /// Return the hex encoded sha256 digest of the written data, and the underlying writer 837 | pub(crate) fn finish(self) -> (Sha256Digest, W) { 838 | let digest = hex::encode(self.sha.finish()); 839 | (Sha256Digest::from_str(&digest).unwrap(), self.inner) 840 | } 841 | } 842 | 843 | impl Write for Sha256Writer 844 | where 845 | W: Write, 846 | { 847 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 848 | let len = self.inner.write(buf)?; 849 | self.sha.update(&buf[..len]); 850 | Ok(len) 851 | } 852 | 853 | fn flush(&mut self) -> std::io::Result<()> { 854 | self.inner.flush() 855 | } 856 | } 857 | 858 | #[cfg(test)] 859 | mod tests { 860 | use cap_std::fs::OpenOptions; 861 | use oci_spec::image::HistoryBuilder; 862 | 863 | use super::*; 864 | 865 | const MANIFEST_DERIVE: &str = r#"{ 866 | "schemaVersion": 2, 867 | "config": { 868 | "mediaType": "application/vnd.oci.image.config.v1+json", 869 | "digest": "sha256:54977ab597b345c2238ba28fe18aad751e5c59dc38b9393f6f349255f0daa7fc", 870 | "size": 754 871 | }, 872 | "layers": [ 873 | { 874 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 875 | "digest": "sha256:ee02768e65e6fb2bb7058282338896282910f3560de3e0d6cd9b1d5985e8360d", 876 | "size": 5462 877 | }, 878 | { 879 | "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", 880 | "digest": "sha256:d203cef7e598fa167cb9e8b703f9f20f746397eca49b51491da158d64968b429", 881 | "size": 214 882 | } 883 | ], 884 | "annotations": { 885 | "ostree.commit": "3cb6170b6945065c2475bc16d7bebcc84f96b4c677811a6751e479b89f8c3770", 886 | "ostree.version": "42.0" 887 | } 888 | } 889 | "#; 890 | 891 | #[test] 892 | fn manifest() -> Result<()> { 893 | let m: oci_image::ImageManifest = serde_json::from_str(MANIFEST_DERIVE)?; 894 | assert_eq!( 895 | m.layers()[0].digest().to_string(), 896 | "sha256:ee02768e65e6fb2bb7058282338896282910f3560de3e0d6cd9b1d5985e8360d" 897 | ); 898 | Ok(()) 899 | } 900 | 901 | #[test] 902 | fn test_build() -> Result<()> { 903 | let td = cap_tempfile::tempdir(cap_std::ambient_authority())?; 904 | let w = OciDir::ensure(td.try_clone()?)?; 905 | let mut layerw = w.create_gzip_layer(None)?; 906 | layerw.write_all(b"pretend this is a tarball")?; 907 | let root_layer = layerw.complete()?; 908 | let root_layer_desc = root_layer.descriptor().build().unwrap(); 909 | assert_eq!( 910 | root_layer.uncompressed_sha256.digest(), 911 | "349438e5faf763e8875b43de4d7101540ef4d865190336c2cc549a11f33f8d7c" 912 | ); 913 | // Nothing referencing this blob yet 914 | assert!(matches!(w.fsck().unwrap_err(), Error::MissingImageIndex)); 915 | assert!(w.has_blob(&root_layer_desc).unwrap()); 916 | 917 | // Check that we don't find nonexistent blobs 918 | assert!( 919 | !w.has_blob(&Descriptor::new( 920 | MediaType::ImageLayerGzip, 921 | root_layer.blob.size, 922 | root_layer.uncompressed_sha256.clone() 923 | )) 924 | .unwrap() 925 | ); 926 | 927 | let mut manifest = w.new_empty_manifest()?.build()?; 928 | let mut config = oci_image::ImageConfigurationBuilder::default() 929 | .build() 930 | .unwrap(); 931 | let annotations: Option> = None; 932 | w.push_layer(&mut manifest, &mut config, root_layer, "root", annotations); 933 | { 934 | let history = config.history().as_ref().unwrap().first().unwrap(); 935 | assert_eq!(history.created_by().as_ref().unwrap(), "root"); 936 | let created = history.created().as_deref().unwrap(); 937 | let ts = chrono::DateTime::parse_from_rfc3339(created) 938 | .unwrap() 939 | .to_utc(); 940 | let now = chrono::offset::Utc::now(); 941 | assert_eq!(now.years_since(ts).unwrap(), 0); 942 | } 943 | let config = w.write_config(config)?; 944 | manifest.set_config(config); 945 | w.replace_with_single_manifest(manifest.clone(), oci_image::Platform::default())?; 946 | assert_eq!(w.read_index().unwrap().manifests().len(), 1); 947 | assert_eq!(w.fsck().unwrap(), 3); 948 | // Also verify that corrupting a blob is found 949 | { 950 | let root_layer_sha256 = root_layer_desc.as_digest_sha256().unwrap(); 951 | let mut f = w.dir.open_with( 952 | format!("blobs/sha256/{root_layer_sha256}"), 953 | OpenOptions::new().write(true), 954 | )?; 955 | let l = f.metadata()?.len(); 956 | f.seek(std::io::SeekFrom::End(0))?; 957 | f.write_all(b"\0")?; 958 | assert!(w.fsck().is_err()); 959 | f.set_len(l)?; 960 | assert_eq!(w.fsck().unwrap(), 3); 961 | } 962 | 963 | let idx = w.read_index()?; 964 | let manifest_desc = idx.manifests().first().unwrap(); 965 | let read_manifest = w.read_json_blob(manifest_desc).unwrap(); 966 | assert_eq!(&read_manifest, &manifest); 967 | 968 | let desc: Descriptor = 969 | w.insert_manifest(manifest, Some("latest"), oci_image::Platform::default())?; 970 | assert!(w.has_manifest(&desc).unwrap()); 971 | // There's more than one now 972 | assert_eq!(w.read_index().unwrap().manifests().len(), 2); 973 | 974 | assert!(w.find_manifest_with_tag("noent").unwrap().is_none()); 975 | let found_via_tag = w.find_manifest_with_tag("latest").unwrap().unwrap(); 976 | assert_eq!(found_via_tag, read_manifest); 977 | 978 | let mut layerw = w.create_gzip_layer(None)?; 979 | layerw.write_all(b"pretend this is an updated tarball")?; 980 | let root_layer = layerw.complete()?; 981 | let mut manifest = w.new_empty_manifest()?.build()?; 982 | let mut config = oci_image::ImageConfigurationBuilder::default() 983 | .build() 984 | .unwrap(); 985 | w.push_layer(&mut manifest, &mut config, root_layer, "root", None); 986 | let _: Descriptor = w.insert_manifest_and_config( 987 | manifest, 988 | config, 989 | Some("latest"), 990 | oci_image::Platform::default(), 991 | )?; 992 | assert_eq!(w.read_index().unwrap().manifests().len(), 2); 993 | assert_eq!(w.fsck().unwrap(), 6); 994 | Ok(()) 995 | } 996 | 997 | #[test] 998 | fn test_complete_verified_as() -> Result<()> { 999 | let td = cap_tempfile::tempdir(cap_std::ambient_authority())?; 1000 | let oci_dir = OciDir::ensure(td.try_clone()?)?; 1001 | 1002 | // Test a successful write 1003 | let empty_json_digest = oci_image::DescriptorBuilder::default() 1004 | .media_type(MediaType::EmptyJSON) 1005 | .size(2u32) 1006 | .digest(Sha256Digest::from_str( 1007 | "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", 1008 | )?) 1009 | .build()?; 1010 | 1011 | let mut empty_json_blob = oci_dir.create_blob()?; 1012 | empty_json_blob.write_all(b"{}")?; 1013 | let blob = empty_json_blob.complete_verified_as(&empty_json_digest)?; 1014 | assert_eq!(blob.sha256().digest(), empty_json_digest.digest().digest()); 1015 | 1016 | // And a checksum mismatch 1017 | let test_descriptor = oci_image::DescriptorBuilder::default() 1018 | .media_type(MediaType::EmptyJSON) 1019 | .size(3u32) 1020 | .digest(Sha256Digest::from_str( 1021 | "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", 1022 | )?) 1023 | .build()?; 1024 | let mut invalid_blob = oci_dir.create_blob()?; 1025 | invalid_blob.write_all(b"foo")?; 1026 | match invalid_blob 1027 | .complete_verified_as(&test_descriptor) 1028 | .err() 1029 | .unwrap() 1030 | { 1031 | Error::DigestMismatch { expected, found } => { 1032 | assert_eq!( 1033 | expected.as_ref(), 1034 | "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" 1035 | ); 1036 | assert_eq!( 1037 | found.as_ref(), 1038 | "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" 1039 | ); 1040 | } 1041 | o => panic!("Unexpected error {o}"), 1042 | } 1043 | 1044 | Ok(()) 1045 | } 1046 | 1047 | #[test] 1048 | fn test_new_empty_manifest() -> Result<()> { 1049 | let td = cap_tempfile::tempdir(cap_std::ambient_authority())?; 1050 | let w = OciDir::ensure(td.try_clone()?)?; 1051 | 1052 | let manifest = w.new_empty_manifest()?.build()?; 1053 | let desc: Descriptor = 1054 | w.insert_manifest(manifest, Some("latest"), oci_image::Platform::default())?; 1055 | assert!(w.has_manifest(&desc).unwrap()); 1056 | 1057 | // We expect two validated blobs: the manifest and the image configuration 1058 | assert_eq!(w.fsck()?, 2); 1059 | Ok(()) 1060 | } 1061 | 1062 | #[test] 1063 | fn test_push_layer_with_history() -> Result<()> { 1064 | let td = cap_tempfile::tempdir(cap_std::ambient_authority())?; 1065 | let w = OciDir::ensure(td.try_clone()?)?; 1066 | 1067 | let mut manifest = w.new_empty_manifest()?.build()?; 1068 | let mut config = oci_image::ImageConfigurationBuilder::default() 1069 | .build() 1070 | .unwrap(); 1071 | let mut layerw = w.create_gzip_layer(None)?; 1072 | layerw.write_all(b"pretend this is a tarball")?; 1073 | let root_layer = layerw.complete()?; 1074 | 1075 | let history = HistoryBuilder::default() 1076 | .created_by("/bin/pretend-tar") 1077 | .build() 1078 | .unwrap(); 1079 | w.push_layer_with_history(&mut manifest, &mut config, root_layer, Some(history)); 1080 | { 1081 | let history = config.history().as_ref().unwrap().first().unwrap(); 1082 | assert_eq!(history.created_by().as_deref().unwrap(), "/bin/pretend-tar"); 1083 | assert_eq!(history.created().as_ref(), None); 1084 | } 1085 | Ok(()) 1086 | } 1087 | } 1088 | --------------------------------------------------------------------------------