├── .cargo └── config ├── .github └── workflows │ └── build.yml ├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md ├── crates └── wagi-provider │ ├── Cargo.toml │ └── src │ ├── lib.rs │ ├── pod.rs │ ├── pod │ ├── completed.rs │ ├── initializing.rs │ ├── running.rs │ └── starting.rs │ └── runtime.rs ├── demos └── wagi │ ├── error.wat │ ├── hello.wasm │ ├── http-example.wasm │ ├── index.wat │ ├── k8s.yaml │ └── log.wasm ├── justfile ├── scripts ├── bootstrap.ps1 └── bootstrap.sh └── src └── krustlet-wagi.rs /.cargo/config: -------------------------------------------------------------------------------- 1 | [target.armv7-unknown-linux-gnueabihf] 2 | linker = "arm-linux-gnueabihf-gcc" 3 | 4 | [target.aarch64-unknown-linux-gnu] 5 | linker = "aarch64-linux-gnu-gcc" 6 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | build: 8 | runs-on: ${{ matrix.config.os }} 9 | env: ${{ matrix.config.env }} 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | config: 14 | - { 15 | os: "ubuntu-latest", 16 | arch: "amd64", 17 | args: "", 18 | url: "https://github.com/casey/just/releases/download/v0.5.11/just-v0.5.11-x86_64-unknown-linux-musl.tar.gz", 19 | name: "just", 20 | pathInArchive: "just", 21 | env: {}, 22 | } 23 | - { 24 | os: "ubuntu-latest", 25 | arch: "aarch64", 26 | args: "--target aarch64-unknown-linux-gnu", 27 | url: "https://github.com/casey/just/releases/download/v0.5.11/just-v0.5.11-x86_64-unknown-linux-musl.tar.gz", 28 | name: "just", 29 | pathInArchive: "just", 30 | env: { OPENSSL_DIR: "/usr/local/openssl-aarch64" }, 31 | } 32 | - { 33 | os: "macos-latest", 34 | arch: "amd64", 35 | args: "", 36 | url: "https://github.com/casey/just/releases/download/v0.5.11/just-v0.5.11-x86_64-apple-darwin.tar.gz", 37 | name: "just", 38 | pathInArchive: "just", 39 | env: {}, 40 | } 41 | steps: 42 | - uses: actions/checkout@v2 43 | - uses: engineerd/configurator@v0.0.7 44 | with: 45 | name: ${{ matrix.config.name }} 46 | url: ${{ matrix.config.url }} 47 | pathInArchive: ${{ matrix.config.pathInArchive }} 48 | # hack(bacongobbler): install rustfmt to work around darwin toolchain issues 49 | - name: "(macOS) install dev tools" 50 | if: runner.os == 'macOS' 51 | run: | 52 | rustup component add rustfmt --toolchain stable-x86_64-apple-darwin 53 | rustup component add clippy --toolchain stable-x86_64-apple-darwin 54 | rustup update stable 55 | - name: setup for cross-compile builds 56 | if: matrix.config.arch == 'aarch64' 57 | run: | 58 | sudo apt update 59 | sudo apt install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu 60 | cd /tmp 61 | git clone https://github.com/openssl/openssl 62 | cd openssl 63 | git checkout OpenSSL_1_1_1h 64 | sudo mkdir -p $OPENSSL_DIR 65 | ./Configure linux-aarch64 --prefix=$OPENSSL_DIR --openssldir=$OPENSSL_DIR shared 66 | make CC=aarch64-linux-gnu-gcc 67 | sudo make install 68 | rustup target add aarch64-unknown-linux-gnu 69 | - name: Build 70 | run: | 71 | just build ${{ matrix.config.args }} 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = [ 3 | "Matt Butcher ", 4 | "Matthew Fisher ", 5 | "Radu Matei ", 6 | "Taylor Thomas ", 7 | "Brian Ketelsen ", 8 | "Brian Hardock ", 9 | "Ryan Levick ", 10 | "Kevin Flansburg ", 11 | ] 12 | default-run = "krustlet-wagi" 13 | description = "A Krustlet Provider implementation in Rust for running WAGI modules in Kubernetes" 14 | edition = "2018" 15 | exclude = ["docs/*", "demos/*", ".github/*", ".gitignore"] 16 | keywords = ["wasm", "wasi", "wagi", "webassembly", "kubernetes"] 17 | license = "Apache-2.0" 18 | name = "krustlet-wagi-provider" 19 | readme = "README.md" 20 | repository = "https://github.com/deislabs/krustlet-wagi-provider" 21 | version = "0.1.0" 22 | 23 | [badges] 24 | maintenance = {status = "actively-developed"} 25 | 26 | [features] 27 | default = ["native-tls"] 28 | native-tls = [ 29 | "kube/native-tls", 30 | "krator/kube-native-tls", 31 | "kubelet/kube-native-tls", 32 | "wagi-provider/native-tls", 33 | "oci-distribution/native-tls", 34 | ] 35 | rustls-tls = [ 36 | "kube/rustls-tls", 37 | "krator/rustls-tls", 38 | "kubelet/rustls-tls", 39 | "wagi-provider/rustls-tls", 40 | "oci-distribution/rustls-tls", 41 | ] 42 | 43 | [dependencies] 44 | anyhow = "1.0" 45 | dirs = {package = "dirs-next", version = "2.0.0"} 46 | futures = "0.3" 47 | hostname = "0.3" 48 | k8s-openapi = {version = "0.12", default-features = false, features = ["v1_21"]} 49 | krator = {version = "0.4", default-features = false} 50 | kube = {version = "0.58", default-features = false} 51 | kubelet = {git = "https://github.com/krustlet/krustlet", tag = "v1.0.0-alpha.1", version = "1.0.0-alpha.1", default-features = false, features = ["cli"]} 52 | oci-distribution = {git = "https://github.com/krustlet/krustlet", tag = "v1.0.0-alpha.1", version = "0.7", default-features = false} 53 | regex = "1.3" 54 | serde = "1.0" 55 | tokio = {version = "1.0", features = ["macros", "rt-multi-thread", "time"]} 56 | tracing-subscriber = "0.2" 57 | wagi-provider = {path = "./crates/wagi-provider", version = "0.1.0", default-features = false} 58 | notify = "=5.0.0-pre.10" 59 | 60 | [workspace] 61 | members = [ 62 | "crates/wagi-provider", 63 | ] 64 | 65 | [[bin]] 66 | name = "krustlet-wagi" 67 | path = "src/krustlet-wagi.rs" 68 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Krustlet WAGI Provider 2 | 3 | **WARNING:** This is experimental code. 4 | It is not considered production-grade by its developers, neither is it "supported" software. 5 | 6 | This is a [Krustlet](https://github.com/deislabs/krustlet) [Provider](https://docs.krustlet.dev/topics/architecture/#providers) implementation for the [WAGI](https://github.com/deislabs/wagi) runtime. 7 | 8 | ## Documentation 9 | 10 | If you're new to Krustlet, get started with [the 11 | introduction](https://github.com/deislabs/krustlet/blob/master/docs/intro/README.md) documentation. 12 | For more in-depth information about Krustlet, plunge right into the [topic 13 | guides](https://github.com/deislabs/krustlet/blob/master/docs/topics/README.md). 14 | 15 | # Running the Demo 16 | 17 | In a new terminal, start the WAGI provider: 18 | ``` 19 | $ just run 20 | ``` 21 | 22 | In another terminal: 23 | ``` 24 | $ kubectl apply -f demos/wagi/k8s.yaml 25 | ``` 26 | 27 | Once the pod is ready: 28 | ``` 29 | $ curl 127.0.0.1:3000/hello 30 | hello world 31 | ``` 32 | -------------------------------------------------------------------------------- /crates/wagi-provider/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["Matt Butcher ", "Matthew Fisher ", "Radu Matei ", "Taylor Thomas ", "Brian Ketelsen ", "Brian Hardock ", "Ryan Levick ", "Kevin Flansburg "] 3 | edition = "2018" 4 | name = "wagi-provider" 5 | publish = false 6 | version = "0.1.0" 7 | 8 | [features] 9 | default = ["native-tls"] 10 | native-tls = ["kube/native-tls", "kubelet/kube-native-tls", "krator/kube-native-tls"] 11 | rustls-tls = ["kube/rustls-tls", "kubelet/rustls-tls", "krator/rustls-tls"] 12 | 13 | [dependencies] 14 | anyhow = "1.0" 15 | async-trait = "0.1" 16 | backtrace = "0.3" 17 | cap-std = "0.13" 18 | futures = "0.3" 19 | serde = "1.0" 20 | serde_derive = "1.0" 21 | serde_json = "1.0" 22 | tempfile = "3.1" 23 | wasmtime = "0.30.0" 24 | toml = "0.5" 25 | indexmap = { version = "^1.6.2", features = ["serde"] } 26 | wagi = { git = "https://github.com/deislabs/wagi", version = "0.4.0", rev = "9abecade" } 27 | tokio = { version = "1.0", features = ["fs", "macros", "io-util", "sync"] } 28 | chrono = { version = "0.4", features = ["serde"] } 29 | krator = { version = "0.4", default-features = false } 30 | kube = { version = "0.58", default-features = false } 31 | kubelet = { git = "https://github.com/krustlet/krustlet", tag = "v1.0.0-alpha.1", version = "1.0.0-alpha.1", features = ["derive"], default-features = false } 32 | tracing = { version = "0.1", features = ["log"] } 33 | hyper = {version = "0.14", features = ["full"]} -------------------------------------------------------------------------------- /crates/wagi-provider/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This code implements a Krustlet provider to schedule and orchestrates WAGI applications deployed to Kubernetes. 2 | 3 | #![deny(missing_docs)] 4 | 5 | use std::collections::HashMap; 6 | use std::convert::TryFrom; 7 | use std::path::{Path, PathBuf}; 8 | use std::sync::Arc; 9 | 10 | use async_trait::async_trait; 11 | use kubelet::node; 12 | use kubelet::node::Builder; 13 | use kubelet::plugin_watcher::PluginRegistry; 14 | use kubelet::pod::state::prelude::SharedState; 15 | use kubelet::pod::{Handle, Pod, PodKey}; 16 | use kubelet::provider::{ 17 | DevicePluginSupport, PluginSupport, Provider, ProviderError, VolumeSupport, 18 | }; 19 | use kubelet::resources::DeviceManager; 20 | use kubelet::state::common::registered::Registered; 21 | use kubelet::state::common::terminated::Terminated; 22 | use kubelet::state::common::{GenericProvider, GenericProviderState}; 23 | use kubelet::store::Store; 24 | use kubelet::volume::VolumeRef; 25 | 26 | use tokio::sync::RwLock; 27 | 28 | use runtime::Runtime; 29 | pub(crate) mod runtime; 30 | 31 | pub(crate) mod pod; 32 | use pod::PodState; 33 | 34 | const TARGET_WASM32_WAGI: &str = "wasm32-wagi"; 35 | const LOG_DIR_NAME: &str = "wagi-logs"; 36 | const VOLUME_DIR: &str = "volumes"; 37 | 38 | /// WagiProvider provides a Kubelet runtime implementation that executes WASM 39 | /// binaries conforming to the WAGI spec. 40 | #[derive(Clone)] 41 | pub struct WagiProvider { 42 | shared: ProviderState, 43 | } 44 | 45 | type PodHandleMap = Arc>>>>; 46 | 47 | /// Provider-level state shared between all pods 48 | #[derive(Clone)] 49 | pub struct ProviderState { 50 | handles: PodHandleMap, 51 | store: Arc, 52 | log_path: PathBuf, 53 | client: kube::Client, 54 | volume_path: PathBuf, 55 | plugin_registry: Arc, 56 | device_plugin_manager: Arc, 57 | } 58 | 59 | #[async_trait] 60 | impl GenericProviderState for ProviderState { 61 | fn client(&self) -> kube::client::Client { 62 | self.client.clone() 63 | } 64 | fn store(&self) -> std::sync::Arc<(dyn Store + Send + Sync + 'static)> { 65 | self.store.clone() 66 | } 67 | async fn stop(&self, pod: &Pod) -> anyhow::Result<()> { 68 | let key = PodKey::from(pod); 69 | let mut handle_writer = self.handles.write().await; 70 | if let Some(handle) = handle_writer.get_mut(&key) { 71 | handle.stop().await 72 | } else { 73 | Ok(()) 74 | } 75 | } 76 | } 77 | 78 | impl VolumeSupport for ProviderState { 79 | fn volume_path(&self) -> Option<&Path> { 80 | Some(self.volume_path.as_ref()) 81 | } 82 | } 83 | 84 | impl PluginSupport for ProviderState { 85 | fn plugin_registry(&self) -> Option> { 86 | Some(self.plugin_registry.clone()) 87 | } 88 | } 89 | 90 | impl DevicePluginSupport for ProviderState { 91 | fn device_plugin_manager(&self) -> Option> { 92 | Some(self.device_plugin_manager.clone()) 93 | } 94 | } 95 | 96 | impl WagiProvider { 97 | /// Create a new wagi provider from a module store and a kubelet config 98 | pub async fn new( 99 | store: Arc, 100 | config: &kubelet::config::Config, 101 | kubeconfig: kube::Config, 102 | plugin_registry: Arc, 103 | device_plugin_manager: Arc, 104 | ) -> anyhow::Result { 105 | let log_path = config.data_dir.join(LOG_DIR_NAME); 106 | let volume_path = config.data_dir.join(VOLUME_DIR); 107 | tokio::fs::create_dir_all(&log_path).await?; 108 | tokio::fs::create_dir_all(&volume_path).await?; 109 | let client = kube::Client::try_from(kubeconfig)?; 110 | Ok(Self { 111 | shared: ProviderState { 112 | handles: Default::default(), 113 | store, 114 | log_path, 115 | volume_path, 116 | client, 117 | plugin_registry, 118 | device_plugin_manager, 119 | }, 120 | }) 121 | } 122 | } 123 | 124 | struct ModuleRunContext { 125 | modules: HashMap>, 126 | volumes: HashMap, 127 | env_vars: HashMap>, 128 | } 129 | 130 | #[async_trait::async_trait] 131 | impl Provider for WagiProvider { 132 | type ProviderState = ProviderState; 133 | type InitialState = Registered; 134 | type TerminatedState = Terminated; 135 | type PodState = PodState; 136 | 137 | const ARCH: &'static str = TARGET_WASM32_WAGI; 138 | 139 | fn provider_state(&self) -> SharedState { 140 | Arc::new(RwLock::new(self.shared.clone())) 141 | } 142 | 143 | async fn node(&self, builder: &mut Builder) -> anyhow::Result<()> { 144 | builder.set_architecture("wasm-wagi"); 145 | builder.add_taint("NoSchedule", "kubernetes.io/arch", Self::ARCH); 146 | builder.add_taint("NoExecute", "kubernetes.io/arch", Self::ARCH); 147 | Ok(()) 148 | } 149 | 150 | async fn initialize_pod_state(&self, pod: &Pod) -> anyhow::Result { 151 | Ok(PodState::new(pod)) 152 | } 153 | 154 | async fn logs( 155 | &self, 156 | namespace: String, 157 | pod_name: String, 158 | container_name: String, 159 | sender: kubelet::log::Sender, 160 | ) -> anyhow::Result<()> { 161 | let mut handles = self.shared.handles.write().await; 162 | let handle = handles 163 | .get_mut(&PodKey::new(&namespace, &pod_name)) 164 | .ok_or_else(|| ProviderError::PodNotFound { 165 | pod_name: pod_name.clone(), 166 | })?; 167 | handle.output(&container_name, sender).await 168 | } 169 | 170 | // Evict all pods upon shutdown 171 | async fn shutdown(&self, node_name: &str) -> anyhow::Result<()> { 172 | node::drain(&self.shared.client, &node_name).await?; 173 | Ok(()) 174 | } 175 | } 176 | 177 | impl GenericProvider for WagiProvider { 178 | type ProviderState = ProviderState; 179 | type PodState = PodState; 180 | type RunState = pod::initializing::Initializing; 181 | 182 | fn validate_pod_runnable(_pod: &Pod) -> anyhow::Result<()> { 183 | Ok(()) 184 | } 185 | 186 | fn validate_container_runnable( 187 | container: &kubelet::container::Container, 188 | ) -> anyhow::Result<()> { 189 | if let Some(image) = container.image()? { 190 | if image.whole().starts_with("k8s.gcr.io/kube-proxy") { 191 | return Err(anyhow::anyhow!("Cannot run kube-proxy")); 192 | } 193 | } 194 | Ok(()) 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /crates/wagi-provider/src/pod.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::sync::Arc; 3 | 4 | use async_trait::async_trait; 5 | use krator::{ObjectState, SharedState}; 6 | use kubelet::backoff::BackoffStrategy; 7 | use kubelet::backoff::ExponentialBackoffStrategy; 8 | use kubelet::pod::Pod; 9 | use kubelet::pod::PodKey; 10 | use kubelet::pod::Status; 11 | use kubelet::state::common::{BackoffSequence, GenericPodState, ThresholdTrigger}; 12 | use tokio::sync::RwLock; 13 | use tracing::error; 14 | 15 | use crate::ModuleRunContext; 16 | use crate::ProviderState; 17 | 18 | pub(crate) mod completed; 19 | pub(crate) mod initializing; 20 | pub(crate) mod running; 21 | pub(crate) mod starting; 22 | 23 | pub const WAGI_ANNOTATION_KEY_DEFAULT_HOST: &str = "alpha.wagi.krustlet.dev/default-host"; 24 | pub const WAGI_ANNOTATION_KEY_MODULES: &str = "alpha.wagi.krustlet.dev/modules"; 25 | 26 | /// State that is shared between pod state handlers. 27 | pub struct PodState { 28 | key: PodKey, 29 | run_context: SharedState, 30 | errors: usize, 31 | image_pull_backoff_strategy: ExponentialBackoffStrategy, 32 | pub(crate) crash_loop_backoff_strategy: ExponentialBackoffStrategy, 33 | } 34 | 35 | #[async_trait] 36 | impl ObjectState for PodState { 37 | type Manifest = Pod; 38 | type Status = Status; 39 | type SharedState = ProviderState; 40 | async fn async_drop(self, provider_state: &mut Self::SharedState) { 41 | { 42 | { 43 | let mut context = self.run_context.write().await; 44 | let unmounts = context.volumes.iter_mut().map(|(k, vol)| async move { 45 | if let Err(e) = vol.unmount().await { 46 | // Just log the error, as there isn't much we can do here 47 | error!(error = %e, volume_name = %k, "Unable to unmount volume"); 48 | } 49 | }); 50 | futures::future::join_all(unmounts).await; 51 | } 52 | let mut handles = provider_state.handles.write().await; 53 | handles.remove(&self.key); 54 | } 55 | } 56 | } 57 | 58 | impl PodState { 59 | pub fn new(pod: &Pod) -> Self { 60 | let run_context = ModuleRunContext { 61 | modules: Default::default(), 62 | volumes: Default::default(), 63 | env_vars: Default::default(), 64 | }; 65 | let key = PodKey::from(pod); 66 | PodState { 67 | key, 68 | run_context: Arc::new(RwLock::new(run_context)), 69 | errors: 0, 70 | image_pull_backoff_strategy: ExponentialBackoffStrategy::default(), 71 | crash_loop_backoff_strategy: ExponentialBackoffStrategy::default(), 72 | } 73 | } 74 | } 75 | 76 | #[async_trait] 77 | impl GenericPodState for PodState { 78 | async fn set_env_vars(&mut self, env_vars: HashMap>) { 79 | let mut run_context = self.run_context.write().await; 80 | run_context.env_vars = env_vars; 81 | } 82 | async fn set_modules(&mut self, modules: HashMap>) { 83 | let mut run_context = self.run_context.write().await; 84 | run_context.modules = modules; 85 | } 86 | async fn set_volumes(&mut self, volumes: HashMap) { 87 | let mut run_context = self.run_context.write().await; 88 | run_context.volumes = volumes; 89 | } 90 | async fn backoff(&mut self, sequence: BackoffSequence) { 91 | let backoff_strategy = match sequence { 92 | BackoffSequence::ImagePull => &mut self.image_pull_backoff_strategy, 93 | BackoffSequence::CrashLoop => &mut self.crash_loop_backoff_strategy, 94 | }; 95 | backoff_strategy.wait().await; 96 | } 97 | async fn reset_backoff(&mut self, sequence: BackoffSequence) { 98 | let backoff_strategy = match sequence { 99 | BackoffSequence::ImagePull => &mut self.image_pull_backoff_strategy, 100 | BackoffSequence::CrashLoop => &mut self.crash_loop_backoff_strategy, 101 | }; 102 | backoff_strategy.reset(); 103 | } 104 | async fn record_error(&mut self) -> ThresholdTrigger { 105 | self.errors += 1; 106 | if self.errors > 3 { 107 | self.errors = 0; 108 | ThresholdTrigger::Triggered 109 | } else { 110 | ThresholdTrigger::Untriggered 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /crates/wagi-provider/src/pod/completed.rs: -------------------------------------------------------------------------------- 1 | use crate::{PodState, ProviderState}; 2 | use kubelet::pod::state::prelude::*; 3 | 4 | /// Pod was deleted. 5 | #[derive(Default, Debug)] 6 | pub struct Completed; 7 | 8 | #[async_trait::async_trait] 9 | impl State for Completed { 10 | async fn next( 11 | self: Box, 12 | _provider_state: SharedState, 13 | _pod_state: &mut PodState, 14 | _pod: Manifest, 15 | ) -> Transition { 16 | Transition::Complete(Ok(())) 17 | } 18 | 19 | async fn status(&self, _pod_state: &mut PodState, _pod: &Pod) -> anyhow::Result { 20 | Ok(make_status(Phase::Succeeded, "Completed")) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /crates/wagi-provider/src/pod/initializing.rs: -------------------------------------------------------------------------------- 1 | use crate::{PodState, ProviderState}; 2 | use kubelet::backoff::BackoffStrategy; 3 | use kubelet::pod::state::prelude::*; 4 | use kubelet::state::common::error::Error; 5 | 6 | use super::starting::Starting; 7 | 8 | #[derive(Default, Debug, TransitionTo)] 9 | #[transition_to(Starting, Error)] 10 | pub struct Initializing; 11 | 12 | #[async_trait::async_trait] 13 | impl State for Initializing { 14 | #[tracing::instrument( 15 | level = "info", 16 | skip(self, _provider_state, pod_state, _pod), 17 | fields(pod_name) 18 | )] 19 | async fn next( 20 | self: Box, 21 | _provider_state: SharedState, 22 | pod_state: &mut PodState, 23 | _pod: Manifest, 24 | ) -> Transition { 25 | pod_state.crash_loop_backoff_strategy.reset(); 26 | Transition::next(self, Starting) 27 | } 28 | 29 | async fn status(&self, _pod_state: &mut PodState, _pmeod: &Pod) -> anyhow::Result { 30 | Ok(make_status(Phase::Running, "Initializing")) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /crates/wagi-provider/src/pod/running.rs: -------------------------------------------------------------------------------- 1 | use tokio::sync::mpsc::Receiver; 2 | 3 | use kubelet::pod::state::prelude::*; 4 | use kubelet::state::common::error::Error; 5 | 6 | use super::completed::Completed; 7 | use crate::{PodState, ProviderState}; 8 | 9 | /// The Kubelet is running the Pod. 10 | #[derive(Debug, TransitionTo)] 11 | #[transition_to(Completed, Error)] 12 | pub struct Running { 13 | rx: Receiver>, 14 | } 15 | 16 | impl Running { 17 | pub fn new(rx: Receiver>) -> Self { 18 | Running { rx } 19 | } 20 | } 21 | 22 | #[async_trait::async_trait] 23 | impl State for Running { 24 | async fn next( 25 | mut self: Box, 26 | _provider_state: SharedState, 27 | _pod_state: &mut PodState, 28 | pod: Manifest, 29 | ) -> Transition { 30 | let pod = pod.latest(); 31 | 32 | while let Some(result) = self.rx.recv().await { 33 | match result { 34 | Ok(()) => { 35 | return Transition::next(self, Completed); 36 | } 37 | Err(e) => { 38 | tracing::error!(error = %e); 39 | return Transition::Complete(Err(e)); 40 | } 41 | } 42 | } 43 | Transition::next( 44 | self, 45 | Error::new(format!( 46 | "Pod {} container result channel hung up.", 47 | pod.name() 48 | )), 49 | ) 50 | } 51 | 52 | async fn status(&self, _pod_state: &mut PodState, _pod: &Pod) -> anyhow::Result { 53 | Ok(make_status(Phase::Running, "Running")) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /crates/wagi-provider/src/pod/starting.rs: -------------------------------------------------------------------------------- 1 | use std::net::SocketAddr; 2 | 3 | use tracing::{info, instrument}; 4 | 5 | use crate::{PodState, ProviderState}; 6 | use kubelet::pod::state::prelude::*; 7 | 8 | use super::running::Running; 9 | use kubelet::state::common::error::Error; 10 | 11 | use crate::pod::{WAGI_ANNOTATION_KEY_DEFAULT_HOST, WAGI_ANNOTATION_KEY_MODULES}; 12 | use crate::runtime::{WagiModuleConfig, WagiModulesConfig}; 13 | 14 | #[derive(Default, Debug, TransitionTo)] 15 | #[transition_to(Running, Error)] 16 | /// The Kubelet is starting the Pod containers 17 | pub(crate) struct Starting; 18 | 19 | #[async_trait::async_trait] 20 | impl State for Starting { 21 | #[instrument( 22 | level = "info", 23 | skip(self, _provider_state, _pod_state, pod), 24 | fields(pod_name) 25 | )] 26 | async fn next( 27 | self: Box, 28 | _provider_state: SharedState, 29 | _pod_state: &mut PodState, 30 | pod: Manifest, 31 | ) -> Transition { 32 | let pod = pod.latest(); 33 | 34 | tracing::Span::current().record("pod_name", &pod.name()); 35 | 36 | info!("Starting WAGI pod"); 37 | 38 | let mut wagi_modules_config = WagiModulesConfig::default(); 39 | 40 | if let Some(default_host_annotation) = 41 | pod.annotations().get(WAGI_ANNOTATION_KEY_DEFAULT_HOST) 42 | { 43 | wagi_modules_config.default_host = Some(default_host_annotation.clone()); 44 | } 45 | 46 | if let Some(modules_annotation) = pod.annotations().get(WAGI_ANNOTATION_KEY_MODULES) { 47 | match serde_json::from_str(&modules_annotation) { 48 | Ok(modules) => { 49 | let modules_map: indexmap::IndexMap = pod 50 | .clone() 51 | .containers() 52 | .into_iter() 53 | .filter_map(|c| { 54 | c.image() 55 | .ok() 56 | .flatten() 57 | .map(|image| (c.name().to_owned(), image.to_string())) 58 | }) 59 | .fold(modules, |mut modules, (name, image)| { 60 | modules 61 | .entry(name) 62 | .and_modify(|m| m.module = format!("oci://{}", image)); 63 | modules 64 | }); 65 | 66 | wagi_modules_config.modules = modules_map.into_iter().map(|(_, m)| m).collect(); 67 | } 68 | Err(parse_err) => { 69 | return Transition::next( 70 | self, 71 | Error::new(format!( 72 | "Error parsing WAGI annotation for key {:?}: {}", 73 | WAGI_ANNOTATION_KEY_MODULES, parse_err, 74 | )), 75 | ); 76 | } 77 | } 78 | } 79 | 80 | let (tx, rx) = tokio::sync::mpsc::channel(1); 81 | tokio::task::spawn(async move { 82 | let default_host = wagi_modules_config 83 | .default_host 84 | .clone() 85 | .unwrap_or("127.0.0.1:3000".to_string()) 86 | .parse::() 87 | .map_err(|e| anyhow::anyhow!("{}", e)); 88 | 89 | let server_addr = match default_host { 90 | Ok(addr) => addr, 91 | Err(error) => { 92 | tracing::error!(error = %error); 93 | return tx.send(Err(error)).await; 94 | } 95 | }; 96 | 97 | match wagi_modules_config.build_wagi_router().await { 98 | Ok(router) => { 99 | use hyper::server::conn::AddrStream; 100 | use hyper::service::{make_service_fn, service_fn}; 101 | use hyper::Server; 102 | 103 | let mk_svc = make_service_fn(move |conn: &AddrStream| { 104 | let addr = conn.remote_addr(); 105 | let r = router.clone(); 106 | async move { 107 | Ok::<_, std::convert::Infallible>(service_fn(move |req| { 108 | let r2 = r.clone(); 109 | async move { r2.route(req, addr).await } 110 | })) 111 | } 112 | }); 113 | 114 | let result = Server::bind(&server_addr) 115 | .serve(mk_svc) 116 | .await 117 | .map_err(|e| anyhow::anyhow!("{}", e)); 118 | 119 | info!("WAGI pod started"); 120 | tx.send(result).await 121 | } 122 | Err(error) => tx.send(Err(anyhow::anyhow!("{}", error))).await, 123 | } 124 | }); 125 | 126 | Transition::next(self, Running::new(rx)) 127 | } 128 | 129 | async fn status(&self, _pod_state: &mut PodState, _pod: &Pod) -> anyhow::Result { 130 | Ok(make_status(Phase::Pending, "Starting")) 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /crates/wagi-provider/src/runtime.rs: -------------------------------------------------------------------------------- 1 | use std::hash::{Hash, Hasher}; 2 | use std::io::Write; 3 | use std::sync::Arc; 4 | 5 | use tempfile::NamedTempFile; 6 | use tokio::sync::mpsc::Sender; 7 | use tokio::task::JoinHandle; 8 | use wasmtime::InterruptHandle; 9 | 10 | use kubelet::container::Status; 11 | use kubelet::handle::StopHandler; 12 | 13 | use serde::{Deserialize, Serialize}; 14 | pub struct Runtime { 15 | handle: JoinHandle>, 16 | interrupt_handle: InterruptHandle, 17 | } 18 | 19 | #[async_trait::async_trait] 20 | impl StopHandler for Runtime { 21 | async fn stop(&mut self) -> anyhow::Result<()> { 22 | self.interrupt_handle.interrupt(); 23 | Ok(()) 24 | } 25 | 26 | async fn wait(&mut self) -> anyhow::Result<()> { 27 | (&mut self.handle).await??; 28 | Ok(()) 29 | } 30 | } 31 | 32 | #[derive(Debug, Clone, Default, Serialize)] 33 | pub struct WagiModulesConfig { 34 | pub default_host: Option, 35 | #[serde(rename = "module")] 36 | pub modules: indexmap::IndexSet, 37 | } 38 | 39 | impl WagiModulesConfig { 40 | pub(crate) async fn build_wagi_router(self) -> anyhow::Result { 41 | // TODO: Maybe add wagi::Router::build_from_serde(...) 42 | let mut temp_file = tempfile::NamedTempFile::new()?; 43 | 44 | { 45 | // debug 46 | let config_data = toml::to_string_pretty(&self)?; 47 | println!("{}", config_data); 48 | } 49 | 50 | let config_data = toml::to_string(&self)?; 51 | write!(temp_file, "{}", config_data)?; 52 | wagi::Router::builder() 53 | .build_from_modules_toml(temp_file.path()) 54 | .await 55 | } 56 | } 57 | 58 | // Configuration for WAGI modules. 59 | #[derive(Debug, Clone, Default, Serialize, Deserialize)] 60 | pub struct WagiModuleConfig { 61 | pub entrypoint: Option, 62 | pub route: String, 63 | pub allowed_hosts: Option>, 64 | #[serde(skip_deserializing)] 65 | pub module: String, 66 | } 67 | 68 | impl PartialEq for WagiModuleConfig { 69 | fn eq(&self, other: &Self) -> bool { 70 | self.route == other.route 71 | } 72 | } 73 | 74 | impl Eq for WagiModuleConfig {} 75 | 76 | impl Hash for WagiModuleConfig { 77 | fn hash(&self, state: &mut H) { 78 | self.route.hash(state); 79 | } 80 | } 81 | 82 | /// Holds our tempfile handle. 83 | pub struct HandleFactory { 84 | temp: Arc, 85 | } 86 | 87 | impl kubelet::log::HandleFactory for HandleFactory { 88 | /// Creates `tokio::fs::File` on demand for log reading. 89 | fn new_handle(&self) -> tokio::fs::File { 90 | tokio::fs::File::from_std(self.temp.reopen().unwrap()) 91 | } 92 | } 93 | 94 | #[tracing::instrument(level = "info", skip(sender, status))] 95 | fn send(sender: &Sender, name: &str, status: Status) { 96 | match sender.blocking_send(status) { 97 | Err(e) => tracing::warn!(error = %e, "error sending wasi status"), 98 | Ok(_) => tracing::debug!("send completed"), 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /demos/wagi/error.wat: -------------------------------------------------------------------------------- 1 | (module 2 | ;; This shows how to return an error 3 | (import "wasi_snapshot_preview1" "fd_write" (func $fd_write (param i32 i32 i32 i32) (result i32))) 4 | (memory 1) 5 | (export "memory" (memory 0)) 6 | 7 | (data (i32.const 8) "content-type: text/plain\nstatus: 404\n\nNot Found\n") 8 | 9 | (func $main (export "_start") 10 | (i32.store (i32.const 0) (i32.const 8)) 11 | (i32.store (i32.const 4) (i32.const 48)) 12 | 13 | (call $fd_write 14 | (i32.const 1) 15 | (i32.const 0) 16 | (i32.const 1) 17 | (i32.const 20) 18 | ) 19 | drop 20 | ) 21 | ) -------------------------------------------------------------------------------- /demos/wagi/hello.wasm: -------------------------------------------------------------------------------- 1 | asm ``#wasi_snapshot_preview1fd_writememory_start 2 | AA6AA%6AAAA +A %content-type:text/plain 3 | 4 | hello world 5 | -------------------------------------------------------------------------------- /demos/wagi/http-example.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deislabs/krustlet-wagi-provider/3a70373e705f8a824293307a70879ac055a1ac79/demos/wagi/http-example.wasm -------------------------------------------------------------------------------- /demos/wagi/index.wat: -------------------------------------------------------------------------------- 1 | (module 2 | ;; This is the example Hello World WAT from the documentation at 3 | ;; https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-tutorial.md 4 | ;; 5 | ;; It has been adapted to send CGI headers. 6 | (import "wasi_snapshot_preview1" "fd_write" (func $fd_write (param i32 i32 i32 i32) (result i32))) 7 | (memory 1) 8 | (export "memory" (memory 0)) 9 | 10 | (data (i32.const 8) "content-type: text/html;charset=UTF-8\n\nOh hi world\n") 11 | 12 | (func $main (export "_start") 13 | (i32.store (i32.const 0) (i32.const 8)) 14 | (i32.store (i32.const 4) (i32.const 51)) 15 | 16 | (call $fd_write 17 | (i32.const 1) 18 | (i32.const 0) 19 | (i32.const 1) 20 | (i32.const 20) 21 | ) 22 | drop 23 | ) 24 | ) -------------------------------------------------------------------------------- /demos/wagi/k8s.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: krustlet-wagi-demo 5 | labels: 6 | app: krustlet-wagi-demo 7 | annotations: 8 | alpha.wagi.krustlet.dev/default-host: "127.0.0.1:3000" 9 | alpha.wagi.krustlet.dev/modules: | 10 | { 11 | "krustlet-wagi-demo-http-example": {"route": "/http-example", "allowed_hosts": ["https://api.brigade.sh"]}, 12 | "krustlet-wagi-demo-hello": {"route": "/hello/..."}, 13 | "krustlet-wagi-demo-error": {"route": "/error"}, 14 | "krustlet-wagi-demo-log": {"route": "/log"}, 15 | "krustlet-wagi-demo-index": {"route": "/"} 16 | } 17 | spec: 18 | containers: 19 | - image: webassembly.azurecr.io/krustlet-wagi-demo-http-example:v1.0.0 20 | imagePullPolicy: Always 21 | name: krustlet-wagi-demo-http-example 22 | - image: webassembly.azurecr.io/krustlet-wagi-demo-hello:v1.0.0 23 | imagePullPolicy: Always 24 | name: krustlet-wagi-demo-hello 25 | - image: webassembly.azurecr.io/krustlet-wagi-demo-index:v1.0.0 26 | imagePullPolicy: Always 27 | name: krustlet-wagi-demo-index 28 | - image: webassembly.azurecr.io/krustlet-wagi-demo-error:v1.0.0 29 | imagePullPolicy: Always 30 | name: krustlet-wagi-demo-error 31 | - image: webassembly.azurecr.io/krustlet-wagi-demo-log:v1.0.0 32 | imagePullPolicy: Always 33 | name: krustlet-wagi-demo-log 34 | tolerations: 35 | - key: "node.kubernetes.io/network-unavailable" 36 | operator: "Exists" 37 | effect: "NoSchedule" 38 | - key: "kubernetes.io/arch" 39 | operator: "Equal" 40 | value: "wasm32-wagi" 41 | effect: "NoExecute" 42 | - key: "kubernetes.io/arch" 43 | operator: "Equal" 44 | value: "wasm32-wagi" 45 | effect: "NoSchedule" 46 | -------------------------------------------------------------------------------- /demos/wagi/log.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deislabs/krustlet-wagi-provider/3a70373e705f8a824293307a70879ac055a1ac79/demos/wagi/log.wasm -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | export RUST_LOG := "wagi_provider=debug,main=debug,kubelet=debug" 2 | export CONFIG_DIR := env_var_or_default('CONFIG_DIR', '$HOME/.krustlet/config') 3 | 4 | build +FLAGS='': 5 | cargo build {{FLAGS}} 6 | 7 | run +FLAGS='': bootstrap 8 | KUBECONFIG=$(eval echo $CONFIG_DIR)/kubeconfig-wagi cargo run --bin krustlet-wagi {{FLAGS}} -- --node-name krustlet-wagi --port 3001 --bootstrap-file $(eval echo $CONFIG_DIR)/bootstrap.conf --cert-file $(eval echo $CONFIG_DIR)/krustlet-wagi.crt --private-key-file $(eval echo $CONFIG_DIR)/krustlet-wagi.key 9 | 10 | bootstrap: 11 | @# This is to get around an issue with the default function returning a string that gets escaped 12 | @mkdir -p $(eval echo $CONFIG_DIR) 13 | @test -f $(eval echo $CONFIG_DIR)/bootstrap.conf || CONFIG_DIR=$(eval echo $CONFIG_DIR) ./scripts/bootstrap.sh 14 | @chmod 600 $(eval echo $CONFIG_DIR)/* -------------------------------------------------------------------------------- /scripts/bootstrap.ps1: -------------------------------------------------------------------------------- 1 | $token_id = -join ((48..57) + (97..122) | Get-Random -Count 6 | ForEach-Object { [char]$_ }) 2 | $token_secret = -join ((48..57) + (97..122) | Get-Random -Count 16 | ForEach-Object { [char]$_ }) 3 | 4 | $expiration = (Get-Date).ToUniversalTime().AddHours(1).ToString("yyyy-MM-ddTHH:mm:ssZ") 5 | 6 | @" 7 | apiVersion: v1 8 | kind: Secret 9 | metadata: 10 | name: bootstrap-token-${token_id} 11 | namespace: kube-system 12 | type: bootstrap.kubernetes.io/token 13 | stringData: 14 | auth-extra-groups: system:bootstrappers:kubeadm:default-node-token 15 | expiration: ${expiration} 16 | token-id: ${token_id} 17 | token-secret: ${token_secret} 18 | usage-bootstrap-authentication: "true" 19 | usage-bootstrap-signing: "true" 20 | "@ | kubectl.exe apply -f - 21 | 22 | if (!$env:CONFIG_DIR -or -not (Test-Path $env:CONFIG_DIR)) { 23 | $config_dir = "$HOME\.krustlet\config" 24 | } 25 | else { 26 | $config_dir = $env:CONFIG_DIR 27 | } 28 | 29 | mkdir $config_dir -ErrorAction SilentlyContinue > $null 30 | 31 | if (!$env:FILE_NAME -or -not (Test-Path $env:FILE_NAME)) { 32 | $file_name = "bootstrap.conf" 33 | } -------------------------------------------------------------------------------- /scripts/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | export LC_CTYPE=C 5 | 6 | token_id="$(/dev/null || 11 | date -v+1H -u "+%Y-%m-%dT%H:%M:%SZ" 2>/dev/null) 12 | 13 | cat <"${KUBECONFIG_FILE}.full.tmp" 50 | 51 | # Switch working context to correct context 52 | kubectl --kubeconfig "${KUBECONFIG_FILE}.full.tmp" config use-context "${CONTEXT}" 53 | 54 | # Minify 55 | kubectl --kubeconfig "${KUBECONFIG_FILE}.full.tmp" \ 56 | config view --flatten --minify >"${KUBECONFIG_FILE}.tmp" 57 | 58 | # Rename context 59 | kubectl config --kubeconfig "${KUBECONFIG_FILE}.tmp" \ 60 | rename-context "${CONTEXT}" "${NEW_CONTEXT}" 61 | 62 | # Create token user 63 | kubectl config --kubeconfig "${KUBECONFIG_FILE}.tmp" \ 64 | set-credentials "${TOKEN_USER}" --token "${TOKEN}" 65 | 66 | # Set context to use token user 67 | kubectl config --kubeconfig "${KUBECONFIG_FILE}.tmp" \ 68 | set-context "${NEW_CONTEXT}" --user "${TOKEN_USER}" 69 | 70 | # Set context to correct namespace 71 | kubectl config --kubeconfig "${KUBECONFIG_FILE}.tmp" \ 72 | set-context "${NEW_CONTEXT}" --namespace "${NAMESPACE}" 73 | 74 | # Flatten/minify kubeconfig 75 | kubectl config --kubeconfig "${KUBECONFIG_FILE}.tmp" \ 76 | view --flatten --minify >"${KUBECONFIG_FILE}" -------------------------------------------------------------------------------- /src/krustlet-wagi.rs: -------------------------------------------------------------------------------- 1 | use kubelet::config::Config; 2 | use kubelet::plugin_watcher::PluginRegistry; 3 | use kubelet::resources::DeviceManager; 4 | use kubelet::store::composite::ComposableStore; 5 | use kubelet::store::oci::FileStore; 6 | use kubelet::Kubelet; 7 | use std::convert::TryFrom; 8 | use std::sync::Arc; 9 | use wagi_provider::WagiProvider; 10 | 11 | #[tokio::main(flavor = "multi_thread")] 12 | async fn main() -> anyhow::Result<()> { 13 | // The provider is responsible for all the "back end" logic. If you are creating 14 | // a new Kubelet, all you need to implement is a provider. 15 | let config = Config::new_from_file_and_flags(env!("CARGO_PKG_VERSION"), None); 16 | 17 | // Initialize the logger 18 | tracing_subscriber::fmt() 19 | .with_writer(std::io::stderr) 20 | .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) 21 | .init(); 22 | 23 | let kubeconfig = kubelet::bootstrap(&config, &config.bootstrap_file, notify_bootstrap).await?; 24 | 25 | let store = make_store(&config); 26 | let plugin_registry = Arc::new(PluginRegistry::new(&config.plugins_dir)); 27 | let device_plugin_manager = Arc::new(DeviceManager::new( 28 | &config.device_plugins_dir, 29 | kube::Client::try_from(kubeconfig.clone())?, 30 | &config.node_name, 31 | )); 32 | 33 | let provider = WagiProvider::new( 34 | store, 35 | &config, 36 | kubeconfig.clone(), 37 | plugin_registry, 38 | device_plugin_manager, 39 | ) 40 | .await?; 41 | let kubelet = Kubelet::new(provider, kubeconfig, config).await?; 42 | kubelet.start().await 43 | } 44 | 45 | fn make_store(config: &Config) -> Arc { 46 | let client = oci_distribution::Client::from_source(config); 47 | let mut store_path = config.data_dir.join(".oci"); 48 | store_path.push("modules"); 49 | let file_store = Arc::new(FileStore::new(client, &store_path)); 50 | 51 | if config.allow_local_modules { 52 | file_store.with_override(Arc::new(kubelet::store::fs::FileSystemStore {})) 53 | } else { 54 | file_store 55 | } 56 | } 57 | 58 | fn notify_bootstrap(message: String) { 59 | println!("BOOTSTRAP: {}", message); 60 | } 61 | --------------------------------------------------------------------------------