├── .dockerignore ├── demo ├── etc │ ├── containerd │ │ └── config.toml │ └── cni │ │ ├── 99-loopback.conf │ │ └── 10-bridge.conf ├── hello.yaml ├── entrypoint └── Dockerfile ├── .gitignore ├── Cargo.toml ├── .github └── workflows │ └── test.yml ├── src ├── states │ ├── error.rs │ ├── running.rs │ ├── registered.rs │ ├── terminated.rs │ ├── image_pull.rs │ ├── mod.rs │ └── starting.rs ├── main.rs └── provider.rs ├── README.md └── LICENSE /.dockerignore: -------------------------------------------------------------------------------- 1 | target/** 2 | -------------------------------------------------------------------------------- /demo/etc/containerd/config.toml: -------------------------------------------------------------------------------- 1 | [plugins] 2 | [plugins.cri.containerd] 3 | snapshotter = "native" 4 | -------------------------------------------------------------------------------- /demo/etc/cni/99-loopback.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "lo", 4 | "type": "loopback" 5 | } 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | 12 | .krustlet 13 | -------------------------------------------------------------------------------- /demo/etc/cni/10-bridge.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "bridge", 4 | "type": "bridge", 5 | "bridge": "cnio0", 6 | "isGateway": true, 7 | "ipMasq": true, 8 | "ipam": { 9 | "type": "host-local", 10 | "ranges": [ 11 | [{"subnet": "10.244.0.0/24"}] 12 | ], 13 | "routes": [{"dst": "0.0.0.0/0"}] 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /demo/hello.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: hello 5 | annotations: 6 | foo: bar 7 | labels: 8 | biz: baz 9 | spec: 10 | containers: 11 | - name: busybox 12 | image: index.docker.io/busybox:latest 13 | imagePullPolicy: IfNotPresent 14 | env: 15 | - name: TEST 16 | value: "foo" 17 | command: ['sh', '-c', 'env; while true; do echo "$(date) Hello World"; sleep 10; done'] 18 | nodeSelector: 19 | kubernetes.io/hostname: krustlet-cri 20 | -------------------------------------------------------------------------------- /demo/entrypoint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | mkdir -p /var/log/containerd 6 | echo "Starting ContainerD" 7 | containerd > /var/log/containerd/stdout 2> /var/log/containerd/stderr & 8 | 9 | echo "Bootstrapping TLS" 10 | KUBECONFIG=/mnt/kube/config bootstrap-tls 11 | 12 | echo "Starting KrustletCRI" 13 | krustlet-cri --hostname krustlet-cri --node-ip 172.17.0.1 --cert-file=/root/.krustlet/config/krustlet.crt --private-key-file=/root/.krustlet/config/krustlet.key --bootstrap-file=/root/.krustlet/config/bootstrap.conf 14 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "krustlet-cri" 3 | version = "0.2.0" 4 | authors = ["kflansburg "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | kubelet = { version = "0.5.0", features = ['cli'] } 9 | tokio = { version = "0.2", features = ["macros", "net"] } 10 | kube = "0.40" 11 | env_logger = "0.7" 12 | anyhow = "*" 13 | async-trait = "0.1" 14 | log = "0.4" 15 | tonic = { version = "0.2" } 16 | prost = "0.6" 17 | tower = "*" 18 | k8s-openapi = { version = "0.9", features = ["v1_17"] } 19 | k8s-cri = "0.2.0" 20 | chrono = "*" 21 | serde_json = "1.0" 22 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | 8 | jobs: 9 | run: 10 | runs-on: ubuntu-latest 11 | if: github.event_name == 'push' 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Format 17 | run: cargo fmt --all -- --check 18 | 19 | - name: Check 20 | run: cargo clippy 21 | 22 | - name: Build 23 | run: cargo build --verbose 24 | 25 | - name: Run tests 26 | run: cargo test --tests --verbose 27 | 28 | # - name: Doctests 29 | # run: cargo test --doc --all 30 | -------------------------------------------------------------------------------- /src/states/error.rs: -------------------------------------------------------------------------------- 1 | use kubelet::state::prelude::*; 2 | 3 | use super::PodState; 4 | 5 | #[derive(Default, Debug)] 6 | /// The Pod failed to run. 7 | pub struct Error { 8 | pub message: String, 9 | } 10 | 11 | #[async_trait::async_trait] 12 | impl State for Error { 13 | async fn next( 14 | self: Box, 15 | _pod_state: &mut PodState, 16 | _pod: &Pod, 17 | ) -> anyhow::Result> { 18 | unimplemented!() 19 | } 20 | 21 | async fn json_status( 22 | &self, 23 | _pod_state: &mut PodState, 24 | _pod: &Pod, 25 | ) -> anyhow::Result { 26 | make_status(Phase::Pending, &self.message) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/states/running.rs: -------------------------------------------------------------------------------- 1 | use super::PodState; 2 | use kubelet::state::prelude::*; 3 | 4 | /// The Kubelet is running the Pod. 5 | #[derive(Default, Debug)] 6 | pub struct Running; 7 | 8 | #[async_trait::async_trait] 9 | impl State for Running { 10 | async fn next( 11 | self: Box, 12 | _pod_state: &mut PodState, 13 | _pod: &Pod, 14 | ) -> anyhow::Result> { 15 | // TODO: Check for container exits. 16 | loop { 17 | tokio::time::delay_for(std::time::Duration::from_secs(10)).await; 18 | } 19 | } 20 | 21 | async fn json_status( 22 | &self, 23 | _pod_state: &mut PodState, 24 | _pod: &Pod, 25 | ) -> anyhow::Result { 26 | make_status(Phase::Running, "Running") 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #![type_length_limit = "1271125"] 2 | mod provider; 3 | mod states; 4 | 5 | use log::debug; 6 | 7 | #[tokio::main] 8 | async fn main() -> anyhow::Result<()> { 9 | let config = kubelet::config::Config::new_from_flags(env!("CARGO_PKG_VERSION")); 10 | 11 | env_logger::init(); 12 | 13 | debug!("Loading Kubeconfig."); 14 | let kubeconfig = 15 | kubelet::bootstrap(&config, &config.bootstrap_file, |s| println!("{}", s)).await?; 16 | 17 | debug!("Creating Provider."); 18 | let provider = provider::Provider::new_from_socket_address( 19 | "/run/containerd/containerd.sock", 20 | kubeconfig.clone(), 21 | ); 22 | 23 | debug!("Creating Kubelet."); 24 | let kubelet = kubelet::Kubelet::new(provider, kubeconfig, config).await?; 25 | 26 | debug!("Running."); 27 | kubelet.start().await 28 | } 29 | -------------------------------------------------------------------------------- /src/states/registered.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use log::info; 3 | 4 | use super::image_pull::ImagePull; 5 | use super::PodState; 6 | use kubelet::state::prelude::*; 7 | 8 | /// The Kubelet is aware of the Pod. 9 | #[derive(Default, Debug)] 10 | pub struct Registered; 11 | 12 | #[async_trait] 13 | impl State for Registered { 14 | async fn next( 15 | self: Box, 16 | _pod_state: &mut PodState, 17 | pod: &Pod, 18 | ) -> anyhow::Result> { 19 | info!( 20 | "ADD called for namespace {} pod {}", 21 | pod.namespace(), 22 | pod.name() 23 | ); 24 | Ok(Transition::next(self, ImagePull)) 25 | } 26 | 27 | async fn json_status( 28 | &self, 29 | _pod_state: &mut PodState, 30 | _pod: &Pod, 31 | ) -> anyhow::Result { 32 | make_status(Phase::Pending, "Registered") 33 | } 34 | } 35 | 36 | impl TransitionTo for Registered {} 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # krustlet-cri 2 | 3 | The goal of this project is to build a fully-featured Kubelet in Rust by leveraging the [Krustlet Project from Deis Labs](https://github.com/deislabs/krustlet). 4 | 5 | * Fully `async` Rust to maximize performance. 6 | * No `panics` and leverage Rust error handling for reliability. 7 | * Use [CNI](https://github.com/containernetworking/cni/blob/master/SPEC.md#network-configuration), [CSI](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/), and [CRI](https://kubernetes.io/blog/2016/12/container-runtime-interface-cri-in-kubernetes/) exclusively to simplify development while maximizing support for existing and future container runtimes and network providers. 8 | 9 | # What Works 10 | * Node registration. 11 | * Basic pod create and delete. 12 | * Container logs. 13 | * Tested with `containerd`. 14 | 15 | # Try It Out 16 | 17 | This example uses Kind to demonstrate KrustletCRI. KrustletCRI will run in a privileged Docker container. 18 | 19 | 1. If you do not already have a Kind cluster running: 20 | 21 | ``` 22 | kind create cluster 23 | ``` 24 | 25 | 2. Ensure that your `kubectl` is configured to use this Kind cluster by default, as it will be used for TLS Bootstrapping. 26 | This should show the Kubernetes master for the Kind cluster: 27 | 28 | ``` 29 | kubectl cluster-info 30 | ``` 31 | 32 | 3. Build the KrustletCRI image. 33 | 34 | ``` 35 | docker build -t krustlet-cri -f demo/Dockerfile . 36 | ``` 37 | 38 | 4. Run KrustletCRI. 39 | 40 | This setup will cache KrustletCRI credentials to a directory mounted from the host, create this directory: 41 | 42 | ``` 43 | mkdir .krustlet 44 | ``` 45 | 46 | This will: 47 | * Launch and background `containerd`. 48 | * Bootstrap Kubelet TLS certificates and configure them with the Kind cluster. (This can take a while the first time.) 49 | * Launch KrustletCRI and follow log output. 50 | 51 | ``` 52 | docker run -it --privileged -p 3000:3000 -v $(pwd)/.krustlet:/root/.krustlet -v $HOME/.kube:/mnt/kube --network host --hostname krustlet-cri krustlet-cri 53 | ``` 54 | 55 | 56 | Once TLS bootstrapping had begun, you will need to approve the KrustletCRI certificate, in another shell: 57 | 58 | ``` 59 | kubectl certificate approve krustlet-cri-tls 60 | ``` 61 | 62 | 6. Verify `krustlet-cri` has joined the node poll. 63 | 64 | ``` 65 | kubectl get nodes 66 | ``` 67 | 68 | 7. Finally, schedule a Pod on KrustletCRI. 69 | 70 | ``` 71 | kubectl apply -f demo/hello.yaml 72 | kubectl logs -f hello 73 | ``` 74 | -------------------------------------------------------------------------------- /demo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.44 as BUILD 2 | 3 | # Build depends 4 | RUN USER=root cargo new --bin app 5 | WORKDIR /app 6 | COPY Cargo.toml . 7 | RUN cargo build --release 8 | 9 | # Build KrustletCRI 10 | RUN rm src/*.rs 11 | RUN rm ./target/release/deps/krustlet_cri* 12 | COPY src/ src/ 13 | RUN cargo build --release 14 | 15 | FROM debian:stable as RUN 16 | 17 | RUN apt update && apt install -y wget iptables && rm -rf /var/lib/apt/lists/* 18 | 19 | WORKDIR / 20 | 21 | # Install ContainerD 22 | ENV CONTAINERD_VERSION 1.3.4 23 | RUN wget -q https://github.com/containerd/containerd/releases/download/v$CONTAINERD_VERSION/containerd-$CONTAINERD_VERSION.linux-amd64.tar.gz && \ 24 | tar -xf containerd-$CONTAINERD_VERSION.linux-amd64.tar.gz && \ 25 | rm containerd-$CONTAINERD_VERSION.linux-amd64.tar.gz 26 | 27 | # Install `crictl`. Useful for debug. 28 | ENV CRICTL_VERSION 1.18.0 29 | RUN wget -q https://github.com/kubernetes-sigs/cri-tools/releases/download/v$CRICTL_VERSION/crictl-v$CRICTL_VERSION-linux-amd64.tar.gz && \ 30 | tar -xf crictl-v$CRICTL_VERSION-linux-amd64.tar.gz && \ 31 | rm crictl-v$CRICTL_VERSION-linux-amd64.tar.gz && \ 32 | mv crictl bin/ 33 | 34 | # Install Kubectl 35 | ENV KUBECTL_VERSION 1.18.0 36 | RUN wget -q https://storage.googleapis.com/kubernetes-release/release/v$KUBECTL_VERSION/bin/linux/amd64/kubectl -O /bin/kubectl && chmod +x /bin/kubectl 37 | 38 | # Install Krustlet TLS Bootstrap Script 39 | RUN wget -q https://raw.githubusercontent.com/deislabs/krustlet/master/docs/howto/assets/bootstrap.sh -O /bin/bootstrap-tls && chmod +x /bin/bootstrap-tls 40 | 41 | # Configure Basic CNI 42 | RUN wget -q https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgz && \ 43 | mkdir -p /etc/cni/net.d /opt/cni/bin && \ 44 | tar -xf cni-plugins-linux-amd64-v0.8.2.tgz -C /opt/cni/bin/ && \ 45 | rm cni-plugins-linux-amd64-v0.8.2.tgz 46 | 47 | # Install RUNC 48 | RUN wget -q https://github.com/opencontainers/runc/releases/download/v1.0.0-rc8/runc.amd64 && \ 49 | mv runc.amd64 /bin/runc && \ 50 | chmod +x /bin/runc 51 | 52 | ADD demo/etc/cni/10-bridge.conf /etc/cni/net.d/10-bridge.conf 53 | ADD demo/etc/cni/99-loopback.conf /etc/cni/net.d/99-loopback.conf 54 | ADD demo/etc/containerd/config.toml /etc/containerd/config.toml 55 | 56 | ENV CONTAINER_RUNTIME_ENDPOINT unix:///run/containerd/containerd.sock 57 | ENV KUBECONFIG /root/.krustlet/config/kubeconfig 58 | 59 | COPY --from=BUILD /app/target/release/krustlet-cri bin/krustlet-cri 60 | 61 | COPY demo/entrypoint /bin/ 62 | 63 | ENTRYPOINT ["entrypoint"] 64 | -------------------------------------------------------------------------------- /src/states/terminated.rs: -------------------------------------------------------------------------------- 1 | use super::PodState; 2 | use async_trait::async_trait; 3 | use k8s_cri::v1alpha2 as cri; 4 | use kubelet::state::prelude::*; 5 | use log::{debug, error, info, warn}; 6 | 7 | /// Pod was deleted. 8 | #[derive(Default, Debug)] 9 | pub struct Terminated; 10 | 11 | pub async fn stop_and_delete_pod_sandbox( 12 | pod_state: &PodState, 13 | pod: kubelet::pod::Pod, 14 | ) -> anyhow::Result<()> { 15 | match pod_state 16 | .shared 17 | .pods 18 | .read() 19 | .await 20 | .get(&(pod.namespace().to_string(), pod.name().to_string())) 21 | { 22 | Some(pod_sandbox) => { 23 | debug!("Stopping pod sandbox {}", pod.name()); 24 | let mut client = match pod_state.shared.client().await { 25 | Ok(client) => client, 26 | Err(e) => { 27 | error!("Error creating client: {:?}", &e); 28 | anyhow::bail!(e); 29 | } 30 | }; 31 | let request = tonic::Request::new(cri::StopPodSandboxRequest { 32 | pod_sandbox_id: pod_sandbox.id.clone(), 33 | }); 34 | debug!("Sending request: {:?}", &request); 35 | let response = match client.stop_pod_sandbox(request).await { 36 | Ok(response) => response, 37 | Err(e) => { 38 | error!("Error making request: {:?}", &e); 39 | anyhow::bail!(e); 40 | } 41 | }; 42 | info!("Stopped pod sandbox {}: {:?}", pod.name(), response); 43 | 44 | debug!("Removing pod sandbox {}", pod.name()); 45 | let request = tonic::Request::new(cri::RemovePodSandboxRequest { 46 | pod_sandbox_id: pod_sandbox.id.clone(), 47 | }); 48 | debug!("Sending request: {:?}", &request); 49 | let response = match client.remove_pod_sandbox(request).await { 50 | Ok(response) => response, 51 | Err(e) => { 52 | error!("Error making request: {:?}", &e); 53 | anyhow::bail!(e); 54 | } 55 | }; 56 | info!("Removed pod sandbox {}: {:?}", pod.name(), response); 57 | Ok(()) 58 | } 59 | None => { 60 | warn!("Unkown pod."); 61 | Ok(()) 62 | } 63 | } 64 | } 65 | 66 | #[async_trait] 67 | impl State for Terminated { 68 | async fn next( 69 | self: Box, 70 | pod_state: &mut PodState, 71 | pod: &Pod, 72 | ) -> anyhow::Result> { 73 | pod_state.shared.refresh_pods().await?; 74 | stop_and_delete_pod_sandbox(&pod_state, pod.clone()).await?; 75 | let dp = kube::api::DeleteParams { 76 | grace_period_seconds: Some(0), 77 | ..Default::default() 78 | }; 79 | let pod_client: kube::Api = kube::Api::namespaced( 80 | kube::client::Client::new(pod_state.shared.kubeconfig.clone()), 81 | pod.namespace(), 82 | ); 83 | // TODO: Handle Either 84 | pod_client.delete(pod.name(), &dp).await?; 85 | Ok(Transition::Complete(Ok(()))) 86 | } 87 | 88 | async fn json_status( 89 | &self, 90 | _pod_state: &mut PodState, 91 | _pod: &Pod, 92 | ) -> anyhow::Result { 93 | make_status(Phase::Succeeded, "Terminated") 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/states/image_pull.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use k8s_cri::v1alpha2 as cri; 3 | use log::{debug, error, info}; 4 | use std::convert::TryFrom; 5 | use tokio::net::UnixStream; 6 | use tonic::transport::{Channel, Endpoint, Uri}; 7 | use tower::service_fn; 8 | 9 | use super::{error::Error, starting::Starting, PodState}; 10 | use kubelet::state::prelude::*; 11 | 12 | /// Kubelet is pulling container images. 13 | #[derive(Default, Debug)] 14 | pub struct ImagePull; 15 | 16 | async fn make_image_client( 17 | path: &'static str, 18 | ) -> anyhow::Result> { 19 | let channel = Endpoint::try_from("lttp://[::]:50051")? 20 | .connect_with_connector(service_fn(move |_: Uri| UnixStream::connect(path))) 21 | .await?; 22 | 23 | let client = cri::image_service_client::ImageServiceClient::new(channel); 24 | Ok(client) 25 | } 26 | 27 | async fn image_present( 28 | image_client: &mut cri::image_service_client::ImageServiceClient, 29 | image: &str, 30 | ) -> anyhow::Result { 31 | let request = tonic::Request::new(cri::ImageStatusRequest { 32 | image: Some(cri::ImageSpec { 33 | image: image.to_string(), 34 | }), 35 | verbose: false, 36 | }); 37 | debug!("Sending request: {:?}", &request); 38 | let response = match image_client.image_status(request).await { 39 | Ok(response) => response.into_inner(), 40 | Err(e) => { 41 | error!("Error making request: {:?}", &e); 42 | anyhow::bail!(e); 43 | } 44 | }; 45 | Ok(response.image.is_some()) 46 | } 47 | 48 | async fn pull_image( 49 | image_client: &mut cri::image_service_client::ImageServiceClient, 50 | image: &str, 51 | sandbox_config: &cri::PodSandboxConfig, 52 | ) -> anyhow::Result<()> { 53 | info!("Pulling image: {}", &image); 54 | let request = tonic::Request::new(cri::PullImageRequest { 55 | image: Some(cri::ImageSpec { 56 | image: image.to_string(), 57 | }), 58 | // TODO support registry auth 59 | auth: None, 60 | sandbox_config: Some(sandbox_config.clone()), 61 | }); 62 | debug!("Sending request: {:?}", &request); 63 | let response = match image_client.pull_image(request).await { 64 | Ok(response) => response.into_inner(), 65 | Err(e) => { 66 | error!("Error making request: {:?}", &e); 67 | anyhow::bail!(e); 68 | } 69 | }; 70 | info!("Pulled image: {:?}", response); 71 | Ok(()) 72 | } 73 | 74 | #[async_trait] 75 | impl State for ImagePull { 76 | async fn next( 77 | self: Box, 78 | pod_state: &mut PodState, 79 | pod: &Pod, 80 | ) -> anyhow::Result> { 81 | let mut image_client = match make_image_client(pod_state.shared.socket_address).await { 82 | Ok(client) => client, 83 | Err(e) => { 84 | let message = format!("Error creating image client: {:?}", &e); 85 | error!("{}", message); 86 | return Ok(Transition::next(self, Error { message })); 87 | } 88 | }; 89 | 90 | for container in pod.containers() { 91 | let image: String = container.image()?.unwrap().into(); 92 | let pull_policy = container.effective_pull_policy()?; 93 | info!("Image pull policy: {:?}", pull_policy); 94 | match pull_policy { 95 | kubelet::container::PullPolicy::Always => { 96 | pull_image(&mut image_client, &image, &pod_state.sandbox_config).await? 97 | } 98 | kubelet::container::PullPolicy::IfNotPresent => { 99 | if !image_present(&mut image_client, &image).await? { 100 | info!("Image not present."); 101 | pull_image(&mut image_client, &image, &pod_state.sandbox_config).await? 102 | } else { 103 | info!("Image present."); 104 | } 105 | } 106 | kubelet::container::PullPolicy::Never => (), 107 | } 108 | } 109 | Ok(Transition::next(self, Starting)) 110 | } 111 | 112 | async fn json_status( 113 | &self, 114 | _pod_state: &mut PodState, 115 | _pod: &Pod, 116 | ) -> anyhow::Result { 117 | make_status(Phase::Pending, "ImagePull") 118 | } 119 | } 120 | 121 | impl TransitionTo for ImagePull {} 122 | impl TransitionTo for ImagePull {} 123 | -------------------------------------------------------------------------------- /src/states/mod.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use k8s_cri::v1alpha2 as cri; 3 | use log::{debug, error, info}; 4 | use std::convert::TryFrom; 5 | use tokio::net::UnixStream; 6 | use tonic::transport::{Channel, Endpoint, Uri}; 7 | use tower::service_fn; 8 | 9 | mod error; 10 | mod image_pull; 11 | mod registered; 12 | mod running; 13 | mod starting; 14 | mod terminated; 15 | 16 | pub(crate) use registered::Registered; 17 | pub(crate) use terminated::Terminated; 18 | 19 | use crate::provider::{ContainerMap, PodMap}; 20 | 21 | #[derive(Clone)] 22 | pub struct SharedPodState { 23 | pub pods: PodMap, 24 | pub containers: ContainerMap, 25 | pub socket_address: &'static str, 26 | pub kubeconfig: kube::Config, 27 | } 28 | 29 | impl SharedPodState { 30 | pub async fn client( 31 | &self, 32 | ) -> anyhow::Result> { 33 | let path = self.socket_address; 34 | let channel = Endpoint::try_from("lttp://[::]:50051")? 35 | .connect_with_connector(service_fn(move |_: Uri| UnixStream::connect(path))) 36 | .await?; 37 | let client = cri::runtime_service_client::RuntimeServiceClient::new(channel); 38 | Ok(client) 39 | } 40 | 41 | pub async fn refresh_containers(&self) -> anyhow::Result<()> { 42 | debug!("Loading containers."); 43 | let request = tonic::Request::new(cri::ListContainersRequest { filter: None }); 44 | debug!("Sending request: {:?}", &request); 45 | let mut client = match self.client().await { 46 | Ok(client) => client, 47 | Err(e) => { 48 | error!("Error creating client: {:?}", &e); 49 | anyhow::bail!(e); 50 | } 51 | }; 52 | let response = match client.list_containers(request).await { 53 | Ok(response) => response.into_inner(), 54 | Err(e) => { 55 | error!("Error making request: {:?}", &e); 56 | anyhow::bail!(e); 57 | } 58 | }; 59 | 60 | debug!("{:?}", &response); 61 | info!("Found {} containerss.", response.containers.len()); 62 | 63 | let mut containers = self.containers.write().await; 64 | *containers = std::collections::HashMap::new(); 65 | for container in response.containers { 66 | if let Some(meta) = container.metadata.clone() { 67 | containers.insert((container.pod_sandbox_id.clone(), meta.name), container); 68 | } 69 | } 70 | Ok(()) 71 | } 72 | 73 | pub async fn refresh_pods(&self) -> anyhow::Result<()> { 74 | debug!("Refreshing Pods."); 75 | let request = tonic::Request::new(cri::ListPodSandboxRequest { filter: None }); 76 | debug!("Sending request: {:?}", &request); 77 | let mut client = match self.client().await { 78 | Ok(client) => client, 79 | Err(e) => { 80 | error!("Error creating client: {:?}", &e); 81 | anyhow::bail!(e); 82 | } 83 | }; 84 | let response = match client.list_pod_sandbox(request).await { 85 | Ok(response) => response.into_inner(), 86 | Err(e) => { 87 | error!("Error making request: {:?}", &e); 88 | anyhow::bail!(e); 89 | } 90 | }; 91 | 92 | debug!("{:?}", &response); 93 | info!("Found {} pods.", response.items.len()); 94 | 95 | let mut pods = self.pods.write().await; 96 | *pods = std::collections::HashMap::new(); 97 | for pod in response.items { 98 | if let Some(meta) = pod.metadata.clone() { 99 | pods.insert((meta.namespace, meta.name), pod); 100 | } 101 | } 102 | Ok(()) 103 | } 104 | } 105 | 106 | pub struct PodState { 107 | pub shared: SharedPodState, 108 | pub sandbox_config: cri::PodSandboxConfig, 109 | } 110 | 111 | impl PodState { 112 | pub fn pod_name(&self) -> String { 113 | self.sandbox_config.metadata.as_ref().unwrap().name.clone() 114 | } 115 | pub fn pod_namespace(&self) -> String { 116 | self.sandbox_config 117 | .metadata 118 | .as_ref() 119 | .unwrap() 120 | .namespace 121 | .clone() 122 | } 123 | } 124 | 125 | #[async_trait] 126 | impl kubelet::state::AsyncDrop for PodState { 127 | async fn async_drop(self) { 128 | self.shared 129 | .pods 130 | .write() 131 | .await 132 | .remove(&(self.pod_namespace(), self.pod_name())); 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/states/starting.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use k8s_cri::v1alpha2 as cri; 3 | use log::{debug, error, info, warn}; 4 | 5 | use super::terminated::stop_and_delete_pod_sandbox; 6 | use super::{running::Running, PodState}; 7 | use kubelet::state::prelude::*; 8 | 9 | /// The Kubelet is starting the Pod. 10 | #[derive(Default, Debug)] 11 | pub struct Starting; 12 | 13 | #[async_trait] 14 | impl State for Starting { 15 | async fn next( 16 | self: Box, 17 | pod_state: &mut PodState, 18 | pod: &Pod, 19 | ) -> anyhow::Result> { 20 | pod_state.shared.refresh_pods().await?; 21 | 22 | let pod_exists = { 23 | pod_state 24 | .shared 25 | .pods 26 | .read() 27 | .await 28 | .contains_key(&(pod.namespace().to_string(), pod.name().to_string())) 29 | }; 30 | 31 | if pod_exists { 32 | stop_and_delete_pod_sandbox(&pod_state, pod.clone()).await?; 33 | } 34 | 35 | debug!("Starting pod sandbox {}", pod.name()); 36 | let request = tonic::Request::new(cri::RunPodSandboxRequest { 37 | config: Some(pod_state.sandbox_config.clone()), 38 | runtime_handler: "".to_string(), 39 | }); 40 | debug!("Sending request: {:?}", &request); 41 | let mut client = match pod_state.shared.client().await { 42 | Ok(client) => client, 43 | Err(e) => { 44 | error!("Error creating client: {:?}", &e); 45 | anyhow::bail!(e); 46 | } 47 | }; 48 | let response = match client.run_pod_sandbox(request).await { 49 | Ok(response) => response.into_inner(), 50 | Err(e) => { 51 | warn!( 52 | "Error creating sandbox: {:?}. Remove existing sandbox and retry.", 53 | e 54 | ); 55 | stop_and_delete_pod_sandbox(&pod_state, pod.clone()).await?; 56 | let request = tonic::Request::new(cri::RunPodSandboxRequest { 57 | config: Some(pod_state.sandbox_config.clone()), 58 | runtime_handler: "".to_string(), 59 | }); 60 | match client.run_pod_sandbox(request).await { 61 | Ok(response) => response.into_inner(), 62 | Err(e) => { 63 | error!("Error making request: {:?}", &e); 64 | anyhow::bail!(e); 65 | } 66 | } 67 | } 68 | }; 69 | info!("Started pod sandbox {}: {:?}", pod.name(), &response); 70 | let pod_sandbox_id = response.pod_sandbox_id; 71 | 72 | for container in pod.containers() { 73 | let image: String = container.image()?.unwrap().into(); 74 | debug!("Creating container: {}", container.name()); 75 | 76 | tokio::fs::create_dir_all(format!( 77 | "/var/log/pods/{}/{}/{}", 78 | pod.namespace(), 79 | pod.name(), 80 | container.name() 81 | )) 82 | .await?; 83 | 84 | let metadata = Some(cri::ContainerMetadata { 85 | name: container.name().to_string(), 86 | attempt: 0, 87 | }); 88 | 89 | let image = Some(cri::ImageSpec { 90 | image: image.clone(), 91 | }); 92 | 93 | let command = container.command().clone().unwrap_or_else(Vec::new); 94 | 95 | let args = container.args().clone().unwrap_or_else(Vec::new); 96 | 97 | let working_dir = container 98 | .working_dir() 99 | .cloned() 100 | .unwrap_or_else(|| "/".to_string()); 101 | 102 | // TODO: Support value_from 103 | let envs = container 104 | .env() 105 | .clone() 106 | .unwrap_or_else(Vec::new) 107 | .into_iter() 108 | .filter_map(|env| match env.value { 109 | Some(value) => Some(k8s_cri::v1alpha2::KeyValue { 110 | key: env.name, 111 | value, 112 | }), 113 | None => None, 114 | }) 115 | .collect(); 116 | 117 | // TODO 118 | let mounts = vec![]; 119 | 120 | // TODO 121 | let devices = vec![]; 122 | 123 | let labels = std::collections::BTreeMap::new(); 124 | 125 | let annotations = std::collections::BTreeMap::new(); 126 | 127 | let log_path = format!("{}/log", container.name()); 128 | 129 | let linux = None; 130 | 131 | let config = Some(cri::ContainerConfig { 132 | metadata, 133 | image, 134 | command, 135 | args, 136 | working_dir, 137 | envs, 138 | mounts, 139 | devices, 140 | labels, 141 | annotations, 142 | log_path, 143 | stdin: false, 144 | stdin_once: false, 145 | tty: false, 146 | linux, 147 | windows: None, 148 | }); 149 | 150 | let request = tonic::Request::new(cri::CreateContainerRequest { 151 | pod_sandbox_id: pod_sandbox_id.clone(), 152 | config, 153 | sandbox_config: Some(pod_state.sandbox_config.clone()), 154 | }); 155 | debug!("Sending request: {:?}", &request); 156 | let response = match client.create_container(request).await { 157 | Ok(response) => response.into_inner(), 158 | Err(e) => { 159 | error!("Error making request: {:?}", &e); 160 | anyhow::bail!(e); 161 | } 162 | }; 163 | debug!("Created container {}: {:?}", container.name(), &response); 164 | let container_id = response.container_id; 165 | 166 | debug!("Starting container: {}", container.name()); 167 | let request = tonic::Request::new(cri::StartContainerRequest { container_id }); 168 | debug!("Sending request: {:?}", &request); 169 | let response = match client.start_container(request).await { 170 | Ok(response) => response.into_inner(), 171 | Err(e) => { 172 | error!("Error making request: {:?}", &e); 173 | anyhow::bail!(e); 174 | } 175 | }; 176 | info!("Started container {}: {:?}", container.name(), &response); 177 | } 178 | Ok(Transition::next(self, Running)) 179 | } 180 | 181 | async fn json_status( 182 | &self, 183 | _pod_state: &mut PodState, 184 | _pod: &Pod, 185 | ) -> anyhow::Result { 186 | make_status(Phase::Pending, "Starting") 187 | } 188 | } 189 | 190 | impl TransitionTo for Starting {} 191 | -------------------------------------------------------------------------------- /src/provider.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use k8s_cri::v1alpha2 as cri; 3 | use log::{debug, error, info}; 4 | use std::sync::Arc; 5 | 6 | use crate::states::{PodState, Registered, SharedPodState, Terminated}; 7 | 8 | type Namespace = String; 9 | type Pod = String; 10 | type Container = String; 11 | type Id = String; 12 | 13 | pub(crate) type PodMap = 14 | Arc>>; 15 | pub(crate) type ContainerMap = 16 | Arc>>; 17 | 18 | pub struct Provider { 19 | shared: SharedPodState, 20 | } 21 | 22 | impl Provider { 23 | pub fn new_from_socket_address(socket_address: &'static str, kubeconfig: kube::Config) -> Self { 24 | Provider { 25 | shared: SharedPodState { 26 | socket_address, 27 | kubeconfig, 28 | pods: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())), 29 | containers: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())), 30 | }, 31 | } 32 | } 33 | 34 | async fn pod_id(&self, namespace: &str, pod: &str) -> anyhow::Result { 35 | let key = (namespace.to_string(), pod.to_string()); 36 | let has_pod = self.shared.pods.read().await.contains_key(&key); 37 | if has_pod { 38 | Ok(self 39 | .shared 40 | .pods 41 | .read() 42 | .await 43 | .get(&key) 44 | .unwrap() 45 | .id 46 | .to_string()) 47 | } else { 48 | self.shared.refresh_pods().await?; 49 | let has_pod = self.shared.pods.read().await.contains_key(&key); 50 | if has_pod { 51 | Ok(self 52 | .shared 53 | .pods 54 | .read() 55 | .await 56 | .get(&key) 57 | .unwrap() 58 | .id 59 | .to_string()) 60 | } else { 61 | error!("Could not find namespace {} pod {}.", namespace, pod); 62 | anyhow::bail!(kubelet::provider::ProviderError::PodNotFound { 63 | pod_name: pod.to_string(), 64 | }); 65 | } 66 | } 67 | } 68 | 69 | async fn container_id( 70 | &self, 71 | namespace: &str, 72 | pod: &str, 73 | container: &str, 74 | ) -> anyhow::Result { 75 | let pod_id = self.pod_id(namespace, pod).await?; 76 | let key = (pod_id, container.to_string()); 77 | let has_container = self.shared.containers.read().await.contains_key(&key); 78 | if has_container { 79 | Ok(self 80 | .shared 81 | .containers 82 | .read() 83 | .await 84 | .get(&key) 85 | .unwrap() 86 | .id 87 | .to_string()) 88 | } else { 89 | self.shared.refresh_containers().await?; 90 | let has_container = self.shared.containers.read().await.contains_key(&key); 91 | if has_container { 92 | Ok(self 93 | .shared 94 | .containers 95 | .read() 96 | .await 97 | .get(&key) 98 | .unwrap() 99 | .id 100 | .to_string()) 101 | } else { 102 | error!( 103 | "Could not find namespace {} pod {} container {}.", 104 | namespace, pod, container 105 | ); 106 | Err(anyhow::anyhow!( 107 | kubelet::provider::ProviderError::ContainerNotFound { 108 | pod_name: pod.to_string(), 109 | container_name: container.to_string(), 110 | } 111 | )) 112 | } 113 | } 114 | } 115 | 116 | async fn describe_container(&self, container_id: Id) -> anyhow::Result { 117 | debug!("Describing container {}.", &container_id); 118 | let request = tonic::Request::new(cri::ContainerStatusRequest { 119 | container_id, 120 | verbose: false, 121 | }); 122 | debug!("Sending request: {:?}", &request); 123 | let mut client = match self.shared.client().await { 124 | Ok(client) => client, 125 | Err(e) => { 126 | error!("Error creating client: {:?}", &e); 127 | anyhow::bail!(e); 128 | } 129 | }; 130 | let response = match client.container_status(request).await { 131 | Ok(response) => response.into_inner(), 132 | Err(e) => { 133 | error!("Error making request: {:?}", &e); 134 | anyhow::bail!(e); 135 | } 136 | }; 137 | debug!("{:?}", &response); 138 | 139 | if let Some(status) = response.status { 140 | Ok(status) 141 | } else { 142 | Err(anyhow::anyhow!( 143 | "Container status response contained no status: {:?}", 144 | &response 145 | )) 146 | } 147 | } 148 | } 149 | 150 | const AMD64: &str = "amd64"; 151 | 152 | #[async_trait] 153 | impl kubelet::provider::Provider for Provider { 154 | type PodState = PodState; 155 | 156 | type InitialState = Registered; 157 | type TerminatedState = Terminated; 158 | 159 | const ARCH: &'static str = AMD64; 160 | 161 | async fn initialize_pod_state( 162 | &self, 163 | pod: &kubelet::pod::Pod, 164 | ) -> anyhow::Result { 165 | let metadata = Some(cri::PodSandboxMetadata { 166 | name: pod.name().to_string(), 167 | namespace: pod.namespace().to_string(), 168 | uid: "".to_string(), 169 | attempt: 0, 170 | }); 171 | 172 | let hostname = pod 173 | .as_kube_pod() 174 | .spec 175 | .clone() 176 | .unwrap_or_default() 177 | .hostname 178 | .unwrap_or_else(|| "".to_string()); 179 | 180 | let log_directory = format!("/var/log/pods/{}/{}/", pod.namespace(), pod.name()); 181 | 182 | // TODO 183 | let dns_config = Some(cri::DnsConfig { 184 | servers: vec![], 185 | searches: vec![], 186 | options: vec![], 187 | }); 188 | 189 | let port_mappings = vec![]; 190 | 191 | let labels = pod.labels().clone(); 192 | 193 | let annotations = pod.annotations().clone(); 194 | 195 | let linux = None; 196 | 197 | let sandbox_config = cri::PodSandboxConfig { 198 | metadata, 199 | hostname, 200 | log_directory, 201 | dns_config, 202 | port_mappings, 203 | labels, 204 | annotations, 205 | linux, 206 | }; 207 | 208 | Ok(PodState { 209 | shared: self.shared.clone(), 210 | sandbox_config, 211 | }) 212 | } 213 | 214 | async fn node(&self, builder: &mut kubelet::node::Builder) -> anyhow::Result<()> { 215 | let request = tonic::Request::new(cri::VersionRequest { 216 | version: "v1alpha2".to_string(), 217 | }); 218 | debug!("Sending request: {:?}", &request); 219 | let mut client = match self.shared.client().await { 220 | Ok(client) => client, 221 | Err(e) => { 222 | error!("Error creating client: {:?}", &e); 223 | anyhow::bail!(e); 224 | } 225 | }; 226 | let response = match client.version(request).await { 227 | Ok(response) => response.into_inner(), 228 | Err(e) => { 229 | error!("Error making request: {:?}", &e); 230 | anyhow::bail!(e); 231 | } 232 | }; 233 | info!("Found container runtime: {:?}", &response); 234 | 235 | builder.set_container_runtime_version(&format!( 236 | "{}://{}", 237 | &response.runtime_name, &response.runtime_version 238 | )); 239 | builder.add_annotation( 240 | "kubeadm.alpha.kubernetes.io/cri-socket", 241 | self.shared.socket_address, 242 | ); 243 | builder.set_architecture("amd64"); 244 | Ok(()) 245 | } 246 | 247 | async fn logs( 248 | &self, 249 | namespace: String, 250 | pod: String, 251 | container: String, 252 | sender: kubelet::log::Sender, 253 | ) -> anyhow::Result<()> { 254 | info!( 255 | "LOGS called for namespace {} pod {} container {}.", 256 | &namespace, &pod, &container 257 | ); 258 | let container_id = self.container_id(&namespace, &pod, &container).await?; 259 | let status = self.describe_container(container_id).await?; 260 | let handle = tokio::fs::File::open(status.log_path).await?; 261 | tokio::spawn(kubelet::log::stream(handle, sender)); 262 | Ok(()) 263 | } 264 | 265 | async fn exec(&self, pod: kubelet::pod::Pod, command: String) -> anyhow::Result> { 266 | info!( 267 | "EXEC called for namespace {} pod {}: {} ", 268 | pod.namespace(), 269 | pod.name(), 270 | &command 271 | ); 272 | Err(kubelet::provider::NotImplementedError.into()) 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------