├── .dockerignore ├── init ├── go.mod └── main.go ├── .gitignore ├── Dockerfile ├── Dockerfile.inlets ├── LICENSE ├── setup-networking.sh ├── boot.sh ├── Makefile └── README.md /.dockerignore: -------------------------------------------------------------------------------- 1 | /rootfs* 2 | -------------------------------------------------------------------------------- /init/go.mod: -------------------------------------------------------------------------------- 1 | module init 2 | 3 | go 1.17 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /extract.tar 2 | /vmlinux 3 | /init/init 4 | /root* 5 | /snapshot_file 6 | /mem_file 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22-alpine as build 2 | 3 | WORKDIR /go/src/github.com/alexellis/firecracker-init-lab/init 4 | 5 | COPY init . 6 | 7 | RUN go build --tags netgo --ldflags '-s -w -extldflags "-lm -lstdc++ -static"' -o init main.go 8 | 9 | FROM alpine:3.20 10 | 11 | RUN apk add --no-cache curl ca-certificates htop 12 | 13 | COPY --from=build /go/src/github.com/alexellis/firecracker-init-lab/init/init /init 14 | 15 | -------------------------------------------------------------------------------- /Dockerfile.inlets: -------------------------------------------------------------------------------- 1 | FROM golang:1.22-alpine as build 2 | 3 | WORKDIR /go/src/github.com/alexellis/firecracker-init-lab/init 4 | 5 | COPY init . 6 | 7 | RUN go build --tags netgo --ldflags '-s -w -extldflags "-lm -lstdc++ -static"' -o init main.go 8 | 9 | FROM alpine:3.20 10 | WORKDIR /root/ 11 | 12 | RUN apk add --no-cache curl && \ 13 | curl -SLs https://get.arkade.dev | sh && \ 14 | arkade get inlets-pro && \ 15 | chmod +x .arkade/bin/inlets-pro && \ 16 | mv .arkade/bin/inlets-pro /usr/local/bin/ 17 | 18 | COPY --from=build /go/src/github.com/alexellis/firecracker-init-lab/init/init /init 19 | 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2023 Alex Ellis, OpenFaaS Ltd 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /setup-networking.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run this once on the host, or upon reboot 4 | 5 | # Add a tap device to act as a bridge between the microVM 6 | # and the host. 7 | sudo ip tuntap add dev ftap0 mode tap 8 | 9 | # The subnet is 172.16.0.0/24 and so the 10 | # host will be 172.16.0.1 and the microVM is going to be set to 11 | # 172.16.0.2 12 | sudo ip addr add 172.16.0.1/24 dev ftap0 13 | sudo ip link set ftap0 up 14 | ip addr show dev ftap0 15 | 16 | # Set up IP forwarding and masquerading 17 | 18 | # Change IFNAME to match your main ethernet adapter, the one that 19 | # accesses the Internet - check "ip addr" or "ifconfig" if you don't 20 | # know which one to use. 21 | IFNAME=wlp1s0 22 | 23 | # Enable IP forwarding 24 | sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward" 25 | 26 | # Enable masquerading / NAT - https://tldp.org/HOWTO/IP-Masquerade-HOWTO/ipmasq-background2.5.html 27 | 28 | 29 | # use iptables-save to verify if any of the rules have been added 30 | # Then, skip if they have been added 31 | 32 | if ! sudo iptables-save | grep "POSTROUTING -o $IFNAME -j MASQUERADE" > /dev/null 2>&1; then 33 | 34 | echo "Adding masquerading rules" 35 | 36 | sudo iptables -t nat -A POSTROUTING -o $IFNAME -j MASQUERADE 37 | sudo iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 38 | sudo iptables -A FORWARD -i ftap0 -o $IFNAME -j ACCEPT 39 | 40 | else 41 | echo "Masquerading rules already exist" 42 | fi 43 | 44 | -------------------------------------------------------------------------------- /init/main.go: -------------------------------------------------------------------------------- 1 | // Copyright Alex Ellis 2023 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "log" 8 | "os" 9 | "os/exec" 10 | 11 | "syscall" 12 | ) 13 | 14 | const paths = "PATH=/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin" 15 | 16 | // main starts an init process that can prepare an environment and start a shell 17 | // after the Kernel has started. 18 | func main() { 19 | fmt.Printf("Lab init booting\nCopyright Alex Ellis 2022, OpenFaaS Ltd\n") 20 | 21 | mount("none", "/proc", "proc", 0) 22 | mount("none", "/dev/pts", "devpts", 0) 23 | mount("none", "/dev/mqueue", "mqueue", 0) 24 | mount("none", "/dev/shm", "tmpfs", 0) 25 | mount("none", "/sys", "sysfs", 0) 26 | mount("none", "/sys/fs/cgroup", "cgroup", 0) 27 | 28 | setHostname("lab-vm") 29 | 30 | fmt.Printf("Lab starting /bin/sh\n") 31 | 32 | cmd := exec.Command("/bin/sh") 33 | 34 | cmd.Env = append(cmd.Env, paths) 35 | cmd.Stdin = os.Stdin 36 | cmd.Stdout = os.Stdout 37 | cmd.Stderr = os.Stderr 38 | 39 | err := cmd.Start() 40 | if err != nil { 41 | panic(fmt.Sprintf("could not start /bin/sh, error: %s", err)) 42 | } 43 | 44 | err = cmd.Wait() 45 | if err != nil { 46 | panic(fmt.Sprintf("could not wait for /bin/sh, error: %s", err)) 47 | } 48 | } 49 | 50 | func setHostname(hostname string) { 51 | err := syscall.Sethostname([]byte(hostname)) 52 | if err != nil { 53 | panic(fmt.Sprintf("cannot set hostname to %s, error: %s", hostname, err)) 54 | } 55 | } 56 | 57 | func mount(source, target, filesystemtype string, flags uintptr) { 58 | 59 | if _, err := os.Stat(target); os.IsNotExist(err) { 60 | err := os.MkdirAll(target, 0755) 61 | if err != nil { 62 | panic(fmt.Sprintf("error creating target folder: %s %s", target, err)) 63 | } 64 | } 65 | 66 | err := syscall.Mount(source, target, filesystemtype, flags, "") 67 | if err != nil { 68 | log.Printf("%s", fmt.Errorf("error mounting %s to %s, error: %s", source, target, err)) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Configure our custom init process, change to /sbin/init to 4 | # use the one that ships with the rootfs 5 | 6 | # Also sets the network IP address statically, via a kernel parameter 7 | 8 | sudo curl --unix-socket /tmp/firecracker.socket -i \ 9 | -X PUT 'http://localhost/boot-source' \ 10 | -H 'Accept: application/json' \ 11 | -H 'Content-Type: application/json' \ 12 | -d "{ 13 | \"kernel_image_path\": \"./vmlinux\", 14 | \"boot_args\": \"console=ttyS0 reboot=k panic=1 pci=off init=/init ip=172.16.0.2::172.16.0.1:255.255.255.0::eth0:off\" 15 | }" 16 | 17 | # Configure memory and vCPUs 18 | sudo curl --unix-socket /tmp/firecracker.socket -i \ 19 | -X PUT 'http://localhost/machine-config' \ 20 | -H 'Accept: application/json' \ 21 | -H 'Content-Type: application/json' \ 22 | -d '{ 23 | "vcpu_count": 2, 24 | "mem_size_mib": 256 25 | }' 26 | 27 | 28 | # Configure a network device, notice the host_dev_name matches what 29 | # we set in setup_networking.sh 30 | 31 | sudo curl -X PUT \ 32 | --unix-socket /tmp/firecracker.socket \ 33 | 'http://localhost/network-interfaces/eth0' \ 34 | -H accept:application/json \ 35 | -H content-type:application/json \ 36 | -d '{ 37 | "iface_id": "eth0", 38 | "host_dev_name": "ftap0" 39 | }' 40 | 41 | # Configure the first drive, pointing at our disk image 42 | sudo curl --unix-socket /tmp/firecracker.socket -i \ 43 | -X PUT 'http://localhost/drives/rootfs' \ 44 | -H 'Accept: application/json' \ 45 | -H 'Content-Type: application/json' \ 46 | -d "{ \ 47 | \"drive_id\": \"rootfs\", 48 | \"path_on_host\": \"./rootfs.img\", 49 | \"is_root_device\": true, 50 | \"is_read_only\": false 51 | }" 52 | 53 | # Boot the VM 54 | sudo curl --unix-socket /tmp/firecracker.socket -i \ 55 | -X PUT 'http://localhost/actions' \ 56 | -H 'Accept: application/json' \ 57 | -H 'Content-Type: application/json' \ 58 | -d '{ 59 | "action_type": "InstanceStart" 60 | }' 61 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: init-local root all start boot extract image 2 | 3 | all: root extract image 4 | 5 | export arch="x86_64" 6 | 7 | # init-local builds a static binary for local testing, but the lab uses a multi-stage 8 | # Dockerfile for this instead - https://docs.docker.com/develop/develop-images/multistage-build/ 9 | init-local: 10 | cd init && \ 11 | go build --tags netgo --ldflags '-s -w -extldflags "-lm -lstdc++ -static"' -o init main.go 12 | 13 | root: 14 | docker build -t alexellis2/custom-init . 15 | 16 | pause: 17 | sudo curl --unix-socket /tmp/firecracker.socket -i \ 18 | -X PATCH 'http://localhost/vm' \ 19 | -H 'Accept: application/json' \ 20 | -H 'Content-Type: application/json' \ 21 | -d '{"state": "Paused"}' 22 | 23 | snapshot: 24 | sudo curl --unix-socket /tmp/firecracker.socket -i \ 25 | -X PUT 'http://localhost/snapshot/create' \ 26 | -H 'Accept: application/json' \ 27 | -H 'Content-Type: application/json' \ 28 | -d '{"snapshot_type": "Full", "snapshot_path": "./snapshot_file", "mem_file_path": "./mem_file"}' 29 | 30 | resume: 31 | sudo curl --unix-socket /tmp/firecracker.socket -i \ 32 | -X PATCH 'http://localhost/vm' \ 33 | -H 'Accept: application/json' \ 34 | -H 'Content-Type: application/json' \ 35 | -d '{"state": "Resumed"}' 36 | 37 | restore: 38 | sudo curl --unix-socket /tmp/firecracker.socket -i \ 39 | -X PUT 'http://localhost/snapshot/load' \ 40 | -H 'Accept: application/json' \ 41 | -H 'Content-Type: application/json' \ 42 | -d "{ \ 43 | \"snapshot_path\": \"./snapshot_file\", \ 44 | \"mem_backend\": { \ 45 | \"backend_path\": \"./mem_file\", \ 46 | \"backend_type\": \"File\" \ 47 | }, \ 48 | \"enable_diff_snapshots\": true, \ 49 | \"resume_vm\": false \ 50 | }" 51 | 52 | # Get the AWS sample image 53 | # change to Image when using aarch64, instead of vmlinux.bin 54 | kernel: 55 | curl -o vmlinux -S -L "https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/v1.10/$(arch)/vmlinux-5.10.223" 56 | 57 | # Extract a root filesystem into a tar 58 | extract: 59 | docker rm -f extract || : 60 | rm -rf rootfs.tar || : 61 | docker create --name extract alexellis2/custom-init 62 | docker export extract -o rootfs.tar 63 | docker rm -f extract 64 | 65 | # Allocate a 5GB disk image, then extract the rootfs.tar from the 66 | # container into it 67 | image: 68 | set -e 69 | rm -rf rootfs.img || : ;\ 70 | sudo fallocate -l 5G ./rootfs.img ;\ 71 | sudo mkfs.ext4 ./rootfs.img ;\ 72 | TMP=$$(mktemp -d) ;\ 73 | echo $$TMP ;\ 74 | sudo mount -o loop ./rootfs.img $$TMP ;\ 75 | sudo tar -xvf rootfs.tar -C $$TMP ;\ 76 | sudo umount $$TMP 77 | 78 | # Start a firecracker process, ready for commands 79 | start: 80 | sudo rm -f /tmp/firecracker.socket || : 81 | sudo firecracker --api-sock /tmp/firecracker.socket 82 | 83 | # Sends commands to boot the firecracker process started via "make start" 84 | boot: 85 | sudo ./boot.sh 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## firecracker-init-lab 2 | 3 | Build a microVM from a container image 4 | 5 | > Many of the examples you'll find are broken due to changes in Firecracker 1.0 - the official quickstart guide doesn't cover the most interesting thing - working Internet access - or extracting a filesystem from a container. This lab extends [the official quickstart](https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md) so that you can explore what an init process does, and add networking. 6 | 7 | ## Pre-reqs 8 | 9 | * A bare-metal Linux host - [where can you get bare metal?](https://github.com/alexellis/awesome-baremetal#bare-metal-cloud) 10 | * Or a VM that supports nested virtualisation such as on [DigitalOcean](https://m.do.co/c/8d4e75e9886f) or GCP. 11 | * Docker installed 12 | 13 | Browse: 14 | 15 | * [Go init process](/init/main.go) 16 | * [Makefile](/Makefile) 17 | * [boot.sh](/boot.sh) - commands to start MicroVM 18 | * [setup-networking.sh](./setup-networking.sh) - configure a tap adapter, IP forwarding and IP masquerading 19 | * [Dockerfile](/Dockerfile) - for building the root filesystem 20 | 21 | ## Usage 22 | 23 | Download and install [Firecracker](https://github.com/firecracker-microvm/firecracker/releases) to `/usr/local/bin/` 24 | 25 | Or, alternatively, [Arkade](https://arkade.dev) can do this for you with: 26 | 27 | ``` 28 | curl -SLs https://get.arkade.dev | sudo sh 29 | sudo arkade system install firecracker 30 | ``` 31 | 32 | Edit the `IFNAME` in `setup-networking.sh` to match your host's network interface. 33 | 34 | Then run the script to create the ftap0 device, and to setup IP masquerading with iptables: 35 | 36 | ```bash 37 | ./setup-networking.sh 38 | ``` 39 | 40 | Download the quickstart Kernel: 41 | 42 | ``` 43 | make kernel 44 | ``` 45 | 46 | Check the Kernel is the correct architecture: 47 | 48 | ```bash 49 | file ./vmlinux 50 | ``` 51 | If you don't have the `file` utility, you can add it with `sudo apt install -qy file` 52 | 53 | Make the init process binary, and package it into a container, extract the container into a rootfs image: 54 | 55 | ```bash 56 | make all 57 | ``` 58 | 59 | In one terminal, start firecracker: 60 | 61 | ```bash 62 | make start 63 | ``` 64 | 65 | In another, instruct it to boot the rootsfs and Kernel: 66 | 67 | ```bash 68 | make boot 69 | ``` 70 | 71 | Play around in the first terminal and explore the system: 72 | 73 | ```bash 74 | free -m 75 | cat /proc/cpuinfo 76 | ip addr 77 | ip route 78 | 79 | ping -c1 1.1.1.1 80 | 81 | echo "nameserver 1.1.1.1" > /etc/resolv.conf 82 | ping -c 4 google.com 83 | 84 | apk add --no-cache curl 85 | 86 | curl -i https://inlets.dev 87 | ``` 88 | 89 | ## Expose a TCP or HTTP service to the Internet 90 | 91 | Once you've got something interesting running like a HTTP server, or an SSHD daemon, you can then get ingress from the public Internet using an [inlets tunnel](https://inlets.dev). Inlets is a static binary, and there are a couple of simple tutorials you can follow depending on what you want to expose. 92 | 93 | * [HTTPS tunnel with Let's Encrypt](https://docs.inlets.dev/tutorial/automated-http-server/) 94 | * [TCP tunnel for SSH, reverse proxy etc](https://docs.inlets.dev/tutorial/ssh-tcp-tunnel/) 95 | 96 | ## Running on a Raspberry Pi 97 | 98 | Edit Makefile, and change `arch` to `aarch64` 99 | 100 | ```Makefile 101 | export arch="x86_64" 102 | ``` 103 | 104 | ## Snapshot and restore 105 | 106 | There is experimental support for snapshot and restore in Firecracker. You will not be able to snapshot on an Intel host and restore on an AMD one, and different generations of CPUs may also be incompatible for snapshot and restore. 107 | 108 | You can read more about the current status of snapshot and restore in the [Firecracker docs](https://github.com/firecracker-microvm/firecracker/blob/main/docs/snapshotting/snapshot-support.md) 109 | 110 | Run the lab and boot the VM as per the instructions above. 111 | 112 | Now, verify the network is up with `ping -c 1 172.16.0.2` 113 | 114 | ```bash 115 | make pause 116 | make snapshot 117 | ``` 118 | 119 | Kill the Firecracker process, and then restore the snapshot: 120 | 121 | ```bash 122 | make start 123 | ``` 124 | 125 | In another terminal, restore the snapshot: 126 | 127 | ```bash 128 | make restore 129 | make resume 130 | ``` 131 | 132 | Run `./setup-networking.sh` again and verify the network is up with `ping -c 1 172.16.0.2`. 133 | 134 | ## Live-event - A cracking time with Richard Case of Weaveworks 135 | 136 | [Richard Case](https://twitter.com/fruit_case) will join me as we explain to you why we're so excited about Firecracker, what use-cases we see and try to show you a little of what can be done with it. Richard's been at the sharp end of this technology for months, and is working on a cutting edge bare-metal Kubernetes project called Liquid Metal. 137 | 138 | [![Live stream](https://img.youtube.com/vi/CYCsa5e2vqg/hqdefault.jpg)](https://www.youtube.com/watch?v=CYCsa5e2vqg) 139 | 140 | > You'll hear more about it on Friday lunch at 12:00pm BST. 141 | 142 | [Subscribe & remind](https://www.youtube.com/watch?v=CYCsa5e2vqg) 143 | 144 | If you can't make it live, then you'll be able to jump onto the replay with your morning coffee. 145 | 146 | # Our real life applications of Firecracker 147 | 148 | Why do I know so much about Firecracker, microVMs and low-level primitives? We've built a couple of products at OpenFaaS Ltd that integrate with it precisely and efficiently to do things that containers cannot. 149 | 150 | ## Slicer - Firecracker via API, or long lived servers 151 | 152 | [SlicerVM.com](https://slicervm.com) was developed as an internal tool for running several or several dozen microVMs with Linux preinstalled on bare-metal, booting in 1-2s. At [openfaas.com](https://openfaas.com) we often need to re-create customer environments to reproduce bugs, write up reference architectures, and to test our own work - either with Kubernetes layered on top or simply with a normal vanilla OS like Ubuntu or Rocky Linux. 153 | 154 | As a result, Slicer is the quickest, best supported, and most user-friendly way to run microVMs on bare-metal machines or cloud VMs using nested virt. 155 | 156 | ## Faster, more secure CI with Firecracker and actuated 157 | 158 | We demoed actuated for fast and secure CI with Firecracker, since then it's being used in production and has launched over 100k VMs so far. 159 | 160 | Read more on the website, on the blog or in the docs at: [actuated.dev](https://actuated.dev). 161 | 162 | Watch a demo: 163 | 164 | * [Actuated for GitHub Actions](https://youtu.be/2o28iUC-J1w?si=FfDcnwemqWPWaDvF) 165 | * [Acuated for GitLab CI](https://www.youtube.com/watch?v=PybSPduDT6s) 166 | --------------------------------------------------------------------------------