├── Makefile ├── README.md ├── docker-to-firecracker.gif ├── main.go └── run.go /Makefile: -------------------------------------------------------------------------------- 1 | # Base path used to install. 2 | DESTDIR=/usr/local 3 | BINARY=docker-to-firecracker 4 | 5 | .PHONY: clean build install uninstall all 6 | 7 | all: clean build 8 | 9 | clean: ## removes the binary 10 | @echo "Cleaning $(BINARY)" 11 | @rm -f $(BINARY) 12 | 13 | build: ## builds the go binary 14 | @echo "Building $(BINARY)" 15 | @go build -o $(BINARY) main.go run.go 16 | 17 | install: ## install binary 18 | @echo "Installing $(BINARY) to $(DESTDIR)/bin" 19 | @mkdir -p $(DESTDIR)/bin 20 | @mv $(BINARY) $(DESTDIR)/bin 21 | 22 | uninstall: ## uninstall binary 23 | @echo "Uninstalling $(BINARY) from $(DESTDIR)/bin" 24 | @rm -f $(addprefix $(DESTDIR)/bin/,$(notdir $(BINARY))) 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [Experiment] Docker To Firecracker 2 | 3 | Experimental CLI that takes a Docker image url and runs it in a Firecracker VM 4 | 5 | Please do not use this in production for anything, _you're gonna have a bad time_. 6 | 7 | ![Docker To Firecracker](https://github.com/pyro/experiment-firecracker-run-docker-image/raw/master/docker-to-firecracker.gif) 8 | 9 | ## How Does This Work? 10 | 11 | - Fetches an image using ContainerD 12 | - Extracts CMD and ENV VARS from image metadata 13 | - Creates an empty ext4 filesystem (mounts it at /mnt but default) 14 | - Dumps the image rootfs in the the empty ext4 filesystem 15 | - Creates an init script with CMD and ENV VARS image metatdata (unmounts /mnt by default) 16 | - Starts a Firecracker VM with a Kernel + Docker Rootfs (Includes Docker ENV VARS + Docker CMD) 17 | 18 | ## Quick(ish) Start 19 | 20 | 1. Dev Environment 21 | 22 | I've been using GCP VMs for this using these instuctions: https://github.com/firecracker-microvm/firecracker/blob/master/docs/dev-machine-setup.md#gcp 23 | 24 | You _could_ also use an i3.metal intance on AWS but its an expensive instance (something like 72 cores and 512GB memory). 25 | 26 | You could also run this on a a local Linux dev box if that's your thing. 27 | 28 | 2. Setup Go 29 | 30 | If you're using Ubuntu (either 1.10 or 1.11 should work): https://github.com/golang/go/wiki/Ubuntu 31 | 32 | If you're using something else I'll leave that up to you, just make sue you've got at least Golang 1.10 33 | 34 | (either 1.10 or 1.11 should work) (either 1.10 or 1.11 should work) (either 1.10 or 1.11 should work) 35 | 36 | 3. Install ContainerD 37 | 38 | Follow the instructions here: https://containerd.io/docs/getting-started/#starting-containerd 39 | 40 | You can also build ContainerD from source. ContainerD needs to be running for this to work. 41 | 42 | 4. Get Source and Build Binary 43 | 44 | ``` 45 | go get github.com/pyro/experiment-firecracker-run-docker-image 46 | cd $GOPATH/src/github.com/pyro/experiment-firecracker-run-docker-image 47 | make 48 | sudo make install 49 | ``` 50 | 51 | _Make sure /usr/local/bin is in your PATH_ 52 | 53 | 4. Download Firecracker Resources 54 | 55 | ```sh 56 | # create a workspace (you can put this anywhere) 57 | mkdir ~/DockerToFirecracker 58 | cd ~/DockerToFirecracker 59 | # Download Firecracker Binary 60 | FC_VERSION=0.14.0 61 | curl -LOJ https://github.com/firecracker-microvm/firecracker/releases/download/v${FC_VERSION}/firecracker-v${FC_VERSION} 62 | mv firecracker-v${FC_VERSION} firecracker 63 | chmod +x firecracker 64 | # Download A Kernel Built For Firecracker 65 | curl -fsSL -o hello-vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/hello/kernel/hello-vmlinux.bin 66 | ``` 67 | 68 | 5. Run A Docker Container On Firecracker 69 | 70 | ```sh 71 | sudo docker-to-firecracker run docker.io/hharnisc/hello:latest 72 | 73 | # you should see a bunch of output and eventually "hello, world!" (its hard to see in the logs) 74 | 75 | [ 0.880238] Write protecting the kernel read-only data: 12288k 76 | [ 0.908489] Freeing unused kernel memory: 2016K 77 | [ 0.922293] Freeing unused kernel memory: 584K 78 | hello, world! 79 | [ 0.942806] Kernel panic - not syncing: Attempted to kill init! exitcode=0x00000000 80 | [ 0.942806] 81 | ``` 82 | 83 | Yes you should see a Kernel panic here because the init script is exiting. :D 84 | 85 | 86 | 87 | You can also run Redis 88 | 89 | ```sh 90 | sudo docker-to-firecracker run docker.io/library/redis:latest 91 | 92 | ... 93 | 94 | [ 0.938889] Freeing unused kernel memory: 584K 95 | [ 0.976555] random: redis-server: uninitialized urandom read (4096 bytes read) 96 | 459:C 28 Jan 2019 16:20:02.571 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo 97 | 459:C 28 Jan 2019 16:20:02.580 # Redis version=5.0.3, bits=64, commit=00000000, modified=0, pid=459, just started 98 | 459:C 28 Jan 2019 16:20:02.591 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf 99 | 459:M 28 Jan 2019 16:20:02.608 * Increased maximum number of open files to 10032 (it was originally set to 1024). 100 | _._ 101 | _.-``__ ''-._ 102 | _.-`` `. `_. ''-._ Redis 5.0.3 (00000000/0) 64 bit 103 | .-`` .-```. ```\/ _.,_ ''-._ 104 | ( ' , .-` | `, ) Running in standalone mode 105 | |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 106 | | `-._ `._ / _.-' | PID: 459 107 | `-._ `-._ `-./ _.-' _.-' 108 | |`-._`-._ `-.__.-' _.-'_.-'| 109 | | `-._`-._ _.-'_.-' | http://redis.io 110 | `-._ `-._`-.__.-'_.-' _.-' 111 | |`-._`-._ `-.__.-' _.-'_.-'| 112 | | `-._`-._ _.-'_.-' | 113 | `-._ `-._`-.__.-'_.-' _.-' 114 | `-._ `-.__.-' _.-' 115 | `-._ _.-' 116 | `-.__.-' 117 | 118 | 459:M 28 Jan 2019 16:20:02.741 # Server initialized 119 | 459:M 28 Jan 2019 16:20:02.747 * Ready to accept connections 120 | [ 1.504370] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x2126dc50dfd, max_idle_ns: 440795251059 ns 121 | INFO[0053] Caught signal terminated 122 | WARN[0053] firecracker exited: signal: terminated 123 | INFO[0053] Start machine was happy 124 | 125 | ``` 126 | There's no networking setup so its basically useless but you can run it. You'll also need to terminate the firecracker vm with `kill`. 127 | 128 | Another caveat is scratch builds -- you'll need to set the init boot arg directly -- since the filesystem generated is basically empty. For example the hello-world container: 129 | 130 | ```sh 131 | sudo docker-to-firecracker run --boot-init-arg=/hello --generate-boot-init=false docker.io/library/hello-world:latest 132 | ``` 133 | 134 | The flags passed to docker-to-firecracker tell it not to generate an init script and instead just call `/hello` directly. 135 | 136 | -------------------------------------------------------------------------------- /docker-to-firecracker.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pyro/experiment-firecracker-run-docker/113abb658af22fc3d682226aebb2cc827acdde04/docker-to-firecracker.gif -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | import ( 3 | "github.com/genuinetools/pkg/cli" 4 | ) 5 | 6 | func main() { 7 | p := cli.NewProgram() 8 | p.Name = "docker-to-firecracker" 9 | p.Description = "Extract a rootfs from a Docker image and run it in a VM" 10 | p.Commands = []cli.Command{ 11 | &runCommand{}, 12 | } 13 | p.Run() 14 | } 15 | -------------------------------------------------------------------------------- /run.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "strings" 7 | "os" 8 | "fmt" 9 | "flag" 10 | "os/exec" 11 | "bufio" 12 | 13 | "github.com/containerd/containerd" 14 | "github.com/containerd/containerd/images" 15 | "github.com/containerd/containerd/platforms" 16 | "github.com/containerd/containerd/content" 17 | "github.com/containerd/containerd/namespaces" 18 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 19 | "github.com/docker/docker/pkg/archive" 20 | "github.com/firecracker-microvm/firecracker-go-sdk" 21 | models "github.com/firecracker-microvm/firecracker-go-sdk/client/models" 22 | log "github.com/sirupsen/logrus" 23 | ) 24 | 25 | const runHelp = `Extract a rootfs from a Docker image and run it in a VM` 26 | func (cmd *runCommand) Name() string { return "run" } 27 | func (cmd *runCommand) Args() string { return "[options] docker.io/organization/name:tag" } 28 | func (cmd *runCommand) ShortHelp() string { return runHelp } 29 | func (cmd *runCommand) LongHelp() string { return runHelp } 30 | func (cmd *runCommand) Hidden() bool { return false } 31 | 32 | func (cmd *runCommand) Register(fs *flag.FlagSet) { 33 | fs.StringVar(&cmd.namespace, "namespace", "docker-to-firecracker", "ContainerD namespace to fetch images [default docker-to-firecracker]") 34 | fs.StringVar(&cmd.containerdSock, "containerdsock", "/run/containerd/containerd.sock", "Path to ContainerD socket [default /run/containerd/containerd.sock]") 35 | fs.StringVar(&cmd.tmpMountPoint, "tmp-mnt", "/mnt", "Path to temporarily mount on the host file system to generate the root filesystem [default /mnt]") 36 | fs.StringVar(&cmd.rootFSPath, "rootfs-path", "./disk.img", "Path to generate a root filesystem [default ./disk.image]") 37 | fs.BoolVar(&cmd.generateBootInit, "generate-boot-init", true, "Generate boot init script and write to root filesystem [default true]") 38 | fs.StringVar(&cmd.bootInitFileName, "boot-init-file-name", "custom.init", "Set the path to the generated boot init script [default custom.init]") 39 | fs.StringVar(&cmd.bootInitArg, "boot-init-arg", "/custom.init", "Set the boot init arg to pass to Firecracker VM [default /custom.init]") 40 | fs.StringVar(&cmd.kernelPath, "kernel-path", "./hello-vmlinux.bin", "Path the Kernel to pass to Firecracker VM [default ./hello-vmlinux.bin]") 41 | fs.StringVar(&cmd.firecrackerPath, "firecracker-path", "./firecracker", "Path to the Firecracker VM binary") 42 | fs.StringVar(&cmd.firecrackerSock, "firecracker-sock", "./firecracker.sock", "Path to a temporary Firecracker socket") 43 | } 44 | 45 | type runCommand struct { 46 | namespace string 47 | containerdSock string 48 | tmpMountPoint string 49 | rootFSPath string 50 | generateBootInit bool 51 | bootInitFileName string 52 | bootInitArg string 53 | kernelPath string 54 | firecrackerPath string 55 | firecrackerSock string 56 | } 57 | 58 | func (cmd *runCommand) Run(ctx context.Context, args []string) (err error) { 59 | if len(args) < 1 { 60 | return fmt.Errorf("Must pass a docker url: docker.io/organization/name:tag") 61 | } 62 | logger := log.New() 63 | imageName := args[0] 64 | 65 | client, err := containerd.New(cmd.containerdSock) 66 | if err != nil { 67 | return err 68 | } 69 | defer client.Close() 70 | log.Printf("Pulling Image: %s\n", imageName) 71 | ctx = namespaces.WithNamespace(ctx, cmd.namespace) 72 | // pull an image 73 | img, err := client.Fetch(ctx, imageName) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | // extract the init command 79 | log.Printf("Extracting init CMD\n") 80 | provider := client.ContentStore() 81 | platform := platforms.Default() 82 | config, err := img.Config(ctx, provider, platform) 83 | configBlob, err := content.ReadBlob(ctx, provider, config) 84 | var imageSpec ocispec.Image 85 | json.Unmarshal(configBlob, &imageSpec) 86 | initCmd := strings.Join(imageSpec.Config.Cmd, " ") 87 | initEnvs := imageSpec.Config.Env 88 | 89 | log.Printf("Creating Root FS\n") 90 | command := exec.Command("dd", "if=/dev/zero", fmt.Sprintf("of=%s", cmd.rootFSPath), "bs=1", "count=0", "seek=1G") 91 | if err := command.Run(); err != nil { 92 | return err 93 | } 94 | 95 | 96 | command = exec.Command("mkfs.ext4", "-F", cmd.rootFSPath) 97 | if err := command.Run(); err != nil { 98 | return err 99 | } 100 | 101 | command = exec.Command("mount", "-o", "loop", cmd.rootFSPath, cmd.tmpMountPoint) 102 | if err := command.Run(); err != nil { 103 | return err 104 | } 105 | 106 | // unpack the image to a root fs 107 | log.Printf("Upacking Image: %s", imageName) 108 | manifest, err := images.Manifest(ctx, client.ContentStore(), img.Target, platforms.Default()) 109 | if err != nil { 110 | return err 111 | } 112 | for _, desc := range manifest.Layers { 113 | log.Printf("Upacking Layer: %s", desc.Digest.String()) 114 | layer, err := client.ContentStore().ReaderAt(ctx, desc) 115 | if err != nil { 116 | return err 117 | } 118 | if err := archive.Untar(content.NewReader(layer), cmd.tmpMountPoint, &archive.TarOptions{ 119 | NoLchown: true, 120 | }); err != nil { 121 | return err 122 | } 123 | } 124 | 125 | if (cmd.generateBootInit) { 126 | log.Printf("Generating Boot Init Script") 127 | // create init script -- this will write over /sbin/init if it already exists 128 | initScriptLocaton := fmt.Sprintf("%s/%s", cmd.tmpMountPoint, cmd.bootInitFileName) 129 | // TODO: handle deeper paths for init script 130 | f, err := os.Create(initScriptLocaton) 131 | if err != nil { 132 | return err 133 | } 134 | writer := bufio.NewWriter(f) 135 | fmt.Fprintf(writer, "#!/bin/sh\n") 136 | for _, env := range initEnvs { 137 | fmt.Fprintf(writer, "export %s\n", env) 138 | } 139 | fmt.Fprintf(writer, "%s\n", initCmd) 140 | writer.Flush() 141 | f.Sync() 142 | f.Close() 143 | mode := int(0755) 144 | os.Chmod(initScriptLocaton, os.FileMode(mode)) 145 | } 146 | 147 | command = exec.Command("umount", cmd.tmpMountPoint) 148 | if err := command.Run(); err != nil { 149 | return err 150 | } 151 | vmmCtx, vmmCancel := context.WithCancel(ctx) 152 | defer vmmCancel() 153 | devices := []models.Drive{} 154 | rootDrive := models.Drive{ 155 | DriveID: firecracker.String("1"), 156 | PathOnHost: &cmd.rootFSPath, 157 | IsRootDevice: firecracker.Bool(true), 158 | IsReadOnly: firecracker.Bool(false), 159 | } 160 | devices = append(devices, rootDrive) 161 | fcCfg := firecracker.Config{ 162 | SocketPath: cmd.firecrackerSock, 163 | KernelImagePath: cmd.kernelPath, 164 | KernelArgs: fmt.Sprintf("console=ttyS0 reboot=k panic=1 pci=off init=\"%s\"", cmd.bootInitArg), 165 | Drives: devices, 166 | MachineCfg: models.MachineConfiguration{ 167 | VcpuCount: 1, 168 | CPUTemplate: models.CPUTemplate("C3"), 169 | HtEnabled: true, 170 | MemSizeMib: 512, 171 | }, 172 | } 173 | machineOpts := []firecracker.Opt{ 174 | firecracker.WithLogger(log.NewEntry(logger)), 175 | } 176 | command = firecracker.VMCommandBuilder{}. 177 | WithBin(cmd.firecrackerPath). 178 | WithSocketPath(fcCfg.SocketPath). 179 | WithStdin(os.Stdin). 180 | WithStdout(os.Stdout). 181 | WithStderr(os.Stderr). 182 | Build(ctx) 183 | machineOpts = append(machineOpts, firecracker.WithProcessRunner(command)) 184 | m, err := firecracker.NewMachine(vmmCtx, fcCfg, machineOpts...) 185 | if err != nil { 186 | return err 187 | } 188 | 189 | if err := m.Start(vmmCtx); err != nil { 190 | return err 191 | } 192 | defer m.StopVMM() 193 | 194 | // wait for the VMM to exit 195 | if err := m.Wait(vmmCtx); err != nil { 196 | return err 197 | } 198 | log.Printf("Start machine was happy") 199 | return nil 200 | } 201 | --------------------------------------------------------------------------------