├── .buildkite ├── al2_cleanup.sh ├── al2_pipeline.yml ├── al2_test.sh ├── al2env.sh ├── hooks │ └── pre-exit ├── pipeline.yml └── setup_al2.sh ├── .dockerignore ├── .github ├── CODEOWNERS ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── build.yaml ├── .gitignore ├── .gitmodules ├── .golangci.yml ├── .headers ├── go.txt └── makefile.txt ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── agent ├── .gitignore ├── Makefile ├── README.md ├── agent_test.go ├── drive_handler.go ├── drive_handler_test.go ├── error.go ├── error_test.go ├── ioproxy_handler.go ├── main.go ├── service.go └── testdata │ ├── block │ ├── vda │ │ ├── dev │ │ └── size │ ├── vdb │ │ ├── dev │ │ └── size │ ├── vdc │ │ ├── dev │ │ └── size │ └── vdd │ │ ├── dev │ │ └── size │ └── dev │ ├── vda │ ├── vdb │ ├── vdc │ └── vdd ├── client └── client.go ├── config ├── config.go ├── config.json.example └── config_test.go ├── docker-credential-mmds ├── .gitignore ├── Makefile ├── README.md ├── go.mod ├── go.sum ├── main.go └── mmds │ ├── client.go │ ├── client_test.go │ ├── helper.go │ └── helper_test.go ├── docs ├── agent.md ├── architecture.md ├── design-approaches.md ├── drive-mounts-proposal.md ├── getting-started.md ├── host-file-isolation.md ├── img │ ├── architecture-diagram.png │ ├── container-launch-sequence-diagram.plantuml │ ├── container-launch-sequence-diagram.svg │ ├── design-overview.png │ ├── design-sequence-diagram.png │ ├── firecracker-containerd-shim-flow.png │ ├── remote-snapshotter-architecture-diagram.png │ ├── remote-snapshotter-launch-sequence.plantuml │ ├── remote-snapshotter-launch-sequence.svg │ └── runtimev2-flow.png ├── logging.md ├── networking.md ├── quickstart.md ├── remote-snapshotter-getting-started.md ├── remote-snapshotter.md ├── root-filesystem.md ├── scaling.md ├── shim-design.md └── snapshotter.md ├── eventbridge ├── eventbridge.go └── eventbridge_test.go ├── examples ├── .gitignore ├── Makefile ├── cmd │ └── remote-snapshotter │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── go.mod │ │ ├── go.sum │ │ └── remote_snapshotter.go ├── etc │ └── containerd │ │ └── firecracker-runtime.json ├── taskworkflow.go └── taskworkflow.md ├── firecracker-control ├── client │ └── client.go ├── cmd │ └── containerd │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── gomod_test.go │ │ └── main.go ├── common.go ├── local.go └── service.go ├── go.mod ├── go.sum ├── internal ├── .gitignore ├── bundle │ ├── bundle.go │ └── doc.go ├── cmd │ └── test-bridged-tap │ │ └── main.go ├── common.go ├── common_test.go ├── cpu_template.go ├── cpu_template_test.go ├── debug │ ├── debug.go │ ├── debug_test.go │ └── error.go ├── event │ └── exchange.go ├── fsutil.go ├── integtest │ ├── config.go │ ├── containerd.go │ ├── firecracker.go │ ├── network.go │ ├── prepare.go │ ├── runtime.go │ └── shim.go ├── network_test_utils.go ├── psutil.go ├── shim │ └── shim.go ├── testutils.go └── vm │ ├── agent.go │ ├── agent_test.go │ ├── dir.go │ ├── dir_test.go │ ├── fifo.go │ ├── ioproxy.go │ ├── ioproxy_test.go │ ├── mount.go │ ├── mount_test.go │ ├── oci.go │ ├── task.go │ ├── task_test.go │ └── vsock.go ├── proto ├── Makefile ├── events.pb.go ├── events.proto ├── firecracker.pb.go ├── firecracker.proto ├── service │ ├── drivemount │ │ ├── Makefile │ │ ├── drivemount.proto │ │ └── ttrpc │ │ │ ├── drivemount.pb.go │ │ │ └── drivemount_ttrpc.pb.go │ ├── fccontrol │ │ ├── Makefile │ │ ├── fccontrol.proto │ │ └── ttrpc │ │ │ ├── fccontrol.pb.go │ │ │ └── fccontrol_ttrpc.pb.go │ └── ioproxy │ │ ├── Makefile │ │ ├── ioproxy.proto │ │ └── ttrpc │ │ ├── ioproxy.pb.go │ │ └── ioproxy_ttrpc.pb.go ├── types.pb.go └── types.proto ├── runtime ├── .gitignore ├── Makefile ├── README.md ├── benchmark_test.go ├── cni_integ_test.go ├── cpuset │ ├── cpuset_builder.go │ └── cpuset_builder_test.go ├── drive_handler.go ├── drive_handler_test.go ├── firecracker-runc-config.json.example ├── firecrackeroci │ ├── annotation.go │ ├── network.go │ └── vm.go ├── helpers.go ├── helpers_test.go ├── integ_test.go ├── jailer.go ├── jailer_integ_test.go ├── jailer_test.go ├── limits_integ_test.go ├── main.go ├── noop_jailer.go ├── runc_jailer.go ├── runc_jailer_test.go ├── service.go ├── service_integ_test.go ├── service_test.go ├── vm │ └── mount.go └── volume_integ_test.go ├── snapshotter ├── .gitignore ├── Makefile ├── README.md ├── app │ └── service.go ├── config │ ├── config.go │ ├── config.toml.example │ └── config_test.go ├── demux │ ├── README.md │ ├── cache │ │ ├── cache.go │ │ ├── cache_test.go │ │ ├── evict.go │ │ └── evict_test.go │ ├── internal │ │ ├── failing_snapshotter.go │ │ └── successful_snapshotter.go │ ├── metrics │ │ ├── discovery │ │ │ └── service_discovery.go │ │ ├── proxy.go │ │ └── proxy_test.go │ ├── proxy │ │ ├── address │ │ │ ├── http_resolver.go │ │ │ ├── http_resolver_test.go │ │ │ └── resolver.go │ │ └── snapshotter.go │ ├── snapshotter.go │ └── snapshotter_test.go ├── internal │ ├── http_address_resolver.go │ ├── integtest │ │ └── stargz │ │ │ └── fs │ │ │ ├── config │ │ │ └── config.go │ │ │ └── source │ │ │ └── source.go │ └── mount │ │ ├── collection.go │ │ └── collection_test.go ├── main.go ├── metrics_integ_test.go ├── service_integ_test.go └── volume_integ_test.go ├── tools ├── critest │ ├── critest_diff.sh │ └── expected_critest_output.out ├── demo │ ├── fc-br0.interface │ └── fcnet.conflist ├── docker │ ├── Dockerfile.integ-test │ ├── Dockerfile.proto-builder │ ├── Dockerfile.runc-builder │ ├── Dockerfile.stargz-builder │ ├── config.toml │ ├── do_not_edit_for_firecracker-control.config.json │ ├── entrypoint.sh │ ├── fc-agent.start │ └── scripts │ │ └── lsblk.sh ├── image-builder │ ├── .dockerignore │ ├── .gitignore │ ├── Dockerfile.debian-image │ ├── Makefile │ ├── README.md │ ├── agent.sh │ ├── files_debootstrap │ │ ├── etc │ │ │ ├── hostname │ │ │ ├── hosts │ │ │ ├── resolv.conf │ │ │ └── systemd │ │ │ │ └── system │ │ │ │ ├── apt-daily-upgrade.timer │ │ │ │ ├── apt-daily.timer │ │ │ │ ├── firecracker-agent.service │ │ │ │ ├── firecracker.target │ │ │ │ ├── firecracker.target.wants │ │ │ │ ├── firecracker-agent.service │ │ │ │ ├── getty.target │ │ │ │ └── haveged.service │ │ │ │ ├── getty-static.service │ │ │ │ ├── haveged.service │ │ │ │ ├── rom-dev.mount │ │ │ │ ├── rom-overlay.mount │ │ │ │ ├── rom.mount │ │ │ │ └── systemd-timesyncd.service │ │ └── sbin │ │ │ └── overlay-init │ └── files_stargz │ │ ├── etc │ │ ├── containerd-stargz-grpc │ │ │ └── config.toml │ │ └── systemd │ │ │ └── system │ │ │ ├── firecracker.target.wants │ │ │ ├── metrics-socat.service │ │ │ ├── socat.service │ │ │ └── stargz-snapshotter.service │ │ │ ├── metrics-socat.service │ │ │ ├── socat.service │ │ │ └── stargz-snapshotter.service │ │ └── root │ │ └── .docker │ │ └── config.json ├── kernel-configs │ ├── microvm-kernel-aarch64-4.14.config │ ├── microvm-kernel-aarch64-5.10.config │ ├── microvm-kernel-x86_64-4.14.config │ └── microvm-kernel-x86_64-5.10.config ├── thinpool.sh └── tidy.sh └── volume ├── .gitignore ├── Makefile ├── cmd └── volume-init │ ├── main.go │ └── main_test.go ├── guest_image.go ├── image.go ├── set.go ├── set_test.go ├── volume.go └── volume_test.go /.buildkite/al2_cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source .buildkite/al2env.sh 3 | 4 | sudo rm -rf $dir 5 | FICD_DM_VOLUME_GROUP=fcci-vg ./tools/thinpool.sh remove $unique_id 6 | -------------------------------------------------------------------------------- /.buildkite/al2_pipeline.yml: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | steps: 15 | 16 | - label: ":docker: Build" 17 | agents: 18 | queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}" 19 | distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}" 20 | hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}" 21 | env: 22 | DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER" 23 | EXTRAGOARGS: "-race" 24 | FICD_DM_VOLUME_GROUP: "fcci-vg" 25 | command: 26 | - ./.buildkite/setup_al2.sh 27 | - docker run --rm -v $PWD:/mnt debian:bullseye-slim rm -rf /mnt/tools/image-builder/rootfs 28 | 29 | - wait 30 | 31 | - label: ":onion: al2 tests" 32 | agents: 33 | queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}" 34 | distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}" 35 | hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}" 36 | env: 37 | NUMBER_OF_VMS: "10" 38 | EXTRAGOARGS: "-v -count=1 -timeout=1h" 39 | artifact_paths: 40 | - "runtime/logs/*" 41 | command: 42 | - ./.buildkite/al2_test.sh 43 | timeout_in_minutes: 10 44 | 45 | - wait: ~ 46 | continue_on_failure: true 47 | 48 | - label: ":onion: cleanup" 49 | agents: 50 | queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}" 51 | distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}" 52 | hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}" 53 | command: 54 | - ./.buildkite/al2_cleanup.sh 55 | timeout_in_minutes: 10 56 | -------------------------------------------------------------------------------- /.buildkite/al2_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | 4 | source ./.buildkite/al2env.sh 5 | 6 | export PATH=$bin_path:$PATH 7 | export FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH=$runtime_config_path 8 | export ENABLE_ISOLATED_TESTS=true 9 | export CONTAINERD_SOCKET=$dir/containerd.sock 10 | 11 | export SHIM_BASE_DIR=$dir 12 | 13 | mkdir -p runtime/logs 14 | 15 | sudo -E PATH=$PATH \ 16 | $bin_path/firecracker-containerd \ 17 | --config $dir/config.toml &>> runtime/logs/containerd.out & 18 | containerd_pid=$! 19 | 20 | sudo $bin_path/firecracker-ctr --address $dir/containerd.sock content fetch docker.io/library/alpine:3.10.1 21 | 22 | TAP_PREFIX=build$BUILDKITE_BUILD_NUMBER \ 23 | sudo -E PATH=$bin_path:$PATH /usr/local/bin/go test -count=1 -run TestMultipleVMs_Isolated ./... -v 24 | 25 | sudo kill $containerd_pid 26 | -------------------------------------------------------------------------------- /.buildkite/al2env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | shim_base=/tmp/shim-base 4 | unique_id=$BUILDKITE_BUILD_NUMBER 5 | dir=$shim_base/$unique_id 6 | bin_path=$dir/bin 7 | devmapper_path=$dir/devmapper 8 | state_path=$dir/state 9 | runtime_config_path=$dir/firecracker-runtime.json 10 | -------------------------------------------------------------------------------- /.buildkite/hooks/pre-exit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo -E PATH=$PATH make -C "$BUILDKITE_BUILD_CHECKOUT_PATH" clean 3 | 4 | # clean up ephemeral files since this will cause an error to build kite due to 5 | # these files being owned by root 6 | sudo -E PATH=$PATH make -C "$BUILDKITE_BUILD_CHECKOUT_PATH/tools/image-builder" distclean 7 | -------------------------------------------------------------------------------- /.buildkite/setup_al2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eux 4 | 5 | source ./.buildkite/al2env.sh 6 | 7 | mkdir -p $dir 8 | mkdir -p $dir/rootfs 9 | mkdir -p $bin_path 10 | mkdir -p $devmapper_path 11 | mkdir -p $state_path 12 | 13 | ./tools/thinpool.sh reset $unique_id 14 | 15 | export INSTALLROOT=$dir 16 | export FIRECRACKER_CONTAINERD_RUNTIME_DIR=$dir 17 | make 18 | cp /var/lib/fc-ci/vmlinux.bin $dir/default-vmlinux.bin 19 | make image firecracker 20 | sudo -E INSTALLROOT=$INSTALLROOT PATH=$PATH \ 21 | make install install-firecracker install-default-rootfs 22 | 23 | cat << EOF > $dir/config.toml 24 | version = 2 25 | disabled_plugins = ["io.containerd.grpc.v1.cri"] 26 | root = "$dir" 27 | state = "$state_path" 28 | [grpc] 29 | address = "$dir/containerd.sock" 30 | [plugins] 31 | [plugins."io.containerd.snapshotter.v1.devmapper"] 32 | pool_name = "fcci--vg-$unique_id" 33 | base_image_size = "10GB" 34 | root_path = "$devmapper_path" 35 | [debug] 36 | level = "debug" 37 | EOF 38 | 39 | cat << EOF > $runtime_config_path 40 | { 41 | "cpu_template": "T2", 42 | "debug": true, 43 | "firecracker_binary_path": "$bin_path/firecracker", 44 | "shim_base_dir": "$dir", 45 | "kernel_image_path": "$dir/default-vmlinux.bin", 46 | "kernel_args": "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules systemd.unified_cgroup_hierarchy=0 systemd.journald.forward_to_console systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init", 47 | "log_levels": ["debug"], 48 | "root_drive": "$dir/default-rootfs.img", 49 | "jailer": { 50 | "runc_binary_path": "$bin_path/runc", 51 | "runc_config_path": "$dir/config.json" 52 | } 53 | } 54 | EOF 55 | 56 | cp ./runtime/firecracker-runc-config.json.example $dir/config.json 57 | 58 | # runc uses /run/runc directory by default. Since our Amazon Linux 2 tests on 59 | # BuildKite don't use Docker, sharing the directory across multiple test 60 | # runs causes a race condition. 61 | # 62 | # This wrapper script gives runc /run/runc-$unique_id instead to avoid 63 | # the race condition. 64 | cp ./_submodules/runc/runc "$bin_path/runc.real" 65 | cat > "$bin_path/runc" < containerd: prepare snapshot 13 | containerd -> snapshotter: prepare snapshot 14 | snapshotter --> containerd: snapshot $foo 15 | containerd --> orchestrator: snapshot $foo 16 | end 17 | 18 | orchestrator -> "control plugin": create VM $id 19 | create runtime 20 | "control plugin" -> runtime: launch VM $id 21 | create Firecracker 22 | runtime -> Firecracker: launch 23 | runtime -> Firecracker ++: configure & boot 24 | create agent 25 | runtime <-> agent: connect 26 | "control plugin" --> orchestrator: VM $id running 27 | 28 | loop "for each container" 29 | orchestrator -> containerd: run container\n$bar with\nsnapshot $foo 30 | containerd -> runtime: run container $bar with snapshot $foo 31 | runtime->Firecracker: attach snapshot\n$foo as device 32 | runtime->agent: mount snapshot $foo 33 | runtime->agent: run container $bar 34 | create "container process" 35 | agent->"container process": start process 36 | activate "container process" 37 | note left of "container process" 38 | After some 39 | time, container 40 | process exits 41 | end note 42 | return 43 | agent -> runtime: container $bar exited 44 | runtime -> containerd: container $bar exited 45 | containerd -> orchestrator: container $bar exited 46 | end 47 | 48 | orchestrator -> "control plugin": stop VM $id 49 | "control plugin" -> Firecracker: stop 50 | deactivate Firecracker 51 | @enduml -------------------------------------------------------------------------------- /docs/img/design-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firecracker-microvm/firecracker-containerd/e6c7a36c34f827a4229d4c898f9b406e9742d7a5/docs/img/design-overview.png -------------------------------------------------------------------------------- /docs/img/design-sequence-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firecracker-microvm/firecracker-containerd/e6c7a36c34f827a4229d4c898f9b406e9742d7a5/docs/img/design-sequence-diagram.png -------------------------------------------------------------------------------- /docs/img/firecracker-containerd-shim-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firecracker-microvm/firecracker-containerd/e6c7a36c34f827a4229d4c898f9b406e9742d7a5/docs/img/firecracker-containerd-shim-flow.png -------------------------------------------------------------------------------- /docs/img/remote-snapshotter-architecture-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firecracker-microvm/firecracker-containerd/e6c7a36c34f827a4229d4c898f9b406e9742d7a5/docs/img/remote-snapshotter-architecture-diagram.png -------------------------------------------------------------------------------- /docs/img/remote-snapshotter-launch-sequence.plantuml: -------------------------------------------------------------------------------- 1 | @startuml 2 | participant orchestrator 3 | participant containerd 4 | participant "control plugin" 5 | participant "demux-snapshotter" 6 | 7 | orchestrator -> "control plugin": create VM $id 8 | create shim 9 | "control plugin" -> shim: Launch VM $id 10 | create Firecracker 11 | shim -> Firecracker: Launch 12 | shim -> Firecracker ++ : configure and boot 13 | create MMDS 14 | Firecracker -> MMDS: start 15 | create agent 16 | Firecracker -> agent: start 17 | create "remote-snapshotter" 18 | Firecracker -> "remote-snapshotter": start 19 | "shim" <-> agent: connect 20 | "control plugin" -> orchestrator: VM running 21 | 22 | orchestrator -> "control plugin": Put registry credentials 23 | "control plugin" -> MMDS: Put registry credentials 24 | 25 | 26 | loop "for each container" 27 | orchestrator -> containerd: prepare snapshot 28 | containerd -> "demux-snapshotter": prepare snapshot 29 | "demux-snapshotter" <-> "remote-snapshotter": connect 30 | "demux-snapshotter" -> "remote-snapshotter": prepare snapshot 31 | create "docker-credential-mmds" 32 | "remote-snapshotter" -> "docker-credential-mmds": Get credentials 33 | "docker-credential-mmds" -> MMDS: Get credentials 34 | MMDS -> "docker-credential-mmds": credentials 35 | "docker-credential-mmds" -> "remote-snapshotter": credentials 36 | 37 | create FUSE 38 | "remote-snapshotter" -> FUSE : spawn 39 | activate FUSE 40 | "remote-snapshotter" -> "demux-snapshotter": snapshot $foo 41 | "demux-snapshotter" -> containerd: snapshot $foo 42 | containerd -> orchestrator: snapshot $foo 43 | orchestrator -> containerd: run container $bar with snapshot $foo 44 | containerd -> shim: run container $bar with snapshot $foo 45 | shim -> agent: run container $bar with snapshot $foo 46 | agent -> FUSE: mount snapshot $foo 47 | create "container process" 48 | agent -> "container process": start process 49 | activate "container process" 50 | "container process" -> "FUSE": access files 51 | FUSE <-> registry: download files 52 | FUSE -> "container process": file content/metadata 53 | note left of "container process" 54 | After some 55 | time, container 56 | process exits 57 | end note 58 | return 59 | agent -> shim: container $bar exited 60 | shim -> containerd: container $bar exited 61 | containerd -> orchestrator: container $bar exited 62 | end 63 | orchestrator -> "control plugin": stop VM $id 64 | "control plugin" -> Firecracker: stop 65 | deactivate Firecracker 66 | @enduml -------------------------------------------------------------------------------- /docs/img/runtimev2-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firecracker-microvm/firecracker-containerd/e6c7a36c34f827a4229d4c898f9b406e9742d7a5/docs/img/runtimev2-flow.png -------------------------------------------------------------------------------- /docs/root-filesystem.md: -------------------------------------------------------------------------------- 1 | # Root Filesystem 2 | 3 | firecracker-containerd requires a kernel and root filesystem to be specified for 4 | the Firecracker virtual machine where containers run. This document specifies 5 | the hard requirements for that root filesystem, the development tools we built, 6 | and some thoughts around where we might land for a production recommendation. 7 | 8 | ## Requirements 9 | 10 | * The firecracker-containerd agent, configured to start on boot 11 | * runc 12 | * Standard Linux filesystem mounts like procfs and sysfs 13 | * A cgroup v1 filesystem configured and mounted, with the controllers you intend 14 | to use for your containers (typical controllers include things like blkio, 15 | cpu, cpuacct, cpuset, devices, freezer, hugetlb, memory, net_cls, net_prio, 16 | perf_event, pids, and rdma, though you may not use all of them) 17 | * Any supporting software you may be interested in leveraging inside the VM 18 | (e.g., a local caching DNS resolver) 19 | * Any supporting dynamically-linked libraries necessary to run the components 20 | inside (e.g., libc, libseccomp, etc) 21 | * (If sharing the root filesystem image among multiple VMs) Ability to run 22 | successfully from a read-only device, in order to prevent a VM from 23 | manipulating the filesystem in use by another VM. 24 | 25 | ## Development tools 26 | 27 | We've built a tool to generate root filesystems that are suitable for 28 | development. The tool can be invoked with `make image` from the root directory 29 | of this repository. It will generate a Debian-based root filesystem as a 30 | squashfs filesystem with the firecracker-containerd agent configured to start 31 | through systemd, a runc binary built from the git submodule in the 32 | `_submodules/runc` folder, and support for multiple VMs by an overlay filesystem 33 | on top of the squashfs base. 34 | 35 | To minimize on-disk overhead, we start with the "minbase" debootstrap 36 | configuration, and only add minimal dependencies on top of it. We ensure that 37 | obviously extraneous files such as the content of `/usr/share/doc` and 38 | `/var/cache/apt` are removed. Further, we reduce startup latency by defining a 39 | custom systemd target that minimizes the services that systemd tries to start at 40 | boot. 41 | 42 | More information on the tool and how to use it can be found in [the README.md 43 | file](../tools/image-builder/README.md). 44 | 45 | ## Further thoughts 46 | 47 | For production use, we want to achieve the following goals: 48 | 49 | * Safe to share between multiple VMs 50 | * Fast to initialize and start the agent 51 | * Reasonably pared-down set of installed software 52 | 53 | The current development filesystem meets the first goal already and has some 54 | initial work toward the second and third goals, but has not been aggressively 55 | optimized yet. We're considering a few things, but are also open to other 56 | ideas. Some ideas we've talked about: 57 | 58 | * Should we use a fully-featured init system like systemd, or something smaller? 59 | * Should we statically- or dynamically-link the software inside the filesystem? 60 | * What is the minimal set of software that is necessary? 61 | * Should we embed runc/libcontainer inside the firecracker-containerd agent to 62 | reduce the requirement for both binaries? 63 | * Should we enforce a size limitation for the writable layer in the VM? 64 | * Should we care about disk I/O performance inside the VM? 65 | -------------------------------------------------------------------------------- /docs/scaling.md: -------------------------------------------------------------------------------- 1 | # Scaling the number of Firecracker microVMs per host 2 | 3 | To scale the number of microVMs past one thousand, one needs to properly configure 4 | the system constraints and provision enough networking resources for the microVMs. 5 | 6 | On Ubuntu 18.04 Linux, one needs to properly configure the number of processes, threads, memory, 7 | open files that may simultaneously exist in a system as well as create enough 8 | virtual bridges (one bridge can serve up to 1023 interfaces). 9 | 10 | To configure the system, one needs to set up the following parameters. 11 | Note that the exact values depend on your setting. 12 | 13 | In `/etc/security/limits.conf`, set `nofile`, `nproc` and `stack` to the 14 | appropriate values for both normal users and root: 15 | ``` 16 | * soft nofile 1000000 17 | * hard nofile 1000000 18 | root soft nofile 1000000 19 | root hard nofile 1000000 20 | * soft nproc 4000000 21 | * hard nproc 4000000 22 | root soft nproc 4000000 23 | root hard nproc 4000000 24 | * soft stack 65536 25 | * hard stack 65536 26 | root soft stack 65536 27 | root hard stack 65536 28 | ``` 29 | 30 | Additionally, one needs to provision the ARP cache to avoid garbage collection. 31 | ``` 32 | sudo sysctl -w net.ipv4.neigh.default.gc_thresh1=1024 33 | sudo sysctl -w net.ipv4.neigh.default.gc_thresh2=2048 34 | sudo sysctl -w net.ipv4.neigh.default.gc_thresh3=4096 35 | sudo sysctl -w net.ipv4.ip_local_port_range="32769 65535" 36 | ``` 37 | 38 | Also, configure the maximum number of processes and threads in the system. 39 | ``` 40 | sudo sysctl -w kernel.pid_max=4194303 41 | sudo sysctl -w kernel.threads-max=999999999 42 | ``` 43 | 44 | Finally, configure the number of tasks. 45 | To configure system-wide, uncomment and set `DefaultTasksMax=infinity`in `/etc/systemd/system.conf`. 46 | One also needs to set `UsersTasksMax=4000000000` in `/etc/systemd/logind.conf` 47 | (note that `infinity` is not a valid value here). 48 | 49 | To configure the bridges for CNI, take a look at `demo-network` target in [Makefile](https://github.com/firecracker-microvm/firecracker-containerd/blob/main/Makefile) 50 | and replicate the code to create enough bridges (1 bridge can have up to 1023 interfaces attached). 51 | -------------------------------------------------------------------------------- /docs/snapshotter.md: -------------------------------------------------------------------------------- 1 | # snapshotter 2 | 3 | The snapshotter is an out-of-process gRPC proxy plugin for containerd that 4 | implements containerd's snapshotter API. This snapshotter creates snapshots as 5 | filesystem images that can be exposed to Firecracker microVMs as devices; this 6 | is necessary as the Firecracker VMM does not enable any filesystem-level sharing 7 | between the microVM and the host. 8 | 9 | ## Current implementation 10 | 11 | We have two current implementations of snapshotters that are device-based and 12 | work with containerd and Firecracker microVMs. 13 | 14 | ### naive snapshotter 15 | 16 | This snapshotter is very simple and naive; it relies on copying data for every 17 | snapshot and does not perform content deduplication. However, it serves as a 18 | useful proof-of-concept for integration with Firecracker and as a sanity-check 19 | to ensure that the other snapshotter is behaving correctly (i.e., other than 20 | performance, we should see exactly the same behavior). 21 | 22 | ### devmapper snapshotter 23 | 24 | This is a snapshotter that leverages 25 | [thin provisioning](https://www.kernel.org/doc/Documentation/device-mapper/thin-provisioning.txt) 26 | of virtual devices with device-mapper. Using this snapshotter, we can achieve 27 | efficient, deduplicated storage between content layers and their associated 28 | images or containers. 29 | 30 | There is a reasonably long history of using device-mapper in this manner in 31 | Docker, with a variety of implications for performance and stability. We do 32 | need a device-based snapshotter for Firecracker, and device-mapper offers a 33 | straightforward path to do so. However, there have been concerns expressed 34 | around file read/write/copy-on-write performance, as well as around provisioning 35 | and deactivation performance. 36 | 37 | ## Plans 38 | 39 | We plan to continue exploring models for device-based, deduplicated snapshot 40 | storage. Along with the devmapper snapshotter that we've already implemented, 41 | we're interested in approaches combining the 42 | [overlay filesystem](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) 43 | inside the microVM with device-based storage attached from the host. -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | taskworkflow 2 | logs 3 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | # Set this to pass additional commandline flags to the go compiler, e.g. "make test EXTRAGOARGS=-v" 15 | EXTRAGOARGS?= 16 | 17 | SOURCES:=$(shell find . -name '*.go') 18 | GOMOD := $(shell go env GOMOD) 19 | GOSUM := $(GOMOD:.mod=.sum) 20 | DOCKER_IMAGE_TAG?=latest 21 | FIRECRACKER_CONTAINERD_TEST_IMAGE?=localhost/firecracker-containerd-test 22 | 23 | all: examples 24 | 25 | examples: taskworkflow remote-snapshotter 26 | 27 | taskworkflow: taskworkflow.go $(GOMOD) $(GOSUM) 28 | go build -o taskworkflow taskworkflow.go 29 | 30 | remote-snapshotter: 31 | $(MAKE) -C cmd/remote-snapshotter $@ 32 | 33 | test: 34 | go test ./... $(EXTRAGOARGS) 35 | 36 | integ-test: 37 | mkdir -p $(CURDIR)/logs 38 | $(CURDIR)/../tools/thinpool.sh reset "$(TEST_POOL)" 39 | docker run --rm -it \ 40 | --privileged \ 41 | --ipc=host \ 42 | --volume /dev:/dev \ 43 | --volume /run/udev/control:/run/udev/control \ 44 | --volume $(CURDIR)/etc/containerd/firecracker-runtime.json:/etc/containerd/firecracker-runtime.json \ 45 | --volume $(CURDIR)/logs:/var/log/firecracker-containerd-test \ 46 | --volume $(CURDIR)/..:/src \ 47 | --env FICD_DM_VOLUME_GROUP=$(FICD_DM_VOLUME_GROUP) \ 48 | --env FICD_DM_POOL=$(TEST_POOL) \ 49 | --env EXTRAGOARGS="${EXTRAGOARGS}" \ 50 | --workdir="/src/examples" \ 51 | $(FIRECRACKER_CONTAINERD_TEST_IMAGE):${DOCKER_IMAGE_TAG} \ 52 | "make examples && make testtap && sleep 3 && ./taskworkflow -ip $(TEST_IP)$(TEST_SUBNET) -gw $(TEST_GATEWAY) -ss devmapper" 53 | 54 | TEST_GATEWAY?=172.16.0.1 55 | TEST_IP?=172.16.0.2 56 | TEST_SUBNET?=/24 57 | FICD_DM_VOLUME_GROUP?= 58 | TEST_POOL?=fc-test-thinpool 59 | testtap: 60 | ip link add br0 type bridge 61 | ip tuntap add tap0 mode tap 62 | ip link set tap0 master br0 63 | ip link set dev tap0 up 64 | ip link set dev br0 up 65 | ip addr add dev br0 $(TEST_GATEWAY)$(TEST_SUBNET) 66 | 67 | clean: 68 | - rm -f taskworkflow 69 | - $(MAKE) -C cmd/remote-snapshotter $@ 70 | 71 | distclean: clean 72 | - rm -rf logs 73 | 74 | install: 75 | 76 | .PHONY: all examples clean distclean install test integ-test testtap remote-snapshotter 77 | -------------------------------------------------------------------------------- /examples/cmd/remote-snapshotter/.gitignore: -------------------------------------------------------------------------------- 1 | remote-snapshotter 2 | -------------------------------------------------------------------------------- /examples/cmd/remote-snapshotter/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | # Set this to pass additional commandline flags to the go compiler, e.g. "make test EXTRAGOARGS=-v" 15 | EXTRAGOARGS?= 16 | 17 | SOURCES:=$(shell find . -name '*.go') 18 | GOMOD := $(shell go env GOMOD) 19 | GOSUM := $(GOMOD:.mod=.sum) 20 | 21 | remote-snapshotter: $(SOURCES) $(GOMOD) $(GOSUM) 22 | go build -o remote-snapshotter $(SOURCES) 23 | 24 | clean: 25 | - rm remote-snapshotter 26 | 27 | .PHONY: clean -------------------------------------------------------------------------------- /examples/cmd/remote-snapshotter/README.md: -------------------------------------------------------------------------------- 1 | # Remote Snapshotter Example 2 | 3 | This example shows how to build a firecracker-containerd client that can launch lazily-loaded estargz images in a firecracker microvm. If you haven't already, please follow the [remote snapshotter getting started guide.](https://github.com/firecracker-microvm/firecracker-containerd/blob/main/docs/remote-snapshotter-getting-started.md) 4 | 5 | # Building 6 | 7 | The example can be built with 8 | 9 | ``` 10 | make remote-snapshotter 11 | ``` 12 | 13 | # Usage 14 | To use the example, run (e.g.) 15 | 16 | ``` 17 | ./remote-snapshotter ghcr.io/firecracker-microvm/firecracker-containerd/amazonlinux:latest-esgz 18 | ``` 19 | 20 | A list of additional, pre-converted images for testing can be found in the [stargz-snapshotter documentation](https://github.com/containerd/stargz-snapshotter/blob/main/docs/pre-converted-images.md) 21 | 22 | 23 | This will launch the container inside a microVM with lazy loading supplied by the stargz-snaphotter and connect stdio to the container. 24 | 25 | Note: stargz-snapshotter has a fallback mechanism to use an embedded overlay snapshotter if an image is not in estargz format. This will not work with firecracker-containerd because the client is responsible for writing content into the snapshot, but the snapshot mountpoints are isolated inside the VM. 26 | 27 | # Credentials 28 | 29 | By default, the example tool will ask for a docker username/password each time it is invoked. It can also read these from environment the variables `$DOCKER_USERNAME` and `$DOCKER_PASSWORD`. 30 | 31 | Note that this means it does not support multiple container registries at the same time, nor credential helpers for retrieving credentials. -------------------------------------------------------------------------------- /examples/cmd/remote-snapshotter/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/firecracker-microvm/firecracker-containerd/example/remote-snapshotter 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/containerd/containerd v1.7.27 7 | github.com/containerd/stargz-snapshotter v0.11.3 8 | github.com/firecracker-microvm/firecracker-containerd v0.0.0-20220430002346-5f6efb9fdce8 9 | ) 10 | 11 | require ( 12 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect 13 | github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect 14 | github.com/Microsoft/go-winio v0.6.2 // indirect 15 | github.com/Microsoft/hcsshim v0.11.7 // indirect 16 | github.com/containerd/cgroups v1.1.0 // indirect 17 | github.com/containerd/containerd/api v1.8.0 // indirect 18 | github.com/containerd/continuity v0.4.4 // indirect 19 | github.com/containerd/errdefs v0.3.0 // indirect 20 | github.com/containerd/fifo v1.1.0 // indirect 21 | github.com/containerd/log v0.1.0 // indirect 22 | github.com/containerd/platforms v0.2.1 // indirect 23 | github.com/containerd/ttrpc v1.2.7 // indirect 24 | github.com/containerd/typeurl/v2 v2.1.1 // indirect 25 | github.com/distribution/reference v0.6.0 // indirect 26 | github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect 27 | github.com/felixge/httpsnoop v1.0.4 // indirect 28 | github.com/go-logr/logr v1.4.2 // indirect 29 | github.com/go-logr/stdr v1.2.2 // indirect 30 | github.com/gogo/protobuf v1.3.2 // indirect 31 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 32 | github.com/golang/protobuf v1.5.4 // indirect 33 | github.com/google/go-cmp v0.6.0 // indirect 34 | github.com/google/uuid v1.6.0 // indirect 35 | github.com/klauspost/compress v1.16.7 // indirect 36 | github.com/moby/locker v1.0.1 // indirect 37 | github.com/moby/sys/mountinfo v0.6.2 // indirect 38 | github.com/moby/sys/sequential v0.5.0 // indirect 39 | github.com/moby/sys/signal v0.7.0 // indirect 40 | github.com/moby/sys/user v0.3.0 // indirect 41 | github.com/moby/sys/userns v0.1.0 // indirect 42 | github.com/opencontainers/go-digest v1.0.0 // indirect 43 | github.com/opencontainers/image-spec v1.1.0 // indirect 44 | github.com/opencontainers/runtime-spec v1.1.0 // indirect 45 | github.com/opencontainers/selinux v1.11.0 // indirect 46 | github.com/pkg/errors v0.9.1 // indirect 47 | github.com/sirupsen/logrus v1.9.3 // indirect 48 | go.opencensus.io v0.24.0 // indirect 49 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect 50 | go.opentelemetry.io/otel v1.27.0 // indirect 51 | go.opentelemetry.io/otel/metric v1.27.0 // indirect 52 | go.opentelemetry.io/otel/trace v1.27.0 // indirect 53 | golang.org/x/net v0.38.0 // indirect 54 | golang.org/x/sync v0.12.0 // indirect 55 | golang.org/x/sys v0.31.0 // indirect 56 | golang.org/x/text v0.23.0 // indirect 57 | google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect 58 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect 59 | google.golang.org/grpc v1.64.1 // indirect 60 | google.golang.org/protobuf v1.35.2 // indirect 61 | ) 62 | 63 | // Workaround for indirect dependency no longer being available. 64 | exclude github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f 65 | 66 | replace github.com/firecracker-microvm/firecracker-containerd => ../../.. 67 | -------------------------------------------------------------------------------- /examples/etc/containerd/firecracker-runtime.json: -------------------------------------------------------------------------------- 1 | { 2 | "firecracker_binary_path": "/usr/local/bin/firecracker", 3 | "kernel_image_path": "/var/lib/firecracker-containerd/runtime/default-vmlinux.bin", 4 | "kernel_args": "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules systemd.unified_cgroup_hierarchy=0 systemd.journald.forward_to_console systemd.unit=firecracker.target init=/sbin/overlay-init", 5 | "root_drive": "/var/lib/firecracker-containerd/runtime/default-rootfs.img", 6 | "cpu_template": "T2", 7 | "log_levels": ["debug"], 8 | "jailer": { 9 | "runc_binary_path": "/usr/local/bin/runc" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /examples/taskworkflow.md: -------------------------------------------------------------------------------- 1 | ### About 2 | The `taskworkfow.go` file contains code to pull and unpack a container image 3 | using the `devmapper` snaphotter, create a container and execute a task using the 4 | `firecracker-containerd` runtime. 5 | 6 | ### Building 7 | After checking out the `firecracker-containerd` repo, you can build this 8 | example using the `make examples` command. 9 | 10 | ### Running 11 | For a basic workflow, without any networking setup, just run the executable 12 | built with the command in the previous section as super user: 13 | ```bash 14 | $ sudo /path/to/firecracker-containerd/examples/taskworkflow 15 | ``` 16 | 17 | For a workflow with networking setup for the container, create a tap device 18 | for the VM by following [these instructions](https://github.com/firecracker-microvm/firecracker/blob/main/docs/network-setup.md). 19 | 20 | This creates a tap device named `tap0`, in the local `172.16.0.1/24` subnet. 21 | Since the example does not rely on a DHCP client running within the VM to 22 | initialize the network interface, `gw` flag should be used to 23 | specify the gateway value. 24 | 25 | The following example sets: 26 | * The IP address (CIDR) to `172.16.0.2/24` 27 | * The gateway IP address to `172.16.0.1` 28 | 29 | ** NOTE: This example will not work if you're running more than 1 container 30 | on a host at the same time ** 31 | 32 | Now, run the example by passing the `-ip` argument: 33 | ```bash 34 | $ sudo /path/to/firecracker-containerd/examples/taskworkflow -ip 172.16.0.2/24 \ 35 | -gw 172.16.0.1 36 | ``` 37 | 38 | You should see output similar to this: 39 | ``` 40 | 2019/02/11 21:02:48.813898 Creating contained client 41 | 2019/02/11 21:02:48.814324 Created contained client 42 | 2019/02/11 21:02:48.981848 Successfully pulled docker.io/library/nginx:latest image 43 | 2019/02/11 21:02:51.202941 Successfully created task: demo for the container 44 | 2019/02/11 21:02:51.202962 Completed waiting for the container task 45 | 2019/02/11 21:02:51.231901 Successfully started the container task 46 | 2019/02/11 21:02:54.232051 Executing http GET on 172.16.0.2 47 | 2019/02/11 21:02:54.233193 Response from [172.16.0.2]: 48 | [ 49 | 50 | 51 | Welcome to nginx! 52 | 59 | 60 | 61 |

Welcome to nginx!

62 |

If you see this page, the nginx web server is successfully installed and 63 | working. Further configuration is required.

64 | 65 |

For online documentation and support please refer to 66 | nginx.org.
67 | Commercial support is available at 68 | nginx.com.

69 | 70 |

Thank you for using nginx.

71 | 72 | 73 | ] 74 | 172.16.0.1 - - [11/Feb/2019:21:02:54 +0000] "GET / HTTP/1.1" 200 612 "-" "Go-http-client/1.1" "-" 75 | ``` 76 | -------------------------------------------------------------------------------- /firecracker-control/client/client.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package client 15 | 16 | import ( 17 | "fmt" 18 | 19 | "github.com/containerd/containerd/pkg/ttrpcutil" 20 | 21 | fccontrol "github.com/firecracker-microvm/firecracker-containerd/proto/service/fccontrol/ttrpc" 22 | ) 23 | 24 | // Client is a helper client for containerd's firecracker-control plugin 25 | type Client struct { 26 | fccontrol.FirecrackerService 27 | 28 | ttrpcClient *ttrpcutil.Client 29 | } 30 | 31 | // New creates a new firecracker-control service client 32 | func New(ttrpcAddress string) (*Client, error) { 33 | ttrpcClient, err := ttrpcutil.NewClient(ttrpcAddress) 34 | if err != nil { 35 | return nil, fmt.Errorf("failed to create ttrpc client: %w", err) 36 | } 37 | 38 | client, err := ttrpcClient.Client() 39 | if err != nil { 40 | return nil, err 41 | } 42 | fcClient := fccontrol.NewFirecrackerClient(client) 43 | 44 | return &Client{ 45 | FirecrackerService: fcClient, 46 | ttrpcClient: ttrpcClient, 47 | }, nil 48 | } 49 | 50 | // Close closes the underlying TTRPC client 51 | func (c *Client) Close() error { 52 | return c.ttrpcClient.Close() 53 | } 54 | -------------------------------------------------------------------------------- /firecracker-control/cmd/containerd/.gitignore: -------------------------------------------------------------------------------- 1 | firecracker-containerd 2 | firecracker-ctr 3 | -------------------------------------------------------------------------------- /firecracker-control/cmd/containerd/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | # Set this to pass additional commandline flags to the go compiler, e.g. "make test EXTRAGOARGS=-v" 15 | EXTRAGOARGS:= 16 | 17 | SOURCES := $(shell find ../../../config ../../../internal ../../../proto ../../ -name '*.go') 18 | GOMOD := $(shell go env GOMOD) 19 | GOSUM := $(GOMOD:.mod=.sum) 20 | REVISION=$(shell git rev-parse HEAD) 21 | VERSION_LDFLAGS="-X github.com/containerd/containerd/version.Revision=$(REVISION)" 22 | 23 | all: build 24 | 25 | build: firecracker-containerd firecracker-ctr 26 | 27 | firecracker-containerd: $(SOURCES) $(GOMOD) $(GOSUM) 28 | go build $(EXTRAGOARGS) \ 29 | -ldflags $(VERSION_LDFLAGS) -o firecracker-containerd 30 | 31 | firecracker-ctr: $(GOMOD) $(GOSUM) 32 | GOBIN=$(CURDIR) go install $(EXTRAGOARGS) \ 33 | -ldflags $(VERSION_LDFLAGS) github.com/containerd/containerd/cmd/ctr 34 | mv ctr firecracker-ctr 35 | 36 | install: firecracker-containerd firecracker-ctr 37 | install -D -o root -g root -m755 -t $(INSTALLROOT)/bin firecracker-containerd 38 | install -D -o root -g root -m755 -t $(INSTALLROOT)/bin firecracker-ctr 39 | 40 | test: 41 | go test ./... $(EXTRAGOARGS) 42 | 43 | # integ tests involving containerd are currently only in runtime package 44 | integ-test: 45 | 46 | clean: 47 | - rm -f firecracker-containerd 48 | - rm -f firecracker-ctr 49 | 50 | distclean: clean 51 | 52 | .PHONY: all build install test integ-test clean distclean 53 | -------------------------------------------------------------------------------- /firecracker-control/cmd/containerd/README.md: -------------------------------------------------------------------------------- 1 | # README 2 | 3 | This a is custom `containerd` binary which embeds the `firecracker-control` GRPC plugin (instead of using it as a golang 4 | plugin binary) in order to simplify the development process. It's not recommended to use this in production. 5 | -------------------------------------------------------------------------------- /firecracker-control/cmd/containerd/gomod_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package main 15 | 16 | import "github.com/containerd/containerd/cmd/ctr/app" 17 | 18 | // Right now, firecracker-ctr command is built by Makefile. 19 | // Go's toolchain is not aware about the command and "go mod tidy" removes 20 | // the command's dependencies. 21 | // This test file makes the ctr command "visible" from Go's toolchain. 22 | var _ = app.New 23 | -------------------------------------------------------------------------------- /firecracker-control/cmd/containerd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "fmt" 18 | "os" 19 | 20 | "github.com/containerd/containerd/cmd/containerd/command" 21 | 22 | "github.com/containerd/containerd/pkg/seed" 23 | 24 | // Register containerd builtins 25 | // See https://github.com/containerd/containerd/blob/main/cmd/containerd/builtins.go 26 | _ "github.com/containerd/containerd/diff/walking/plugin" 27 | _ "github.com/containerd/containerd/events/plugin" 28 | _ "github.com/containerd/containerd/gc/scheduler" 29 | _ "github.com/containerd/containerd/leases/plugin" 30 | _ "github.com/containerd/containerd/metadata/plugin" 31 | _ "github.com/containerd/containerd/pkg/hasher" 32 | _ "github.com/containerd/containerd/pkg/nri/plugin" 33 | _ "github.com/containerd/containerd/plugins/sandbox" 34 | _ "github.com/containerd/containerd/plugins/streaming" 35 | _ "github.com/containerd/containerd/plugins/transfer" 36 | _ "github.com/containerd/containerd/runtime/restart/monitor" 37 | _ "github.com/containerd/containerd/runtime/v2" 38 | _ "github.com/containerd/containerd/services/containers" 39 | _ "github.com/containerd/containerd/services/content" 40 | _ "github.com/containerd/containerd/services/diff" 41 | _ "github.com/containerd/containerd/services/events" 42 | _ "github.com/containerd/containerd/services/healthcheck" 43 | _ "github.com/containerd/containerd/services/images" 44 | _ "github.com/containerd/containerd/services/introspection" 45 | _ "github.com/containerd/containerd/services/leases" 46 | _ "github.com/containerd/containerd/services/namespaces" 47 | _ "github.com/containerd/containerd/services/opt" 48 | _ "github.com/containerd/containerd/services/sandbox" 49 | _ "github.com/containerd/containerd/services/snapshots" 50 | _ "github.com/containerd/containerd/services/streaming" 51 | _ "github.com/containerd/containerd/services/tasks" 52 | _ "github.com/containerd/containerd/services/transfer" 53 | _ "github.com/containerd/containerd/services/version" 54 | _ "github.com/containerd/containerd/services/warning" 55 | 56 | // Linux specific builtins 57 | // See https://github.com/containerd/containerd/blob/main/cmd/containerd/builtins_linux.go 58 | _ "github.com/containerd/containerd/metrics/cgroups" 59 | _ "github.com/containerd/containerd/metrics/cgroups/v2" 60 | _ "github.com/containerd/containerd/runtime/v1/linux" 61 | _ "github.com/containerd/containerd/runtime/v2/runc/options" 62 | 63 | // Snapshotters 64 | _ "github.com/containerd/containerd/snapshots/devmapper/plugin" 65 | _ "github.com/containerd/containerd/snapshots/overlay/plugin" 66 | 67 | // Register cri plugin 68 | _ "github.com/containerd/containerd/pkg/cri" 69 | 70 | // Register fc-control plugin 71 | _ "github.com/firecracker-microvm/firecracker-containerd/firecracker-control" 72 | ) 73 | 74 | func init() { 75 | seed.WithTimeAndRand() 76 | } 77 | 78 | func main() { 79 | app := command.App() 80 | if err := app.Run(os.Args); err != nil { 81 | fmt.Fprintf(os.Stderr, "containerd: %s\n", err) 82 | os.Exit(1) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /firecracker-control/common.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package service 15 | 16 | const ( 17 | localPluginID = "fc-control" 18 | grpcPluginID = "fc-control-service" 19 | ) 20 | -------------------------------------------------------------------------------- /internal/.gitignore: -------------------------------------------------------------------------------- 1 | test-bridged-tap -------------------------------------------------------------------------------- /internal/bundle/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | /* 15 | Package bundle implements some helper functions for firecracker-containerd's interaction 16 | with bundle dirs, both inside the VM and outside on the host. 17 | */ 18 | package bundle 19 | -------------------------------------------------------------------------------- /internal/common_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package internal 15 | 16 | import ( 17 | "bytes" 18 | "io" 19 | "strings" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | ) 24 | 25 | func TestIsStubDrive(t *testing.T) { 26 | cases := []struct { 27 | name string 28 | expected bool 29 | r io.Reader 30 | }{ 31 | { 32 | name: "simple case", 33 | expected: false, 34 | r: strings.NewReader("foo"), 35 | }, 36 | { 37 | name: "lengthy case", 38 | expected: false, 39 | r: strings.NewReader(strings.Repeat("0", 512)), 40 | }, 41 | { 42 | name: "valid max_len+1 case", 43 | expected: true, 44 | r: strings.NewReader(string(append(MagicStubBytes, 0xFF)) + strings.Repeat("0", 0xFF+1)), 45 | }, 46 | { 47 | name: "valid max_len case", 48 | expected: true, 49 | r: strings.NewReader(string(append(MagicStubBytes, 0xFF)) + strings.Repeat("0", 0xFF)), 50 | }, 51 | { 52 | name: "valid case", 53 | expected: true, 54 | r: strings.NewReader(string(append(MagicStubBytes, 3, 100, 200, 255))), 55 | }, 56 | } 57 | 58 | for _, c := range cases { 59 | t.Run(c.name, func(t *testing.T) { 60 | if e, a := c.expected, IsStubDrive(c.r); e != a { 61 | t.Errorf("expected %t, but received %t", e, a) 62 | } 63 | }) 64 | } 65 | } 66 | 67 | func TestGenerateStubContent(t *testing.T) { 68 | driveID := "foo" 69 | stubContent, err := GenerateStubContent(driveID) 70 | assert.NoError(t, err) 71 | 72 | expected := append([]byte{}, MagicStubBytes...) 73 | expected = append(expected, byte(len(driveID))) 74 | expected = append(expected, []byte(driveID)...) 75 | assert.Equal(t, string(expected), stubContent) 76 | } 77 | 78 | func TestGenerateStubContent_LongID(t *testing.T) { 79 | driveID := strings.Repeat("0", 0xFF+1) 80 | _, err := GenerateStubContent(driveID) 81 | assert.Error(t, err) 82 | } 83 | 84 | func TestParseStubContent(t *testing.T) { 85 | expectedDriveID := "foo" 86 | contents := append([]byte{}, MagicStubBytes...) 87 | contents = append(contents, byte(len(expectedDriveID))) 88 | contents = append(contents, []byte(expectedDriveID)...) 89 | contents = append(contents, []byte("junkcontent")...) 90 | 91 | driveID, err := ParseStubContent(bytes.NewBuffer(contents)) 92 | assert.NoError(t, err) 93 | assert.Equal(t, expectedDriveID, driveID) 94 | } 95 | 96 | func TestStubID(t *testing.T) { 97 | const expectedDriveID = "foo" 98 | buf := bytes.NewBuffer(nil) 99 | 100 | stubContent, err := GenerateStubContent(expectedDriveID) 101 | assert.NoError(t, err) 102 | 103 | isStubBuffer := bytes.NewBuffer([]byte(stubContent)) 104 | assert.True(t, IsStubDrive(isStubBuffer)) 105 | 106 | buf.WriteString(stubContent) 107 | driveID, err := ParseStubContent(buf) 108 | assert.NoError(t, err) 109 | assert.Equal(t, expectedDriveID, driveID) 110 | } 111 | -------------------------------------------------------------------------------- /internal/cpu_template.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package internal 15 | 16 | import ( 17 | "bufio" 18 | "io" 19 | "os" 20 | "regexp" 21 | "runtime" 22 | "sync" 23 | ) 24 | 25 | var ( 26 | isIntel bool 27 | isIntelOnce sync.Once 28 | ) 29 | 30 | // SupportCPUTemplate returns true if Firecracker supports CPU templates on 31 | // the current architecture. 32 | func SupportCPUTemplate() (bool, error) { 33 | if runtime.GOARCH != "amd64" { 34 | return false, nil 35 | } 36 | 37 | var err error 38 | isIntelOnce.Do(func() { 39 | isIntel, err = checkIsIntel() 40 | }) 41 | return isIntel, err 42 | } 43 | 44 | var vendorID = regexp.MustCompile(`^vendor_id\s*:\s*(.+)$`) 45 | 46 | func checkIsIntel() (bool, error) { 47 | f, err := os.Open("/proc/cpuinfo") 48 | if err != nil { 49 | return false, err 50 | } 51 | defer f.Close() 52 | 53 | id, err := findFirstVendorID(f) 54 | if err != nil { 55 | return false, err 56 | } 57 | 58 | return id == "GenuineIntel", nil 59 | } 60 | 61 | func findFirstVendorID(r io.Reader) (string, error) { 62 | s := bufio.NewScanner(r) 63 | for s.Scan() { 64 | line := s.Text() 65 | matches := vendorID.FindStringSubmatch(line) 66 | if len(matches) == 2 { 67 | return matches[1], nil 68 | } 69 | } 70 | return "", nil 71 | } 72 | -------------------------------------------------------------------------------- /internal/cpu_template_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package internal 15 | 16 | import ( 17 | "strings" 18 | "testing" 19 | 20 | "github.com/stretchr/testify/assert" 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func TestFindFirstVendorID(t *testing.T) { 25 | cases := []struct { 26 | input string 27 | vendorID string 28 | }{ 29 | {"vendor_id : GenuineIntel", "GenuineIntel"}, 30 | {"vendor_id : AuthenticAMD", "AuthenticAMD"}, 31 | 32 | // aarch64 doesn't have vendor IDs. 33 | {"", ""}, 34 | } 35 | for _, c := range cases { 36 | r := strings.NewReader(c.input) 37 | id, err := findFirstVendorID(r) 38 | require.NoError(t, err) 39 | assert.Equal(t, c.vendorID, id) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /internal/debug/error.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package debug 15 | 16 | import ( 17 | "fmt" 18 | ) 19 | 20 | // InvalidLogLevelError is an error that will be returned in the event that an 21 | // invalid log level was provided 22 | type InvalidLogLevelError struct { 23 | logLevel string 24 | } 25 | 26 | // NewInvalidLogLevelError will constract an InvalidLogLevelError 27 | func NewInvalidLogLevelError(logLevel string) error { 28 | return &InvalidLogLevelError{ 29 | logLevel: logLevel, 30 | } 31 | } 32 | 33 | func (e *InvalidLogLevelError) Error() string { 34 | return fmt.Sprintf("log level %q is an invalid log level", e.logLevel) 35 | } 36 | -------------------------------------------------------------------------------- /internal/event/exchange.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package event 15 | 16 | import ( 17 | "github.com/containerd/containerd/events/exchange" 18 | "github.com/containerd/containerd/runtime/v2/shim" 19 | ) 20 | 21 | var _ shim.Publisher = &ExchangeCloser{} 22 | 23 | // ExchangeCloser wraps an event exchange with an extra no-op Close method 24 | // to make it compatible with the containerd shim.Publisher interface 25 | type ExchangeCloser struct { 26 | *exchange.Exchange 27 | } 28 | 29 | // Close is a no-op, just needed to satisfy shim.Publisher interface 30 | func (*ExchangeCloser) Close() error { 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /internal/integtest/config.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package integtest 15 | 16 | import ( 17 | "encoding/json" 18 | "os" 19 | 20 | "github.com/firecracker-microvm/firecracker-containerd/config" 21 | ) 22 | 23 | const runtimeConfigPath = "/etc/containerd/firecracker-runtime.json" 24 | 25 | // DefaultRuntimeConfig represents a simple firecracker-containerd configuration. 26 | var DefaultRuntimeConfig = config.Config{ 27 | FirecrackerBinaryPath: "/usr/local/bin/firecracker", 28 | KernelImagePath: "/var/lib/firecracker-containerd/runtime/default-vmlinux.bin", 29 | KernelArgs: "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules systemd.unified_cgroup_hierarchy=0 systemd.journald.forward_to_console systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init", 30 | RootDrive: "/var/lib/firecracker-containerd/runtime/default-rootfs.img", 31 | LogLevels: []string{"debug"}, 32 | ShimBaseDir: ShimBaseDir(), 33 | JailerConfig: config.JailerConfig{ 34 | RuncBinaryPath: "/usr/local/bin/runc", 35 | RuncConfigPath: "/etc/containerd/firecracker-runc-config.json", 36 | }, 37 | } 38 | 39 | func writeRuntimeConfig(options ...func(*config.Config)) error { 40 | config := DefaultRuntimeConfig 41 | for _, option := range options { 42 | option(&config) 43 | } 44 | 45 | file, err := os.OpenFile(runtimeConfigPath, os.O_CREATE|os.O_WRONLY, 0600) 46 | if err != nil { 47 | return err 48 | } 49 | defer file.Close() 50 | 51 | bytes, err := json.Marshal(config) 52 | if err != nil { 53 | return err 54 | } 55 | 56 | _, err = file.Write(bytes) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /internal/integtest/containerd.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package integtest 15 | 16 | import ( 17 | "bytes" 18 | "context" 19 | 20 | "github.com/containerd/containerd" 21 | "github.com/containerd/containerd/cio" 22 | ) 23 | 24 | // CommandResult encapsulates the stdout, stderr, and exit code returned 25 | // from a task. 26 | type CommandResult struct { 27 | Stdout string 28 | Stderr string 29 | ExitCode uint32 30 | } 31 | 32 | // RunTask is a utility function for running a task and returning the result. 33 | func RunTask(ctx context.Context, c containerd.Container) (*CommandResult, error) { 34 | var stdout bytes.Buffer 35 | var stderr bytes.Buffer 36 | 37 | task, err := c.NewTask(ctx, cio.NewCreator(cio.WithStreams(nil, &stdout, &stderr))) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | exitCh, err := task.Wait(ctx) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | err = task.Start(ctx) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | select { 53 | case exitStatus := <-exitCh: 54 | if err := exitStatus.Error(); err != nil { 55 | return nil, err 56 | } 57 | 58 | _, err := task.Delete(ctx) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return &CommandResult{ 64 | Stdout: stdout.String(), 65 | Stderr: stderr.String(), 66 | ExitCode: exitStatus.ExitCode(), 67 | }, nil 68 | case <-ctx.Done(): 69 | return nil, ctx.Err() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/integtest/firecracker.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package integtest 15 | 16 | import "github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client" 17 | 18 | // NewFCControlClient returns a Firecracker control client for the given socket. 19 | func NewFCControlClient(socket string) (*client.Client, error) { 20 | return client.New(socket + ".ttrpc") 21 | } 22 | -------------------------------------------------------------------------------- /internal/integtest/network.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package integtest 15 | 16 | import ( 17 | "github.com/firecracker-microvm/firecracker-containerd/config" 18 | "github.com/firecracker-microvm/firecracker-containerd/proto" 19 | ) 20 | 21 | // WithDefaultNetwork is an option to use the default network configuration 22 | // in the runtime configuration for integration testing 23 | func WithDefaultNetwork() func(c *config.Config) { 24 | return func(c *config.Config) { 25 | c.DefaultNetworkInterfaces = []proto.FirecrackerNetworkInterface{ 26 | { 27 | CNIConfig: &proto.CNIConfiguration{ 28 | NetworkName: "fcnet", 29 | InterfaceName: "veth0", 30 | }, 31 | }, 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /internal/integtest/prepare.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package integtest 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/firecracker-microvm/firecracker-containerd/config" 20 | "github.com/firecracker-microvm/firecracker-containerd/internal" 21 | ) 22 | 23 | // Prepare is a common integration test setup function which ensures 24 | // isolation and prepares the runtime configuration for firecracker-containerd 25 | func Prepare(t testing.TB, options ...func(*config.Config)) { 26 | t.Helper() 27 | 28 | internal.RequiresIsolation(t) 29 | 30 | err := writeRuntimeConfig(options...) 31 | if err != nil { 32 | t.Error(err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /internal/integtest/runtime.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package integtest 15 | 16 | import ( 17 | "os" 18 | "strconv" 19 | ) 20 | 21 | const ( 22 | // FirecrackerRuntime is the Firecracker-containerd runtime 23 | FirecrackerRuntime = "aws.firecracker" 24 | 25 | containerdSockPathEnvVar = "CONTAINERD_SOCKET" 26 | numberOfVmsEnvVar = "NUMBER_OF_VMS" 27 | ) 28 | 29 | var ( 30 | // ContainerdSockPath is the default Firecracker-containerd socket path 31 | ContainerdSockPath = "/run/firecracker-containerd/containerd.sock" 32 | 33 | // NumberOfVms is the number of VMs used in integration testing set 34 | // by either environment variable read or defaults to 5 VMs. 35 | NumberOfVms = 5 36 | ) 37 | 38 | func init() { 39 | if v := os.Getenv(containerdSockPathEnvVar); v != "" { 40 | ContainerdSockPath = v 41 | } 42 | if v := os.Getenv(numberOfVmsEnvVar); v != "" { 43 | NumberOfVms, _ = strconv.Atoi(v) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /internal/integtest/shim.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package integtest 15 | 16 | import "os" 17 | 18 | const shimBaseDirEnvVar = "SHIM_BASE_DIR" 19 | const defaultShimBaseDir = "/srv/firecracker_containerd_tests" 20 | 21 | // ShimBaseDir checks the "SHIM_BASE_DIR" environment variable and returns its value 22 | // if it exists, else returns the default value 23 | func ShimBaseDir() string { 24 | if v := os.Getenv(shimBaseDirEnvVar); v != "" { 25 | return v 26 | } 27 | 28 | return defaultShimBaseDir 29 | } 30 | -------------------------------------------------------------------------------- /internal/shim/shim.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package shim 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/containerd/containerd/runtime/v2/shim" 20 | ) 21 | 22 | // FCControlSocketAddress is a unix socket address at which a shim 23 | // with the given namespace, the containerd socket address, and the vmID will listen and serve 24 | // the fccontrol api 25 | func FCControlSocketAddress(namespacedCtx context.Context, socketPath, vmID string) (string, error) { 26 | shimSocketAddr, err := shim.SocketAddress(namespacedCtx, socketPath, vmID) 27 | if err != nil { 28 | return "", err 29 | } 30 | 31 | return shimSocketAddr + "-fccontrol", nil 32 | } 33 | -------------------------------------------------------------------------------- /internal/testutils.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package internal 15 | 16 | import ( 17 | "os" 18 | "testing" 19 | ) 20 | 21 | const ( 22 | rootDisableEnvName = "DISABLE_ROOT_TESTS" 23 | enableIsolatedTestsEnvName = "ENABLE_ISOLATED_TESTS" 24 | ) 25 | 26 | var ( 27 | rootDisabled bool 28 | isolatedTestsEnabled bool 29 | ) 30 | 31 | func init() { 32 | if v := os.Getenv(rootDisableEnvName); len(v) != 0 { 33 | rootDisabled = true 34 | } 35 | 36 | if v := os.Getenv(enableIsolatedTestsEnvName); len(v) != 0 { 37 | isolatedTestsEnabled = true 38 | } 39 | } 40 | 41 | // RequiresRoot will ensure that tests that require root access are actually 42 | // root. In addition, this will skip root tests if the DISABLE_ROOT_TESTS is 43 | // set to true 44 | func RequiresRoot(t testing.TB) { 45 | if rootDisabled { 46 | t.Skip("skipping test that requires root") 47 | } 48 | 49 | if e, a := 0, os.Getuid(); e != a { 50 | t.Fatalf("This test must be run as root. To disable tests that "+ 51 | "require root, run the tests with the %s environment variable set.", 52 | rootDisableEnvName) 53 | } 54 | } 55 | 56 | // RequiresIsolation will ensure that tests that must be run in an isolated 57 | // environment are being run with the explicit ENABLE_ISOLATED_TESTS env var. 58 | // If the env var has not been set, the test will be skipped 59 | func RequiresIsolation(t testing.TB) { 60 | if !isolatedTestsEnabled { 61 | t.Skip("skipping test that requires isolation") 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /internal/vm/agent.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "net/url" 18 | 19 | "github.com/sirupsen/logrus" 20 | ) 21 | 22 | // IsAgentOnlyIO checks whether Stdout target is agent only in order to allow 23 | // log redirection entirely within the VM (currently cio.BinaryIO and cio.LogFile) 24 | func IsAgentOnlyIO(stdout string, logger *logrus.Entry) bool { 25 | parsed, err := url.Parse(stdout) 26 | if err != nil { 27 | logger.WithError(err).Debugf("invalid URL %q", stdout) 28 | return false 29 | } 30 | 31 | switch parsed.Scheme { 32 | case "binary", "file": 33 | return true 34 | default: 35 | return false 36 | } 37 | } 38 | 39 | // nullIOProxy represents an empty IOProxy implementation for cases when there is no need to 40 | // stream logs from the Agent through vsock. 41 | type nullIOProxy struct{} 42 | 43 | // NewNullIOProxy creates an empty IOProxy 44 | func NewNullIOProxy() IOProxy { 45 | return &nullIOProxy{} 46 | } 47 | 48 | func (*nullIOProxy) start(_ *vmProc) (ioInitDone <-chan error, ioCopyDone <-chan error) { 49 | initCh := make(chan error) 50 | close(initCh) 51 | 52 | copyCh := make(chan error) 53 | close(copyCh) 54 | 55 | return initCh, copyCh 56 | } 57 | 58 | // Close is no-op. 59 | func (*nullIOProxy) Close() { 60 | } 61 | 62 | // IsOpen always returns true, since this proxy implementation is no-op. 63 | func (*nullIOProxy) IsOpen() bool { 64 | return true 65 | } 66 | -------------------------------------------------------------------------------- /internal/vm/agent_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/containerd/containerd/cio" 20 | "github.com/containerd/log" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestAgentOnlyIO(t *testing.T) { 25 | assert.True(t, IsAgentOnlyIO(creator(t, cio.BinaryIO("/root/binary", nil)), log.L)) 26 | assert.True(t, IsAgentOnlyIO(creator(t, cio.LogFile("/log.txt")), log.L)) 27 | } 28 | 29 | func TestNonAgentIO(t *testing.T) { 30 | assert.False(t, IsAgentOnlyIO("fifo://test", log.L)) 31 | assert.False(t, IsAgentOnlyIO("/tmp/path", log.L)) 32 | assert.False(t, IsAgentOnlyIO("", log.L)) 33 | } 34 | 35 | func creator(t *testing.T, creator cio.Creator) string { 36 | t.Helper() 37 | 38 | io, err := creator("!") 39 | assert.NoError(t, err) 40 | 41 | return io.Config().Stdout 42 | } 43 | -------------------------------------------------------------------------------- /internal/vm/dir_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "path" 18 | "strings" 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | var invalidContainerIDs = []string{"", "id?", "*", "id/1", "id\\"} 25 | 26 | func TestShimDir(t *testing.T) { 27 | runDir := t.TempDir() 28 | 29 | tests := []struct { 30 | name string 31 | ns string 32 | id string 33 | outDir string 34 | outErr string 35 | }{ 36 | {name: "empty ns", outErr: "invalid namespace: identifier must not be empty"}, 37 | {name: "ns with /", ns: "/", outErr: `invalid namespace: identifier "/" must match`}, 38 | {name: "ns with ?", ns: "?", outErr: `invalid namespace: identifier "?" must match`}, 39 | {name: "ns with *", ns: "*", outErr: `invalid namespace: identifier "*" must match`}, 40 | {name: "ns with ,", ns: ",", outErr: `invalid namespace: identifier "," must match`}, 41 | {name: "empty id", ns: "test", outErr: "invalid vm id: identifier must not be empty"}, 42 | {name: "id with /", ns: "test", id: "/", outErr: `invalid vm id: identifier "/" must match`}, 43 | {name: "id with ?", ns: "test", id: "?", outErr: `invalid vm id: identifier "?" must match`}, 44 | {name: "id with *", ns: "test", id: "*", outErr: `invalid vm id: identifier "*" must match`}, 45 | {name: "id with ,", ns: "test", id: ",", outErr: `invalid vm id: identifier "," must match`}, 46 | {name: "valid", ns: "ns", id: "1", outDir: "ns#1"}, 47 | {name: "valid with dashes", ns: "test-123", id: "123-456", outDir: "test-123#123-456"}, 48 | {name: "valid with dots", ns: "test.123", id: "123.456", outDir: "test.123#123.456"}, 49 | {name: "ns with aaa", ns: "aaa", id: "bbb-ccc", outDir: "aaa#bbb-ccc"}, 50 | {name: "ns with aaa-bbb", ns: "aaa-bbb", id: "ccc", outDir: "aaa-bbb#ccc"}, 51 | } 52 | 53 | for _, tc := range tests { 54 | test := tc 55 | t.Run(test.name, func(t *testing.T) { 56 | dir, err := shimDir(runDir, test.ns, test.id) 57 | 58 | if test.outErr != "" { 59 | assert.Error(t, err) 60 | assert.True(t, strings.Contains(err.Error(), test.outErr), err.Error()) 61 | } else { 62 | assert.NoError(t, err) 63 | assert.EqualValues(t, dir, path.Join(runDir, test.outDir)) 64 | } 65 | }) 66 | } 67 | } 68 | 69 | func TestBundleLink(t *testing.T) { 70 | var root Dir = "/root/" 71 | 72 | dir, err := root.BundleLink("1") 73 | assert.NoError(t, err) 74 | assert.EqualValues(t, "/root/1", dir) 75 | 76 | dir, err = root.BundleLink("test-1") 77 | assert.NoError(t, err) 78 | assert.EqualValues(t, "/root/test-1", dir) 79 | } 80 | 81 | func TestBundleLinkInvalidID(t *testing.T) { 82 | var ( 83 | root Dir = "/root/" 84 | err error 85 | ) 86 | 87 | for _, id := range invalidContainerIDs { 88 | _, err = root.BundleLink(id) 89 | assert.Error(t, err) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /internal/vm/fifo.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "context" 18 | "syscall" 19 | 20 | "github.com/containerd/fifo" 21 | "github.com/sirupsen/logrus" 22 | ) 23 | 24 | // fifoConnector adapts containerd's fifo package to the IOConnector interface 25 | func fifoConnector(path string, flag int) IOConnector { 26 | return func(procCtx context.Context, _ *logrus.Entry) <-chan IOConnectorResult { 27 | returnCh := make(chan IOConnectorResult, 1) 28 | defer close(returnCh) 29 | 30 | // We open the FIFO synchronously to ensure that the FIFO is created (via O_CREAT) before 31 | // it is passed to any task service. O_NONBLOCK ensures that we don't block on the syscall 32 | // level (as documented in the fifo pkg). 33 | fifo, err := fifo.OpenFifo(procCtx, path, syscall.O_CREAT|syscall.O_NONBLOCK|flag, 0300) 34 | returnCh <- IOConnectorResult{ 35 | ReadWriteCloser: fifo, 36 | Err: err, 37 | } 38 | 39 | return returnCh 40 | } 41 | } 42 | 43 | // ReadFIFOConnector returns a FIFO which is open for reading 44 | func ReadFIFOConnector(path string) IOConnector { 45 | return fifoConnector(path, syscall.O_RDONLY) 46 | } 47 | 48 | // WriteFIFOConnector returns a FIFO which is open for writing 49 | func WriteFIFOConnector(path string) IOConnector { 50 | return fifoConnector(path, syscall.O_WRONLY) 51 | } 52 | -------------------------------------------------------------------------------- /internal/vm/ioproxy_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "context" 18 | "os" 19 | "path/filepath" 20 | "testing" 21 | 22 | "github.com/sirupsen/logrus" 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func fileConnector(path string, flag int) IOConnector { 28 | return func(_ context.Context, _ *logrus.Entry) <-chan IOConnectorResult { 29 | returnCh := make(chan IOConnectorResult, 1) 30 | defer close(returnCh) 31 | 32 | file, err := os.OpenFile(path, flag, 0600) 33 | returnCh <- IOConnectorResult{ 34 | ReadWriteCloser: file, 35 | Err: err, 36 | } 37 | 38 | return returnCh 39 | } 40 | } 41 | 42 | func TestProxy(t *testing.T) { 43 | dir := t.TempDir() 44 | 45 | ctx := context.Background() 46 | content := "hello world" 47 | 48 | err := os.WriteFile(filepath.Join(dir, "input"), []byte(content), 0600) 49 | require.NoError(t, err) 50 | 51 | pair := &IOConnectorPair{ 52 | ReadConnector: fileConnector(filepath.Join(dir, "input"), os.O_RDONLY), 53 | WriteConnector: fileConnector(filepath.Join(dir, "output"), os.O_CREATE|os.O_WRONLY), 54 | } 55 | initCh, copyCh := pair.proxy(ctx, logrus.WithFields(logrus.Fields{}), 0) 56 | 57 | assert.Nil(t, <-initCh) 58 | assert.Nil(t, <-copyCh) 59 | 60 | bytes, err := os.ReadFile(filepath.Join(dir, "output")) 61 | require.NoError(t, err) 62 | assert.Equal(t, content, string(bytes)) 63 | } 64 | -------------------------------------------------------------------------------- /internal/vm/mount.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "strings" 18 | 19 | "github.com/containerd/containerd/api/types" 20 | "github.com/containerd/containerd/mount" 21 | ) 22 | 23 | const vmLocalMountTypePrefix = "vm:" 24 | 25 | // IsLocalMount returns true if the mount source is inside the VM as opposed to the 26 | // default assumption that the mount source is a block device on the host. 27 | func IsLocalMount(mount *types.Mount) bool { 28 | return mount != nil && strings.HasPrefix(mount.Type, vmLocalMountTypePrefix) 29 | } 30 | 31 | // AddLocalMountIdentifier adds an identifier to a mount so that it can be detected by IsLocalMount. 32 | // This is intended to be used by cooperating snapshotters to mark mounts as inside the VM so they 33 | // can be plumbed at the proper points inside/outside the VM. 34 | func AddLocalMountIdentifier(mnt mount.Mount) mount.Mount { 35 | return mount.Mount{ 36 | Type: vmLocalMountTypePrefix + mnt.Type, 37 | Source: mnt.Source, 38 | Options: mnt.Options, 39 | } 40 | } 41 | 42 | // StripLocalMountIdentifier removes the identifier that signals that a mount 43 | // is inside the VM. This is used before passing the mount information to runc 44 | func StripLocalMountIdentifier(mnt *types.Mount) *types.Mount { 45 | options := make([]string, len(mnt.Options)) 46 | copy(options, mnt.Options) 47 | return &types.Mount{ 48 | Type: strings.Replace(mnt.Type, vmLocalMountTypePrefix, "", 1), 49 | Source: mnt.Source, 50 | Target: mnt.Target, 51 | Options: options, 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /internal/vm/mount_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/containerd/containerd/api/types" 20 | "github.com/containerd/containerd/mount" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestIsLocalMount(t *testing.T) { 25 | mnt := types.Mount{ 26 | Type: "bind", 27 | Source: "/tmp/snapshots/1/fs", 28 | Options: []string{"rbind"}, 29 | } 30 | assert.False(t, IsLocalMount(&mnt), "Standard mount was considered vm local") 31 | } 32 | 33 | func TestAddLocalMountIdentifier(t *testing.T) { 34 | mnt := mount.Mount{ 35 | Type: "bind", 36 | Source: "/tmp/snapshots/1/fs", 37 | Options: []string{"rbind"}, 38 | } 39 | localMnt := AddLocalMountIdentifier(mnt) 40 | 41 | mntProto := types.Mount{ 42 | Type: mnt.Type, 43 | Source: mnt.Source, 44 | Options: mnt.Options, 45 | } 46 | localMntProto := types.Mount{ 47 | Type: localMnt.Type, 48 | Source: localMnt.Source, 49 | Options: localMnt.Options, 50 | } 51 | 52 | assert.False(t, IsLocalMount(&mntProto), "Standard mount was considered vm local") 53 | assert.True(t, IsLocalMount(&localMntProto), "Mount was not vm local after adding the local mount identifier") 54 | } 55 | 56 | func TestStripLocalMountIdentifier(t *testing.T) { 57 | mnt := mount.Mount{ 58 | Type: "bind", 59 | Source: "/tmp/snapshots/1/fs", 60 | Options: []string{"rbind"}, 61 | } 62 | localMnt := AddLocalMountIdentifier(mnt) 63 | localMntProto := &types.Mount{ 64 | Type: localMnt.Type, 65 | Source: localMnt.Source, 66 | Options: localMnt.Options, 67 | } 68 | assert.True(t, IsLocalMount(localMntProto), "Mount was not vm local after adding the local mount identifier") 69 | localMntProto = StripLocalMountIdentifier(localMntProto) 70 | assert.False(t, IsLocalMount(localMntProto), "Mount was vm local after stripping the local mount identifier") 71 | } 72 | -------------------------------------------------------------------------------- /internal/vm/vsock.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "context" 18 | "time" 19 | 20 | "github.com/firecracker-microvm/firecracker-go-sdk/vsock" 21 | "github.com/sirupsen/logrus" 22 | ) 23 | 24 | // VSockDialConnector returns an IOConnector for establishing vsock connections 25 | // that are dialed from the host to a guest listener. 26 | func VSockDialConnector(timeout time.Duration, udsPath string, port uint32) IOConnector { 27 | return func(procCtx context.Context, logger *logrus.Entry) <-chan IOConnectorResult { 28 | returnCh := make(chan IOConnectorResult) 29 | 30 | go func() { 31 | defer close(returnCh) 32 | timeoutCtx, cancel := context.WithTimeout(procCtx, timeout) 33 | defer cancel() 34 | 35 | conn, err := vsock.DialContext(timeoutCtx, udsPath, port, vsock.WithLogger(logger)) 36 | returnCh <- IOConnectorResult{ 37 | ReadWriteCloser: conn, 38 | Err: err, 39 | } 40 | }() 41 | 42 | return returnCh 43 | } 44 | } 45 | 46 | // VSockAcceptConnector provides an IOConnector that establishes the connection by listening 47 | // on the provided guest-side vsock port and accepting the first connection that comes in. 48 | func VSockAcceptConnector(port uint32) IOConnector { 49 | return func(procCtx context.Context, logger *logrus.Entry) <-chan IOConnectorResult { 50 | returnCh := make(chan IOConnectorResult) 51 | 52 | listener, err := vsock.Listener(procCtx, logger, port) 53 | if err != nil { 54 | returnCh <- IOConnectorResult{ 55 | Err: err, 56 | } 57 | close(returnCh) 58 | return returnCh 59 | } 60 | 61 | go func() { 62 | defer close(returnCh) 63 | defer listener.Close() 64 | 65 | conn, err := listener.Accept() 66 | returnCh <- IOConnectorResult{ 67 | ReadWriteCloser: conn, 68 | Err: err, 69 | } 70 | }() 71 | 72 | return returnCh 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /proto/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | PROTO_SRC := $(wildcard *.proto) 15 | PROTO_GEN_SRC := $(PROTO_SRC:.proto=.pb.go) 16 | DOCKER_IMAGE_TAG?=latest 17 | UID?=$(shell id -u) 18 | GID?=$(shell id -g) 19 | 20 | $(PROTO_GEN_SRC): $(PROTO_SRC) 21 | protoc \ 22 | --go_out=:. \ 23 | $^ 24 | 25 | proto: $(PROTO_GEN_SRC) 26 | PROTOPATH=$(CURDIR) $(MAKE) -C service/fccontrol proto 27 | PROTOPATH=$(CURDIR) $(MAKE) -C service/drivemount proto 28 | PROTOPATH=$(CURDIR) $(MAKE) -C service/ioproxy proto 29 | 30 | proto-docker: 31 | docker run --rm \ 32 | -v $(CURDIR):/protobuf \ 33 | --user $(UID):$(GID) \ 34 | localhost/proto-builder:${DOCKER_IMAGE_TAG} 35 | 36 | clean: 37 | - rm -f $(PROTO_GEN_SRC) 38 | - $(MAKE) -C service/fccontrol clean 39 | - $(MAKE) -C service/drivemount clean 40 | - $(MAKE) -C service/ioproxy clean 41 | 42 | .PHONY: clean proto proto-docker 43 | -------------------------------------------------------------------------------- /proto/events.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = ".;proto"; 4 | 5 | message VMStart { 6 | string VMID = 1; 7 | } 8 | 9 | message VMStop { 10 | string VMID = 1; 11 | } 12 | -------------------------------------------------------------------------------- /proto/service/drivemount/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | PROTO_SRC := $(wildcard *.proto) 15 | PROTO_GEN_SRC := $(PROTO_SRC:.proto=.pb.go) 16 | PROTO_GEN_SRC_TTRPC := $(addprefix ttrpc/,$(PROTO_GEN_SRC)) 17 | 18 | $(PROTO_GEN_SRC_TTRPC): $(PROTO_SRC) 19 | protoc -I. -I$(PROTOPATH)\ 20 | --go_out=:ttrpc \ 21 | $^ 22 | protoc -I. -I$(PROTOPATH)\ 23 | --go-ttrpc_out=:ttrpc \ 24 | $^ 25 | 26 | 27 | proto: $(PROTO_GEN_SRC_TTRPC) 28 | 29 | clean: 30 | - rm -f $(PROTO_GEN_SRC_TTRPC) 31 | 32 | .PHONY: clean proto 33 | -------------------------------------------------------------------------------- /proto/service/drivemount/drivemount.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/empty.proto"; 4 | 5 | option go_package = ".;drivemount"; 6 | 7 | service DriveMounter { 8 | rpc MountDrive(MountDriveRequest) returns (google.protobuf.Empty); 9 | rpc UnmountDrive(UnmountDriveRequest) returns (google.protobuf.Empty); 10 | } 11 | 12 | message MountDriveRequest { 13 | string DriveID = 1; 14 | string DestinationPath = 2; 15 | string FilesytemType = 3; 16 | repeated string Options = 4; 17 | } 18 | 19 | message UnmountDriveRequest { 20 | string DriveID = 1; 21 | } -------------------------------------------------------------------------------- /proto/service/drivemount/ttrpc/drivemount_ttrpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-ttrpc. DO NOT EDIT. 2 | // source: drivemount.proto 3 | package drivemount 4 | 5 | import ( 6 | context "context" 7 | ttrpc "github.com/containerd/ttrpc" 8 | empty "github.com/golang/protobuf/ptypes/empty" 9 | ) 10 | 11 | type DriveMounterService interface { 12 | MountDrive(context.Context, *MountDriveRequest) (*empty.Empty, error) 13 | UnmountDrive(context.Context, *UnmountDriveRequest) (*empty.Empty, error) 14 | } 15 | 16 | func RegisterDriveMounterService(srv *ttrpc.Server, svc DriveMounterService) { 17 | srv.RegisterService("DriveMounter", &ttrpc.ServiceDesc{ 18 | Methods: map[string]ttrpc.Method{ 19 | "MountDrive": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { 20 | var req MountDriveRequest 21 | if err := unmarshal(&req); err != nil { 22 | return nil, err 23 | } 24 | return svc.MountDrive(ctx, &req) 25 | }, 26 | "UnmountDrive": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { 27 | var req UnmountDriveRequest 28 | if err := unmarshal(&req); err != nil { 29 | return nil, err 30 | } 31 | return svc.UnmountDrive(ctx, &req) 32 | }, 33 | }, 34 | }) 35 | } 36 | 37 | type drivemounterClient struct { 38 | client *ttrpc.Client 39 | } 40 | 41 | func NewDriveMounterClient(client *ttrpc.Client) DriveMounterService { 42 | return &drivemounterClient{ 43 | client: client, 44 | } 45 | } 46 | 47 | func (c *drivemounterClient) MountDrive(ctx context.Context, req *MountDriveRequest) (*empty.Empty, error) { 48 | var resp empty.Empty 49 | if err := c.client.Call(ctx, "DriveMounter", "MountDrive", req, &resp); err != nil { 50 | return nil, err 51 | } 52 | return &resp, nil 53 | } 54 | 55 | func (c *drivemounterClient) UnmountDrive(ctx context.Context, req *UnmountDriveRequest) (*empty.Empty, error) { 56 | var resp empty.Empty 57 | if err := c.client.Call(ctx, "DriveMounter", "UnmountDrive", req, &resp); err != nil { 58 | return nil, err 59 | } 60 | return &resp, nil 61 | } 62 | -------------------------------------------------------------------------------- /proto/service/fccontrol/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | PROTO_SRC := $(wildcard *.proto) 15 | PROTO_GEN_SRC := $(PROTO_SRC:.proto=.pb.go) 16 | PROTO_GEN_SRC_TTRPC := $(addprefix ttrpc/,$(PROTO_GEN_SRC)) 17 | 18 | $(PROTO_GEN_SRC_TTRPC): $(PROTO_SRC) 19 | protoc -I. -I$(PROTOPATH)\ 20 | --go_out=Mfirecracker.proto=github.com/firecracker-microvm/firecracker-containerd/proto,Mtypes.proto=github.com/firecracker-microvm/firecracker-containerd/proto:ttrpc \ 21 | $^ 22 | protoc -I. -I$(PROTOPATH)\ 23 | --go-ttrpc_out=Mfirecracker.proto=github.com/firecracker-microvm/firecracker-containerd/proto,Mtypes.proto=github.com/firecracker-microvm/firecracker-containerd/proto:ttrpc \ 24 | $^ 25 | 26 | proto: $(PROTO_GEN_SRC_TTRPC) 27 | 28 | clean: 29 | - rm -f $(PROTO_GEN_SRC_TTRPC) 30 | 31 | .PHONY: clean proto 32 | -------------------------------------------------------------------------------- /proto/service/fccontrol/fccontrol.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/empty.proto"; 4 | 5 | import "firecracker.proto"; 6 | 7 | option go_package = ".;fccontrol"; 8 | 9 | service Firecracker { 10 | // Runs new Firecracker VM instance 11 | rpc CreateVM(CreateVMRequest) returns (CreateVMResponse); 12 | 13 | // Pauses a VM 14 | rpc PauseVM(PauseVMRequest) returns (google.protobuf.Empty); 15 | 16 | // Resumes a VM 17 | rpc ResumeVM(ResumeVMRequest) returns (google.protobuf.Empty); 18 | 19 | // Stops existing Firecracker instance by VM ID 20 | rpc StopVM(StopVMRequest) returns (google.protobuf.Empty); 21 | 22 | // Returns VM info by VM ID 23 | rpc GetVMInfo(GetVMInfoRequest) returns (GetVMInfoResponse); 24 | 25 | // Sets VM's instance metadata 26 | rpc SetVMMetadata(SetVMMetadataRequest) returns (google.protobuf.Empty); 27 | 28 | // Update Vm's instance metadata 29 | rpc UpdateVMMetadata(UpdateVMMetadataRequest) returns (google.protobuf.Empty); 30 | 31 | // Get Vm's instance metadata 32 | rpc GetVMMetadata(GetVMMetadataRequest) returns (GetVMMetadataResponse); 33 | 34 | // Get balloon configuration 35 | rpc GetBalloonConfig(GetBalloonConfigRequest) returns (GetBalloonConfigResponse); 36 | 37 | // Update balloon's memory 38 | rpc UpdateBalloon(UpdateBalloonRequest) returns (google.protobuf.Empty); 39 | 40 | // Gets a balloon device statistics 41 | rpc GetBalloonStats(GetBalloonStatsRequest) returns(GetBalloonStatsResponse); 42 | 43 | // Updates a balloon device statistics polling interval. 44 | rpc UpdateBalloonStats(UpdateBalloonStatsRequest) returns(google.protobuf.Empty); 45 | } 46 | -------------------------------------------------------------------------------- /proto/service/ioproxy/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | PROTO_SRC := $(wildcard *.proto) 15 | PROTO_GEN_SRC := $(PROTO_SRC:.proto=.pb.go) 16 | PROTO_GEN_SRC_TTRPC := $(addprefix ttrpc/,$(PROTO_GEN_SRC)) 17 | 18 | $(PROTO_GEN_SRC_TTRPC): $(PROTO_SRC) 19 | protoc -I. -I$(PROTOPATH)\ 20 | --go_out=:ttrpc \ 21 | $^ 22 | protoc -I. -I$(PROTOPATH)\ 23 | --go-ttrpc_out=:ttrpc \ 24 | $^ 25 | 26 | 27 | proto: $(PROTO_GEN_SRC_TTRPC) 28 | 29 | clean: 30 | - rm -f $(PROTO_GEN_SRC_TTRPC) 31 | 32 | .PHONY: clean proto 33 | -------------------------------------------------------------------------------- /proto/service/ioproxy/ioproxy.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/empty.proto"; 4 | 5 | option go_package = ".;ioproxy"; 6 | 7 | service IOProxy { 8 | rpc State(StateRequest) returns (StateResponse); 9 | rpc Attach(AttachRequest) returns (google.protobuf.Empty); 10 | } 11 | 12 | message StateRequest { 13 | string ID = 1; 14 | string ExecID = 2; 15 | } 16 | 17 | message StateResponse { 18 | bool IsOpen = 1; 19 | } 20 | 21 | message AttachRequest { 22 | string ID = 1; 23 | string ExecID = 2; 24 | uint32 StdinPort = 3; 25 | uint32 StdoutPort = 4; 26 | uint32 StderrPort = 5; 27 | } 28 | -------------------------------------------------------------------------------- /proto/service/ioproxy/ttrpc/ioproxy_ttrpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-ttrpc. DO NOT EDIT. 2 | // source: ioproxy.proto 3 | package ioproxy 4 | 5 | import ( 6 | context "context" 7 | ttrpc "github.com/containerd/ttrpc" 8 | empty "github.com/golang/protobuf/ptypes/empty" 9 | ) 10 | 11 | type IOProxyService interface { 12 | State(context.Context, *StateRequest) (*StateResponse, error) 13 | Attach(context.Context, *AttachRequest) (*empty.Empty, error) 14 | } 15 | 16 | func RegisterIOProxyService(srv *ttrpc.Server, svc IOProxyService) { 17 | srv.RegisterService("IOProxy", &ttrpc.ServiceDesc{ 18 | Methods: map[string]ttrpc.Method{ 19 | "State": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { 20 | var req StateRequest 21 | if err := unmarshal(&req); err != nil { 22 | return nil, err 23 | } 24 | return svc.State(ctx, &req) 25 | }, 26 | "Attach": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { 27 | var req AttachRequest 28 | if err := unmarshal(&req); err != nil { 29 | return nil, err 30 | } 31 | return svc.Attach(ctx, &req) 32 | }, 33 | }, 34 | }) 35 | } 36 | 37 | type ioproxyClient struct { 38 | client *ttrpc.Client 39 | } 40 | 41 | func NewIOProxyClient(client *ttrpc.Client) IOProxyService { 42 | return &ioproxyClient{ 43 | client: client, 44 | } 45 | } 46 | 47 | func (c *ioproxyClient) State(ctx context.Context, req *StateRequest) (*StateResponse, error) { 48 | var resp StateResponse 49 | if err := c.client.Call(ctx, "IOProxy", "State", req, &resp); err != nil { 50 | return nil, err 51 | } 52 | return &resp, nil 53 | } 54 | 55 | func (c *ioproxyClient) Attach(ctx context.Context, req *AttachRequest) (*empty.Empty, error) { 56 | var resp empty.Empty 57 | if err := c.client.Call(ctx, "IOProxy", "Attach", req, &resp); err != nil { 58 | return nil, err 59 | } 60 | return &resp, nil 61 | } 62 | -------------------------------------------------------------------------------- /runtime/.gitignore: -------------------------------------------------------------------------------- 1 | containerd-shim-aws-firecracker 2 | containerd-firecracker-runtime 3 | runtime 4 | -------------------------------------------------------------------------------- /runtime/README.md: -------------------------------------------------------------------------------- 1 | # containerd-firecracker-runtime 2 | 3 | This is the runtime component enabling containerd to control the Firecracker 4 | VMM. This component runs on the host, outside your microVM. In general, it 5 | strives for 6 | [OCI runtime](https://github.com/opencontainers/runtime-spec/blob/main/spec.md) 7 | compliance within the bounds of Firecracker's feature set. 8 | 9 | ## Building 10 | 11 | `make` 12 | 13 | This will generate a `containerd-shim-aws-firecracker` binary in the current 14 | working directory. 15 | 16 | ## Installation 17 | 18 | Ensure that you have [containerd](https://github.com/containerd/containerd) 19 | installed and configured. 20 | 21 | Copy `containerd-shim-aws-firecracker` to /bin (or something else on the PATH) 22 | following the naming guidelines for a containerd 23 | [runtime v2](https://github.com/containerd/containerd/blob/main/runtime/v2/README.md) 24 | shim: 25 | 26 | * If the runtime is invoked as aws.firecracker 27 | * Then the binary name needs to be containerd-shim-aws-firecracker 28 | 29 | ## Configuration 30 | 31 | The runtime expects a JSON-formatted configuration file to be located either in 32 | `/etc/containerd/firecracker-runtime.json` or in a location defined by the 33 | `FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH` environment variable. The 34 | configuration file has the following fields: 35 | 36 | * `firecracker_binary_path` (optional) - A path to locate the `firecracker` 37 | executable. If left undefined, the runtime looks for an executable named 38 | `firecracker` located in its working directory. A fully-qualified path to the 39 | `firecracker` binary is recommended, as the working directory typically 40 | changes every execution when run by containerd. 41 | * `kernel_image_path` (optional) - A path where the kernel image file is 42 | located. A fully-qualified path is recommended. If left undefined, the 43 | runtime looks for a file named 44 | `/var/lib/firecracker-containerd/runtime/default-vmlinux.bin`. 45 | * `kernel_args` (optional) - Arguments for the kernel command line. If left 46 | undefined, the runtime specifies "console=ttyS0 noapic reboot=k panic=1 47 | pci=off nomodules rw". 48 | * `root_drive` (optional) - A path where the root drive image file is located. A 49 | fully-qualified path is recommended. If left undefined, the runtime looks for 50 | a file named `/var/lib/firecracker-containerd/runtime/default-rootfs.img`. 51 | * `cpu_template` (required) - The Firecracker CPU emulation template. Supported 52 | values are "C3" and "T2". 53 | * `additional_drives` (unused) 54 | * `log_fifo` (optional) - Named pipe where Firecracker logs should be delivered. 55 | * `log_level` (optional) - Log level for the Firecracker logs 56 | * `metrics_fifo` (optional) - Named pipe where Firecracker metrics should be 57 | delivered. 58 | * `ht_enabled` (unused) - Reserved for future use. 59 | * `debug` (optional) - Enable debug-level logging from the runtime. 60 | 61 | ## Usage 62 | See our [Getting Started Guide](../docs/getting-started.md) for details on how to use 63 | the aws.firecracker runtime. 64 | -------------------------------------------------------------------------------- /runtime/firecracker-runc-config.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "ociVersion": "1.0.1", 3 | "process": { 4 | "terminal": false, 5 | "user": { 6 | "uid": 0, 7 | "gid": 0 8 | }, 9 | "args": [ 10 | "/firecracker", 11 | "--api-sock", 12 | "api.socket" 13 | ], 14 | "env": [ 15 | "PATH=/" 16 | ], 17 | "cwd": "/", 18 | "capabilities": { 19 | "effective": [ 20 | ], 21 | "bounding": [ 22 | ], 23 | "inheritable": [ 24 | ], 25 | "permitted": [ 26 | ], 27 | "ambient": [ 28 | ] 29 | }, 30 | "rlimits": [ 31 | { 32 | "type": "RLIMIT_NOFILE", 33 | "hard": 1024, 34 | "soft": 1024 35 | } 36 | ], 37 | "noNewPrivileges": true 38 | }, 39 | "root": { 40 | "path": "rootfs", 41 | "readonly": false 42 | }, 43 | "hostname": "runc", 44 | "mounts": [ 45 | { 46 | "destination": "/proc", 47 | "type": "proc", 48 | "source": "proc" 49 | } 50 | ], 51 | "linux": { 52 | "devices": [ 53 | { 54 | "path": "/dev/kvm", 55 | "type": "c", 56 | "major": 10, 57 | "minor": 232, 58 | "fileMode": 438, 59 | "uid": 0, 60 | "gid": 0 61 | }, 62 | { 63 | "path": "/dev/net/tun", 64 | "type": "c", 65 | "major": 10, 66 | "minor": 200, 67 | "fileMode": 438, 68 | "uid": 0, 69 | "gid": 0 70 | } 71 | ], 72 | "resources": { 73 | "devices": [ 74 | { 75 | "allow": false, 76 | "access": "rwm" 77 | }, 78 | { 79 | "allow": true, 80 | "major": 10, 81 | "minor": 232, 82 | "access": "rwm" 83 | }, 84 | { 85 | "allow": true, 86 | "major": 10, 87 | "minor": 200, 88 | "access": "rwm" 89 | } 90 | ] 91 | }, 92 | "namespaces": [ 93 | { 94 | "type": "cgroup" 95 | }, 96 | { 97 | "type": "pid" 98 | }, 99 | { 100 | "type": "network" 101 | }, 102 | { 103 | "type": "ipc" 104 | }, 105 | { 106 | "type": "uts" 107 | }, 108 | { 109 | "type": "mount" 110 | } 111 | ], 112 | "maskedPaths": [ 113 | "/proc/asound", 114 | "/proc/kcore", 115 | "/proc/latency_stats", 116 | "/proc/timer_list", 117 | "/proc/timer_stats", 118 | "/proc/sched_debug", 119 | "/sys/firmware", 120 | "/proc/scsi" 121 | ], 122 | "readonlyPaths": [ 123 | "/proc/bus", 124 | "/proc/fs", 125 | "/proc/irq", 126 | "/proc/sys", 127 | "/proc/sysrq-trigger" 128 | ] 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /runtime/firecrackeroci/annotation.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package firecrackeroci 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/containerd/containerd/containers" 20 | "github.com/containerd/containerd/oci" 21 | ) 22 | 23 | const ( 24 | // VMIDAnnotationKey is the key specified in an OCI-runtime config annotation section 25 | // specifying the ID of the VM in which the container should be spun up. 26 | VMIDAnnotationKey = "aws.firecracker.vm.id" 27 | ) 28 | 29 | // WithVMID annotates a containerd client's container object with a given firecracker VMID. 30 | func WithVMID(vmID string) oci.SpecOpts { 31 | return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { 32 | if s.Annotations == nil { 33 | s.Annotations = make(map[string]string) 34 | } 35 | 36 | s.Annotations[VMIDAnnotationKey] = vmID 37 | return nil 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /runtime/firecrackeroci/network.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package firecrackeroci 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/containerd/containerd/containers" 20 | "github.com/containerd/containerd/oci" 21 | "github.com/opencontainers/runtime-spec/specs-go" 22 | ) 23 | 24 | var _ oci.SpecOpts = WithVMNetwork 25 | 26 | // WithVMNetwork modifies a container to use its host network settings (host netns, host utsns, host 27 | // /etc/resolv.conf and host /etc/hosts). It's intended to configure Firecracker-containerd containers 28 | // to have access to the network (if any) their VM was configured with. 29 | func WithVMNetwork(ctx context.Context, cli oci.Client, ctr *containers.Container, spec *oci.Spec) error { 30 | for _, opt := range []oci.SpecOpts{ 31 | oci.WithHostNamespace(specs.NetworkNamespace), 32 | oci.WithHostNamespace(specs.UTSNamespace), 33 | oci.WithHostResolvconf, 34 | oci.WithHostHostsFile, 35 | } { 36 | err := opt(ctx, cli, ctr, spec) 37 | if err != nil { 38 | return err 39 | } 40 | } 41 | 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /runtime/integ_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | package main 14 | 15 | import ( 16 | "os" 17 | "path/filepath" 18 | "strings" 19 | 20 | "github.com/firecracker-microvm/firecracker-containerd/internal" 21 | "github.com/firecracker-microvm/firecracker-containerd/internal/integtest" 22 | ) 23 | 24 | func init() { 25 | flag, err := internal.SupportCPUTemplate() 26 | if err != nil { 27 | panic(err) 28 | } 29 | 30 | if flag { 31 | integtest.DefaultRuntimeConfig.CPUTemplate = "T2" 32 | } 33 | } 34 | 35 | // devmapper is the only snapshotter we can use with Firecracker 36 | const defaultSnapshotterName = "devmapper" 37 | 38 | var testNameToVMIDReplacer = strings.NewReplacer("/", "-", "_", "-") 39 | 40 | func testNameToVMID(s string) string { 41 | return testNameToVMIDReplacer.Replace(s) 42 | } 43 | 44 | func cgroupExists(name string) bool { 45 | // cgroups v1 46 | _, err := os.Stat(filepath.Join("/sys/fs/cgroup/cpu", name)) 47 | if err == nil { 48 | return true 49 | } 50 | 51 | // cgroups v2 52 | _, err = os.Stat(filepath.Join("/sys/fs/cgroup", name)) 53 | return err == nil 54 | } 55 | -------------------------------------------------------------------------------- /runtime/limits_integ_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | package main 14 | 15 | import ( 16 | "context" 17 | "testing" 18 | 19 | "github.com/containerd/containerd" 20 | "github.com/containerd/containerd/namespaces" 21 | "github.com/containerd/containerd/oci" 22 | "github.com/firecracker-microvm/firecracker-containerd/internal/integtest" 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func TestDiskLimit_Isolated(t *testing.T) { 28 | integtest.Prepare(t) 29 | 30 | ctx := namespaces.WithNamespace(context.Background(), "default") 31 | 32 | client, err := containerd.New(integtest.ContainerdSockPath, containerd.WithDefaultRuntime(firecrackerRuntime)) 33 | require.NoError(t, err, "unable to create client to containerd service at %s, is containerd running?", integtest.ContainerdSockPath) 34 | defer client.Close() 35 | 36 | image, err := alpineImage(ctx, client, defaultSnapshotterName) 37 | require.NoError(t, err, "failed to get alpine image") 38 | 39 | // Right now, both naive snapshotter and devmapper snapshotter are configured to have 1024MB image size. 40 | // The former is hard-coded since the snapshotter is not for production. The latter is configured in tools/docker/entrypoint.sh. 41 | sh := containerd.WithNewSpec( 42 | oci.WithProcessArgs("dd", "if=/dev/zero", "of=/tmp/fill", "bs=1M", "count=2000"), 43 | oci.WithDefaultPathEnv, 44 | ) 45 | 46 | container, err := client.NewContainer(ctx, 47 | "container", 48 | containerd.WithSnapshotter(defaultSnapshotterName), 49 | containerd.WithNewSnapshot("snapshot", image), 50 | sh, 51 | ) 52 | defer func() { 53 | err = container.Delete(ctx, containerd.WithSnapshotCleanup) 54 | require.NoError(t, err, "failed to delete a container") 55 | }() 56 | 57 | result, err := integtest.RunTask(ctx, container) 58 | require.NoError(t, err, "failed to create a container") 59 | 60 | assert.Equal(t, uint32(1), result.ExitCode, "writing 2GB must fail") 61 | assert.Equal(t, `952+0 records in 62 | 951+0 records out 63 | `, result.Stderr, "but it must be able to write ~1024MB") 64 | } 65 | -------------------------------------------------------------------------------- /runtime/main.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "os" 18 | 19 | "github.com/containerd/containerd/runtime/v2/shim" 20 | "github.com/containerd/log" 21 | "github.com/sirupsen/logrus" 22 | ) 23 | 24 | const shimID = "aws.firecracker" 25 | 26 | var revision string 27 | 28 | func init() { 29 | logrus.SetFormatter(&logrus.TextFormatter{ 30 | TimestampFormat: log.RFC3339NanoFixed, 31 | FullTimestamp: true, 32 | }) 33 | 34 | logrus.SetOutput(os.Stdout) 35 | } 36 | 37 | func main() { 38 | shim.Run(shimID, NewService, func(cfg *shim.Config) { 39 | cfg.NoSetupLogger = true 40 | 41 | // Just let child processes get reparented to init 42 | // (or the nearest subreaper). Enabling reaping 43 | // creates races with `os.Exec` commands that expect 44 | // to be able to wait on their child processes. 45 | cfg.NoSubreaper = true 46 | cfg.NoReaper = true 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /runtime/vm/mount.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package vm 15 | 16 | import ( 17 | "github.com/containerd/containerd/mount" 18 | "github.com/firecracker-microvm/firecracker-containerd/internal/vm" 19 | ) 20 | 21 | // AddLocalMountIdentifier adds an identifier to a mount so that it can be detected by IsLocalMount. 22 | // This is intended to be used by cooperating snapshotters to mark mounts as inside the VM so they 23 | // can be plumbed at the proper points inside/outside the VM. 24 | func AddLocalMountIdentifier(mnt mount.Mount) mount.Mount { 25 | return vm.AddLocalMountIdentifier(mnt) 26 | } 27 | -------------------------------------------------------------------------------- /snapshotter/.gitignore: -------------------------------------------------------------------------------- 1 | demux-snapshotter 2 | http-address-resolver 3 | logs/ 4 | -------------------------------------------------------------------------------- /snapshotter/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | EXTRAGOARGS?= 15 | NUMBER_OF_VMS?= 16 | 17 | SOURCES := $(shell find . -name '*.go') 18 | GOMOD := $(shell go env GOMOD) 19 | GOSUM := $(GOMOD:.mod=.sum) 20 | DOCKER_IMAGE_TAG?=latest 21 | GO_CACHE_VOLUME_NAME?=gocache 22 | FIRECRACKER_CONTAINERD_TEST_IMAGE?=localhost/firecracker-containerd-test 23 | FICD_DM_POOL?=fc-test-thinpool 24 | 25 | REVISION := $(shell git rev-parse HEAD) 26 | 27 | INTEG_TEST_SUFFIX := _Isolated 28 | INTEG_TESTNAMES=$(shell docker run --rm \ 29 | --network=none \ 30 | --volume $(CURDIR)/..:/src \ 31 | --volume $(GO_CACHE_VOLUME_NAME):/go \ 32 | --entrypoint=/bin/bash \ 33 | --workdir=/src/snapshotter \ 34 | $(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \ 35 | -c "go test -list . | sed '$$d' | grep $(INTEG_TEST_SUFFIX)") 36 | 37 | all: demux-snapshotter http-address-resolver 38 | 39 | demux-snapshotter: $(SOURCES) $(GOMOD) $(GOSUM) 40 | go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@ 41 | 42 | http-address-resolver: $(SOURCES) $(GOMOD) $(GOSUM) 43 | go build $(EXTRAGOARGS) -ldflags "-X main.revision=$(REVISION)" -o $@ internal/http_address_resolver.go 44 | 45 | install: demux-snapshotter http-address-resolver 46 | install -D -o root -g root -m755 -t $(INSTALLROOT)/bin $^ 47 | 48 | test: 49 | go test ./... $(EXTRAGOARGS) 50 | 51 | integ-test: 52 | $(MAKE) $(addprefix integ-test-,$(INTEG_TESTNAMES)) 53 | 54 | integ-test-%: logs 55 | $(CURDIR)/../tools/thinpool.sh reset "$(FICD_DM_POOL)" 56 | docker run --rm -it \ 57 | --privileged \ 58 | --ipc=host \ 59 | --network=bridge \ 60 | --volume /dev:/dev \ 61 | --volume /run/udev/control:/run/udev/control \ 62 | --volume $(CURDIR)/logs:/var/log/firecracker-containerd-test \ 63 | --volume $(CURDIR)/..:/src \ 64 | --volume $(GO_CACHE_VOLUME_NAME):/go \ 65 | --env ENABLE_ISOLATED_TESTS=1 \ 66 | --env GOPROXY=direct \ 67 | --env GOSUMDB=off \ 68 | --env GO111MODULE=on \ 69 | --env NUMBER_OF_VMS=$(NUMBER_OF_VMS) \ 70 | --env FICD_DM_VOLUME_GROUP=$(FICD_DM_VOLUME_GROUP) \ 71 | --env FICD_DM_POOL=$(FICD_DM_POOL) \ 72 | --workdir="/src/snapshotter" \ 73 | --init \ 74 | $(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \ 75 | "go test $(EXTRAGOARGS) -run '^$(subst integ-test-,,$@)$$'" 76 | 77 | logs: 78 | mkdir logs 79 | 80 | clean: 81 | - rm -f demux-snapshotter 82 | - rm -f http-address-resolver 83 | 84 | distclean: clean 85 | - rm -rf logs 86 | 87 | .PHONY: all clean distclean install test integ-test 88 | -------------------------------------------------------------------------------- /snapshotter/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package config 15 | 16 | import ( 17 | "io" 18 | "os" 19 | 20 | "github.com/pelletier/go-toml" 21 | ) 22 | 23 | // Config contains metadata necessary to forward snapshotter requests 24 | // to the destined remote snapshotter. 25 | // 26 | // The default location for the configuration file is '/etc/demux-snapshotter/config.toml' 27 | type Config struct { 28 | Snapshotter snapshotter `toml:"snapshotter"` 29 | 30 | Debug debug `toml:"debug"` 31 | } 32 | 33 | type snapshotter struct { 34 | Listener listener `toml:"listener"` 35 | Proxy proxy `toml:"proxy"` 36 | Dialer dialer `toml:"dialer"` 37 | Cache cache `toml:"cache"` 38 | Metrics metrics `toml:"metrics"` 39 | } 40 | 41 | type dialer struct { 42 | Timeout string `toml:"timeout" default:"5s"` 43 | RetryInterval string `toml:"retry_interval" default:"100ms"` 44 | } 45 | 46 | type listener struct { 47 | Network string `toml:"network" default:"unix"` 48 | Address string `toml:"address" default:"/var/lib/demux-snapshotter/snapshotter.sock"` 49 | } 50 | 51 | type proxy struct { 52 | Address address `toml:"address"` 53 | } 54 | 55 | type address struct { 56 | Resolver resolver `toml:"resolver"` 57 | } 58 | 59 | type resolver struct { 60 | Type string `toml:"type"` 61 | Address string `toml:"address"` 62 | } 63 | 64 | type cache struct { 65 | EvictOnConnectionFailure bool `toml:"evict_on_connection_failure" default:"true"` 66 | PollConnectionFrequency string `toml:"poll_connection_frequency" default:"60s"` 67 | } 68 | 69 | type debug struct { 70 | LogLevel string `toml:"logLevel" default:"info"` 71 | } 72 | 73 | type metrics struct { 74 | Enable bool `toml:"enable" default:"false"` 75 | Host string `toml:"host"` 76 | PortRange string `toml:"port_range"` 77 | ServiceDiscoveryPort int `toml:"service_discovery_port"` 78 | } 79 | 80 | // Load parses application configuration from a specified file path. 81 | func Load(filePath string) (Config, error) { 82 | file, err := os.Open(filePath) 83 | if err != nil { 84 | return Config{}, err 85 | } 86 | defer file.Close() 87 | 88 | fileInfo, err := file.Stat() 89 | if err != nil { 90 | return Config{}, err 91 | } 92 | 93 | return load(file, fileInfo.Size()) 94 | } 95 | 96 | func load(reader io.Reader, readSize int64) (Config, error) { 97 | buffer := make([]byte, readSize) 98 | readBytes, err := reader.Read(buffer) 99 | if int64(readBytes) != readSize || err != nil { 100 | return Config{}, err 101 | } 102 | 103 | config := Config{} 104 | if err = toml.Unmarshal(buffer, &config); err != nil { 105 | return Config{}, err 106 | } 107 | return config, nil 108 | } 109 | -------------------------------------------------------------------------------- /snapshotter/config/config.toml.example: -------------------------------------------------------------------------------- 1 | [snapshotter.listener] 2 | type = "unix" 3 | address = "/var/lib/demux-snapshotter/snapshotter.sock" 4 | 5 | [snapshotter.proxy.address.resolver] 6 | type = "http" 7 | address = "127.0.0.1:10001" 8 | 9 | [snapshotter.dialer] 10 | timeout = "5s" 11 | retry_interval = "500ms" 12 | 13 | [snapshotter.cache] 14 | evict_on_connection_failure = true 15 | poll_connection_frequency = "60s" 16 | 17 | [snapshotter.metrics] 18 | enable = true 19 | port_range = "9000-9999" 20 | 21 | [snapshotter.metrics.service_discovery] 22 | enable = true 23 | port = 8080 24 | 25 | [debug] 26 | logLevel = "info" 27 | -------------------------------------------------------------------------------- /snapshotter/demux/README.md: -------------------------------------------------------------------------------- 1 | # snapshotter 2 | 3 | ## Building an esgz Formatted Image 4 | 5 | The demux snapshotter uses a publicly available [eStargz](https://github.com/containerd/stargz-snapshotter/blob/main/docs/estargz.md) formatted image in its integration testing pipeline. 6 | This image has the reference `ghcr.io/firecracker-microvm/firecracker-containerd/amazonlinux:latest-esgz`. 7 | 8 | ### Optimize an Amazon Linux Container Image for esgz 9 | 10 | The `ctr-remote` binary is needed to optimize an image for the esgz image format. Building the stargz-snapshotter submodule provides `ctr-remote`, a `ctr` wrapper, in `_submodules/stargz-snapshotter/out/ctr-remote`. 11 | Refer to the [containerd stargz-snapshotter documentation on `ctr-remote`](https://github.com/containerd/stargz-snapshotter/blob/main/docs/ctr-remote.md) for additional details. 12 | 13 | The root level Makefile provides `make esgz-test-image` for pulling the latest Amazon Linux image locally and optimizing it for eStargz using `ctr-remote` into an image `ghcr.io/firecracker-microvm/firecracker-containerd/amazonlinux:latest-esgz`. 14 | Code owners of `firecracker-containerd` then push the eStargz optimized image as this publicly available reference using `make push-esgz-test-image` with proper credential environment variables `GH_USER` and `GH_PERSONAL_ACCESS_TOKEN` set. 15 | 16 | ### Example optimization and push to GHCR 17 | 18 | Using default base and esgz images: 19 | 20 | ```bash 21 | $ make esgz-test-image 22 | $ GH_USER=xxx 23 | # set GH_PERSONAL_ACCESS_TOKEN with command substitution such that it does not show in shell history 24 | $ make \ 25 | GH_USER=$GH_USER \ 26 | GH_PERSONAL_ACCESS_TOKEN=`cat` 27 | push-esgz-test-image 28 | # enter personal access token and CTRL^D 29 | ``` 30 | 31 | To pull, convert, and push to a repo where the user has permissions with custom base and esgz images: 32 | 33 | ```bash 34 | $ DEFAULT_BASE_IMAGE=public.ecr.aws/amazonlinux/amazonlinux:2022 35 | $ DEFAULT_ESGZ_IMAGE=ghcr.io/$DESTINATION_REPO/amazonlinux:2022-esgz 36 | $ make \ 37 | DEFAULT_BASE_IMAGE=$DEFAULT_BASE_IMAGE \ 38 | DEFAULT_ESGZ_IMAGE=$DEFAULT_ESGZ_IMAGE \ 39 | esgz-test-image 40 | $ GH_USER=xxx 41 | # set GH_PERSONAL_ACCESS_TOKEN with command substitution such that it does not show in shell history 42 | $ make \ 43 | GH_USER=$GH_USER \ 44 | GH_PERSONAL_ACCESS_TOKEN=`cat` 45 | DEFAULT_ESGZ_IMAGE=$DEFAULT_ESGZ_IMAGE \ 46 | push-esgz-test-image 47 | # enter personal access token and CTRL^D 48 | ``` 49 | 50 | -------------------------------------------------------------------------------- /snapshotter/demux/cache/evict.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package cache 15 | 16 | import ( 17 | "context" 18 | "time" 19 | 20 | "github.com/firecracker-microvm/firecracker-containerd/snapshotter/demux/proxy" 21 | ) 22 | 23 | // EvictionPolicy defines the interface for enforcing a cache eviction policy. 24 | type EvictionPolicy interface { 25 | // Enforce monitors for policy failures. 26 | Enforce(key string) 27 | } 28 | 29 | type evictionPolicy struct { 30 | evict chan string 31 | stop chan struct{} 32 | } 33 | 34 | // EvictOnConnectionFailurePolicy defines an eviction policy where entries are evicted 35 | // from cache after a failed connection attempt occurs. 36 | type EvictOnConnectionFailurePolicy struct { 37 | evictionPolicy 38 | 39 | dialer proxy.Dialer 40 | 41 | frequency time.Duration 42 | } 43 | 44 | // NewEvictOnConnectionFailurePolicy creates a new policy to evict on remote snapshotter 45 | // connection failure on a specified frequency duration. 46 | func NewEvictOnConnectionFailurePolicy(evictChan chan string, dialer proxy.Dialer, frequency time.Duration, stopCondition chan struct{}) EvictionPolicy { 47 | return &EvictOnConnectionFailurePolicy{evictionPolicy: evictionPolicy{evict: evictChan, stop: stopCondition}, dialer: dialer, frequency: frequency} 48 | } 49 | 50 | // Enforce launches a go routine which periodically attempts to dial the cached entry 51 | // using the provided dial function. 52 | // 53 | // On connection failure, the entry will be evicted from cache. 54 | func (p EvictOnConnectionFailurePolicy) Enforce(key string) { 55 | go func() { 56 | ticker := time.NewTicker(p.frequency) 57 | defer ticker.Stop() 58 | 59 | for { 60 | select { 61 | case <-ticker.C: 62 | ctx, cancel := context.WithTimeout(context.Background(), p.dialer.Timeout) 63 | conn, err := p.dialer.Dial(ctx, key) 64 | if err != nil { 65 | p.evict <- key 66 | cancel() 67 | return 68 | } 69 | cancel() 70 | conn.Close() 71 | case <-p.stop: 72 | return 73 | } 74 | } 75 | }() 76 | } 77 | -------------------------------------------------------------------------------- /snapshotter/demux/internal/failing_snapshotter.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package internal 15 | 16 | import ( 17 | "context" 18 | "errors" 19 | 20 | "github.com/containerd/containerd/mount" 21 | "github.com/containerd/containerd/snapshots" 22 | ) 23 | 24 | // FailingSnapshotter mocks containerd snapshots.Snapshotter interface 25 | // return non-nil errors on calls. 26 | type FailingSnapshotter struct{} 27 | 28 | // Stat mocks a failing remote call with a non-nil error. 29 | func (s *FailingSnapshotter) Stat(_ context.Context, _ string) (snapshots.Info, error) { 30 | return snapshots.Info{}, errors.New("mock Stat error from remote snapshotter") 31 | } 32 | 33 | // Update mocks a failing remote call with a non-nil error. 34 | func (s *FailingSnapshotter) Update(_ context.Context, _ snapshots.Info, _ ...string) (snapshots.Info, error) { 35 | return snapshots.Info{}, errors.New("mock Update error from remote snapshotter") 36 | } 37 | 38 | // Usage mocks a failing remote call with a non-nil error. 39 | func (s *FailingSnapshotter) Usage(_ context.Context, _ string) (snapshots.Usage, error) { 40 | return snapshots.Usage{}, errors.New("mock Usage error from remote snapshotter") 41 | } 42 | 43 | // Mounts mocks a failing remote call with a non-nil error. 44 | func (s *FailingSnapshotter) Mounts(_ context.Context, _ string) ([]mount.Mount, error) { 45 | return []mount.Mount{}, errors.New("mock Mounts error from remote snapshotter") 46 | } 47 | 48 | // Prepare mocks a failing remote call with a non-nil error. 49 | func (s *FailingSnapshotter) Prepare(_ context.Context, _ string, _ string, _ ...snapshots.Opt) ([]mount.Mount, error) { 50 | return []mount.Mount{}, errors.New("mock Prepare error from remote snapshotter") 51 | } 52 | 53 | // View mocks a failing remote call with a non-nil error. 54 | func (s *FailingSnapshotter) View(_ context.Context, _ string, _ string, _ ...snapshots.Opt) ([]mount.Mount, error) { 55 | return []mount.Mount{}, errors.New("mock View error from remote snapshotter") 56 | } 57 | 58 | // Commit mocks a failing remote call with a non-nil error. 59 | func (s *FailingSnapshotter) Commit(_ context.Context, _ string, _ string, _ ...snapshots.Opt) error { 60 | return errors.New("mock Commit error from remote snapshotter") 61 | } 62 | 63 | // Remove mocks a failing remote call with a non-nil error. 64 | func (s *FailingSnapshotter) Remove(_ context.Context, _ string) error { 65 | return errors.New("mock Remove error from remote snapshotter") 66 | } 67 | 68 | // Walk mocks a failing remote call with a non-nil error. 69 | func (s *FailingSnapshotter) Walk(_ context.Context, _ snapshots.WalkFunc, _ ...string) error { 70 | return errors.New("mock Walk error from remote snapshotter") 71 | } 72 | 73 | // Close mocks a failing remote call with a non-nil error. 74 | func (s *FailingSnapshotter) Close() error { 75 | return errors.New("mock Close error from remote snapshotter") 76 | } 77 | 78 | // Cleanup mocks a failing remote call with a non-nil error. 79 | func (s *FailingSnapshotter) Cleanup(_ context.Context) error { 80 | return errors.New("mock Cleanup error from remote snapshotter") 81 | } 82 | -------------------------------------------------------------------------------- /snapshotter/demux/internal/successful_snapshotter.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package internal 15 | 16 | import ( 17 | "context" 18 | 19 | "github.com/containerd/containerd/mount" 20 | "github.com/containerd/containerd/snapshots" 21 | ) 22 | 23 | // SuccessfulSnapshotter mocks containerd snapshots.Snapshotter interface 24 | // returning nil errors on calls. 25 | type SuccessfulSnapshotter struct{} 26 | 27 | // Stat mocks a successful remote call with a nil error. 28 | func (s *SuccessfulSnapshotter) Stat(_ context.Context, _ string) (snapshots.Info, error) { 29 | return snapshots.Info{}, nil 30 | } 31 | 32 | // Update mocks a successful remote call with a nil error. 33 | func (s *SuccessfulSnapshotter) Update(_ context.Context, _ snapshots.Info, _ ...string) (snapshots.Info, error) { 34 | return snapshots.Info{}, nil 35 | } 36 | 37 | // Usage mocks a successful remote call with a nil error. 38 | func (s *SuccessfulSnapshotter) Usage(_ context.Context, _ string) (snapshots.Usage, error) { 39 | return snapshots.Usage{}, nil 40 | } 41 | 42 | // Mounts mocks a successful remote call with a nil error. 43 | func (s *SuccessfulSnapshotter) Mounts(_ context.Context, _ string) ([]mount.Mount, error) { 44 | return []mount.Mount{}, nil 45 | } 46 | 47 | // Prepare mocks a successful remote call with a nil error. 48 | func (s *SuccessfulSnapshotter) Prepare(_ context.Context, _ string, _ string, _ ...snapshots.Opt) ([]mount.Mount, error) { 49 | return []mount.Mount{}, nil 50 | } 51 | 52 | // View mocks a successful remote call with a nil error. 53 | func (s *SuccessfulSnapshotter) View(_ context.Context, _ string, _ string, _ ...snapshots.Opt) ([]mount.Mount, error) { 54 | return []mount.Mount{}, nil 55 | } 56 | 57 | // Commit mocks a successful remote call with a nil error. 58 | func (s *SuccessfulSnapshotter) Commit(_ context.Context, _ string, _ string, _ ...snapshots.Opt) error { 59 | return nil 60 | } 61 | 62 | // Remove mocks a successful remote call with a nil error. 63 | func (s *SuccessfulSnapshotter) Remove(_ context.Context, _ string) error { 64 | return nil 65 | } 66 | 67 | // Walk mocks a successful remote call with a nil error. 68 | func (s *SuccessfulSnapshotter) Walk(_ context.Context, _ snapshots.WalkFunc, _ ...string) error { 69 | return nil 70 | } 71 | 72 | // Close mocks a successful remote call with a nil error. 73 | func (s *SuccessfulSnapshotter) Close() error { 74 | return nil 75 | } 76 | 77 | // Cleanup mocks a successful remote call with a nil error. 78 | func (s *SuccessfulSnapshotter) Cleanup(_ context.Context) error { 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /snapshotter/demux/proxy/address/http_resolver.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package address 15 | 16 | import ( 17 | "encoding/json" 18 | "fmt" 19 | "io" 20 | "net/http" 21 | ) 22 | 23 | // HTTPClient defines the interface for the client used 24 | // for HTTP communications in the resolver. 25 | type HTTPClient interface { 26 | Get(string) (*http.Response, error) 27 | } 28 | 29 | // ResponseReader defines the reading function interface. 30 | type ResponseReader func(io.Reader) ([]byte, error) 31 | 32 | // HTTPResolver implements a proxy address resolver via HTTP. 33 | type HTTPResolver struct { 34 | url string 35 | client HTTPClient 36 | } 37 | 38 | // NewHTTPResolver creates a new instance of HttpResolver with specified the URL. 39 | func NewHTTPResolver(url string) HTTPResolver { 40 | return HTTPResolver{url: url, client: http.DefaultClient} 41 | } 42 | 43 | func requestURL(url string, namespace string) string { 44 | return fmt.Sprintf("%s/address?namespace=%s", url, namespace) 45 | } 46 | 47 | // Get queries the proxy network type and address for the specified namespace. 48 | func (h HTTPResolver) Get(namespace string) (Response, error) { 49 | url := requestURL(h.url, namespace) 50 | 51 | httpResponse, err := h.client.Get(url) 52 | if err != nil { 53 | return Response{}, err 54 | } 55 | defer httpResponse.Body.Close() 56 | 57 | body, err := io.ReadAll(httpResponse.Body) 58 | if err != nil { 59 | return Response{}, err 60 | } 61 | 62 | code := httpResponse.StatusCode 63 | if code != 200 { 64 | return Response{}, fmt.Errorf("failed to GET %s: status=%d, body=%s", url, code, body) 65 | } 66 | 67 | var response Response 68 | err = json.Unmarshal(body, &response) 69 | if err != nil { 70 | return Response{}, err 71 | } 72 | return response, nil 73 | } 74 | -------------------------------------------------------------------------------- /snapshotter/demux/proxy/address/resolver.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package address 15 | 16 | // Response to the proxy network address resolver. 17 | type Response struct { 18 | // Network type used in net.Dial. 19 | // 20 | // Reference: https://pkg.go.dev/net#Dial 21 | Network string `json:"network"` 22 | 23 | // Network address used in net.Dial. 24 | Address string `json:"address"` 25 | 26 | // SnapshotterPort is the port used in vsock.DialContext for sending snapshotter API requests to the remote snapshotter. 27 | SnapshotterPort string `json:"snapshotter_port"` 28 | 29 | // MetricsPort is the port used in vsock.DialContext for sending metrics requests to the remote snapshotter. 30 | MetricsPort string `json:"metrics_port"` 31 | 32 | // Labels is a map used for applying labels to metrics. 33 | Labels map[string]string `json:"labels"` 34 | } 35 | 36 | // Resolver for the proxy network address. 37 | type Resolver interface { 38 | // Fetch the network address used to forward snapshotter 39 | // requests by namespace lookup from a remote service call. 40 | Get(namespace string) (Response, error) 41 | } 42 | -------------------------------------------------------------------------------- /snapshotter/internal/integtest/stargz/fs/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Copied from https://github.com/containerd/stargz-snapshotter/blob/v0.11.4/fs/config/config.go 15 | // 16 | // Taking Stargz as a code dependency required dropping support for Go 1.16. 17 | // Used to add Stargz annotations to pull requests. 18 | 19 | /* 20 | Copyright The containerd Authors. 21 | 22 | Licensed under the Apache License, Version 2.0 (the "License"); 23 | you may not use this file except in compliance with the License. 24 | You may obtain a copy of the License at 25 | 26 | http://www.apache.org/licenses/LICENSE-2.0 27 | 28 | Unless required by applicable law or agreed to in writing, software 29 | distributed under the License is distributed on an "AS IS" BASIS, 30 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 31 | See the License for the specific language governing permissions and 32 | limitations under the License. 33 | */ 34 | 35 | /* 36 | Copyright 2019 The Go Authors. All rights reserved. 37 | Use of this source code is governed by a BSD-style 38 | license that can be found in the NOTICE.md file. 39 | */ 40 | 41 | package config 42 | 43 | const ( 44 | // TargetSkipVerifyLabel is a snapshot label key that indicates to skip content 45 | // verification for the layer. 46 | TargetSkipVerifyLabel = "containerd.io/snapshot/remote/stargz.skipverify" 47 | 48 | // TargetPrefetchSizeLabel is a snapshot label key that indicates size to prefetch 49 | // the layer. If the layer is eStargz and contains prefetch landmarks, these config 50 | // will be respeced. 51 | TargetPrefetchSizeLabel = "containerd.io/snapshot/remote/stargz.prefetch" 52 | ) 53 | -------------------------------------------------------------------------------- /snapshotter/internal/mount/collection.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package mount 15 | 16 | import "github.com/containerd/containerd/mount" 17 | 18 | // Map returns the slice of results after applying the given function 19 | // to each item of a given slice. 20 | func Map(mounts []mount.Mount, f func(mount.Mount) mount.Mount) []mount.Mount { 21 | fms := make([]mount.Mount, len(mounts)) 22 | for i, mount := range mounts { 23 | fms[i] = f(mount) 24 | } 25 | return fms 26 | } 27 | -------------------------------------------------------------------------------- /snapshotter/internal/mount/collection_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package mount 15 | 16 | import ( 17 | "fmt" 18 | "testing" 19 | 20 | "github.com/containerd/containerd/mount" 21 | ) 22 | 23 | func assertEqual(actual, expected []mount.Mount) error { 24 | if len(actual) != len(expected) { 25 | return fmt.Errorf("expected %s actual %s", expected, actual) 26 | } 27 | for k, v := range actual { 28 | if v.Type != expected[k].Type && v.Source != expected[k].Source { 29 | return fmt.Errorf("expected %s actual %s at index %d", expected[k], v, k) 30 | } 31 | } 32 | return nil 33 | } 34 | 35 | func okOnEmptySlice() error { 36 | expected := []mount.Mount{} 37 | actual := Map([]mount.Mount{}, nil) 38 | return assertEqual(actual, expected) 39 | } 40 | 41 | func applyFunctionToSlice() error { 42 | pre := []mount.Mount{ 43 | {Type: "pre-type-1", Source: "pre-source-1"}, 44 | {Type: "pre-type-2", Source: "pre-source-2"}, 45 | } 46 | transform := func(m mount.Mount) mount.Mount { 47 | switch m.Type { 48 | case "pre-type-1": 49 | return mount.Mount{Type: "post-type-1", Source: "post-source-1"} 50 | case "pre-type-2": 51 | return mount.Mount{Type: "post-type-2", Source: "post-source-2"} 52 | default: 53 | return mount.Mount{} 54 | } 55 | } 56 | 57 | expected := []mount.Mount{ 58 | {Type: "post-type-1", Source: "post-source-1"}, 59 | {Type: "post-type-2", Source: "post-source-2"}, 60 | } 61 | actual := Map(pre, transform) 62 | return assertEqual(actual, expected) 63 | } 64 | 65 | func TestMap(t *testing.T) { 66 | t.Parallel() 67 | 68 | tests := []struct { 69 | name string 70 | run func() error 71 | }{ 72 | {"EmptySlice", okOnEmptySlice}, 73 | {"ApplyFoo", applyFunctionToSlice}, 74 | } 75 | 76 | for _, test := range tests { 77 | if err := test.run(); err != nil { 78 | t.Fatalf("%s: %s", test.name, err.Error()) 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /snapshotter/main.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "flag" 18 | "os" 19 | 20 | "github.com/sirupsen/logrus" 21 | 22 | "github.com/firecracker-microvm/firecracker-containerd/snapshotter/app" 23 | "github.com/firecracker-microvm/firecracker-containerd/snapshotter/config" 24 | ) 25 | 26 | func main() { 27 | var filepath string 28 | flag.StringVar(&filepath, "config", "/etc/demux-snapshotter/config.toml", "Configuration filepath") 29 | 30 | if !flag.Parsed() { 31 | flag.Parse() 32 | } 33 | 34 | config, err := config.Load(filepath) 35 | if err != nil { 36 | logger := logrus.New().WithField("config", filepath) 37 | logger.WithError(err).Error("error reading config") 38 | return 39 | } 40 | 41 | logLevel, err := logrus.ParseLevel(config.Debug.LogLevel) 42 | if err != nil { 43 | logLevel = logrus.InfoLevel 44 | logrus.New().WithError(err).Warn("Fallback to default log level") 45 | } 46 | logrus.SetLevel(logLevel) 47 | 48 | if err := app.Run(config); err != nil { 49 | os.Exit(1) 50 | } 51 | os.Exit(0) 52 | } 53 | -------------------------------------------------------------------------------- /tools/critest/critest_diff.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 6 | # not use this file except in compliance with the License. A copy of the 7 | # License is located at 8 | # 9 | # http://aws.amazon.com/apache2.0/ 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | 16 | 17 | # Create temporary critest output file 18 | critest_output="$(> "$critest_output_file" 21 | 22 | set -eu 23 | 24 | # Remove up until report summary 25 | sed -i -E '0,/^Summarizing [0-9][0-9]? Failure[s]?:$/d' "$critest_output_file" # Remove empty lines 26 | sed -i '/^$/d' "$critest_output_file" 27 | 28 | # Remove unnecessary error messages 29 | sed -i '/^\/.*[0-9]$/d' "$critest_output_file" 30 | sed -i '/^Ran [0-9][0-9] of [0-9][0-9] Specs in .*seconds$/d' "$critest_output_file" 31 | sed -i '/^--- FAIL: TestCRISuite.*$/d' "$critest_output_file" 32 | sed -i '/^FAIL.*$/d' "$critest_output_file" 33 | sed -i '/^Ran.*$/d' "$critest_output_file" 34 | 35 | # Diff expected vs. actual 36 | diff -y <(sort critest/expected_critest_output.out) <(sort "$critest_output_file") -------------------------------------------------------------------------------- /tools/demo/fc-br0.interface: -------------------------------------------------------------------------------- 1 | # This interfaces(5) configuration file is intended to work with the 2 | # ifupdown interface management framework provided by Debian 10. It 3 | # may require modification to work elsewhere. 4 | auto fc-br0 5 | iface fc-br0 inet static 6 | address 192.168.1.1 7 | network 192.168.1.0 8 | netmask 255.255.255.0 9 | broadcast 192.168.1.255 10 | gateway 192.168.1.1 11 | bridge_ports none 12 | -------------------------------------------------------------------------------- /tools/demo/fcnet.conflist: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "1.0.0", 3 | "name": "fcnet", 4 | "plugins": [ 5 | { 6 | "type": "bridge", 7 | "bridge": "fc-br0", 8 | "isDefaultGateway": true, 9 | "forceAddress": false, 10 | "ipMasq": true, 11 | "hairpinMode": true, 12 | "mtu": 1500, 13 | "ipam": { 14 | "type": "host-local", 15 | "subnet": "192.168.1.0/24", 16 | "resolvConf": "/etc/resolv.conf" 17 | } 18 | }, 19 | { 20 | "type": "firewall" 21 | }, 22 | { 23 | "type": "tc-redirect-tap" 24 | }, 25 | { 26 | "type": "loopback" 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /tools/docker/Dockerfile.integ-test: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:experimental 2 | # Test image that starts up containerd and the devmapper snapshotter. The default CMD will drop to a bash shell. Overrides 3 | # to CMD will be provided appended to /bin/bash -c 4 | FROM public.ecr.aws/docker/library/golang:1.23-bullseye 5 | ENV PATH="/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/go/bin" 6 | ENV INSTALLROOT="/usr/local" 7 | ENV DEBIAN_FRONTEND="noninteractive" 8 | 9 | ARG FIRECRACKER_TARGET=x86_64-unknown-linux-musl 10 | ENV FICD_LOG_DIR="/var/log/firecracker-containerd-test" 11 | ENV FICD_CONTAINERD_OUTFILE="${FICD_LOG_DIR}/containerd.out" 12 | 13 | RUN apt-get update && apt-get install --yes --no-install-recommends \ 14 | build-essential \ 15 | ca-certificates \ 16 | curl \ 17 | git \ 18 | iptables \ 19 | iperf3 \ 20 | libdevmapper-dev \ 21 | libseccomp-dev \ 22 | tcpdump \ 23 | iproute2 \ 24 | rng-tools # used for rngtest 25 | 26 | RUN mkdir -p \ 27 | /var/run/firecracker-containerd \ 28 | /src \ 29 | /srv/firecracker_containerd_tests \ 30 | ${FICD_LOG_DIR} \ 31 | /etc/cni/net.d 32 | 33 | # Install containerd to have runc shim. 34 | ENV CTRD_VERSION="1.7.16" 35 | RUN ARCH=`go env GOARCH` && \ 36 | wget --quiet -O- https://github.com/containerd/containerd/releases/download/v$CTRD_VERSION/containerd-$CTRD_VERSION-linux-${ARCH}.tar.gz | tar zxf - -C /tmp/ && \ 37 | install -D -o root -g root -m755 -t /usr/local/bin /tmp/bin/containerd-shim-runc-v2 && \ 38 | rm -rf /tmp/bin 39 | 40 | # Install critest. 41 | ENV CRITEST_VERSION="1.23.0" 42 | RUN ARCH=`go env GOARCH` && \ 43 | wget --quiet -O- https://github.com/kubernetes-sigs/cri-tools/releases/download/v$CRITEST_VERSION/critest-v$CRITEST_VERSION-linux-${ARCH}.tar.gz | tar zxf - -C /tmp/ && \ 44 | install -D -o root -g root -m755 -t /usr/local/bin /tmp/critest && \ 45 | rm -f /tmp/critest 46 | 47 | # Pull the images the tests need into the content store so we don't need internet 48 | # access during the tests themselves. This runs as a seperate step before the other 49 | # installs so we can minimize re-runs of the time-expensive downloading of images. 50 | COPY tools/docker/config.toml /etc/containerd/config.toml 51 | COPY tools/docker/do_not_edit_for_firecracker-control.config.json /etc/containerd/firecracker-runtime.json 52 | COPY tools/demo/fcnet.conflist /etc/cni/net.d/10-fcnet.conflist 53 | 54 | RUN --mount=type=bind,target=/src \ 55 | make -C /src install && \ 56 | ln -sv /usr/local/bin/firecracker-containerd /usr/local/bin/containerd && \ 57 | ln -sv /usr/local/bin/firecracker-ctr /usr/local/bin/ctr 58 | 59 | RUN containerd 2>/dev/null & \ 60 | ctr --address /run/firecracker-containerd/containerd.sock content fetch docker.io/library/alpine:3.10.1 >/dev/null && \ 61 | ctr --address /run/firecracker-containerd/containerd.sock content fetch docker.io/mlabbe/iperf3:3.6-r0 >/dev/null && \ 62 | ctr --address /run/firecracker-containerd/containerd.sock content fetch docker.io/library/postgres:14.3 >/dev/null 63 | 64 | ADD bin/firecracker /usr/local/bin 65 | 66 | # Install everything we need in this image. Due to the bind-mount, if the host has already 67 | # up-to-date versions of everything built, this step will be a very quick copy 68 | RUN --mount=type=bind,target=/src make -C /src \ 69 | install \ 70 | install-runc \ 71 | install-kernel \ 72 | install-default-rootfs \ 73 | install-default-runc-jailer-config \ 74 | install-test-cni-bins \ 75 | install-test-rootfs \ 76 | install-stargz-rootfs \ 77 | demo-network 78 | 79 | RUN --mount=type=bind,target=/src make -C /src/snapshotter install 80 | 81 | COPY tools/docker/entrypoint.sh /entrypoint 82 | ENTRYPOINT ["/entrypoint"] 83 | CMD ["exec /bin/bash"] 84 | -------------------------------------------------------------------------------- /tools/docker/Dockerfile.proto-builder: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | FROM public.ecr.aws/docker/library/golang:1.23-bullseye 15 | 16 | RUN apt-get update && apt-get install --yes --no-install-recommends \ 17 | libprotobuf-dev=3.12.4-1+deb11u1 \ 18 | protobuf-compiler=3.12.4-1+deb11u1 \ 19 | && go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.33 \ 20 | && go install github.com/containerd/ttrpc/cmd/protoc-gen-go-ttrpc@v1.2.3 \ 21 | && mkdir /protobuf 22 | 23 | WORKDIR /protobuf 24 | ENTRYPOINT ["/usr/bin/make"] 25 | CMD ["proto"] 26 | -------------------------------------------------------------------------------- /tools/docker/Dockerfile.runc-builder: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | FROM public.ecr.aws/docker/library/golang:1.23-bullseye 15 | 16 | RUN apt-get update && \ 17 | DEBIAN_FRONTEND=noninteractive apt-get -y install libseccomp-dev pkg-config 18 | -------------------------------------------------------------------------------- /tools/docker/Dockerfile.stargz-builder: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | FROM public.ecr.aws/docker/library/golang:1.23-bullseye 15 | -------------------------------------------------------------------------------- /tools/docker/config.toml: -------------------------------------------------------------------------------- 1 | imports = ["/etc/containerd/snapshotter/*.toml", "/etc/containerd/cri/*.toml"] 2 | [grpc] 3 | address = "/run/firecracker-containerd/containerd.sock" 4 | 5 | [proxy_plugins] 6 | [proxy_plugins.demux] 7 | type = "snapshot" 8 | address = "/var/lib/demux-snapshotter/snapshotter.sock" 9 | -------------------------------------------------------------------------------- /tools/docker/do_not_edit_for_firecracker-control.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "shim_base_dir": "/srv/firecracker_containerd_tests" 3 | } 4 | -------------------------------------------------------------------------------- /tools/docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | chmod a+rwx ${FICD_LOG_DIR} 5 | 6 | mkdir -p /etc/containerd/snapshotter 7 | mkdir -p /etc/containerd/cri 8 | 9 | if [[ -z "$FICD_DM_VOLUME_GROUP" ]]; then 10 | pool_name="${FICD_DM_POOL}" 11 | else 12 | pool_name="$(echo "$FICD_DM_VOLUME_GROUP" | sed s/-/--/g)-${FICD_DM_POOL}" 13 | fi 14 | 15 | cat > /etc/containerd/snapshotter/devmapper.toml < /etc/containerd/cri/criconfig.toml < /etc/demux-snapshotter/config.toml <> ${FICD_CONTAINERD_OUTFILE} & 70 | 71 | /usr/local/bin/http-address-resolver &>> ${FICD_LOG_DIR}/http-address-resolver.out & 72 | /usr/local/bin/demux-snapshotter &>> ${FICD_LOG_DIR}/demux-snapshotter.out & 73 | 74 | exec /bin/bash -c "$@" 75 | -------------------------------------------------------------------------------- /tools/docker/fc-agent.start: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | mkdir -p /container 3 | exec > /container/agent-debug.log # Debug logs from the agent 4 | exec 2>&1 5 | touch /container/runtime 6 | 7 | cd /container 8 | 9 | exec /usr/local/bin/agent -debug 10 | -------------------------------------------------------------------------------- /tools/docker/scripts/lsblk.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # tiny lsblk(8) equivalent to make integration tests distro-agnostic 3 | set -euo pipefail 4 | 5 | echo 'NAME MAJ:MIN RM SIZE RO | MAGIC' 6 | 7 | for name in $(ls /sys/block) 8 | do 9 | # Ignore loop devices 10 | case "$name" in 11 | loop*) 12 | continue 13 | ;; 14 | esac 15 | 16 | # The size entry returns the number of sectors, 17 | # not the number of bytes. 18 | # https://unix.stackexchange.com/a/301403 19 | bytes=$(($(cat /sys/block/$name/size) * 512)) 20 | 21 | # Show the first 8 bytes, where our stub device files have own signatures 22 | # https://github.com/firecracker-microvm/firecracker-containerd/blob/2578f3df9d899aa48decb39c9f7f23fa41635ede/internal/common.go#L67 23 | magic=$(head -c 8 /dev/$name | od -A n -t u1) 24 | 25 | printf "%-4s %-7s %2d %10dB %2d | %s\n" \ 26 | "$name" \ 27 | $(cat /sys/block/$name/dev) \ 28 | $(cat /sys/block/$name/removable) \ 29 | "$bytes" \ 30 | $(cat /sys/block/$name/ro) \ 31 | "$magic" 32 | done 33 | -------------------------------------------------------------------------------- /tools/image-builder/.dockerignore: -------------------------------------------------------------------------------- 1 | rootfs 2 | -------------------------------------------------------------------------------- /tools/image-builder/.gitignore: -------------------------------------------------------------------------------- 1 | files_ephemeral 2 | *.img 3 | rootfs 4 | *stamp 5 | -------------------------------------------------------------------------------- /tools/image-builder/Dockerfile.debian-image: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | FROM public.ecr.aws/docker/library/debian:bullseye-slim 15 | 16 | RUN apt-get update && \ 17 | DEBIAN_FRONTEND=noninteractive apt-get -y install \ 18 | --no-install-recommends \ 19 | make squashfs-tools debootstrap 20 | 21 | VOLUME ["/src"] 22 | WORKDIR "/src" 23 | ENTRYPOINT ["make"] 24 | -------------------------------------------------------------------------------- /tools/image-builder/agent.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # This script adds multiple failure modes to firecracker-containerd's in-VM agent. 3 | set -euo pipefail 4 | 5 | agent=/usr/local/bin/agent.real 6 | 7 | if [[ "$(cat /proc/cmdline)" =~ failure=([a-z-]+) ]]; 8 | then 9 | failure="${BASH_REMATCH[1]}" 10 | else 11 | failure='none' 12 | fi 13 | 14 | case "$failure" in 15 | slow-boot) 16 | sleep 30 17 | $agent 18 | ;; 19 | slow-reboot) 20 | $agent 21 | sleep 30 22 | ;; 23 | no-agent) 24 | exit 1 25 | ;; 26 | none) 27 | $agent 28 | ;; 29 | *) 30 | echo "unknown failure mode: $failure" 31 | exit 1 32 | esac 33 | 34 | -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/hostname: -------------------------------------------------------------------------------- 1 | microvm -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/hosts: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost 2 | ::1 localhost ip6-localhost ip6-loopback 3 | ff02::1 ip6-allnodes 4 | ff02::2 ip6-allrouters 5 | -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/resolv.conf: -------------------------------------------------------------------------------- 1 | /proc/net/pnp -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/apt-daily-upgrade.timer: -------------------------------------------------------------------------------- 1 | /dev/null -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/apt-daily.timer: -------------------------------------------------------------------------------- 1 | /dev/null -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/firecracker-agent.service: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | [Unit] 14 | Description=Firecracker VM agent 15 | StartLimitIntervalSec=2 16 | After=local-fs.target 17 | Requires=local-fs.target 18 | SuccessAction=reboot 19 | 20 | [Service] 21 | Type=simple 22 | WorkingDirectory=/container 23 | ExecStart=/usr/local/bin/agent --debug 24 | -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/firecracker.target: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | [Unit] 14 | Description=Firecracker containerd VM 15 | Requires=basic.target 16 | Conflicts=rescue.service rescue.target 17 | After=basic.target rescue.service rescue.target 18 | AllowIsolate=yes 19 | -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/firecracker.target.wants/firecracker-agent.service: -------------------------------------------------------------------------------- 1 | /etc/systemd/system/firecracker-agent.service -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/firecracker.target.wants/getty.target: -------------------------------------------------------------------------------- 1 | /dev/null -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/firecracker.target.wants/haveged.service: -------------------------------------------------------------------------------- 1 | /etc/systemd/system/haveged.service -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/getty-static.service: -------------------------------------------------------------------------------- 1 | /dev/null -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/haveged.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Entropy Daemon based on the HAVEGE algorithm 3 | Documentation=man:haveged(8) http://www.issihosts.com/haveged/ 4 | DefaultDependencies=no 5 | ConditionVirtualization=!container 6 | After=local-fs.target 7 | Before=firecracker.target sysinit.target shutdown.target 8 | 9 | [Service] 10 | ExecStart=/usr/sbin/haveged --Foreground --verbose=1 -w 1024 11 | SuccessExitStatus=143 12 | SecureBits=noroot-locked 13 | NoNewPrivileges=yes 14 | CapabilityBoundingSet=CAP_SYS_ADMIN 15 | PrivateTmp=yes 16 | PrivateDevices=yes 17 | PrivateNetwork=yes 18 | ProtectSystem=full 19 | ProtectHome=yes 20 | 21 | [Install] 22 | WantedBy=default.target 23 | -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/rom-dev.mount: -------------------------------------------------------------------------------- 1 | /dev/null -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/rom-overlay.mount: -------------------------------------------------------------------------------- 1 | /dev/null -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/rom.mount: -------------------------------------------------------------------------------- 1 | /dev/null -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/etc/systemd/system/systemd-timesyncd.service: -------------------------------------------------------------------------------- 1 | /dev/null -------------------------------------------------------------------------------- /tools/image-builder/files_debootstrap/sbin/overlay-init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 5 | # not use this file except in compliance with the License. A copy of the 6 | # License is located at 7 | # 8 | # http://aws.amazon.com/apache2.0/ 9 | # 10 | # or in the "license" file accompanying this file. This file is distributed 11 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | # express or implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | 15 | # Parameters: 16 | # 1. rw_root -- path where the read/write root is mounted 17 | # 2. work_dir -- path to the overlay workdir (must be on same filesystem as rw_root) 18 | # Overlay will be set up on /mnt, original root on /mnt/rom 19 | pivot() { 20 | local rw_root work_dir 21 | rw_root="$1" 22 | work_dir="$2" 23 | /bin/mount \ 24 | -o noatime,lowerdir=/,upperdir=${rw_root},workdir=${work_dir} \ 25 | -t overlay "overlayfs:${rw_root}" /mnt 26 | pivot_root /mnt /mnt/rom 27 | } 28 | 29 | # Overlay is configured under /overlay 30 | # Global variable $overlay_root is expected to be set to either: 31 | # "ram", which configures a tmpfs as the rw overlay layer (this is 32 | # the default, if the variable is unset) 33 | # - or - 34 | # A block device name, relative to /dev, in which case it is assumed 35 | # to contain an ext4 filesystem suitable for use as a rw overlay 36 | # layer. e.g. "vdb" 37 | do_overlay() { 38 | local overlay_dir="/overlay" 39 | if [ "$overlay_root" = ram ] || 40 | [ -z "$overlay_root" ]; then 41 | /bin/mount -t tmpfs -o noatime,mode=0755 tmpfs /overlay 42 | else 43 | /bin/mount -t ext4 "/dev/$overlay_root" /overlay 44 | fi 45 | mkdir -p /overlay/root /overlay/work 46 | pivot /overlay/root /overlay/work 47 | } 48 | 49 | # If we're given an overlay, ensure that it really exists. Panic if not. 50 | if [ -n "$overlay_root" ] && 51 | [ "$overlay_root" != ram ] && 52 | [ ! -b "/dev/$overlay_root" ]; then 53 | echo -n "FATAL: " 54 | echo "Overlay root given as $overlay_root but /dev/$overlay_root does not exist" 55 | exit 1 56 | fi 57 | 58 | do_overlay 59 | 60 | # firecracker-containerd itself doesn't need /volumes but volume package 61 | # uses that to share files between in-VM snapshotters. 62 | mkdir /volumes 63 | 64 | # invoke the actual system init program and procede with the boot 65 | # process. 66 | exec /usr/sbin/init $@ 67 | -------------------------------------------------------------------------------- /tools/image-builder/files_stargz/etc/containerd-stargz-grpc/config.toml: -------------------------------------------------------------------------------- 1 | metrics_address = "localhost:8234" 2 | no_prometheus = false -------------------------------------------------------------------------------- /tools/image-builder/files_stargz/etc/systemd/system/firecracker.target.wants/metrics-socat.service: -------------------------------------------------------------------------------- 1 | /etc/systemd/system/metrics-socat.service -------------------------------------------------------------------------------- /tools/image-builder/files_stargz/etc/systemd/system/firecracker.target.wants/socat.service: -------------------------------------------------------------------------------- 1 | /etc/systemd/system/socat.service -------------------------------------------------------------------------------- /tools/image-builder/files_stargz/etc/systemd/system/firecracker.target.wants/stargz-snapshotter.service: -------------------------------------------------------------------------------- 1 | /etc/systemd/system/stargz-snapshotter.service -------------------------------------------------------------------------------- /tools/image-builder/files_stargz/etc/systemd/system/metrics-socat.service: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | [Unit] 14 | Description=Metrics Socat 15 | StartLimitIntervalSec=2 16 | After=stargz-snapshotter.service 17 | Requires=stargz-snapshotter.service 18 | SuccessAction=reboot 19 | 20 | [Service] 21 | Type=simple 22 | ExecStart=socat VSOCK-LISTEN:10002,reuseaddr,fork TCP:localhost:8234 23 | -------------------------------------------------------------------------------- /tools/image-builder/files_stargz/etc/systemd/system/socat.service: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | [Unit] 14 | Description=Socat 15 | StartLimitIntervalSec=2 16 | After=stargz-snapshotter.service 17 | Requires=stargz-snapshotter.service 18 | SuccessAction=reboot 19 | 20 | [Service] 21 | Type=simple 22 | ExecStart=socat VSOCK-LISTEN:10000,reuseaddr,fork UNIX-CONNECT:/run/containerd-stargz-grpc/containerd-stargz-grpc.sock 23 | -------------------------------------------------------------------------------- /tools/image-builder/files_stargz/etc/systemd/system/stargz-snapshotter.service: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | [Unit] 14 | Description=Stargz Snapshotter 15 | StartLimitIntervalSec=2 16 | After=local-fs.target 17 | Requires=local-fs.target 18 | 19 | [Service] 20 | Type=simple 21 | ExecStartPre=/bin/mkdir -p /var/lib/containerd-stargz-grpc 22 | ExecStartPre=/bin/mount -t tmpfs -o noatime,mode=0755 stargz /var/lib/containerd-stargz-grpc 23 | ExecStart=/usr/local/bin/containerd-stargz-grpc --log-level=debug 24 | -------------------------------------------------------------------------------- /tools/image-builder/files_stargz/root/.docker/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "credsStore": "mmds" 3 | } -------------------------------------------------------------------------------- /tools/tidy.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 6 | # not use this file except in compliance with the License. A copy of the 7 | # License is located at 8 | # 9 | # http://aws.amazon.com/apache2.0/ 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | 16 | set -euo pipefail 17 | 18 | for path in $(find . -name 'go.mod' | grep -v '^\./_submodules/') 19 | do 20 | (cd $(dirname $path) && pwd && go mod tidy) 21 | done -------------------------------------------------------------------------------- /volume/.gitignore: -------------------------------------------------------------------------------- 1 | volume-init 2 | -------------------------------------------------------------------------------- /volume/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | # not use this file except in compliance with the License. A copy of the 5 | # License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | volume-init: $(shell find ./cmd -name '*.go') 15 | go build -o $@ ./cmd/volume-init 16 | 17 | test: 18 | go test ./... 19 | 20 | clean: 21 | rm -f volume-init 22 | 23 | install: 24 | 25 | .PHONY: test clean install 26 | -------------------------------------------------------------------------------- /volume/cmd/volume-init/main.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "encoding/json" 18 | "fmt" 19 | "io" 20 | "os" 21 | 22 | "github.com/containerd/continuity/fs" 23 | "github.com/firecracker-microvm/firecracker-containerd/volume" 24 | ) 25 | 26 | func copyVolume(in volume.GuestVolumeImageInput) error { 27 | for _, v := range in.Volumes { 28 | to, err := fs.RootPath(in.To, v) 29 | if err != nil { 30 | return fmt.Errorf("failed to join %q and %q: %w", in.To, v, err) 31 | } 32 | 33 | from, err := fs.RootPath(in.From, v) 34 | if err != nil { 35 | return fmt.Errorf("failed to join %q and %q: %w", in.From, v, err) 36 | } 37 | 38 | err = os.MkdirAll(to, 0700) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | err = fs.CopyDir(to, from) 44 | if err != nil { 45 | return err 46 | } 47 | } 48 | return nil 49 | } 50 | 51 | func realMain() error { 52 | b, err := io.ReadAll(os.Stdin) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | var in volume.GuestVolumeImageInput 58 | err = json.Unmarshal(b, &in) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | return copyVolume(in) 64 | } 65 | 66 | func main() { 67 | err := realMain() 68 | if err != nil { 69 | out := volume.GuestVolumeImageOutput{Error: err.Error()} 70 | b, err := json.Marshal(out) 71 | if err != nil { 72 | fmt.Fprintf(os.Stderr, "failed to unmarshal %+v: %s", out, err) 73 | os.Exit(2) 74 | } 75 | fmt.Printf("%s", string(b)) 76 | os.Exit(1) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /volume/cmd/volume-init/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "os" 18 | "path/filepath" 19 | "testing" 20 | 21 | "github.com/firecracker-microvm/firecracker-containerd/volume" 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func mkdirAndTouch(path, content string) error { 26 | err := os.MkdirAll(filepath.Dir(path), 0700) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | return os.WriteFile(path, []byte(content), 0600) 32 | } 33 | 34 | func TestCopy(t *testing.T) { 35 | from := t.TempDir() 36 | to := t.TempDir() 37 | 38 | err := mkdirAndTouch(filepath.Join(from, "/etc/foobar/hello"), "helloworld") 39 | require.NoError(t, err) 40 | 41 | err = copyVolume(volume.GuestVolumeImageInput{ 42 | From: from, 43 | To: to, 44 | Volumes: []string{"/etc/foobar"}, 45 | }) 46 | require.NoError(t, err) 47 | 48 | b, err := os.ReadFile(filepath.Join(to, "/etc/foobar/hello")) 49 | require.NoError(t, err) 50 | require.Equal(t, "helloworld", string(b)) 51 | } 52 | -------------------------------------------------------------------------------- /volume/set_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package volume 15 | 16 | import ( 17 | "context" 18 | "testing" 19 | 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | type testProvider struct { 24 | name string 25 | } 26 | 27 | func (p *testProvider) CreateVolumesUnder(_ context.Context, _ string) ([]*Volume, error) { 28 | return nil, nil 29 | } 30 | func (p *testProvider) Delete(_ context.Context) error { 31 | return nil 32 | } 33 | func (p *testProvider) Name() string { 34 | return p.name 35 | } 36 | 37 | func TestNewSet(t *testing.T) { 38 | provider := &testProvider{name: "foobar"} 39 | 40 | ctx := context.Background() 41 | vs := NewSet("") 42 | err := vs.AddFrom(ctx, provider) 43 | require.NoError(t, err) 44 | } 45 | -------------------------------------------------------------------------------- /volume/volume.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // Package volume provides volumes like Docker and Amazon ECS. 15 | // Volumes are specicial directories that could be shared by multiple containers. 16 | // 17 | // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#volumes 18 | package volume 19 | 20 | // Volume is a special directory that could be shared between containers. 21 | type Volume struct { 22 | name string 23 | hostPath string 24 | vmPath string 25 | containerPath string 26 | } 27 | 28 | // New returns an empty volume. 29 | func New(name string) *Volume { 30 | return &Volume{name: name} 31 | } 32 | 33 | // FromHost returns a volume which has the files from the given path. 34 | func FromHost(name, path string) *Volume { 35 | return &Volume{name: name, hostPath: path} 36 | } 37 | -------------------------------------------------------------------------------- /volume/volume_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may 4 | // not use this file except in compliance with the License. A copy of the 5 | // License is located at 6 | // 7 | // http://aws.amazon.com/apache2.0/ 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed 10 | // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package volume 15 | 16 | import ( 17 | "context" 18 | "os" 19 | "testing" 20 | 21 | "github.com/containerd/containerd/mount" 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | const mib = 1024 * 1024 26 | 27 | func TestCreateDiskImage(t *testing.T) { 28 | ctx := context.Background() 29 | 30 | vs := &Set{} 31 | 32 | _, err := vs.createDiskImage(ctx, 10) 33 | require.Errorf(t, err, "10 bytes is too small to have a valid %s image", fsType) 34 | 35 | path, err := vs.createDiskImage(ctx, 100*mib) 36 | require.NoError(t, err) 37 | 38 | defer os.Remove(path) 39 | 40 | uid := os.Getuid() 41 | if uid != 0 { 42 | t.Logf("skip mounting %s since uid=%d", path, uid) 43 | return 44 | } 45 | 46 | // Make sure that the disk image is valid. 47 | target := t.TempDir() 48 | err = mountDiskImage(path, target) 49 | require.NoError(t, err) 50 | 51 | err = mount.Unmount(target, 0) 52 | require.NoError(t, err) 53 | } 54 | --------------------------------------------------------------------------------