├── .gitignore ├── pkg ├── deb │ └── DEBIAN │ │ ├── prerm │ │ ├── control │ │ └── postinst ├── pre-deploy-test-deb.sh └── pre-deploy-deb.sh ├── k8s ├── ukontainer-runtimeclass.yaml ├── Dockerfile ├── kind-cluster.yaml ├── kind-cluster-calico.yaml ├── alpine-runu.yaml ├── hello-world.yaml └── README.md ├── utils_darwin.go ├── exec.go ├── test ├── goreport.sh ├── k8s-test.sh ├── containerd-nerdctl-test.sh ├── common.sh ├── containerd-ctr-test.sh ├── standalone-test.sh ├── docker-more-test.sh ├── docker-volume-test.sh └── docker-oci-test.sh ├── delete.go ├── mount.go ├── mount_unsupported.go ├── niu.Makefile ├── devices_darwin.go ├── run.go ├── README.md ├── go.mod ├── start.go ├── cmd └── containerd-shim-runu-v1 │ ├── main.go │ ├── platform.go │ ├── reaper.go │ └── service.go ├── log.go ├── kill.go ├── 9pfs.go ├── state.go ├── devices.go ├── mount_linux.go ├── boot.go ├── main.go ├── spec.go ├── create.go ├── .travis.yml ├── devices_linux.go ├── utils_linux.go ├── utils_linux_test.go ├── LICENSE ├── .github └── workflows │ └── ci.yml └── utils.go /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | GPATH 3 | GTAGS 4 | GRTAGS 5 | runu 6 | config.json -------------------------------------------------------------------------------- /pkg/deb/DEBIAN/prerm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # echo "Pre-Removal Macro" 5 | -------------------------------------------------------------------------------- /k8s/ukontainer-runtimeclass.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: node.k8s.io/v1 2 | kind: RuntimeClass 3 | metadata: 4 | name: ukontainer 5 | labels: 6 | addonmanager.kubernetes.io/mode: Reconcile 7 | handler: runu -------------------------------------------------------------------------------- /k8s/Dockerfile: -------------------------------------------------------------------------------- 1 | # XXX v1.27.0 is the last version that `containerd-shim` file is included 2 | FROM kindest/node:v1.27.0 3 | 4 | COPY ./runu /usr/bin/runu 5 | COPY ./libc.so /usr/lib/runu/ 6 | COPY ./lkick /usr/lib/runu/ 7 | -------------------------------------------------------------------------------- /pkg/deb/DEBIAN/control: -------------------------------------------------------------------------------- 1 | Package: docker-runu 2 | Version: __VERSION__ 3 | Depends: libc6, jq 4 | Architecture: __ARCH__ 5 | Maintainer: Hajime Tazaki 6 | Description: runu OCI runtime for docker (Build date: __DATE__) 7 | -------------------------------------------------------------------------------- /pkg/pre-deploy-test-deb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | sudo apt-get install ./$PACKAGE_FILENAME 4 | sudo systemctl restart docker 5 | sudo cat /etc/docker/daemon.json 6 | DOCKER_RUN_ARGS="run --rm -i -e RUMP_VERBOSE=1 -e DEBUG=1 --runtime=runu --net=none $DOCKER_RUN_ARGS_ARCH" 7 | 8 | docker $DOCKER_RUN_ARGS $DOCKER_RUN_EXT_ARGS alpine uname -a 9 | -------------------------------------------------------------------------------- /k8s/kind-cluster.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | containerdConfigPatches: 4 | - |- 5 | [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runu] 6 | runtime_type = "io.containerd.runc.v2" 7 | [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runu.options] 8 | BinaryName = "/usr/bin/runu" 9 | -------------------------------------------------------------------------------- /k8s/kind-cluster-calico.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | networking: 4 | disableDefaultCNI: true # disable kindnet 5 | containerdConfigPatches: 6 | - |- 7 | [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runu] 8 | runtime_type = "io.containerd.runc.v2" 9 | [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runu.options] 10 | BinaryName = "/usr/bin/runu" 11 | -------------------------------------------------------------------------------- /utils_darwin.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/opencontainers/runtime-spec/specs-go" 5 | "github.com/sirupsen/logrus" 6 | ) 7 | 8 | // /usr/lib is not writable on recent darwin 9 | var runuAuxFileDir = "/usr/local/lib/runu" 10 | 11 | func setupNetwork(spec *specs.Spec) (*lklIfInfo, []lklRoute, error) { 12 | logrus.Infof("no netns detected: no addr configuration, skipping") 13 | return nil, nil, nil 14 | } 15 | -------------------------------------------------------------------------------- /pkg/deb/DEBIAN/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | echo "Configuring daemon.json ..." 5 | 6 | if [ ! -s /etc/docker/daemon.json ] ; then 7 | sudo mv -f /etc/docker/daemon.json /etc/docker/daemon.json.org || true 8 | fi 9 | 10 | (sudo cat /etc/docker/daemon.json 2>/dev/null || echo '{}') | \ 11 | jq '.runtimes."runu" |= {"path":"/usr/bin/runu","runtimeArgs":[]}' > \ 12 | /tmp/tmp.json 13 | 14 | sudo mv /tmp/tmp.json /etc/docker/daemon.json 15 | -------------------------------------------------------------------------------- /exec.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | "github.com/urfave/cli" 6 | ) 7 | 8 | var execCommand = cli.Command{ 9 | Name: "exec", 10 | Usage: "execute new process inside the container", 11 | ArgsUsage: ` [command options] || -p process.json `, 12 | Flags: []cli.Flag{}, 13 | Action: func(context *cli.Context) error { 14 | logrus.Debug("exec called\n") 15 | return nil 16 | }, 17 | SkipArgReorder: true, 18 | } 19 | -------------------------------------------------------------------------------- /test/goreport.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | set -x 5 | 6 | gofmt -s -d . 7 | gometalinter --deadline=180s --disable-all --enable=gofmt 8 | gometalinter --deadline=180s --disable-all --enable=vet 9 | 10 | # gocyclo 11 | gometalinter --deadline=180s --disable-all --enable=gocyclo --cyclo-over=15 12 | # golint 13 | gometalinter --deadline=180s --disable-all --enable=golint --min-confidence=0.85 --vendor 14 | 15 | # ineffassign 16 | gometalinter --deadline=180s --disable-all --enable=ineffassign 17 | 18 | # misspell 19 | gometalinter --deadline=180s --disable-all --enable=misspell 20 | -------------------------------------------------------------------------------- /k8s/alpine-runu.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 2 | kind: Deployment 3 | metadata: 4 | name: alpine-runu 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: alpine-runu 9 | replicas: 1 10 | template: 11 | metadata: 12 | labels: 13 | app: alpine-runu 14 | spec: 15 | runtimeClassName: ukontainer 16 | containers: 17 | - name: alpine-runu 18 | image: alpine:latest 19 | imagePullPolicy: Always 20 | args: ["busybox", "ping", "-c", "50", "8.8.8.8"] 21 | env: 22 | - name: RUMP_VERBOSE 23 | value: "1" 24 | -------------------------------------------------------------------------------- /delete.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/urfave/cli" 6 | ) 7 | 8 | var deleteCommand = cli.Command{ 9 | Name: "delete", 10 | ArgsUsage: ``, 11 | Flags: []cli.Flag{ 12 | cli.BoolFlag{ 13 | Name: "force, f", 14 | Usage: "Forcibly deletes the container if it is still running", 15 | }, 16 | }, 17 | Action: func(context *cli.Context) error { 18 | args := context.Args() 19 | if args.Present() == false { 20 | return fmt.Errorf("Missing container ID") 21 | } 22 | 23 | container := context.Args().First() 24 | return deleteContainer(context.GlobalString("root"), container) 25 | }, 26 | } 27 | -------------------------------------------------------------------------------- /mount.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/opencontainers/runtime-spec/specs-go" 8 | ) 9 | 10 | func checkFsFlags(spec *specs.Spec) (bool, bool, error) { 11 | use9pFs := false 12 | hasRootFs := false 13 | 14 | for _, env := range spec.Process.Env { 15 | if strings.HasPrefix(env, "LKL_USE_9PFS=") { 16 | use9pFs = true 17 | } 18 | if strings.HasPrefix(env, "LKL_ROOTFS=") { 19 | hasRootFs = true 20 | } 21 | } 22 | if hasRootFs && use9pFs { 23 | return false, false, fmt.Errorf("LKL_ROOTFS and LKL_USE_9PFS cannot be specified at the same time") 24 | } 25 | return use9pFs, hasRootFs, nil 26 | } 27 | -------------------------------------------------------------------------------- /k8s/hello-world.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 2 | kind: Deployment 3 | metadata: 4 | name: helloworld-runu 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: helloworld-runu 9 | replicas: 1 10 | template: 11 | metadata: 12 | labels: 13 | app: helloworld-runu 14 | spec: 15 | runtimeClassName: ukontainer 16 | containers: 17 | - name: helloworld-runu 18 | image: ukontainer/runu-base:$DOCKER_IMG_VERSION 19 | imagePullPolicy: Always 20 | args: ["ping", "-c", "50", "8.8.8.8"] 21 | env: 22 | - name: RUMP_VERBOSE 23 | value: "1" 24 | -------------------------------------------------------------------------------- /mount_unsupported.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | // +build !linux 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | goruntime "runtime" 9 | 10 | "github.com/opencontainers/runtime-spec/specs-go" 11 | ) 12 | 13 | func doMounts(spec *specs.Spec) (bool, error) { 14 | 15 | for _, m := range spec.Mounts { 16 | switch m.Type { 17 | case "bind": 18 | doContinue := false 19 | for _, d := range []string{"/etc/hosts", "/etc/hostname", "/etc/resolv.conf", "/dev/shm"} { 20 | if m.Destination == d { 21 | doContinue = true 22 | break 23 | } 24 | } 25 | 26 | if doContinue { 27 | continue 28 | } 29 | 30 | return false, fmt.Errorf("volume mount is not supported on %s: %v", goruntime.GOOS, m) 31 | } 32 | } 33 | return false, nil 34 | } 35 | -------------------------------------------------------------------------------- /niu.Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | 3 | GO := go 4 | INSTALL := install 5 | PREFIX := /usr/local/ 6 | SOURCES := $(shell find . 2>&1 | grep -E '.*\.(c|h|go)$$') 7 | COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true) 8 | COMMIT := $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}") 9 | #VERSION := ${shell cat ./VERSION} 10 | EXTRA_FLAGS := -gcflags "-N -l" 11 | 12 | .DEFAULT: runu 13 | 14 | runu: $(SOURCES) Makefile 15 | $(GO) build -buildmode=pie $(EXTRA_FLAGS) -ldflags \ 16 | "-X main.gitCommit=${COMMIT} -X main.version=${VERSION} $(EXTRA_LDFLAGS)" \ 17 | -tags "$(BUILDTAGS)" -o runu . 18 | 19 | all: runu 20 | 21 | install: all 22 | $(INSTALL) -d $(DESTDIR)$(PREFIX)/bin/ 23 | $(INSTALL) -m 755 runu $(DESTDIR)$(PREFIX)/bin/ 24 | -------------------------------------------------------------------------------- /devices_darwin.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "C" 5 | "os" 6 | _ "syscall" 7 | _ "unsafe" 8 | 9 | "github.com/sirupsen/logrus" 10 | "golang.org/x/sys/unix" 11 | ) 12 | 13 | // Device information for macOS 14 | var ( 15 | DefaultDevices = []*Device{ 16 | // /dev/urandom 17 | { 18 | Path: "/dev/urandom", 19 | Type: 'c', 20 | Major: 14, 21 | Minor: 1, 22 | Permissions: "rwm", 23 | FileMode: 0666, 24 | }, 25 | } 26 | ) 27 | 28 | func openNetFd(ifname string, specEnv []string) (*os.File, bool) { 29 | tapDev, err := os.OpenFile("/dev/"+ifname, os.O_RDWR|unix.O_NONBLOCK, 0666) 30 | if err != nil { 31 | logrus.Errorf("open %s error: /dev/%s\n", ifname, err) 32 | panic(err) 33 | } 34 | 35 | return tapDev, true 36 | } 37 | -------------------------------------------------------------------------------- /run.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/urfave/cli" 7 | ) 8 | 9 | // default action is to start a container 10 | var runCommand = cli.Command{ 11 | Name: "run", 12 | ArgsUsage: ``, 13 | Usage: "create and run a container", 14 | Flags: []cli.Flag{ 15 | cli.StringFlag{ 16 | Name: "bundle, b", 17 | Value: "", 18 | Usage: `path to the root of the bundle directory, defaults to the current directory`, 19 | }, 20 | }, 21 | Action: func(context *cli.Context) error { 22 | args := context.Args() 23 | if args.Present() == false { 24 | return fmt.Errorf("Missing container ID") 25 | } 26 | 27 | // XXX: create + start 28 | container := context.Args().Get(0) 29 | err := cmdCreateUkon(context, false) 30 | if err != nil { 31 | return err 32 | } 33 | return resumeUkontainer(context, container) 34 | 35 | }, 36 | } 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CI](https://github.com/ukontainer/runu/actions/workflows/ci.yml/badge.svg)](https://github.com/ukontainer/runu/actions/workflows/ci.yml) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/libos-nuse/runu)](https://goreportcard.com/report/github.com/libos-nuse/runu) 3 | 4 | 5 | # runu 6 | OCI runtime for frankenlibc unikernel 7 | 8 | # Installation 9 | 10 | ``` 11 | make 12 | sudo cp runu /usr/local/bin/runu 13 | ``` 14 | 15 | add an entry to `/etc/docker/daemon.json` 16 | 17 | ``` 18 | "runu": { 19 | "path": "/usr/local/bin/runu", 20 | "runtimeArgs": [ 21 | ] 22 | }, 23 | ``` 24 | 25 | Optionally, you can install debian package from the repository. 26 | 27 | ``` 28 | # register apt repository 29 | curl -s https://packagecloud.io/install/repositories/ukontainer/runu/script.deb.sh | sudo bash 30 | # install the package 31 | sudo apt-get install docker-runu 32 | ``` -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ukontainer/runu 2 | 3 | go 1.16 4 | 5 | replace github.com/containerd/containerd => github.com/ukontainer/containerd v1.5.1-0.20220121000121-190b8b350994 6 | 7 | replace github.com/docker/go-p9p => github.com/ukontainer/go-p9p v0.0.0-20211006131049-f1e80d0d54ed 8 | 9 | require ( 10 | github.com/containerd/console v1.0.3 11 | github.com/containerd/containerd v1.6.0-rc.1 12 | github.com/containerd/go-runc v1.0.0 13 | github.com/containerd/typeurl v1.0.2 14 | github.com/docker/go-p9p v0.0.0-20191112112554-37d97cf40d03 15 | github.com/gogo/protobuf v1.3.2 16 | github.com/opencontainers/image-spec v1.0.2 // indirect 17 | github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 18 | github.com/pkg/errors v0.9.1 19 | github.com/sirupsen/logrus v1.8.1 20 | github.com/urfave/cli v1.22.2 21 | github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 22 | github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f 23 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e 24 | ) 25 | -------------------------------------------------------------------------------- /start.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/opencontainers/runtime-spec/specs-go" 7 | "github.com/sirupsen/logrus" 8 | "github.com/urfave/cli" 9 | "os" 10 | "syscall" 11 | ) 12 | 13 | var startCommand = cli.Command{ 14 | Name: "start", 15 | ArgsUsage: ``, 16 | Flags: []cli.Flag{}, 17 | Action: func(context *cli.Context) error { 18 | args := context.Args() 19 | if args.Present() == false { 20 | return fmt.Errorf("Missing container ID") 21 | } 22 | 23 | container := context.Args().Get(0) 24 | resumeUkontainer(context, container) 25 | saveState(specs.StateRunning, container, context) 26 | return nil 27 | }, 28 | } 29 | 30 | func resumeUkontainer(context *cli.Context, container string) error { 31 | // wake the process 32 | pidI, err := readPidFile(context, pidFilePriv) 33 | if err != nil { 34 | return fmt.Errorf("couldn't find pid %d", pidI) 35 | } 36 | proc, err := os.FindProcess(pidI) 37 | if err != nil { 38 | return fmt.Errorf("couldn't find pid %d", pidI) 39 | } 40 | 41 | logrus.Debugf("proc %p, pid=%d", proc, pidI) 42 | proc.Signal(syscall.Signal(syscall.SIGCONT)) 43 | 44 | return nil 45 | } 46 | -------------------------------------------------------------------------------- /cmd/containerd-shim-runu-v1/main.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | // +build darwin 3 | 4 | /* 5 | Copyright The containerd Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "github.com/containerd/containerd/runtime/v2/shim" 24 | ) 25 | 26 | const ( 27 | // RunuRoot is root directory for runtime execution 28 | RunuRoot = "/var/run/containerd/runu" 29 | // RuntimeV1 is the name of runtime 30 | RuntimeV1 = "io.containerd.runu.v1" 31 | ) 32 | 33 | func main() { 34 | shim.Run(RuntimeV1, New, func(cfg *shim.Config) { 35 | cfg.NoSetupLogger = false 36 | // We have own reaper implementation in shim 37 | cfg.NoSubreaper = true 38 | cfg.NoReaper = true 39 | }) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/pre-deploy-deb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . $(dirname "${BASH_SOURCE[0]}")/../test/common.sh 4 | 5 | # Prepare supplement files for runu 6 | URL="https://github.com/ukontainer/frankenlibc/releases/download/latest/frankenlibc-${TRAVIS_ARCH}-${TRAVIS_OS_NAME}.tar.gz" 7 | curl -L $URL -o /tmp/frankenlibc.tar.gz 8 | tar xfz /tmp/frankenlibc.tar.gz -C /tmp/ 9 | 10 | 11 | # Replace version and build number with the Debian control file 12 | sed -i "s/__VERSION__/$BUILD_VERSION/g" pkg/deb/DEBIAN/control 13 | sed -i "s/__DATE__/$BUILD_DATE/g" pkg/deb/DEBIAN/control 14 | sed -i "s/__ARCH__/$DEB_ARCH/g" pkg/deb/DEBIAN/control 15 | 16 | # Create the Debian package 17 | mkdir -p pkg/deb/usr/bin 18 | mkdir -p pkg/deb/usr/lib/runu/ 19 | 20 | cp -f /tmp/opt/rump/bin/lkick pkg/deb/usr/lib/runu/ 21 | cp -f /tmp/opt/rump/lib/libc.so pkg/deb/usr/lib/runu/ 22 | 23 | GOPATH=`go env GOPATH` 24 | 25 | if [ -f $GOPATH/bin/runu ] ; then 26 | cp $GOPATH/bin/runu pkg/deb/usr/bin/ 27 | elif [ -f $GOPATH/bin/${GOOS}_${GOARCH}/runu ] ; then 28 | cp $GOPATH/bin/${GOOS}_${GOARCH}/runu pkg/deb/usr/bin/ 29 | fi 30 | 31 | dpkg-deb --build pkg/deb 32 | mv pkg/deb.deb $PACKAGE_FILENAME 33 | 34 | # Output detail on the resulting package for debugging purpose 35 | ls -l $PACKAGE_FILENAME 36 | dpkg --contents $PACKAGE_FILENAME 37 | md5sum $PACKAGE_FILENAME 38 | -------------------------------------------------------------------------------- /cmd/containerd-shim-runu-v1/platform.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | // +build darwin 3 | 4 | /* 5 | Copyright The containerd Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "context" 24 | "sync" 25 | 26 | "github.com/containerd/console" 27 | ) 28 | 29 | type unixPlatform struct { 30 | } 31 | 32 | func (p *unixPlatform) CopyConsole(ctx context.Context, console console.Console, id, stdin, stdout, stderr string, wg *sync.WaitGroup) (console.Console, error) { 33 | return nil, nil 34 | } 35 | 36 | func (p *unixPlatform) ShutdownConsole(ctx context.Context, cons console.Console) error { 37 | return nil 38 | } 39 | 40 | func (p *unixPlatform) Close() error { 41 | return nil 42 | } 43 | 44 | func (s *service) initPlatform() error { 45 | s.platform = &unixPlatform{} 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /test/k8s-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 4 | 5 | DST_ADDR=$1 6 | 7 | # XXX: need multi-arch image build 8 | if [ $TRAVIS_ARCH != "amd64" ] || [ $TRAVIS_OS_NAME != "linux" ] ; then 9 | echo "This now only builds linux/amd64 image. Skipping" 10 | exit 0 11 | fi 12 | 13 | fold_start k8s.test.2 "k8s: kind setup" 14 | 15 | # install runtime class 16 | kubectl apply -f k8s/ukontainer-runtimeclass.yaml 17 | 18 | fold_end k8s.test.2 "" 19 | 20 | fold_start k8s.test.3 "k8s: hello world" 21 | # install runu pod 22 | ## XXX: github action runners don't allow to pass ICMP at firewall 23 | cat k8s/hello-world.yaml | sed "s/\$DOCKER_IMG_VERSION/$DOCKER_IMG_VERSION/" \ 24 | | sed "s/8.8.8.8/$DST_ADDR/" \ 25 | | kubectl apply -f - 26 | 27 | kubectl get nodes -o wide -A 28 | sleep 20 29 | set -x 30 | kubectl get pods -o wide -A 31 | kubectl describe deployment/helloworld-runu 32 | kubectl logs deployment/helloworld-runu |& tee /tmp/log.txt 33 | grep "icmp_req=1" /tmp/log.txt 34 | 35 | fold_end k8s.test.3 "" 36 | 37 | fold_start k8s.test.4 "k8s: alpine hello world" 38 | # install runu pod 39 | cat k8s/alpine-runu.yaml | sed "s/8.8.8.8/$DST_ADDR/" \ 40 | | kubectl apply -f - 41 | 42 | kubectl get nodes -o wide -A 43 | sleep 20 44 | set -x 45 | kubectl get pods -o wide -A 46 | kubectl describe deployment/alpine-runu 47 | kubectl logs deployment/alpine-runu |& tee /tmp/log.txt 48 | grep "seq=1" /tmp/log.txt 49 | 50 | fold_end k8s.test.4 "" 51 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log/syslog" 5 | "os" 6 | "time" 7 | 8 | "github.com/sirupsen/logrus" 9 | lSyslog "github.com/sirupsen/logrus/hooks/syslog" 10 | ) 11 | 12 | const ( 13 | name = "runu" 14 | ) 15 | 16 | type sysLogHook struct { 17 | shook *lSyslog.SyslogHook 18 | formatter logrus.Formatter 19 | } 20 | 21 | func (h *sysLogHook) Levels() []logrus.Level { 22 | return h.shook.Levels() 23 | } 24 | 25 | // Fire is responsible for adding a log entry to the system log. It switches 26 | // formatter before adding the system log entry, then reverts the original log 27 | // formatter. 28 | func (h *sysLogHook) Fire(e *logrus.Entry) (err error) { 29 | formatter := e.Logger.Formatter 30 | 31 | e.Logger.Formatter = h.formatter 32 | 33 | err = h.shook.Fire(e) 34 | 35 | e.Logger.Formatter = formatter 36 | 37 | return err 38 | } 39 | 40 | func newSystemLogHook(network, raddr string) (*sysLogHook, error) { 41 | hook, err := lSyslog.NewSyslogHook(network, raddr, syslog.LOG_INFO, name) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | return &sysLogHook{ 47 | formatter: &logrus.TextFormatter{ 48 | TimestampFormat: time.RFC3339Nano, 49 | }, 50 | shook: hook, 51 | }, nil 52 | } 53 | 54 | func handleSystemLog(network, raddr string) error { 55 | hook, err := newSystemLogHook(network, raddr) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | log := logrus.WithFields(logrus.Fields{ 61 | "name": name, 62 | "source": "runtime", 63 | "arch": arch, 64 | "pid": os.Getpid(), 65 | }) 66 | 67 | log.Logger.Hooks.Add(hook) 68 | 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /kill.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/opencontainers/runtime-spec/specs-go" 7 | "github.com/sirupsen/logrus" 8 | "github.com/urfave/cli" 9 | "os" 10 | "strconv" 11 | "syscall" 12 | ) 13 | 14 | func killFromPidFile(context *cli.Context, pidFile string, signal int) error { 15 | pid, err := readPidFile(context, pidFile) 16 | if err != nil { 17 | // logrus.Warnf("couldn't find pid %d(%s)", pidI, err) 18 | return nil 19 | } 20 | 21 | proc, err := os.FindProcess(pid) 22 | if err != nil { 23 | logrus.Infof("couldn't find pid %d(%s)", pid, err) 24 | return err 25 | } 26 | 27 | err = proc.Signal(syscall.Signal(signal)) 28 | if err != nil { 29 | logrus.Warnf("couldn't signal to pid %d(%s)", pid, err) 30 | return err 31 | } 32 | 33 | return nil 34 | } 35 | 36 | var killCommand = cli.Command{ 37 | Name: "kill", 38 | ArgsUsage: ``, 39 | Flags: []cli.Flag{ 40 | cli.BoolFlag{ 41 | Name: "all", 42 | }, 43 | }, 44 | Action: func(context *cli.Context) error { 45 | args := context.Args() 46 | if args.Present() == false { 47 | return fmt.Errorf("Missing container ID") 48 | } 49 | 50 | container := context.Args().Get(0) 51 | signal, _ := strconv.Atoi(context.Args().Get(1)) 52 | 53 | // kill 9pfs server 54 | err := killFromPidFile(context, pidFile9p, signal) 55 | if err != nil { 56 | logrus.Warnf("killing 9pfs error %s", err) 57 | } 58 | 59 | // kill main process 60 | err = killFromPidFile(context, pidFilePriv, signal) 61 | if err != nil { 62 | return fmt.Errorf("killing main process error %s", err) 63 | } 64 | 65 | saveState(specs.StateStopped, container, context) 66 | return nil 67 | }, 68 | } 69 | -------------------------------------------------------------------------------- /k8s/README.md: -------------------------------------------------------------------------------- 1 | ## Container runtimeClass for uKontainer 2 | 3 | uKontainer (runu) supports runtimeClass to use alternate runtime 4 | instead of runc over CRI mechanism. We only support (and tested) 5 | containerd CRI; thus the default dockershim need to be stopped and 6 | containerd should be used instead. 7 | 8 | 9 | ### Installation 10 | 11 | 1. containerd configuration 12 | 13 | /etc/containerd/config.toml 14 | ``` 15 | [plugins.cri.containerd] 16 | (snip) 17 | [plugins.cri.containerd.default_runtime] 18 | runtime_type = "io.containerd.runtime.v1.linux" 19 | runtime_engine = "/usr/bin/runu" 20 | runtime_root = "" 21 | [plugins.cri.containerd.runtimes.runu] 22 | runtime_type = "io.containerd.runtime.v1.linux" 23 | 24 | ``` 25 | 26 | The last two lines are added one. Let's restart containerd afterward 27 | by `systemctl restart containerd`. 28 | 29 | 30 | 2. Installing RuntimeClass resoure 31 | 32 | Need to install a runtime class resource by: 33 | 34 | ``` 35 | kubectl apply -f ./k8s/ukontainer-runtimeclass.yaml 36 | ``` 37 | 38 | 3. Install runu binary 39 | 40 | See the instruction described at https://bintray.com/ukontainer/debian. 41 | Alternatively, you can use a KinD (k8s in Docker) image which contains runu 42 | binary (https://hub.docker.com/r/ukontainer/node-runu). 43 | 44 | ### Usage 45 | 46 | 47 | 48 | The pod configuration (.yaml file) should look like the following: 49 | 50 | 51 | ``` 52 | spec: 53 | runtimeClassName: ukontainer 54 | containers: 55 | - name: test-pod 56 | image: test-pod:1.0 57 | imagePullPolicy: Always 58 | ports: 59 | - containerPort: 8080 60 | ``` 61 | 62 | ### Reference 63 | 64 | https://kubernetes.io/docs/concepts/containers/runtime-class/ 65 | -------------------------------------------------------------------------------- /9pfs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net" 7 | "os" 8 | 9 | p9p "github.com/docker/go-p9p" 10 | "github.com/docker/go-p9p/ufs" 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | const ( 15 | addr9p = "127.0.0.1:5640" 16 | p9Timeout = 30 17 | ) 18 | 19 | type contextKey string 20 | 21 | // 9pfs client side 22 | func connect9pfs() (*os.File, bool) { 23 | tcpAddr, err := net.ResolveTCPAddr("tcp", addr9p) 24 | if err != nil { 25 | panic(err) 26 | } 27 | 28 | conn, err := net.DialTCP("tcp", nil, tcpAddr) 29 | if err != nil { 30 | logrus.Errorf("open %s error: %s\n", tcpAddr, err) 31 | panic(err) 32 | } 33 | 34 | fd, err := conn.File() 35 | if err != nil { 36 | logrus.Errorf("File() %s error: %s\n", conn.RemoteAddr(), err) 37 | panic(err) 38 | } 39 | 40 | conn.Close() 41 | 42 | // XXX: 9pfs sv requires blocking socket ?? 43 | return fd, false 44 | } 45 | 46 | // 9pfs server side 47 | func start9pfsServer(path string) error { 48 | ctx := context.Background() 49 | l, err := net.Listen("tcp", addr9p) 50 | if err != nil { 51 | panic(err) 52 | } 53 | defer l.Close() 54 | 55 | for { 56 | statusChan := make(chan error) 57 | c, err := l.Accept() 58 | if err != nil { 59 | panic(err) 60 | } 61 | 62 | go func(conn net.Conn) error { 63 | ctx := context.WithValue(ctx, contextKey("conn"), conn) 64 | session, err := ufs.NewSession(ctx, path) 65 | if err != nil { 66 | logrus.Println("error creating session", err) 67 | statusChan <- err 68 | } 69 | 70 | if err := p9p.ServeConn(ctx, conn, p9p.Dispatch(session), p9Timeout); err != nil && err != io.EOF { 71 | logrus.Printf("error serving conn: (path=%s) %+v", path, err) 72 | statusChan <- err 73 | } 74 | return nil 75 | }(c) 76 | return <-statusChan 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /state.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/opencontainers/runtime-spec/specs-go" 7 | "github.com/sirupsen/logrus" 8 | "github.com/urfave/cli" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | var stateCommand = cli.Command{ 15 | Name: "state", 16 | ArgsUsage: ``, 17 | Action: func(context *cli.Context) error { 18 | args := context.Args() 19 | if args.Present() == false { 20 | return fmt.Errorf("Missing container ID") 21 | } 22 | 23 | root := context.GlobalString("root") 24 | name := context.Args().First() 25 | stateFile := filepath.Join(root, name, stateJSON) 26 | stateData, _ := ioutil.ReadFile(stateFile) 27 | 28 | os.Stdout.Write(stateData) 29 | logrus.Debug(string(stateData)) 30 | return nil 31 | }, 32 | } 33 | 34 | func saveState(status specs.ContainerState, container string, context *cli.Context) error { 35 | root := context.GlobalString("root") 36 | absRoot, _ := filepath.Abs(root) 37 | 38 | spec, err := setupSpec(context) 39 | if err != nil { 40 | return fmt.Errorf("setupSepc err: %v", err) 41 | } 42 | 43 | rootfs, _ := filepath.Abs(spec.Root.Path) 44 | stateFile := filepath.Join(absRoot, container, stateJSON) 45 | cs := &specs.State{ 46 | Version: spec.Version, 47 | ID: context.Args().Get(0), 48 | Status: status, 49 | Bundle: rootfs, 50 | } 51 | stateData, _ := json.MarshalIndent(cs, "", "\t") 52 | 53 | if err := ioutil.WriteFile(stateFile, stateData, 0666); err != nil { 54 | panic(err) 55 | } 56 | 57 | return nil 58 | } 59 | 60 | func createContainer(container, bundle, stateRoot string, spec *specs.Spec) error { 61 | // Prepare container state directory 62 | stateDir := filepath.Join(stateRoot, container) 63 | _, err := os.Stat(stateDir) 64 | if err == nil { 65 | logrus.Errorf("Container %s exists", container) 66 | return fmt.Errorf("Container %s exists", container) 67 | } 68 | err = os.MkdirAll(stateDir, 0755) 69 | if err != nil { 70 | fmt.Printf("%s\n", err.Error()) 71 | return err 72 | } 73 | 74 | return nil 75 | } 76 | 77 | func deleteContainer(root, container string) error { 78 | return os.RemoveAll(filepath.Join(root, container)) 79 | } 80 | -------------------------------------------------------------------------------- /devices.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/sirupsen/logrus" 9 | "golang.org/x/sys/unix" 10 | ) 11 | 12 | // Device information 13 | type Device struct { 14 | // Device type, block, char, etc. 15 | Type rune `json:"type"` 16 | 17 | // Path to the device. 18 | Path string `json:"path"` 19 | 20 | // Major is the device's major number. 21 | Major int64 `json:"major"` 22 | 23 | // Minor is the device's minor number. 24 | Minor int64 `json:"minor"` 25 | 26 | // Cgroup permissions format, rwm. 27 | Permissions string `json:"permissions"` 28 | 29 | // FileMode permission bits for the device. 30 | FileMode os.FileMode `json:"file_mode"` 31 | 32 | // Uid of the device. 33 | Uid uint32 `json:"uid"` 34 | 35 | // Gid of the device. 36 | Gid uint32 `json:"gid"` 37 | 38 | // Write the file to the allowed list 39 | Allow bool `json:"allow"` 40 | } 41 | 42 | func createDeviceNode(rootfs string, node *Device) error { 43 | dest := filepath.Join(rootfs, node.Path) 44 | if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { 45 | return err 46 | } 47 | 48 | return mknodDevice(dest, node) 49 | } 50 | 51 | func mknodDevice(dest string, node *Device) error { 52 | fileMode := node.FileMode 53 | switch node.Type { 54 | case 'c', 'u': 55 | fileMode |= unix.S_IFCHR 56 | case 'b': 57 | fileMode |= unix.S_IFBLK 58 | case 'p': 59 | fileMode |= unix.S_IFIFO 60 | default: 61 | return fmt.Errorf("%c is not a valid device type for device %s", node.Type, node.Path) 62 | } 63 | 64 | dNum := int((node.Major << 8) | (node.Minor & 0xff) | ((node.Minor & 0xfff00) << 12)) 65 | if err := unix.Mknod(dest, uint32(fileMode), dNum); err != nil { 66 | return err 67 | } 68 | return unix.Chown(dest, int(node.Uid), int(node.Gid)) 69 | } 70 | 71 | func openRootfsFd(file string) (*os.File, bool) { 72 | fd, err := os.OpenFile(file, os.O_RDWR, 0666) 73 | if err != nil { 74 | logrus.Errorf("open %s error: /dev/%s\n", file, err) 75 | panic(err) 76 | } 77 | 78 | return fd, true 79 | } 80 | 81 | func openJsonFd(file string) (*os.File, bool) { 82 | fd, err := os.OpenFile(file, os.O_RDONLY, unix.S_IRUSR|unix.S_IWUSR) 83 | if err != nil { 84 | logrus.Errorf("open %s error: %s\n", file, err) 85 | panic(err) 86 | } 87 | 88 | return fd, false 89 | } 90 | -------------------------------------------------------------------------------- /test/containerd-nerdctl-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 4 | 5 | if [ $TRAVIS_OS_NAME != "osx" ] ; then 6 | echo "containerd and ctr runtime test only support with osx host. Skipped" 7 | exit 0 8 | fi 9 | 10 | CTR_GLOBAL_OPT="--debug -a /tmp/ctrd/run/containerd/containerd.sock --snapshotter=sparsebundle" 11 | NERDCTL_ARGS="--rm --env RUMP_VERBOSE=1 --net=none" 12 | 13 | 14 | ### 15 | ### nerdctl tests 16 | ### 17 | 18 | # preparation of nerdctl 19 | 20 | # test hello-world (nerdctl) 21 | fold_start test.nerdctl.1 "test hello (nerdctl)" 22 | sudo nerdctl $CTR_GLOBAL_OPT run $NERDCTL_ARGS \ 23 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION hello 24 | fold_end test.nerdctl.1 25 | 26 | # test ping (nerdctl) 27 | fold_start test.nerdctl.2 "test ping (nerdctl)" 28 | sudo nerdctl $CTR_GLOBAL_OPT run $NERDCTL_ARGS \ 29 | --env LKL_ROOTFS=imgs/python.iso \ 30 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION \ 31 | ping -c5 127.0.0.1 32 | fold_end test.nerdctl.2 33 | 34 | # test python (nerdctl) 35 | # XXX: PYTHONHASHSEED=1 is workaround for slow read of getrandom() on 4.19 36 | # (4.16 doesn't have such) 37 | fold_start test.nerdctl.3 "test python (nerdctl)" 38 | sudo nerdctl $CTR_GLOBAL_OPT run $NERDCTL_ARGS \ 39 | --env HOME=/ --env PYTHONHOME=/python \ 40 | --env LKL_ROOTFS=imgs/python.img \ 41 | --env PYTHONHASHSEED=1 \ 42 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION \ 43 | python -c "print(\"hello world from python(docker-runu)\")" 44 | fold_end test.nerdctl.3 45 | 46 | # test nginx (nerdctl) 47 | fold_start test.nerdctl.4 "test nginx (nerdctl)" 48 | sudo nerdctl $CTR_GLOBAL_OPT run $NERDCTL_ARGS \ 49 | --env LKL_ROOTFS=imgs/data.iso \ 50 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION \ 51 | nginx & 52 | sleep 3 53 | sudo killall -9 nerdctl 54 | fold_end test.nerdctl.4 55 | 56 | # test alpine (nerdctl) 57 | create_runu_aux_dir 58 | fold_start test.nerdctl.5 "test alpine Linux on darwin (nerdctl)" 59 | sudo nerdctl $CTR_GLOBAL_OPT run $NERDCTL_ARGS \ 60 | --env RUNU_AUX_DIR=$RUNU_AUX_DIR --env LKL_USE_9PFS=1 \ 61 | --platform=linux/amd64 \ 62 | library/alpine:latest /bin/busybox ls -l 63 | fold_end test.nerdctl.5 64 | -------------------------------------------------------------------------------- /mount_linux.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/opencontainers/runtime-spec/specs-go" 10 | "golang.org/x/sys/unix" 11 | ) 12 | 13 | func prepareMount(src, dest string) error { 14 | srcStat, err := os.Stat(src) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | if _, err := os.Stat(dest); err != nil { 20 | if os.IsNotExist(err) { 21 | if srcStat.IsDir() { 22 | return os.MkdirAll(dest, 0755) 23 | } 24 | if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { 25 | return err 26 | } 27 | f, err := os.OpenFile(dest, os.O_CREATE, 0755) 28 | if err != nil { 29 | return err 30 | } 31 | f.Close() 32 | } 33 | } 34 | return nil 35 | } 36 | 37 | func doMount(src, dest string, flags uintptr) error { 38 | 39 | if err := unix.Mount(src, dest, "bind", flags, ""); err != nil { 40 | return fmt.Errorf("mount %s on %s failed", src, dest) 41 | } 42 | if err := unix.Mount("", dest, "bind", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { 43 | return fmt.Errorf("mount private on %s failed", dest) 44 | } 45 | if flags&unix.MS_RDONLY != 0 { 46 | if err := unix.Mount(src, dest, "bind", flags|unix.MS_REMOUNT, ""); err != nil { 47 | return fmt.Errorf("remount %s on %s failed", src, dest) 48 | } 49 | } 50 | return nil 51 | } 52 | 53 | func doMounts(spec *specs.Spec) (bool, error) { 54 | rootfs := spec.Root.Path 55 | volumeMounted := false 56 | _, hasRootFs, err := checkFsFlags(spec) 57 | if err != nil { 58 | return false, err 59 | } 60 | 61 | for _, m := range spec.Mounts { 62 | var ( 63 | dest = m.Destination 64 | flags uintptr = unix.MS_REC 65 | ) 66 | if !strings.HasPrefix(dest, rootfs) { 67 | dest = filepath.Join(rootfs, dest) 68 | } 69 | 70 | for _, f := range m.Options { 71 | switch f { 72 | case "rbind": 73 | flags = flags | unix.MS_BIND 74 | case "ro": 75 | flags = flags | unix.MS_RDONLY 76 | } 77 | } 78 | 79 | switch m.Type { 80 | case "bind": 81 | doBreak := false 82 | for _, d := range []string{"/etc/hosts", "/etc/hostname", "/etc/resolv.conf", "/dev/shm"} { 83 | if m.Destination == d { 84 | doBreak = true 85 | break 86 | } 87 | } 88 | if doBreak { 89 | break 90 | } 91 | 92 | if hasRootFs { 93 | return false, fmt.Errorf("LKL_ROOTFS cannot be used with -v options") 94 | } 95 | if err := prepareMount(m.Source, dest); err != nil { 96 | return false, err 97 | } 98 | if err := doMount(m.Source, dest, flags); err != nil { 99 | return false, err 100 | } 101 | volumeMounted = true 102 | } 103 | } 104 | 105 | return volumeMounted, nil 106 | } 107 | -------------------------------------------------------------------------------- /test/common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | travis_nanoseconds() { 4 | local cmd='date' 5 | local format='+%s%N' 6 | 7 | if hash gdate >/dev/null 2>&1; then 8 | cmd='gdate' 9 | elif [[ "${TRAVIS_OS_NAME}" == osx ]]; then 10 | format='+%s000000000' 11 | fi 12 | 13 | "${cmd}" -u "${format}" 14 | } 15 | 16 | travis_time_start() { 17 | TRAVIS_TIMER_ID="$(printf %08x $((RANDOM * RANDOM)))" 18 | TRAVIS_TIMER_START_TIME="$(travis_nanoseconds)" 19 | export TRAVIS_TIMER_ID TRAVIS_TIMER_START_TIME 20 | echo -en "travis_time:start:$TRAVIS_TIMER_ID\\r${ANSI_CLEAR}" 21 | } 22 | 23 | 24 | travis_time_finish() { 25 | local result="${?}" 26 | local travis_timer_end_time 27 | local event="${1}" 28 | travis_timer_end_time="$(travis_nanoseconds)" 29 | local duration 30 | duration="$((travis_timer_end_time - TRAVIS_TIMER_START_TIME))" 31 | echo -en "travis_time:end:${TRAVIS_TIMER_ID}:start=${TRAVIS_TIMER_START_TIME},finish=${travis_timer_end_time},duration=${duration},event=${event}\\r${ANSI_CLEAR}" 32 | return "${result}" 33 | } 34 | 35 | fold_start() { 36 | echo -e "$1 $2" 37 | set -x 38 | } 39 | 40 | fold_end() { 41 | echo -e "$1" 42 | set +x 43 | echo "=====================================================================" 44 | } 45 | 46 | 47 | create_osx_chroot() { 48 | ROOTFS=$1 49 | if [ $TRAVIS_OS_NAME == "osx" ]; then 50 | sudo mount -t devfs devfs $ROOTFS/dev 51 | fi 52 | } 53 | 54 | create_runu_aux_dir() { 55 | export RUNU_AUX_DIR="/tmp/runu" 56 | mkdir -p $RUNU_AUX_DIR 57 | 58 | if [ -a $RUNU_AUX_DIR/lkick ] ; then 59 | return 60 | fi 61 | 62 | # download pre-built frankenlibc 63 | URL="https://github.com/ukontainer/frankenlibc/releases/download/latest/frankenlibc-$ARCH-$TRAVIS_OS_NAME.tar.gz" 64 | URL_LINUX="https://github.com/ukontainer/frankenlibc/releases/download/latest/frankenlibc-amd64-linux.tar.gz" 65 | curl -L $URL -o /tmp/frankenlibc.tar.gz 66 | tar xfz /tmp/frankenlibc.tar.gz -C /tmp/ 67 | cp /tmp/opt/rump/bin/rexec $RUNU_AUX_DIR/rexec 68 | cp /tmp/opt/rump/bin/lkick $RUNU_AUX_DIR/lkick 69 | if [ $TRAVIS_OS_NAME == "osx" ]; then 70 | curl -L $URL_LINUX -o /tmp/frankenlibc-linux.tar.gz 71 | tar xfz /tmp/frankenlibc-linux.tar.gz -C /tmp opt/rump/lib/libc.so 72 | fi 73 | if [ -f /tmp/opt/rump/lib/libc.so ] ; then 74 | cp /tmp/opt/rump/lib/libc.so $RUNU_AUX_DIR/libc.so 75 | fi 76 | } 77 | 78 | # common variables 79 | OSNAME=$(uname -s) 80 | if [ -z $TRAVIS_OS_NAME ] ; then 81 | if [ $OSNAME = "Linux" ] ; then 82 | TRAVIS_OS_NAME="linux" 83 | elif [ $OSNAME = "Darwin" ] ; then 84 | TRAVIS_OS_NAME="osx" 85 | fi 86 | fi 87 | 88 | PNAME=$(uname -m) 89 | if [ -z $ARCH ] ; then 90 | if [ $PNAME = "x86_64" ] ; then 91 | ARCH="amd64" 92 | elif [ $DEB_ARCH = "arm64" ] ; then 93 | ARCH="arm64" 94 | elif [ $DEB_ARCH = "armhf" ] ; then 95 | ARCH="arm" 96 | fi 97 | fi 98 | -------------------------------------------------------------------------------- /boot.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | goruntime "runtime" 9 | "strconv" 10 | 11 | "github.com/opencontainers/runtime-spec/specs-go" 12 | "github.com/sirupsen/logrus" 13 | "github.com/urfave/cli" 14 | "golang.org/x/sys/unix" 15 | ) 16 | 17 | var bootCommand = cli.Command{ 18 | Name: "boot", 19 | Usage: "(internal use only) boot a container", 20 | Flags: []cli.Flag{ 21 | cli.StringFlag{ 22 | Name: "bundle, b", 23 | Value: "", 24 | Usage: `path to the root of the bundle directory, defaults to the current directory`, 25 | }, 26 | cli.StringFlag{ 27 | Name: "pid-file", 28 | Usage: "specify the file to write the process id to", 29 | }, 30 | }, 31 | Action: func(context *cli.Context) error { 32 | return bootContainer(context, false) 33 | }, 34 | } 35 | 36 | func bootContainer(context *cli.Context, attach bool) error { 37 | container := context.Args().First() 38 | cmd, err := prepareUkontainer(context) 39 | if err != nil { 40 | return fmt.Errorf("failed to prepare container: %v", err) 41 | } 42 | 43 | // write pid file for containerd-shim 44 | if pidf := context.String("pid-file"); pidf != "" { 45 | // 0) pid file for containerd 46 | f, err := os.OpenFile(pidf, 47 | os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666) 48 | if err != nil { 49 | return fmt.Errorf("pid-file: %s\n", err) 50 | } 51 | // 52 | // XXX: 53 | // linux should tell containerd with a child process (subreaper) 54 | // while darwin should with a grandchild process (ReapMore) 55 | // 56 | if goruntime.GOOS == "linux" { 57 | _, _ = fmt.Fprintf(f, "%d", os.Getpid()) 58 | } else if goruntime.GOOS == "darwin" { 59 | _, _ = fmt.Fprintf(f, "%d", cmd.Process.Pid) 60 | } 61 | f.Close() 62 | } 63 | 64 | saveState(specs.StateCreated, container, context) 65 | 66 | envInitPipe := os.Getenv("_LIBCONTAINER_INITPIPE") 67 | pipefd, err := strconv.Atoi(envInitPipe) 68 | if err != nil { 69 | return fmt.Errorf("unable to convert _LIBCONTAINER_INITPIPE=%s to int: %s", envInitPipe, err) 70 | } 71 | 72 | // notify to `runu create` 73 | pipe := os.NewFile(uintptr(pipefd), "pipe") 74 | logrus.Debugf("writing pipe %s _LIBCONTAINER_INITPIPE=%s", pipe.Name(), envInitPipe) 75 | pipe.Write([]byte("1")) 76 | pipe.Close() 77 | 78 | // catch child errors if possible 79 | err = cmd.Wait() 80 | if err != nil { 81 | logrus.Warnf("wait error %s", err) 82 | } 83 | 84 | // stop 9pfs server 85 | err9 := killFromPidFile(context, pidFile9p, 15) 86 | if err9 != nil { 87 | logrus.Warnf("killing 9pfs error %s", err9) 88 | } 89 | 90 | // signal to containerd-shim to request exit 91 | bundle := context.String("bundle") 92 | file := filepath.Join(bundle, "shim.pid") 93 | pid, _ := ioutil.ReadFile(file) 94 | pidI, _ := strconv.Atoi(string(pid)) 95 | unix.Kill(pidI, unix.SIGCHLD) 96 | logrus.Debugf("sending SIGCHLD to parent %d", pidI) 97 | 98 | saveState(specs.StateStopped, container, context) 99 | logrus.Debugf("process stopped %s", cmd.Args) 100 | 101 | return err 102 | } 103 | -------------------------------------------------------------------------------- /test/containerd-ctr-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 4 | 5 | if [ $TRAVIS_OS_NAME != "osx" ] ; then 6 | echo "containerd and ctr runtime test only support with osx host. Skipped" 7 | exit 0 8 | fi 9 | 10 | CTR_ARGS="--rm --snapshotter=sparsebundle --runtime=io.containerd.runu.v1 --fifo-dir /tmp/ctrd --env RUMP_VERBOSE=1" 11 | CTR_GLOBAL_OPT="--debug -a /tmp/ctrd/run/containerd/containerd.sock" 12 | 13 | sudo rm -rf /tmp/ctrd/ 14 | 15 | # prepare containerd 16 | fold_start test.containerd.0 "boot containerd" 17 | git clone https://gist.github.com/aba357f73da4e14bc3f5cbeb00aeaea4.git \ 18 | /tmp/containerd-config || true 19 | cp /tmp/containerd-config/config.toml /tmp/ 20 | sed "s/501/$UID/" /tmp/config.toml > /tmp/a 21 | mv /tmp/a /tmp/config.toml 22 | 23 | mkdir -p /tmp/containerd-shim 24 | sudo killall containerd || true 25 | containerd -l debug -c /tmp/config.toml & 26 | sleep 3 27 | killall containerd 28 | sudo containerd -l debug -c /tmp/config.toml > /tmp/containerd.log 2>&1 & 29 | sleep 3 30 | chmod 755 /tmp/ctrd 31 | 32 | containerd-darwin-snapshotter-grpc /tmp/ctrd/run/containerd-darwin-snapshotter.sock /tmp/ctrd/var/lib/containerd/darwin > /tmp/darwin-snapshotter.log 2>&1 & 33 | 34 | 35 | ctr $CTR_GLOBAL_OPT version 36 | nerdctl $CTR_GLOBAL_OPT version 37 | nerdctl $CTR_GLOBAL_OPT info 38 | fold_end test.containerd.0 "" 39 | 40 | 41 | # pull an image 42 | fold_start test.containerd.0 "pull image" 43 | ctr -a /tmp/ctrd/run/containerd/containerd.sock i pull \ 44 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION 45 | ctr -a /tmp/ctrd/run/containerd/containerd.sock i pull \ 46 | --platform=linux/amd64 docker.io/library/alpine:latest 47 | fold_end test.containerd.0 "pull image" 48 | 49 | # test hello-world 50 | fold_start test.containerd.1 "test hello" 51 | sudo ctr $CTR_GLOBAL_OPT run $CTR_ARGS \ 52 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION hello hello 53 | fold_end test.containerd.1 54 | 55 | # test ping 56 | fold_start test.containerd.2 "test ping" 57 | sudo ctr $CTR_GLOBAL_OPT run $CTR_ARGS \ 58 | --env LKL_ROOTFS=imgs/python.iso \ 59 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION hello \ 60 | ping -c5 127.0.0.1 61 | fold_end test.containerd.2 62 | 63 | # test python 64 | # XXX: PYTHONHASHSEED=1 is workaround for slow read of getrandom() on 4.19 65 | # (4.16 doesn't have such) 66 | fold_start test.containerd.3 "test python" 67 | sudo ctr $CTR_GLOBAL_OPT run $CTR_ARGS \ 68 | --env HOME=/ --env PYTHONHOME=/python \ 69 | --env LKL_ROOTFS=imgs/python.img \ 70 | --env PYTHONHASHSEED=1 \ 71 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION hello \ 72 | python -c "print(\"hello world from python(docker-runu)\")" 73 | fold_end test.containerd.3 74 | 75 | # test nginx 76 | fold_start test.containerd.4 "test nginx" 77 | sudo ctr $CTR_GLOBAL_OPT run $CTR_ARGS \ 78 | --env LKL_ROOTFS=imgs/data.iso \ 79 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION hello \ 80 | nginx & 81 | sleep 3 82 | sudo killall -9 ctr 83 | fold_end test.containerd.4 84 | 85 | # test alpine 86 | # prepare RUNU_AUX_DIR 87 | create_runu_aux_dir 88 | 89 | fold_start test.containerd.5 "test alpine Linux on darwin" 90 | sudo ctr $CTR_GLOBAL_OPT run $CTR_ARGS \ 91 | --env RUNU_AUX_DIR=$RUNU_AUX_DIR --env LKL_USE_9PFS=1 \ 92 | docker.io/library/alpine:latest alpine1 /bin/busybox ls -l 93 | fold_end test.containerd.5 94 | -------------------------------------------------------------------------------- /test/standalone-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mkdir -p $HOME/tmp/bundle/rootfs/dev 4 | mkdir -p /tmp/runu-root 5 | 6 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 7 | 8 | fold_start test.0 "preparation test" 9 | # get script from moby 10 | curl https://raw.githubusercontent.com/moby/moby/7608e42da5abdd56c4d7b209384a6e512928d054/contrib/download-frozen-image-v2.sh \ 11 | -o /tmp/download-frozen-image-v2.sh 12 | 13 | # get image runu-base 14 | mkdir -p /tmp/runu 15 | if [ $TRAVIS_OS_NAME = "osx" ] ; then 16 | export OS_NAME="darwin" 17 | elif [ $TRAVIS_OS_NAME = "linux" ] ; then 18 | export OS_NAME="linux" 19 | fi 20 | DIGEST=`curl -s "https://registry.hub.docker.com/v2/repositories/ukontainer/runu-base/tags/$DOCKER_IMG_VERSION?page_size=100" | jq ".images | .[] | select(.os == \"$OS_NAME\" and .architecture == \"$ARCH\") | .digest " | sed "s/\\"//g" ` 21 | bash /tmp/download-frozen-image-v2.sh /tmp/runu/ ukontainer/runu-base:$DOCKER_IMG_VERSION@$DIGEST 22 | 23 | # extract images from layers 24 | for layer in `find /tmp/runu -name layer.tar` 25 | do 26 | tar xvfz $layer -C $HOME/tmp/bundle/rootfs 27 | done 28 | 29 | # sync /usr/lib for chrooted env 30 | create_osx_chroot $HOME/tmp/bundle/rootfs/ 31 | 32 | # prepare RUNU_AUX_DIR 33 | create_runu_aux_dir 34 | 35 | rm -f config.json 36 | runu spec 37 | 38 | fold_end test.0 39 | 40 | run_test() 41 | { 42 | bundle=$1 43 | RUNU=`which runu` 44 | 45 | sudo ${RUNU} --log="$HOME/runu.log" --debug --root=/tmp/runu-root run --bundle=$bundle foo 46 | sleep 5 47 | sudo ${RUNU} --log="$HOME/runu.log" --debug --root=/tmp/runu-root kill foo 9 || true 48 | sudo ${RUNU} --log="$HOME/runu.log" --debug --root=/tmp/runu-root delete foo 49 | } 50 | 51 | # test hello-world 52 | fold_start test.1 "test hello" 53 | cat config.json | jq '.process.args |=["hello"] ' > $HOME/tmp/bundle/config.json 54 | run_test $HOME/tmp/bundle 55 | fold_end test.1 56 | 57 | # test ping 58 | fold_start test.2 "test ping" 59 | cat config.json | jq '.process.args |=["ping","127.0.0.1"] ' > $HOME/tmp/bundle/config.json 60 | run_test $HOME/tmp/bundle 61 | fold_end test.2 62 | 63 | # test python 64 | # XXX: PYTHONHASHSEED=1 is workaround for slow read of getrandom() on 4.19 65 | # (4.16 doesn't have such) 66 | fold_start test.3 "test python" 67 | cat config.json | \ 68 | jq '.process.args |=["python", "-c", "print(\"hello world from python(runu)\")"] ' | \ 69 | jq '.process.env |= .+["LKL_ROOTFS=imgs/python.img", "RUMP_VERBOSE=1", "HOME=/", "PYTHONHOME=/python", "PYTHONHASHSEED=1"]' > $HOME/tmp/bundle/config.json 70 | run_test $HOME/tmp/bundle 71 | fold_end test.3 72 | 73 | #test nginx 74 | fold_start test.4 "test nginx" 75 | cat config.json | \ 76 | jq '.process.args |=["nginx"]' | \ 77 | jq '.process.env |= .+["LKL_ROOTFS=imgs/data.iso"]' \ 78 | > $HOME/tmp/bundle/config.json 79 | RUMP_VERBOSE=1 run_test $HOME/tmp/bundle 80 | fold_end test.4 81 | 82 | 83 | # download alpine image 84 | fold_start test.0 "test alpine" 85 | mkdir -p /tmp/alpine 86 | mkdir -p $HOME/tmp/alpine/bundle/rootfs/dev 87 | bash /tmp/download-frozen-image-v2.sh /tmp/alpine alpine:latest 88 | for layer in `find /tmp/alpine -name layer.tar` 89 | do 90 | tar xfz $layer -C $HOME/tmp/alpine/bundle/rootfs 91 | done 92 | 93 | # prepare RUNU_AUX_DIR 94 | create_runu_aux_dir 95 | 96 | #test alpine 97 | cat config.json | \ 98 | jq '.process.args |=["/bin/busybox","ls", "-l", "/"]' | \ 99 | jq '.process.env |= .+["RUNU_AUX_DIR='$RUNU_AUX_DIR'", "RUMP_VERBOSE=1", "LKL_USE_9PFS=1"]' \ 100 | > $HOME/tmp/alpine/bundle/config.json 101 | RUMP_VERBOSE=1 run_test $HOME/tmp/alpine/bundle 102 | fold_end test.0 103 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/opencontainers/runtime-spec/specs-go" 9 | "github.com/sirupsen/logrus" 10 | "github.com/urfave/cli" 11 | goruntime "runtime" 12 | ) 13 | 14 | const ( 15 | specConfig = "config.json" 16 | stateJSON = "state.json" 17 | usage = "runu run [ -b bundle ] " 18 | arch = goruntime.GOARCH 19 | pidFilePriv = "runu.pid" 20 | pidFile9p = "runu-9p.pid" 21 | ) 22 | 23 | var ( 24 | version = "" 25 | ) 26 | 27 | func main() { 28 | app := cli.NewApp() 29 | app.Name = "runu" 30 | app.Usage = usage 31 | 32 | var v []string 33 | v = append(v, version) 34 | v = append(v, fmt.Sprintf("spec: %s", specs.Version)) 35 | app.Version = strings.Join(v, "\n") 36 | 37 | app.Flags = []cli.Flag{ 38 | cli.BoolFlag{ 39 | Name: "debug", 40 | Usage: "enable debug output for logging", 41 | }, 42 | cli.StringFlag{ 43 | Name: "log", 44 | Value: "/tmp/runu.log", 45 | Usage: "set the log file path where internal debug information is written", 46 | }, 47 | cli.StringFlag{ 48 | Name: "debug-log", 49 | Usage: "set the log file path where debug information is written", 50 | Value: "", 51 | }, 52 | cli.StringFlag{ 53 | Name: "log-format", 54 | Value: "text", 55 | Usage: "set the format used by logs ('text' (default), or 'json')", 56 | }, 57 | cli.StringFlag{ 58 | Name: "root", 59 | Value: "/run/runu", 60 | Usage: "root directory for storage of container state (this should be located in tmpfs)", 61 | }, 62 | cli.StringFlag{ 63 | Name: "9ps", 64 | Usage: "start 9pfs server", 65 | Value: "", 66 | }, 67 | cli.BoolFlag{ 68 | Name: "systemd-cgroup", 69 | Usage: "unsupported flag", 70 | }, 71 | } 72 | app.Commands = []cli.Command{ 73 | createCommand, 74 | runCommand, 75 | specCommand, 76 | startCommand, 77 | stateCommand, 78 | execCommand, 79 | killCommand, 80 | deleteCommand, 81 | bootCommand, 82 | } 83 | 84 | app.Before = func(context *cli.Context) error { 85 | if rootfs := context.GlobalString("9ps"); rootfs != "" { 86 | logrus.Debugf("Runu called with args: %v\n", os.Args) 87 | return start9pfsServer(rootfs) 88 | } 89 | if path := context.GlobalString("log"); path != "" { 90 | f, err := os.OpenFile(path, 91 | os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 92 | 0666) 93 | if err != nil { 94 | fmt.Printf("%s\n", err) 95 | return err 96 | } 97 | logrus.SetOutput(f) 98 | } 99 | if path := context.GlobalString("debug-log"); path != "" { 100 | f, err := os.OpenFile(path, 101 | os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 102 | 0666) 103 | if err != nil { 104 | fmt.Printf("%s\n", err) 105 | return err 106 | } 107 | logrus.SetLevel(logrus.DebugLevel) 108 | logrus.SetOutput(f) 109 | } 110 | if context.GlobalBool("debug") { 111 | logrus.SetLevel(logrus.DebugLevel) 112 | logrus.SetOutput(os.Stdout) 113 | } 114 | if os.Getenv("DEBUG") != "" { 115 | logrus.SetLevel(logrus.DebugLevel) 116 | } 117 | switch context.GlobalString("log-format") { 118 | case "text": 119 | // retain logrus's default. 120 | case "json": 121 | logrus.SetFormatter(new(logrus.JSONFormatter)) 122 | default: 123 | return fmt.Errorf("unknown log-format %q", 124 | context.GlobalString("log-format")) 125 | } 126 | 127 | err := handleSystemLog("", "") 128 | if err != nil { 129 | return err 130 | } 131 | logrus.Debugf("Runu called with args: %v", os.Args) 132 | return nil 133 | } 134 | 135 | if err := app.Run(os.Args); err != nil { 136 | panic(err) 137 | } 138 | 139 | } 140 | -------------------------------------------------------------------------------- /test/docker-more-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 4 | 5 | if [ $TRAVIS_ARCH != "amd64" ] || [ $TRAVIS_OS_NAME != "linux" ] ; then 6 | echo "those tests contain tap device creation, which is only supported on amd64/linux instance. Skipped" 7 | exit 0 8 | fi 9 | 10 | # prepare RUNU_AUX_DIR 11 | create_runu_aux_dir 12 | 13 | DOCKER_RUN_ARGS="run --rm -i --runtime=runu-dev --net=none $DOCKER_RUN_ARGS_ARCH" 14 | 15 | # 0. tap config 16 | # 1. local json / ip addr config 17 | # 2. local json / no ip addr config 18 | # 3. no json 19 | # 4. image json 20 | # and more.. 21 | 22 | # 0. tap config 23 | TAP_IFNAME=tap0 24 | set +e 25 | set -x 26 | TAP_EXIST=`ip link |grep $TAP_IFNAME` 27 | set -e 28 | 29 | if [ -z "$TAP_EXIST" ] ; then 30 | sudo ip tuntap add $TAP_IFNAME mode tap user $USER 31 | sudo ifconfig $TAP_IFNAME up 32 | sudo brctl addif docker0 $TAP_IFNAME 33 | 34 | ip addr 35 | fi 36 | set +x 37 | 38 | # 1. local json / ip addr config 39 | fold_start test.docker.conf.1 "docker+: local json / ip addr config" 40 | cat > /tmp/lkl.json < /tmp/lkl.json < /tmp/lkl.json <`, 18 | Flags: []cli.Flag{ 19 | cli.StringFlag{ 20 | Name: "bundle, b", 21 | Value: "", 22 | Usage: `path to the root of the bundle directory, defaults to the current directory`, 23 | }, 24 | cli.StringFlag{ 25 | Name: "pid-file", 26 | Usage: "specify the file to write the process id to", 27 | }, 28 | }, 29 | Action: func(context *cli.Context) error { 30 | args := context.Args() 31 | if args.Present() == false { 32 | return fmt.Errorf("Missing container ID") 33 | } 34 | 35 | return cmdCreateUkon(context, false) 36 | }, 37 | } 38 | 39 | func prepareBootCommand(context *cli.Context, container string, childPipe *os.File, volumeMounted bool) (*exec.Cmd, error) { 40 | root := context.GlobalString("root") 41 | const stdioFdCount = 3 42 | // call `runu boot` to create new process 43 | self, err := os.Executable() 44 | if err != nil { 45 | return nil, fmt.Errorf("could not identify who am I: %v", err) 46 | } 47 | 48 | args := []string{} 49 | if val := context.GlobalString("log-format"); val != "" { 50 | args = append(args, "-log-format") 51 | args = append(args, context.GlobalString("log-format")) 52 | } 53 | if val := context.GlobalString("log"); val != "" { 54 | args = append(args, "-log") 55 | args = append(args, context.GlobalString("log")) 56 | } 57 | if val := context.GlobalString("root"); val != "" { 58 | args = append(args, "-root") 59 | args = append(args, context.GlobalString("root")) 60 | } 61 | if context.GlobalBool("debug") { 62 | args = append(args, "-debug") 63 | } 64 | args = append(args, "boot") 65 | if val := context.String("bundle"); val != "" { 66 | args = append(args, "-bundle") 67 | args = append(args, context.String("bundle")) 68 | } 69 | if val := context.String("pid-file"); val != "" { 70 | args = append(args, "-pid-file") 71 | args = append(args, context.String("pid-file")) 72 | } 73 | args = append(args, container) 74 | 75 | cmd := exec.Command(self, args...) 76 | 77 | cmd.ExtraFiles = append(cmd.ExtraFiles, childPipe) 78 | cmd.Env = append(cmd.Env, 79 | fmt.Sprintf("_LIBCONTAINER_INITPIPE=%d", stdioFdCount+len(cmd.ExtraFiles)-1), 80 | ) 81 | 82 | if volumeMounted { 83 | cmd.Env = append(cmd.Env, "LKL_USE_9PFS=1") 84 | } 85 | 86 | cwd, _ := os.Getwd() 87 | logrus.Debugf("Starting command %s, cwd=%s, root=%s", 88 | cmd.Args, cwd, root) 89 | cmd.Stdin = os.Stdin 90 | cmd.Stdout = os.Stdout 91 | cmd.Stderr = os.Stderr 92 | cmd.SysProcAttr = &syscall.SysProcAttr{ 93 | Setpgid: false, 94 | } 95 | return cmd, nil 96 | } 97 | 98 | func cmdCreateUkon(context *cli.Context, attach bool) error { 99 | root := context.GlobalString("root") 100 | bundle := context.String("bundle") 101 | container := context.Args().First() 102 | ocffile := filepath.Join(bundle, specConfig) 103 | spec, err := loadSpec(ocffile) 104 | 105 | if err != nil { 106 | return fmt.Errorf("load config failed: %v", err) 107 | } 108 | if container == "" { 109 | return fmt.Errorf("no container id provided") 110 | } 111 | 112 | err = createContainer(container, bundle, root, spec) 113 | if err != nil { 114 | return fmt.Errorf("failed to create container: %v", err) 115 | } 116 | 117 | volumeMounted, err := doMounts(spec) 118 | if err != nil { 119 | return err 120 | } 121 | // wait for init complete 122 | parentPipe, childPipe, err := os.Pipe() 123 | if err != nil { 124 | return fmt.Errorf("create: pipe failure (%s)", err) 125 | } 126 | cmd, err := prepareBootCommand(context, container, childPipe, volumeMounted) 127 | 128 | if err := cmd.Start(); err != nil { 129 | panic(err) 130 | } 131 | 132 | go func() { 133 | err = cmd.Wait() 134 | if err != nil { 135 | fmt.Printf("failed to wait a process: %s", cmd.Args) 136 | panic(err) 137 | } 138 | }() 139 | 140 | buf := make([]byte, 1) 141 | logrus.Debugf("Waiting for pipe to complete boot %s", cmd.Args) 142 | if _, err := parentPipe.Read(buf); err != nil { 143 | fmt.Printf("pipe read: %s", err) 144 | } 145 | parentPipe.Close() 146 | childPipe.Close() 147 | logrus.Debugf("Waiting pipe done") 148 | 149 | return nil 150 | } 151 | -------------------------------------------------------------------------------- /test/docker-volume-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 4 | 5 | # prepare RUNU_AUX_DIR 6 | create_runu_aux_dir 7 | 8 | DOCKER_RUN_ARGS="run --rm -i --runtime=runu-dev --net=none $DOCKER_RUN_ARGS_ARCH" 9 | 10 | if ! [ $TRAVIS_OS_NAME = "osx" ] ; then 11 | fold_start test.docker.0 "docker-volume: -v option should fail when specified with -e LKL_ROOTFS" 12 | 13 | MNT_SRC=$PWD/host_dir 14 | MNT_DST=/mnt 15 | mkdir -p $MNT_SRC 16 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 17 | -e LKL_ROOTFS=imgs/python.img \ 18 | -v $MNT_SRC:$MNT_DST \ 19 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION \ 20 | hello 2> fail_log || true 21 | cat fail_log 22 | cat fail_log | grep "OCI runtime create failed" >/dev/null 23 | rm -rf $MNT_SRC fail_log 24 | 25 | fold_end test.docker.0 26 | fi 27 | 28 | if [ $TRAVIS_OS_NAME = "linux" ] ; then 29 | 30 | fold_start test.docker.1 "docker-volume: naive -v option for directory" 31 | 32 | MNT_SRC=$PWD/host_dir 33 | MNT_DST=/mnt 34 | mkdir -p $MNT_SRC 35 | touch $MNT_SRC/foo 36 | touch $MNT_SRC/bar 37 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 38 | -v $MNT_SRC:$MNT_DST \ 39 | ${REGISTRY}ukontainer/runu-python:$DOCKER_IMG_VERSION \ 40 | python -c "import os; print(os.listdir('/mnt'))" | egrep "foo.*bar|bar.*foo" 41 | rm -rf $MNT_SRC 42 | 43 | fold_end test.docker.1 44 | 45 | fold_start test.docker.2 "docker-volume: naive -v option for file" 46 | 47 | MNT_SRC=/tmp/hello.txt 48 | MNT_DST=/mnt/hello.txt 49 | echo "hello_world" > $MNT_SRC 50 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 51 | -v $MNT_SRC:$MNT_DST \ 52 | -e RUNU_AUX_DIR=$RUNU_AUX_DIR alpine /bin/busybox cat /mnt/hello.txt | grep 'hello_world' 53 | 54 | rm -f $MNT_SRC 55 | fold_end test.docker.2 56 | 57 | fold_start test.docker.3 "docker-volume: naive -v option for named volume" 58 | 59 | docker run --rm -v named_vol:/mnt alpine touch /mnt/foo /mnt/bar 60 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 61 | -v named_vol:/mnt \ 62 | ${REGISTRY}ukontainer/runu-python:$DOCKER_IMG_VERSION \ 63 | python -c "import os; print(os.listdir('/mnt'))" | egrep "foo.*bar|bar.*foo" 64 | docker volume rm named_vol 65 | fold_end test.docker.3 66 | 67 | fold_start test.docker.4 "docker-volume: read only -v option for directory" 68 | 69 | 70 | MNT_SRC=$PWD/host_dir 71 | MNT_DST=/mnt 72 | mkdir -p $MNT_SRC 73 | touch $MNT_SRC/foo 74 | touch $MNT_SRC/bar 75 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 76 | -v $MNT_SRC:$MNT_DST:ro \ 77 | ${REGISTRY}ukontainer/runu-python:$DOCKER_IMG_VERSION \ 78 | python -c "import os; print(os.listdir('/mnt'))" | egrep "foo.*bar|bar.*foo" 79 | 80 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 81 | -v $MNT_SRC:$MNT_DST:ro \ 82 | ${REGISTRY}ukontainer/runu-python:$DOCKER_IMG_VERSION \ 83 | python -c "f=open('${MNT_DST}/hello.txt', 'w'), print('hello',file=f);close(f)" 2> fail_log || true 84 | cat fail_log 85 | cat fail_log | grep "OSError" >/dev/null 86 | rm -rf $MNT_SRC 87 | 88 | 2> fail_log || true 89 | 90 | fold_end test.docker.4 91 | 92 | fold_start test.docker.5 "docker-volume: read only -v option for file" 93 | MNT_SRC=/tmp/hello.txt 94 | MNT_DST=/mnt/hello.txt 95 | echo "hello_world" > $MNT_SRC 96 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 97 | -v $MNT_SRC:$MNT_DST:ro \ 98 | -e RUNU_AUX_DIR=$RUNU_AUX_DIR alpine /bin/busybox cat $MNT_DST | grep 'hello_world' 99 | 100 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 101 | -v $MNT_SRC:$MNT_DST:ro \ 102 | ${REGISTRY}ukontainer/runu-python:$DOCKER_IMG_VERSION \ 103 | python -c "f=open('${MNT_DST}', 'w'), print('hello',file=f);close(f)" 2> fail_log || true 104 | cat fail_log 105 | cat fail_log | grep "OSError" >/dev/null 106 | rm -f $MNT_SRC 107 | fold_end test.docker.5 108 | 109 | fold_start test.docker.6 "docker-volume: read only -v option for named volume" 110 | 111 | docker run --rm -v named_vol:/mnt alpine touch /mnt/foo /mnt/bar 112 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 113 | -v named_vol:/mnt:ro \ 114 | ${REGISTRY}ukontainer/runu-python:$DOCKER_IMG_VERSION \ 115 | python -c "import os; print(os.listdir('/mnt'))" | egrep "foo.*bar|bar.*foo" 116 | 117 | docker $DOCKER_RUN_ARGS -e RUMP_VERBOSE=1 \ 118 | -v named_vol:/mnt:ro \ 119 | ${REGISTRY}ukontainer/runu-python:$DOCKER_IMG_VERSION \ 120 | python -c "f=open('/mnt/hello.txt', 'w'), print('hello',file=f);close(f)" 2> fail_log || true 121 | cat fail_log 122 | cat fail_log | grep "OSError" >/dev/null 123 | 124 | docker volume rm named_vol 125 | 126 | fold_end test.docker.6 127 | fi 128 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: bionic 2 | language: go 3 | go: 4 | - "1.13.x" 5 | 6 | cache: 7 | directories: 8 | - $HOME/.ccache 9 | 10 | env: 11 | global: 12 | - PACKAGE_NAME=docker-runu 13 | - RELEASE_VERSION=0.2 14 | - BINTRAY_REPO_NAME=debian 15 | - BINTRAY_ORG=ukontainer 16 | - BINTRAY_LICENSE=Apache-2.0 17 | - GO111MODULE=on 18 | 19 | jobs: 20 | include: 21 | - os: linux 22 | cache: 23 | directories: 24 | - $HOME/.cache/go-build 25 | env: 26 | - DEB_ARCH=amd64 27 | before_install: 28 | - sudo apt-get install jq bridge-utils 29 | - os: linux 30 | cache: 31 | directories: 32 | - $HOME/.cache/go-build 33 | arch: arm64 34 | env: 35 | - DEB_ARCH=armhf 36 | before_install: 37 | - sudo dpkg --add-architecture armhf 38 | - sudo apt-get update && sudo apt-get install jq libc6:armhf crossbuild-essential-armhf 39 | - export CC=arm-linux-gnueabihf-gcc 40 | - export CGO_ENABLED=1 41 | - export GOARCH=arm 42 | - export RUNU_PATH="linux_arm/" 43 | - export PATH=$GOPATH/bin/linux_arm:$PATH 44 | - export DOCKER_RUN_ARGS_ARCH="--platform=linux/arm" 45 | - os: linux 46 | cache: 47 | directories: 48 | - $HOME/.cache/go-build 49 | arch: arm64 50 | env: 51 | - DEB_ARCH=arm64 52 | before_install: 53 | - sudo apt-get update && sudo apt-get install jq 54 | - export CGO_ENABLED=1 55 | - export GOARCH=arm64 56 | - os: osx 57 | osx_image: xcode12.5 58 | cache: 59 | directories: 60 | - $HOME/Library/Caches/go-build 61 | before_install: 62 | - HOMEBREW_NO_AUTO_UPDATE=1 brew install jq 63 | - HOMEBREW_NO_AUTO_UPDATE=1 brew install ukontainer/lkl/dockerd-darwin 64 | - HOMEBREW_NO_AUTO_UPDATE=1 brew install ukontainer/lkl/nerdctl 65 | - mkdir -p ~/.local/bin 66 | - export PATH=/usr/local/opt/ccache/libexec:$HOME/.local/bin:$PATH 67 | - ln -sf /usr/local/bin/gsha256sum ~/.local/bin/sha256sum 68 | 69 | #- os: osx 70 | # osx_image: xcode10.1 71 | # cache: 72 | # directories: 73 | # - $HOME/Library/Caches/go-build 74 | # before_install: 75 | # - HOMEBREW_NO_AUTO_UPDATE=1 brew install jq 76 | # - HOMEBREW_NO_AUTO_UPDATE=1 brew cask info tuntap 77 | # - HOMEBREW_NO_AUTO_UPDATE=1 brew install ukontainer/lkl/dockerd-darwin 78 | # - mkdir -p ~/.local/bin 79 | # - export PATH=/usr/local/opt/ccache/libexec:$HOME/.local/bin:$PATH 80 | # - ln -sf /usr/local/bin/gsha256sum ~/.local/bin/sha256sum 81 | #- os: osx 82 | # osx_image: xcode10.2 83 | # cache: 84 | # directories: 85 | # - $HOME/Library/Caches/go-build 86 | # before_install: 87 | # - HOMEBREW_NO_AUTO_UPDATE=1 brew install jq 88 | # - HOMEBREW_NO_AUTO_UPDATE=1 brew cask info tuntap 89 | # - HOMEBREW_NO_AUTO_UPDATE=1 brew install ukontainer/lkl/dockerd-darwin 90 | # - mkdir -p ~/.local/bin 91 | # - export PATH=/usr/local/opt/ccache/libexec:$HOME/.local/bin:$PATH 92 | # - ln -sf /usr/local/bin/gsha256sum ~/.local/bin/sha256sum 93 | 94 | before_script: 95 | - export -f travis_nanoseconds 96 | - export -f travis_fold 97 | - export -f travis_time_start 98 | - export -f travis_time_finish 99 | - export GO111MODULE=auto 100 | - go get -u github.com/gojp/goreportcard/cmd/goreportcard-cli 101 | - GO111MODULE=off go get -u github.com/alecthomas/gometalinter 102 | - go get -u github.com/gordonklaus/ineffassign 103 | - go get -u github.com/fzipp/gocyclo/cmd/gocyclo 104 | - go get -u github.com/client9/misspell/cmd/misspell 105 | - go get -u golang.org/x/lint/golint 106 | # Export variables containing versions and filename 107 | - export BUILD_VERSION=$RELEASE_VERSION.$TRAVIS_BUILD_NUMBER 108 | - export BUILD_DATE=$(date "+%Y%m%d") 109 | - export PACKAGE_NAME_VERSION=$PACKAGE_NAME-$BUILD_VERSION-$DEB_ARCH.deb 110 | # TODO: for dockerd which use runtime name 'io.containerd.runtime.v1.linux' 111 | - if [ $TRAVIS_OS_NAME = "osx" ] ; then cp $GOPATH/bin/containerd-shim-runu-v1 ~/.local/bin/containerd-shim-v1-linux ; fi 112 | 113 | script: 114 | - if [ $TRAVIS_ARCH = "amd64" ] && [ $TRAVIS_OS_NAME = "linux" ] ; then GO111MODULE=on goreportcard-cli -t 100.0 -v ; fi 115 | - bash -e test/standalone-test.sh 116 | - bash -e test/docker-oci-test.sh 117 | - bash -e test/containerd-ctr-test.sh 118 | - bash -e test/docker-more-test.sh 119 | - bash -e test/docker-volume-test.sh 120 | - bash -e test/k8s-test.sh 121 | 122 | after_failure: 123 | - cat /tmp/dockerd.log 124 | - cat /tmp/containerd.log 125 | - cat /tmp/docker-manifest.log 126 | 127 | 128 | ## package deploy to github release (TODO: look for bintray alt.) 129 | before_deploy: 130 | - bash -ex pkg/pre-deploy-deb.sh 131 | - bash -ex pkg/pre-deploy-test-deb.sh 132 | 133 | deploy: 134 | provider: releases 135 | api_key: $GITHUB_TOKEN 136 | file: $PACKAGE_NAME_VERSION 137 | skip_cleanup: true 138 | on: 139 | condition: $TRAVIS_OS_NAME = linux 140 | branch: master 141 | 142 | after_deploy: 143 | - bash -ex pkg/post-deploy-deb.sh 144 | - bash -ex pkg/post-deploy-kind-img.sh 145 | -------------------------------------------------------------------------------- /test/docker-oci-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . $(dirname "${BASH_SOURCE[0]}")/common.sh 4 | 5 | DOCKER_RUN_ARGS="run --rm -i -e RUMP_VERBOSE=1 -e DEBUG=1 --runtime=runu-dev --net=none $DOCKER_RUN_ARGS_ARCH" 6 | 7 | # prepare RUNU_AUX_DIR 8 | create_runu_aux_dir 9 | 10 | # update daemon.json 11 | fold_start test.dockerd.0 "boot dockerd" 12 | if [ $TRAVIS_OS_NAME = "linux" ] ; then 13 | 14 | (sudo cat /etc/docker/daemon.json 2>/dev/null || echo '{}') | \ 15 | jq '.runtimes."runu-dev" |= {"path":"'`which runu`'","runtimeArgs":[]}' | \ 16 | jq '. |= .+{"experimental":true}' | \ 17 | jq '. |= .+{"ipv6":true}' | \ 18 | jq '. |= .+{"fixed-cidr-v6": "2001:db8::/64"}' | \ 19 | tee /tmp/tmp.json 20 | sudo mv /tmp/tmp.json /etc/docker/daemon.json 21 | sudo service docker restart 22 | 23 | elif [ $TRAVIS_OS_NAME = "osx" ] ; then 24 | 25 | set -x 26 | sudo mkdir -p /etc/docker/ 27 | git clone https://gist.github.com/aba357f73da4e14bc3f5cbeb00aeaea4.git \ 28 | /tmp/containerd-config-dockerd || true 29 | sudo cp /tmp/containerd-config-dockerd/daemon.json /etc/docker/ 30 | 31 | # prepare dockerd 32 | mkdir -p /tmp/containerd-shim 33 | sudo killall containerd || true 34 | sudo dockerd --config-file /etc/docker/daemon.json > /tmp/dockerd.log 2>&1 & 35 | sleep 3 36 | sudo chmod 666 /tmp/var/run/docker.sock 37 | sudo chmod 777 /tmp/var/run/ 38 | sudo ln -sf /tmp/var/run/docker.sock /var/run/docker.sock 39 | 40 | # build docker (client) 41 | if [ -z "$(which docker)" ] ; then 42 | curl -O https://download.docker.com/mac/static/stable/x86_64/docker-18.09.0.tgz 43 | tar xfz docker-18.09.0.tgz 44 | cp -f docker/docker ~/.local/bin 45 | chmod +x ~/.local/bin/docker 46 | fi 47 | 48 | docker version 49 | docker info 50 | 51 | DOCKER_RUN_EXT_ARGS="--platform=linux/amd64 -e LKL_USE_9PFS=1" 52 | # XXX: this is required when we use 9pfs rootfs (e.g., on mac) 53 | # see #3 issue more detail https://github.com/ukontainer/runu/issues/3 54 | ALPINE_PREFIX="/bin/busybox" 55 | fi 56 | fold_end test.dockerd.0 "" 57 | 58 | # test hello-world 59 | fold_start test.docker.0 "docker hello" 60 | docker $DOCKER_RUN_ARGS ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION hello 61 | fold_end test.docker.0 62 | 63 | # test ping 64 | fold_start test.docker.1 "docker ping" 65 | docker $DOCKER_RUN_ARGS ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION \ 66 | ping -c5 127.0.0.1 67 | fold_end test.docker.1 68 | 69 | # test python 70 | # XXX: PYTHONHASHSEED=1 is workaround for slow read of getrandom() on 4.19 71 | # (4.16 doesn't have such) 72 | fold_start test.docker.2 "docker python" 73 | docker $DOCKER_RUN_ARGS -e HOME=/ \ 74 | -e PYTHONHOME=/python -e LKL_ROOTFS=imgs/python.img \ 75 | -e PYTHONHASHSEED=1 \ 76 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION \ 77 | python -c "print(\"hello world from python(docker-runu)\")" 78 | fold_end test.docker.2 79 | 80 | # osx cannot use 9pfs as rootfs (issue #4) 81 | if [ $TRAVIS_OS_NAME = "linux" ] ; then 82 | fold_start test.docker.2.2 "docker python-slim" 83 | docker $DOCKER_RUN_ARGS \ 84 | ${REGISTRY}ukontainer/runu-python:$DOCKER_IMG_VERSION-slim \ 85 | python -c "print(\"hello world from python(docker-runu)\")" 86 | fold_end test.docker.2.2 87 | fi 88 | 89 | # test nginx 90 | fold_start test.docker.3 "docker nginx" 91 | CID=`docker $DOCKER_RUN_ARGS -d \ 92 | -e LKL_ROOTFS=imgs/data.iso \ 93 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION \ 94 | nginx` 95 | sleep 2 96 | docker ps -a 97 | docker logs $CID 98 | docker kill $CID 99 | fold_end test.docker.3 100 | 101 | fold_start test.docker.3.2 "docker nginx-slim" 102 | CID=`docker $DOCKER_RUN_ARGS -d \ 103 | -e LKL_ROOTFS=imgs/data.iso \ 104 | ${REGISTRY}ukontainer/runu-nginx:$DOCKER_IMG_VERSION-slim \ 105 | nginx` 106 | sleep 2 107 | docker ps -a 108 | docker logs $CID 109 | docker kill $CID 110 | fold_end test.docker.3.2 111 | 112 | 113 | # alipine image test 114 | fold_start test.docker.4 "docker alpine" 115 | docker $DOCKER_RUN_ARGS $DOCKER_RUN_EXT_ARGS \ 116 | -e RUNU_AUX_DIR=$RUNU_AUX_DIR alpine $ALPINE_PREFIX uname -a 117 | 118 | docker $DOCKER_RUN_ARGS $DOCKER_RUN_EXT_ARGS \ 119 | -e RUNU_AUX_DIR=$RUNU_AUX_DIR alpine $ALPINE_PREFIX \ 120 | ping -c 5 127.0.0.1 121 | 122 | docker $DOCKER_RUN_ARGS $DOCKER_RUN_EXT_ARGS \ 123 | -e RUNU_AUX_DIR=$RUNU_AUX_DIR alpine $ALPINE_PREFIX dmesg | head 124 | 125 | docker $DOCKER_RUN_ARGS $DOCKER_RUN_EXT_ARGS \ 126 | -e RUNU_AUX_DIR=$RUNU_AUX_DIR alpine $ALPINE_PREFIX ls -l / 127 | 128 | if [ $TRAVIS_OS_NAME = "linux" ] ; then 129 | # XXX: df -ha gives core dumps. remove above if statement this once fixed 130 | docker $DOCKER_RUN_ARGS $DOCKER_RUN_EXT_ARGS \ 131 | -e RUNU_AUX_DIR=$RUNU_AUX_DIR alpine $ALPINE_PREFIX df -ha 132 | fi 133 | fold_end test.docker.4 134 | 135 | # test named 136 | fold_start test.docker.5 "docker named" 137 | CID=named-docker 138 | docker $DOCKER_RUN_ARGS -d --name $CID \ 139 | -e LKL_ROOTFS=imgs/named.img \ 140 | ${REGISTRY}ukontainer/runu-base:$DOCKER_IMG_VERSION \ 141 | named -c /etc/bind/named.conf -g 142 | 143 | sleep 10 144 | docker ps -a 145 | docker logs $CID 146 | docker kill $CID 147 | fold_end test.docker.5 148 | -------------------------------------------------------------------------------- /devices_linux.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "C" 5 | "net" 6 | "os" 7 | "strings" 8 | "syscall" 9 | "unsafe" 10 | 11 | "github.com/sirupsen/logrus" 12 | "golang.org/x/sys/unix" 13 | ) 14 | 15 | // Device information for linux 16 | var ( 17 | DefaultDevices = []*Device{ 18 | // /dev/urandom 19 | { 20 | Path: "/dev/urandom", 21 | Type: 'c', 22 | Major: 1, 23 | Minor: 9, 24 | Permissions: "rwm", 25 | FileMode: 0666, 26 | }, 27 | // /dev/net/tun 28 | { 29 | Path: "/dev/net/tun", 30 | Type: 'c', 31 | Major: 10, 32 | Minor: 200, 33 | Permissions: "rwm", 34 | FileMode: 0666, 35 | }, 36 | } 37 | ) 38 | 39 | const ( 40 | tunDevice = "/dev/net/tun" 41 | tunFCsum = 0x01 42 | tunFTso4 = 0x02 43 | virtioNetHdrSize = 12 44 | ) 45 | 46 | const ( 47 | _ETHTOOL_GTXCSUM = 0x00000016 // linux/ethtool.h 48 | _ETHTOOL_STXCSUM = 0x00000017 // linux/ethtool.h 49 | ) 50 | 51 | type ifReq struct { 52 | Name [syscall.IFNAMSIZ]byte 53 | Flags uint16 54 | } 55 | 56 | type ifReqData struct { 57 | Name [syscall.IFNAMSIZ]byte 58 | Data uintptr 59 | } 60 | 61 | // linux/ethtool.h 'struct ethtool_value' 62 | type ethtoolValue struct { 63 | Cmd uint32 64 | Data uint32 65 | } 66 | 67 | func openNetTapFd(ifname string, specEnv []string) (*os.File, bool) { 68 | var ifr ifReq 69 | var vnetHdrSz int 70 | var offload string 71 | 72 | for _, v := range specEnv { 73 | if strings.HasPrefix(v, "LKL_OFFLOAD=") { 74 | offload = "1" 75 | break 76 | } 77 | } 78 | 79 | tapDev, err := os.OpenFile(tunDevice, os.O_RDWR|unix.O_NONBLOCK, 0666) 80 | if err != nil { 81 | logrus.Errorf("open %s error: %s\n", tunDevice, err) 82 | panic(err) 83 | } 84 | 85 | copy(ifr.Name[:(syscall.IFNAMSIZ-1)], ifname) 86 | ifr.Flags = syscall.IFF_TAP | syscall.IFF_NO_PI 87 | 88 | if offload != "" { 89 | ifr.Flags |= syscall.IFF_VNET_HDR 90 | vnetHdrSz = virtioNetHdrSize 91 | } 92 | 93 | _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, 94 | uintptr(tapDev.Fd()), 95 | uintptr(syscall.TUNSETIFF), 96 | uintptr(unsafe.Pointer(&ifr)), 97 | ) 98 | if errno != 0 { 99 | panic(errno) 100 | } 101 | 102 | if offload != "" { 103 | /* XXX: offload feature should be configurable */ 104 | _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, 105 | uintptr(tapDev.Fd()), 106 | uintptr(syscall.TUNSETVNETHDRSZ), 107 | uintptr(unsafe.Pointer(&vnetHdrSz)), 108 | ) 109 | if errno != 0 { 110 | panic(errno) 111 | } 112 | 113 | tapArg := tunFCsum | tunFTso4 114 | _, _, errno = syscall.Syscall(syscall.SYS_IOCTL, 115 | uintptr(tapDev.Fd()), 116 | uintptr(syscall.TUNSETOFFLOAD), 117 | uintptr(tapArg), 118 | ) 119 | if errno != 0 { 120 | panic(errno) 121 | } 122 | } 123 | 124 | return tapDev, true 125 | } 126 | 127 | func htons(i uint16) uint16 { 128 | return (i<<8)&0xff00 | i>>8 129 | } 130 | 131 | func disableTxCsumOffloadForRawsock(ifname string) error { 132 | // XXX: disable tx csum offload (may need vnet_hdr impl in raw sock) 133 | iocSock, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0) 134 | if err != nil { 135 | logrus.Errorf("socket for ioctl failure error: %s\n", err) 136 | panic(err) 137 | } 138 | defer syscall.Close(iocSock) 139 | 140 | value := ethtoolValue{Cmd: _ETHTOOL_STXCSUM, Data: 0} 141 | request := ifReqData{Data: uintptr(unsafe.Pointer(&value))} 142 | copy(request.Name[:], ifname) 143 | 144 | _, _, errno := syscall.RawSyscall(syscall.SYS_IOCTL, uintptr(iocSock), 145 | uintptr(unix.SIOCETHTOOL), uintptr(unsafe.Pointer(&request))) 146 | if errno != 0 { 147 | logrus.Errorf("disabling csum offload failure: %d\n", errno) 148 | panic("ETHTOOL_STXCSUM") 149 | } 150 | 151 | value.Cmd = _ETHTOOL_GTXCSUM 152 | _, _, errno = syscall.RawSyscall(syscall.SYS_IOCTL, uintptr(iocSock), 153 | uintptr(unix.SIOCETHTOOL), uintptr(unsafe.Pointer(&request))) 154 | if errno != 0 { 155 | logrus.Errorf("disabling csum offload failure: %d\n", errno) 156 | panic("ETHTOOL_GTXCSUM") 157 | } 158 | logrus.Debugf("ifreq rx csum flag is %d\n", value.Data) 159 | 160 | return nil 161 | } 162 | 163 | func openNetRawsockFd(ifname string, specEnv []string) (*os.File, bool) { 164 | fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, 165 | int(htons(syscall.ETH_P_ALL))) 166 | if err != nil { 167 | logrus.Errorf("open raw socket error: %s\n", err) 168 | panic(err) 169 | } 170 | 171 | // bind to ifname 172 | ifi, err := net.InterfaceByName(ifname) 173 | if err != nil { 174 | logrus.Errorf("can't find interface %s: %s\n", ifname, err) 175 | panic(err) 176 | } 177 | 178 | sa := &unix.SockaddrLinklayer{ 179 | Protocol: htons(unix.ETH_P_ALL), 180 | Ifindex: ifi.Index, 181 | } 182 | err = unix.Bind(fd, sa) 183 | if err != nil { 184 | logrus.Errorf("can't bind to interface %s: %s\n", ifname, err) 185 | panic(err) 186 | } 187 | 188 | // set promisc 189 | mreq := unix.PacketMreq{ 190 | Ifindex: int32(ifi.Index), 191 | Type: unix.PACKET_MR_PROMISC, 192 | } 193 | if err = unix.SetsockoptPacketMreq(int(fd), unix.SOL_PACKET, 194 | unix.PACKET_ADD_MEMBERSHIP, &mreq); err != nil { 195 | logrus.Errorf("set nonblocking error: %s\n", err) 196 | panic(err) 197 | } 198 | 199 | // vnethdr 200 | var on uint64 201 | if err = unix.SetsockoptUint64(int(fd), unix.SOL_PACKET, 202 | unix.PACKET_VNET_HDR, on); err != nil { 203 | logrus.Errorf("set vnethdr sockopt error: %s\n", err) 204 | panic(err) 205 | } 206 | 207 | // set nonblock sock 208 | if err = unix.SetNonblock(fd, true); err != nil { 209 | logrus.Errorf("set nonblocking error: %s\n", err) 210 | panic(err) 211 | } 212 | 213 | return os.NewFile(uintptr(fd), "eth-packet-socket"), true 214 | } 215 | 216 | func openNetFd(ifname string, specEnv []string) (*os.File, bool) { 217 | if strings.HasPrefix(ifname, "eth") { 218 | return openNetRawsockFd(ifname, specEnv) 219 | } else if strings.HasPrefix(ifname, "tap") { 220 | return openNetTapFd(ifname, specEnv) 221 | } 222 | 223 | return nil, false 224 | } 225 | -------------------------------------------------------------------------------- /utils_linux.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "strings" 7 | 8 | "github.com/opencontainers/runtime-spec/specs-go" 9 | "github.com/sirupsen/logrus" 10 | "github.com/vishvananda/netlink" 11 | "github.com/vishvananda/netns" 12 | ) 13 | 14 | var runuAuxFileDir = "/usr/lib/runu" 15 | 16 | func getVethHost(spec *specs.Spec) *net.Interface { 17 | var netnsPath string 18 | 19 | if spec.Linux != nil { 20 | for _, v := range spec.Linux.Namespaces { 21 | if v.Type == specs.NetworkNamespace { 22 | netnsPath = v.Path 23 | break 24 | } 25 | } 26 | } 27 | // should look like '/var/run/netns/cni-6d46d1b2-c836-3b4e-87a0-88641242aa5e' 28 | if strings.Index(netnsPath, "/var/run/netns/") == -1 { 29 | return nil 30 | } 31 | 32 | origns, _ := netns.Get() 33 | defer origns.Close() 34 | 35 | netnsPath = strings.Replace(netnsPath, "/var/run/netns/", "", 1) 36 | 37 | nsh, err := netns.GetFromName(netnsPath) 38 | if err != nil { 39 | logrus.Errorf("unable to get netns handle %s(%s)", 40 | netnsPath, err) 41 | return nil 42 | } 43 | 44 | if err := netns.Set(nsh); err != nil { 45 | logrus.Errorf("unable to get set netns %s", err) 46 | return nil 47 | } 48 | 49 | // Look for the ifindex of veth pair of the host interface 50 | vEth, err := netlink.LinkByName("eth0") 51 | if err != nil { 52 | logrus.Errorf("unable to get guest veth %s", err) 53 | return nil 54 | } 55 | 56 | ifIndex, err := netlink.VethPeerIndex(&netlink.Veth{LinkAttrs: *vEth.Attrs()}) 57 | if err != nil { 58 | logrus.Errorf("unable to get ifindex of veth pair %s", err) 59 | return nil 60 | } 61 | 62 | // Switch back to the original namespace 63 | netns.Set(origns) 64 | iface, err := net.InterfaceByIndex(ifIndex) 65 | if err != nil { 66 | logrus.Errorf("unable to get interface of veth pair (idx=%d)(%s)", 67 | ifIndex, err) 68 | return nil 69 | } 70 | 71 | return iface 72 | } 73 | 74 | func retrieveRouteInfo() ([]lklRoute, error) { 75 | var routeInfo []lklRoute 76 | // listing routes 77 | // We only care about eth0 78 | link, err := netlink.LinkByName("eth0") 79 | if err != nil { 80 | return nil, fmt.Errorf("cannot find link eth0: (%v)", err) 81 | } 82 | routes, err := netlink.RouteList(link, netlink.FAMILY_V4) 83 | if err != nil { 84 | return nil, fmt.Errorf("cannot list routes: (%v)", err) 85 | } 86 | 87 | for _, route := range routes { 88 | var r lklRoute 89 | 90 | // skip default gateway 91 | if route.Dst == nil { 92 | continue 93 | } 94 | logrus.Debugf("route= %+v", route) 95 | r.Destination = route.Dst.String() 96 | if route.Gw != nil { 97 | r.Nexthop = route.Gw.String() 98 | } else { 99 | // onlink route 100 | ifgw, err := netlink.LinkByIndex(route.LinkIndex) 101 | if err != nil { 102 | logrus.Infof("incomplete route (%+v)", route) 103 | } 104 | r.Nexthop = ifgw.Attrs().Name 105 | } 106 | routeInfo = append(routeInfo, r) 107 | } 108 | return routeInfo, nil 109 | } 110 | 111 | // XXX: Only treat ipv4 information 112 | func getVethInfo(spec *specs.Spec) (*lklIfInfo, []lklRoute, error) { 113 | ifInfo := new(lklIfInfo) 114 | 115 | defer func() { 116 | logrus.Infof("ifInfo %+v", ifInfo) 117 | }() 118 | 119 | // default gateway 120 | v4gw, err := netlink.RouteGet(net.ParseIP("8.8.8.8")) 121 | if err != nil { 122 | return nil, nil, fmt.Errorf("Could not determine single default route (got %v)", 123 | len(v4gw)) 124 | } 125 | ifInfo.v4Gw = v4gw[0].Gw 126 | logrus.Infof("gw %v", v4gw) 127 | 128 | routeInfo, err := retrieveRouteInfo() 129 | if err != nil { 130 | return nil, nil, fmt.Errorf("Error during route retrival (got %v)", err) 131 | } 132 | 133 | ifaces, _ := net.Interfaces() 134 | logrus.Debugf("ifaces= %+v", ifaces) 135 | for _, iface := range ifaces { 136 | if iface.Name != "eth0" { 137 | continue 138 | } 139 | 140 | allAddrs, _ := iface.Addrs() 141 | for _, ifaddr := range allAddrs { 142 | ipNet, ok := ifaddr.(*net.IPNet) 143 | if !ok { 144 | return nil, nil, fmt.Errorf("address is not IPNet: %+v", ifaddr) 145 | } 146 | 147 | logrus.Infof("ifaddr= %s, ipnet=%s, gw=%s", ifaddr, ipNet, ifInfo.v4Gw) 148 | // XXX: We only treat IPv4 at the moment 149 | // may need to work for IPv6 support 150 | if ipNet.IP.To4() == nil { 151 | continue 152 | } 153 | 154 | ifInfo.ifAddrs = append(ifInfo.ifAddrs, *ipNet) 155 | ifInfo.ifName = "eth0" 156 | 157 | // Get the link for the interface. 158 | ifaceLink, err := netlink.LinkByName(iface.Name) 159 | if err != nil { 160 | return nil, nil, fmt.Errorf("getting link for interface %q: %v", iface.Name, err) 161 | } 162 | 163 | // Steal IP address from NIC. 164 | r4addr, err := netlink.ParseAddr(ipNet.String()) 165 | if err != nil { 166 | return nil, nil, fmt.Errorf("parse address error %v: %v", iface.Name, err) 167 | } 168 | 169 | logrus.Debugf("r4addr= %s", r4addr) 170 | // XXX: delete only the main container (works fine but dunno ?) 171 | if spec.Process.Args[0] != "/pause" { 172 | if err := netlink.AddrDel(ifaceLink, r4addr); err != nil { 173 | return nil, nil, fmt.Errorf("removing address %v from device %q: %v", 174 | iface.Name, ipNet, err) 175 | } 176 | } 177 | break 178 | } 179 | break 180 | } 181 | 182 | return ifInfo, routeInfo, nil 183 | } 184 | 185 | func setupNetwork(spec *specs.Spec) (*lklIfInfo, []lklRoute, error) { 186 | var netnsPath string 187 | 188 | // disable HW offload if raw socket 189 | vethHostIf := getVethHost(spec) 190 | // only pause container on k8s retures non-nil value 191 | if vethHostIf != nil { 192 | // XXX: this disable can be eliminated when vnethdr on raw sock 193 | // is implemented. 194 | disableTxCsumOffloadForRawsock(vethHostIf.Name) 195 | } 196 | 197 | if spec.Linux != nil { 198 | for _, v := range spec.Linux.Namespaces { 199 | if v.Type == specs.NetworkNamespace { 200 | netnsPath = v.Path 201 | break 202 | } 203 | } 204 | } 205 | 206 | // if there is no path, then runu assumes 207 | // it runs on docker (not on k8s) 208 | if netnsPath == "" { 209 | logrus.Infof("no netns detected: no addr configuration, skipped") 210 | return nil, nil, nil 211 | } 212 | 213 | logrus.Infof("nspath= %s", netnsPath) 214 | nsh, err := netns.GetFromPath(netnsPath) 215 | if err != nil { 216 | return nil, nil, fmt.Errorf("unable to get netns handle %s", err) 217 | } 218 | 219 | if err := netns.Set(nsh); err != nil { 220 | return nil, nil, fmt.Errorf("unable to get set netns %s", err) 221 | } 222 | 223 | // now traverse in netns 224 | return getVethInfo(spec) 225 | } 226 | -------------------------------------------------------------------------------- /utils_linux_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net" 8 | "os" 9 | "testing" 10 | "time" 11 | 12 | "github.com/opencontainers/runtime-spec/specs-go" 13 | "github.com/sirupsen/logrus" 14 | "github.com/vishvananda/netlink" 15 | "github.com/vishvananda/netns" 16 | ) 17 | 18 | const ( 19 | nsName1 = "runu-test1" 20 | vethHost = "runu-host" 21 | vethPeer = "runu-peer" 22 | vethPeerLast = "eth0" 23 | ) 24 | 25 | var ( 26 | origNs netns.NsHandle 27 | ) 28 | 29 | func createVethPair(t *testing.T) { 30 | veth := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: vethHost}, 31 | PeerName: vethPeer} 32 | t.Logf("veth = %v", veth) 33 | 34 | // add link 35 | if err := netlink.LinkAdd(veth); err != nil { 36 | t.Fatal(err) 37 | } 38 | 39 | } 40 | 41 | func configGuest(t *testing.T, newns netns.NsHandle, v4Addr, v4Gw string) { 42 | vethG, err := netlink.LinkByName(vethPeer) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | if err := netlink.LinkSetNsFd(vethG, int(newns)); err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | // revert to newns 51 | if err := netns.Set(newns); err != nil { 52 | t.Fatal(err) 53 | } 54 | 55 | // rename veth guest 56 | if err := netlink.LinkSetName(vethG, vethPeerLast); err != nil { 57 | t.Fatal(err) 58 | } 59 | 60 | if err := netlink.LinkSetUp(vethG); err != nil { 61 | t.Fatal(err) 62 | } 63 | 64 | // assign IP address to veth guest 65 | // add ipv4 addr 66 | addr, _ := netlink.ParseAddr(v4Addr) 67 | if err := netlink.AddrAdd(vethG, addr); err != nil { 68 | t.Fatal(err) 69 | } 70 | t.Logf("IPv4 address: %v\n", addr) 71 | 72 | // add ipv4 default gw 73 | gw, err := netlink.ParseAddr(v4Gw) 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | 78 | // XXX: calico case 79 | // if v4Gw and v4Addr are in same subnet, it adds 80 | // an onlink route in addition to default route 81 | if !addr.Contains(gw.IPNet.IP) { 82 | t.Logf("gw is not in v4addr mask (%s: %s)", addr, gw.IPNet.IP) 83 | 84 | route := netlink.Route{ 85 | LinkIndex: vethG.Attrs().Index, 86 | Dst: gw.IPNet, 87 | } 88 | if err := netlink.RouteAdd(&route); err != nil { 89 | t.Fatal(err) 90 | } 91 | t.Logf("Gateway(extra): %v\n", route) 92 | } 93 | route := netlink.Route{ 94 | LinkIndex: vethG.Attrs().Index, 95 | Gw: gw.IPNet.IP, 96 | Flags: int(netlink.FLAG_ONLINK), 97 | Dst: nil, 98 | } 99 | if err := netlink.RouteAdd(&route); err != nil { 100 | t.Logf("Failure/Gateway: %v\n", route) 101 | t.Fatalf("adding route failed: %s", err) 102 | } 103 | t.Logf("Gateway: %v\n", route) 104 | 105 | // print created interfaces 106 | ifaces, _ := net.Interfaces() 107 | t.Logf("Interfaces: %v\n", ifaces) 108 | 109 | addrs, err := netlink.AddrList(vethG, netlink.FAMILY_ALL) 110 | if err != nil { 111 | t.Fatal(err) 112 | } 113 | t.Logf("Address: %v\n", addrs) 114 | 115 | } 116 | 117 | func createNetNs(t *testing.T, v4Addr, v4Gw string) { 118 | // Clean up previous names 119 | destroyNetNs(t) 120 | 121 | // create eth0 pair 122 | createVethPair(t) 123 | 124 | // store origin ns 125 | origns, err := netns.Get() 126 | if err != nil { 127 | t.Fatal(err) 128 | } 129 | 130 | origNs = origns 131 | 132 | // ip netns add NAME 133 | newns, err := netns.NewNamed(nsName1) 134 | if err != nil { 135 | t.Fatal(err) 136 | } 137 | 138 | parent, err := netns.Get() 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | 143 | t.Logf("netns = %v", newns) 144 | t.Logf("parent netns = %v", parent) 145 | t.Logf("origin netns = %v", origns) 146 | 147 | // set ns to veth guest 148 | if err := netns.Set(origns); err != nil { 149 | t.Fatal(err) 150 | } 151 | 152 | // Link up 153 | vethH, err := netlink.LinkByName(vethHost) 154 | if err != nil { 155 | t.Fatal(err) 156 | } 157 | if err := netlink.LinkSetUp(vethH); err != nil { 158 | t.Fatal(err) 159 | } 160 | 161 | configGuest(t, newns, v4Addr, v4Gw) 162 | 163 | } 164 | 165 | func destroyNetNs(t *testing.T) { 166 | t.Log("Cleaning up the netns") 167 | 168 | ns, err := netns.Get() 169 | if err != nil { 170 | t.Fatal(err) 171 | } 172 | ns.Close() 173 | netns.DeleteNamed(nsName1) 174 | 175 | // restore origin ns 176 | netns.Set(origNs) 177 | 178 | // delete veth pair 179 | vethH, err := netlink.LinkByName(vethHost) 180 | if err != nil { 181 | t.Log(err) 182 | return 183 | } 184 | 185 | if err := netlink.LinkDel(vethH); err != nil { 186 | t.Fatal(err) 187 | } 188 | } 189 | 190 | func testSetupSpec(t *testing.T) *specs.Spec { 191 | spec := Example() 192 | 193 | t.Logf("spec = %v", spec.Linux) 194 | 195 | spec.Linux.Namespaces = []specs.LinuxNamespace{ 196 | { 197 | Type: specs.NetworkNamespace, 198 | Path: fmt.Sprintf("/var/run/netns/%s", nsName1), 199 | }, 200 | } 201 | 202 | return spec 203 | } 204 | 205 | func validateJson(t *testing.T, lklJson, v4Gw string) { 206 | var config lklConfig 207 | 208 | bytes, err := ioutil.ReadFile(lklJson) 209 | if err != nil { 210 | t.Fatalf("failed to read JSON file: %s (%s)", 211 | lklJson, err) 212 | } 213 | 214 | // decode json 215 | if err := json.Unmarshal(bytes, &config); err != nil { 216 | t.Fatalf("failed to decode JSON file: %s (%s)", 217 | lklJson, err) 218 | } 219 | 220 | t.Logf("%+v", config) 221 | gw, _ := netlink.ParseAddr(v4Gw) 222 | if config.V4Gateway != gw.IP.String() { 223 | t.Fatalf("gateway address is invalid (expected: %s, value: %s)", gw.IP.String(), config.V4Gateway) 224 | } 225 | } 226 | 227 | func testIPAddressAuto(t *testing.T, v4Addr, v4Gw string) { 228 | tmp, err := ioutil.TempFile("/tmp/", "lkl-json") 229 | if err != nil { 230 | t.Fatal(err) 231 | } 232 | 233 | t.Cleanup(func() { 234 | os.Remove(tmp.Name()) 235 | }) 236 | jsonOut := tmp.Name() 237 | t.Logf("jsonOut = %v", jsonOut) 238 | 239 | createNetNs(t, v4Addr, v4Gw) 240 | 241 | spec := testSetupSpec(t) 242 | json, err := generateLklJsonFile("", &jsonOut, spec) 243 | if err != nil { 244 | t.Fatal(err) 245 | } 246 | 247 | validateJson(t, jsonOut, v4Gw) 248 | 249 | t.Logf("json = %+v", json) 250 | } 251 | 252 | func TestIPAddressAuto(t *testing.T) { 253 | t.Cleanup(func() { 254 | time.Sleep(time.Second * 0) 255 | destroyNetNs(t) 256 | }) 257 | 258 | if testing.Verbose() { 259 | logrus.SetLevel(logrus.DebugLevel) 260 | } 261 | 262 | // flannel case 263 | t.Log("==================================") 264 | testIPAddressAuto(t, "192.168.39.2/24", "192.168.39.1/24") 265 | // calico case 266 | t.Log("==================================") 267 | testIPAddressAuto(t, "192.168.39.2/32", "172.16.0.1/32") 268 | } 269 | -------------------------------------------------------------------------------- /cmd/containerd-shim-runu-v1/reaper.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | // +build darwin 3 | 4 | /* 5 | Copyright The containerd Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "context" 24 | "os" 25 | "os/exec" 26 | "os/signal" 27 | "path/filepath" 28 | "strings" 29 | "sync" 30 | "syscall" 31 | "time" 32 | 33 | "github.com/containerd/containerd/pkg/process" 34 | runc "github.com/containerd/go-runc" 35 | "github.com/pkg/errors" 36 | "github.com/sirupsen/logrus" 37 | "golang.org/x/sys/unix" 38 | ) 39 | 40 | // ErrNoSuchProcess is returned when the process no longer exists 41 | var ErrNoSuchProcess = errors.New("no such process") 42 | 43 | const bufferSize = 32 44 | 45 | type subscriber struct { 46 | sync.Mutex 47 | c chan runc.Exit 48 | closed bool 49 | } 50 | 51 | func (s *subscriber) close() { 52 | s.Lock() 53 | if s.closed { 54 | s.Unlock() 55 | return 56 | } 57 | close(s.c) 58 | s.closed = true 59 | s.Unlock() 60 | } 61 | 62 | func (s *subscriber) do(fn func()) { 63 | s.Lock() 64 | fn() 65 | s.Unlock() 66 | } 67 | 68 | // Reap should be called when the process receives an SIGCHLD. Reap will reap 69 | // all exited processes and close their wait channels 70 | func Reap() error { 71 | now := time.Now() 72 | exits, err := reap(false) 73 | if err != nil { 74 | return err 75 | } 76 | exits, err = reapOS(exits) 77 | for _, e := range exits { 78 | done := Default.notify(runc.Exit{ 79 | Timestamp: now, 80 | Pid: e.Pid, 81 | Status: e.Status, 82 | }) 83 | 84 | select { 85 | case <-done: 86 | case <-time.After(1 * time.Second): 87 | } 88 | } 89 | return err 90 | } 91 | 92 | // Default is the default monitor initialized for the package 93 | var Default = &Monitor{ 94 | subscribers: make(map[chan runc.Exit]*subscriber), 95 | } 96 | 97 | // Monitor monitors the underlying system for process status changes 98 | type Monitor struct { 99 | sync.Mutex 100 | 101 | subscribers map[chan runc.Exit]*subscriber 102 | } 103 | 104 | // Start starts the command a registers the process with the reaper 105 | func (m *Monitor) Start(c *exec.Cmd) (chan runc.Exit, error) { 106 | ec := m.Subscribe() 107 | if err := c.Start(); err != nil { 108 | m.Unsubscribe(ec) 109 | return nil, err 110 | } 111 | return ec, nil 112 | } 113 | 114 | // Wait blocks until a process is signal as dead. 115 | // User should rely on the value of the exit status to determine if the 116 | // command was successful or not. 117 | func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) { 118 | for e := range ec { 119 | if e.Pid == c.Process.Pid { 120 | // make sure we flush all IO 121 | c.Wait() 122 | m.Unsubscribe(ec) 123 | return e.Status, nil 124 | } 125 | } 126 | // return no such process if the ec channel is closed and no more exit 127 | // events will be sent 128 | return -1, ErrNoSuchProcess 129 | } 130 | 131 | // Subscribe to process exit changes 132 | func (m *Monitor) Subscribe() chan runc.Exit { 133 | c := make(chan runc.Exit, bufferSize) 134 | m.Lock() 135 | m.subscribers[c] = &subscriber{ 136 | c: c, 137 | } 138 | m.Unlock() 139 | return c 140 | } 141 | 142 | // Unsubscribe to process exit changes 143 | func (m *Monitor) Unsubscribe(c chan runc.Exit) { 144 | m.Lock() 145 | s, ok := m.subscribers[c] 146 | if !ok { 147 | m.Unlock() 148 | return 149 | } 150 | s.close() 151 | delete(m.subscribers, c) 152 | m.Unlock() 153 | } 154 | 155 | func (m *Monitor) getSubscribers() map[chan runc.Exit]*subscriber { 156 | out := make(map[chan runc.Exit]*subscriber) 157 | m.Lock() 158 | for k, v := range m.subscribers { 159 | out[k] = v 160 | } 161 | m.Unlock() 162 | return out 163 | } 164 | 165 | func (m *Monitor) notify(e runc.Exit) chan struct{} { 166 | const timeout = 1 * time.Millisecond 167 | var ( 168 | done = make(chan struct{}, 1) 169 | timer = time.NewTimer(timeout) 170 | success = make(map[chan runc.Exit]struct{}) 171 | ) 172 | stop(timer, true) 173 | 174 | go func() { 175 | defer close(done) 176 | 177 | for { 178 | var ( 179 | failed int 180 | subscribers = m.getSubscribers() 181 | ) 182 | for _, s := range subscribers { 183 | s.do(func() { 184 | if s.closed { 185 | return 186 | } 187 | if _, ok := success[s.c]; ok { 188 | return 189 | } 190 | timer.Reset(timeout) 191 | recv := true 192 | select { 193 | case s.c <- e: 194 | success[s.c] = struct{}{} 195 | case <-timer.C: 196 | recv = false 197 | failed++ 198 | } 199 | stop(timer, recv) 200 | }) 201 | } 202 | // all subscribers received the message 203 | if failed == 0 { 204 | return 205 | } 206 | } 207 | }() 208 | return done 209 | } 210 | 211 | func stop(timer *time.Timer, recv bool) { 212 | if !timer.Stop() && recv { 213 | <-timer.C 214 | } 215 | } 216 | 217 | // exit is the wait4 information from an exited process 218 | type exit struct { 219 | Pid int 220 | Status int 221 | } 222 | 223 | // reap reaps all child processes for the calling process and returns their 224 | // exit information 225 | func reap(wait bool) (exits []exit, err error) { 226 | var ( 227 | ws unix.WaitStatus 228 | rus unix.Rusage 229 | ) 230 | flag := unix.WNOHANG 231 | if wait { 232 | flag = 0 233 | } 234 | for { 235 | pid, err := unix.Wait4(-1, &ws, flag, &rus) 236 | if err != nil { 237 | if err == unix.ECHILD { 238 | return exits, nil 239 | } 240 | return exits, err 241 | } 242 | if pid <= 0 { 243 | return exits, nil 244 | } 245 | exits = append(exits, exit{ 246 | Pid: pid, 247 | Status: exitStatus(ws), 248 | }) 249 | } 250 | } 251 | 252 | // reapOS is additional reap process upon receipt of SIGCHLD. 253 | // Since macOS doesn't raise SIGCHLD on orphaned children's exit, 254 | // reapOS polls the status of registered process and terminate it 255 | // if it's already exited. 256 | func reapOS(exits []exit) ([]exit, error) { 257 | pid, err := runc.ReadPidFile(filepath.Join("", process.InitPidFile)) 258 | if pid <= 0 { 259 | return exits, errors.Errorf("can't find pid=%d %s", pid, err) 260 | } 261 | 262 | process, err := os.FindProcess(pid) 263 | // ensure the process is running 264 | if process != nil { 265 | // from kill(2): 266 | // A value of 0, however, will cause error checking to be 267 | // performed (with no signal being sent). 268 | // This can be used to check the validity of pid. 269 | err = process.Signal(syscall.Signal(0)) 270 | } 271 | logrus.Debugf("checking pid=%d proc=%v err=%v", pid, process, err) 272 | 273 | // if process exists && already finished 274 | if err != nil && strings.Contains(err.Error(), "os: process already finished") { 275 | exits = append(exits, exit{ 276 | Pid: pid, 277 | Status: 0, // XXX 278 | }) 279 | 280 | logrus.Debugf("reapOS: detect exited, pid=%d", pid) 281 | } 282 | 283 | return exits, nil 284 | } 285 | 286 | const exitSignalOffset = 128 287 | 288 | // exitStatus returns the correct exit status for a process based on if it 289 | // was signaled or exited cleanly 290 | func exitStatus(status unix.WaitStatus) int { 291 | if status.Signaled() { 292 | return exitSignalOffset + int(status.Signal()) 293 | } 294 | return status.ExitStatus() 295 | } 296 | 297 | // SetupReaperSignals initializes of signal handlings 298 | func SetupReaperSignals(ctx context.Context, logger *logrus.Entry) error { 299 | signals := make(chan os.Signal, 32) 300 | signal.Notify(signals, unix.SIGCHLD) 301 | go handleSignals(ctx, logger, signals) 302 | return nil 303 | } 304 | 305 | // copied from containerd code 306 | func handleSignals(ctx context.Context, logger *logrus.Entry, signals chan os.Signal) error { 307 | for { 308 | select { 309 | case <-ctx.Done(): 310 | return ctx.Err() 311 | case s := <-signals: 312 | switch s { 313 | case unix.SIGCHLD: 314 | if err := Reap(); err != nil { 315 | logger.WithError(err).Error("reap exit status") 316 | } 317 | case unix.SIGPIPE: 318 | } 319 | } 320 | } 321 | } 322 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | Copyright 2019 IIJ Research Laboratory 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - '**' 6 | release: 7 | types: 8 | - created 9 | - edited 10 | repository_dispatch: 11 | types: [trigger-test] 12 | workflow_dispatch: 13 | inputs: 14 | debug_enabled: 15 | description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' 16 | required: false 17 | default: false 18 | image_version: 19 | description: 'Specify docker image version to test' 20 | required: false 21 | 22 | jobs: 23 | build: 24 | name: test (${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.runs_on }}) 25 | runs-on: ${{ matrix.runs_on }} 26 | timeout-minutes: 60 27 | strategy: 28 | fail-fast: false 29 | matrix: 30 | include: 31 | - os: linux 32 | arch: amd64 33 | os_alias: linux 34 | arch_alias: amd64 35 | runs_on: ubuntu-20.04 36 | goos: linux 37 | - os: macos 38 | arch: amd64 39 | os_alias: osx 40 | arch_alias: amd64 41 | goos: darwin 42 | runs_on: macos-12 43 | - os: linux 44 | arch: arm32 45 | os_alias: linux 46 | arch_alias: armhf 47 | goos: linux 48 | goarch: arm 49 | goarm: "7" 50 | runs_on: ubuntu-20.04 51 | cc: arm-linux-gnueabihf-gcc 52 | cgo_enabled: 1 53 | - os: linux 54 | arch: arm64 55 | os_alias: linux 56 | arch_alias: arm64 57 | goos: linux 58 | goarch: arm64 59 | runs_on: ubuntu-20.04 60 | cc: aarch64-linux-gnu-gcc 61 | cgo_enabled: 1 62 | env: 63 | TRAVIS_OS_NAME: ${{ matrix.os_alias }} 64 | TRAVIS_ARCH: ${{ matrix.arch }} 65 | REGISTRY: ghcr.io/ 66 | KIND_IMG_VERSION: v1.27.0 67 | KIND_VERSION: v0.19.0 68 | CC: ${{ matrix.cc }} 69 | CGO_ENABLED: ${{ matrix.cgo_enabled }} 70 | GOOS: ${{matrix.goos}} 71 | GOARCH: ${{matrix.goarch}} 72 | GOARM: ${{matrix.goarm}} 73 | DOCKER_IMG_VERSION_DEFAULT: 0.8 74 | steps: 75 | - uses: actions/setup-go@v5 76 | with: 77 | go-version: 1.17.1 78 | cache: false 79 | - uses: actions/checkout@v4 80 | with: 81 | fetch-depth: 0 82 | - uses: actions/cache@v4 83 | with: 84 | # In order: 85 | # * Module download cache 86 | # * Build cache (Linux) 87 | # * Build cache (Mac) 88 | # * Build cache (Windows) 89 | path: | 90 | ~/go/pkg/mod 91 | ~/.cache/go-build 92 | ~/Library/Caches/go-build 93 | %LocalAppData%\go-build 94 | key: ${{ runner.os }}-${{ matrix.arch }}-ccache-build-${{ hashFiles('**/go.sum') }} 95 | restore-keys: ${{ runner.os }}-${{ matrix.arch }}-ccache-build- 96 | 97 | - name: Event Information 98 | run: | 99 | echo "Event '${{ github.event.action }}' from '${{ github.event.client_payload.repository }}'" 100 | 101 | - name: Set env 102 | run: | 103 | echo "$HOME/.local/bin:${{ github.workspace }}/bin" >> $GITHUB_PATH 104 | echo "export PATH=$HOME/.local/bin:${{ github.workspace }}/bin:$PATH" >> $HOME/.bashrc 105 | RELEASE_VERSION=`git describe --tags --abbrev=0 | sed "s/^v//"` 106 | if [ `git rev-list -n 1 v"$RELEASE_VERSION"` == `git rev-list -n 1 HEAD` ] ; then 107 | BUILD_VERSION=${RELEASE_VERSION} 108 | else 109 | BUILD_VERSION=${RELEASE_VERSION}-next 110 | fi 111 | PACKAGE_FILENAME=docker-runu_${BUILD_VERSION}_${{ matrix.arch_alias }}.deb 112 | echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV 113 | echo "BUILD_VERSION=$BUILD_VERSION" >> $GITHUB_ENV 114 | echo "BUILD_DATE="`date "+%Y%m%d"` >> $GITHUB_ENV 115 | echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV 116 | echo "DEB_ARCH=${{ matrix.arch_alias }}" >> $GITHUB_ENV 117 | # image version 118 | if [ -n "${{ github.event.inputs.image_version }}" ] ; then 119 | echo "DOCKER_IMG_VERSION=${{ github.event.inputs.image_version }}" >> $GITHUB_ENV 120 | elif [ -n "${{ github.event.client_payload.img_version }}" ] ; then 121 | # TODO: not implemented yet 122 | echo "DOCKER_IMG_VERSION=${{ github.event.client_payload.img_version }}" >> $GITHUB_ENV 123 | else 124 | echo "DOCKER_IMG_VERSION=${{ env.DOCKER_IMG_VERSION_DEFAULT }}" >> $GITHUB_ENV 125 | fi 126 | echo "GO_FLAGS=-ldflags \"-X main.version="${BUILD_VERSION}"\"" >> $GITHUB_ENV 127 | 128 | - name: package installation (linux) 129 | if: runner.os == 'linux' && matrix.arch == 'amd64' 130 | run: | 131 | sudo apt update -y 132 | sudo apt install -y bridge-utils 133 | 134 | - name: package installation (linux-cross) 135 | if: runner.os == 'linux' && ( matrix.arch == 'arm32' || matrix.arch == 'arm64') 136 | run: | 137 | sudo apt-get update -y 138 | sudo apt-get install -y crossbuild-essential-${{ matrix.arch_alias }} 139 | 140 | - name: package installation (mac) 141 | if: runner.os == 'macos' 142 | run: | 143 | mkdir -p ~/.local/bin 144 | # use pre-installed golang instead of brew 145 | ln -s `which go` /usr/local/Homebrew/Library/Homebrew/shims/mac/super/go 146 | brew install ukontainer/lkl/darwin-snapshotter ukontainer/lkl/containerd ukontainer/lkl/nerdctl --ignore-dependencies 147 | brew install coreutils 148 | ln -sf /usr/local/bin/gsha256sum ~/.local/bin/sha256sum 149 | 150 | - name: Build 151 | run: | 152 | go install ${{ env.GO_FLAGS }} -v . 153 | - name: Build shim 154 | if: runner.os == 'macos' 155 | run: | 156 | go install ${{ env.GO_FLAGS }} -v ./cmd/containerd-shim-runu-v1 157 | - name: Go Test 158 | if: matrix.arch == 'amd64' 159 | run: | 160 | sudo go test -v . 161 | runu -v 162 | 163 | - name: goreportcard 164 | if: runner.os == 'linux' && matrix.arch == 'amd64' && matrix.runs_on == 'ubuntu-18.04' 165 | run: | 166 | cd /tmp 167 | go get -u github.com/gojp/goreportcard/cmd/goreportcard-cli 168 | GO111MODULE=off go get -u github.com/alecthomas/gometalinter 169 | go get -u github.com/gordonklaus/ineffassign 170 | go get -u github.com/fzipp/gocyclo/cmd/gocyclo 171 | go get -u github.com/client9/misspell/cmd/misspell 172 | go get -u golang.org/x/lint/golint 173 | cd ${{ github.workspace }} 174 | # Do checks 175 | GO111MODULE=on goreportcard-cli -t 100.0 -v 176 | 177 | # TODO: run qemu for arm32/arm64 tests 178 | - name: Test (standalone) 179 | if: matrix.arch != 'arm32' && matrix.arch != 'arm64' 180 | run: bash -e ${{ github.workspace }}/test/standalone-test.sh 181 | - name: Test (containerd/ctr) 182 | run: bash -e ${{ github.workspace }}/test/containerd-ctr-test.sh 183 | - name: Test (containerd/nerdctl) 184 | run: bash -e ${{ github.workspace }}/test/containerd-nerdctl-test.sh 185 | # TODO: run qemu for arm32/arm64 tests 186 | - name: Test (dockerd) 187 | if: runner.os == 'linux' && matrix.arch == 'amd64' 188 | run: bash -e ${{ github.workspace }}/test/docker-oci-test.sh 189 | - name: Test (docker more) 190 | if: runner.os == 'linux' && matrix.arch == 'amd64' 191 | run: bash -e ${{ github.workspace }}/test/docker-more-test.sh 192 | - name: Test (docker volume) 193 | if: runner.os == 'linux' && matrix.arch == 'amd64' 194 | run: bash -e ${{ github.workspace }}/test/docker-volume-test.sh 195 | 196 | - name: KinD image preparation 197 | if: runner.os == 'linux' && matrix.arch == 'amd64' 198 | run: | 199 | . ${{ github.workspace }}/test/common.sh 200 | # prepare RUNU_AUX_DIR 201 | create_runu_aux_dir 202 | cp $RUNU_AUX_DIR/libc.so k8s/ 203 | cp $RUNU_AUX_DIR/lkick k8s/ 204 | 205 | # Build kind node docker image 206 | cp `which runu` k8s/ 207 | cd k8s 208 | docker build -t ukontainer/node-runu:$KIND_IMG_VERSION . 209 | cd .. 210 | - uses: engineerd/setup-kind@v0.5.0 211 | name: kind setup with default CNI 212 | if: runner.os == 'linux' && matrix.arch == 'amd64' 213 | with: 214 | version: "${{ env.KIND_VERSION }}" 215 | config: "k8s/kind-cluster.yaml" 216 | image: "ukontainer/node-runu:${{ env.KIND_IMG_VERSION }}" 217 | 218 | - name: Test (k8s) 219 | if: runner.os == 'linux' && matrix.arch == 'amd64' 220 | run: | 221 | bash -e ${{ github.workspace }}/test/k8s-test.sh 127.0.0.1 222 | 223 | - run: kind delete cluster 224 | if: runner.os == 'linux' && matrix.arch == 'amd64' 225 | 226 | - uses: engineerd/setup-kind@v0.5.0 227 | name: kind setup with calico 228 | if: runner.os == 'linux' && matrix.arch == 'amd64' 229 | with: 230 | version: "${{ env.KIND_VERSION }}" 231 | config: "k8s/kind-cluster-calico.yaml" 232 | image: "ukontainer/node-runu:${{ env.KIND_IMG_VERSION }}" 233 | wait: "0s" 234 | 235 | - name: Test (k8s/calico) 236 | if: runner.os == 'linux' && matrix.arch == 'amd64' 237 | run: | 238 | set -x 239 | NET=`kubectl cluster-info dump | grep -- --cluster-cidr | cut -d'=' -f2 | sed s/\"\,//` 240 | # install calico 241 | curl -s https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml | sed "s,192.168.0.0/16,$NET," | kubectl apply -f - 242 | sleep 30 243 | DST=`kubectl get node -o wide | grep control-plane | awk '{print $6}'` 244 | bash -e ${{ github.workspace }}/test/k8s-test.sh $DST 245 | 246 | 247 | - name: Build Debian package 248 | if: runner.os == 'linux' 249 | run: | 250 | bash -ex pkg/pre-deploy-deb.sh 251 | - name: Verify Debian package 252 | if: runner.os == 'linux' 253 | run: | 254 | dpkg --info *.deb 255 | dpkg --contents *.deb 256 | - name: Test Debian package 257 | if: runner.os == 'linux' && matrix.arch == 'amd64' 258 | run: | 259 | bash -ex pkg/pre-deploy-test-deb.sh 260 | 261 | - name: upload artifact 262 | if: runner.os == 'linux' 263 | uses: actions/upload-artifact@v4 264 | with: 265 | path: ${{ env.PACKAGE_FILENAME }} 266 | name: ${{ env.PACKAGE_FILENAME }} 267 | 268 | # TODO: prepare homebrew bottle 269 | - name: Github Releases 270 | if: runner.os == 'linux' && gitHub.event_name == 'release' 271 | uses: softprops/action-gh-release@v2 272 | with: 273 | tag_name: v${{ env.RELEASE_VERSION }} 274 | prerelease: true 275 | token: ${{ secrets.GITHUB_TOKEN }} 276 | files: | 277 | ${{ env.PACKAGE_FILENAME }} 278 | - uses: ruby/setup-ruby@v1 279 | with: 280 | ruby-version: 2.7 281 | - name: Release to packagecloud.io 282 | if: runner.os == 'linux' && gitHub.event_name == 'release' 283 | run: | 284 | # Instsall packagecloud CLI 285 | gem install package_cloud 286 | # push all versions and delete it in advance 287 | for distro_version in ${{ env.DISTRO_LIST }} ; do 288 | package_cloud yank ukontainer/runu/$distro_version ${{ env.PACKAGE_FILENAME }} || true 289 | package_cloud push ukontainer/runu/$distro_version ${{ env.PACKAGE_FILENAME }} 290 | done 291 | env: 292 | PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} 293 | DISTRO_LIST: "ubuntu/focal ubuntu/jammy ubuntu/noble" 294 | - name: Test Released Debian package 295 | if: runner.os == 'linux' && matrix.arch == 'amd64' && gitHub.event_name == 'release' 296 | run: | 297 | sudo apt-get remove docker-runu 298 | #set -x 299 | # wait for complete indexing at pkgcloud 300 | sleep 60 301 | curl -s https://packagecloud.io/install/repositories/ukontainer/runu/script.deb.sh | sudo bash 302 | sudo apt-get install docker-runu 303 | docker run --rm -i --runtime=runu-dev alpine uname -a 304 | 305 | - name: Log in to docker.io 306 | if: runner.os == 'linux' 307 | uses: docker/login-action@v3.2.0 308 | with: 309 | username: ${{ secrets.DOCKER_USERNAME }} 310 | password: ${{ secrets.DOCKER_PASSWORD }} 311 | - name: Log in to the ghcr.io 312 | if: runner.os == 'linux' 313 | uses: docker/login-action@v3.2.0 314 | with: 315 | registry: ghcr.io 316 | username: ${{ github.actor }} 317 | password: ${{ secrets.GITHUB_TOKEN }} 318 | - name: Build and push Docker image 319 | if: gitHub.event_name == 'release' && runner.os == 'linux' && matrix.arch == 'amd64' 320 | uses: docker/build-push-action@v2 321 | with: 322 | context: k8s 323 | platforms: linux/amd64 324 | push: true 325 | tags: | 326 | docker.io/ukontainer/node-runu:${{ env.KIND_IMG_VERSION }} 327 | ghcr.io/ukontainer/node-runu:${{ env.KIND_IMG_VERSION }} 328 | 329 | - name: Log 330 | if: always() 331 | run: | 332 | cat /tmp/dockerd.log || true 333 | cat /tmp/containerd.log || true 334 | cat /tmp/darwin-snapshotter.log || true 335 | 336 | - run: kind export logs logs 337 | if: runner.os == 'linux' && matrix.arch == 'amd64' && always() 338 | - uses: actions/upload-artifact@v4 339 | if: runner.os == 'linux' && matrix.arch == 'amd64' && always() 340 | with: 341 | name: KinD-log 342 | path: logs 343 | 344 | - name: Setup tmate session 345 | uses: mxschmitt/action-tmate@v3 346 | if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled && always ()}} 347 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net" 9 | "os" 10 | "os/exec" 11 | "path/filepath" 12 | "regexp" 13 | goruntime "runtime" 14 | "sort" 15 | "strconv" 16 | "strings" 17 | "syscall" 18 | "time" 19 | 20 | "github.com/opencontainers/runtime-spec/specs-go" 21 | "github.com/sirupsen/logrus" 22 | "github.com/urfave/cli" 23 | ) 24 | 25 | const ( 26 | exactArgs = iota 27 | minArgs 28 | maxArgs 29 | ) 30 | 31 | var ( 32 | fdInfoConfigJson = "__RUMP_FDINFO_CONFIGJSON" 33 | fdInfoEnvPrefixNet = "__RUMP_FDINFO_NET_" 34 | fdInfoEnvPrefixDisk = "__RUMP_FDINFO_DISK_" 35 | fdInfoEnvPrefixRoot = "__RUMP_FDINFO_ROOT" 36 | ) 37 | 38 | func checkArgs(context *cli.Context, expected, checkType int) error { 39 | var err error 40 | cmdName := context.Command.Name 41 | switch checkType { 42 | case exactArgs: 43 | if context.NArg() != expected { 44 | err = fmt.Errorf( 45 | "%s: %q requires exactly %d argument(s)", 46 | os.Args[0], cmdName, expected) 47 | } 48 | case minArgs: 49 | if context.NArg() < expected { 50 | err = fmt.Errorf( 51 | "%s: %q requires a minimum of %d argument(s)", 52 | os.Args[0], cmdName, expected) 53 | } 54 | case maxArgs: 55 | if context.NArg() > expected { 56 | err = fmt.Errorf( 57 | "%s: %q requires a maximum of %d argument(s)", 58 | os.Args[0], cmdName, expected) 59 | } 60 | } 61 | 62 | if err != nil { 63 | fmt.Printf("Incorrect Usage.\n\n") 64 | cli.ShowCommandHelp(context, cmdName) 65 | return err 66 | } 67 | return nil 68 | } 69 | 70 | func readPidFile(context *cli.Context, pidFile string) (int, error) { 71 | root := context.GlobalString("root") 72 | container := context.Args().Get(0) 73 | file := filepath.Join(root, container, pidFile) 74 | pid, err := ioutil.ReadFile(file) 75 | if err != nil { 76 | return 0, err 77 | } 78 | pidI, err := strconv.Atoi(string(pid)) 79 | if err != nil { 80 | return 0, err 81 | } 82 | 83 | return pidI, nil 84 | 85 | } 86 | 87 | func copyFile(src, dst string, mode os.FileMode) error { 88 | b, err := ioutil.ReadFile(src) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | err = ioutil.WriteFile(dst, b, mode) 94 | if err != nil { 95 | return err 96 | } 97 | return nil 98 | } 99 | 100 | func isAlpineImage(rootfs string) bool { 101 | osRelease := rootfs + "/etc/os-release" 102 | 103 | f, err := os.Open(osRelease) 104 | if err != nil { 105 | return false 106 | } 107 | defer f.Close() 108 | 109 | scanner := bufio.NewScanner(f) 110 | for scanner.Scan() { 111 | matched, _ := regexp.MatchString("Alpine Linux", 112 | scanner.Text()) 113 | if matched { 114 | return true 115 | } 116 | } 117 | 118 | return false 119 | } 120 | 121 | func changeLdso(spec *specs.Spec, rootfs string) error { 122 | for _, env := range spec.Process.Env { 123 | if strings.HasPrefix(env, "RUNU_AUX_DIR=") { 124 | runuAuxFileDir = strings.TrimLeft(env, "RUNU_AUX_DIR=") 125 | } 126 | } 127 | 128 | // XXX: only for alpine 129 | // install frankenlibc-ed libc.so to the system one 130 | if goruntime.GOARCH == "amd64" { 131 | if err := copyFile(runuAuxFileDir+"/libc.so", 132 | rootfs+"/lib/ld-musl-x86_64.so.1", 0755); err != nil { 133 | return err 134 | } 135 | } else if goruntime.GOARCH == "arm" { 136 | if err := copyFile(runuAuxFileDir+"/libc.so", 137 | rootfs+"/lib/ld-musl-armhf.so.1", 0755); err != nil { 138 | return err 139 | } 140 | } else if goruntime.GOARCH == "arm64" { 141 | if err := copyFile(runuAuxFileDir+"/libc.so", 142 | rootfs+"/lib/ld-musl-aarch64.so.1", 0755); err != nil { 143 | return err 144 | } 145 | } 146 | 147 | // install frankenlibc-ed libc.so to the system one 148 | if err := copyFile(runuAuxFileDir+"/lkick", 149 | rootfs+"/bin/lkick", 0755); err != nil { 150 | return err 151 | } 152 | 153 | return nil 154 | } 155 | 156 | type lklInterface struct { 157 | MacAddr string `json:"mac,omitempty"` 158 | V4Addr string `json:"ip"` 159 | V4MaskLen string `json:"masklen"` 160 | V6Addr string `json:"ipv6,omitempty"` 161 | V6MaskLen string `json:"masklen6,omitempty"` 162 | Name string `json:"name"` 163 | Iftype string `json:"type"` 164 | Offload string `json:"offload,omitempty"` 165 | } 166 | 167 | type lklRoute struct { 168 | Destination string `json:"destination"` 169 | Nexthop string `json:"nexthop"` 170 | } 171 | 172 | type lklConfig struct { 173 | V4Gateway string `json:"gateway,omitempty"` 174 | Interfaces []lklInterface `json:"interfaces,omitempty"` 175 | Debug string `json:"debug,omitempty"` 176 | DelayMain string `json:"delay_main,omitempty"` 177 | SingleCpu string `json:"singlecpu,omitempty"` 178 | Sysctl string `json:"sysctl,omitempty"` 179 | Routes string `json:"route,omitempty"` 180 | } 181 | 182 | type lklIfInfo struct { 183 | ifAddrs []net.IPNet 184 | ifName string 185 | v4Gw net.IP 186 | } 187 | 188 | func generateLklJsonFile(lklJson string, lklJsonOut *string, spec *specs.Spec) (*lklConfig, error) { 189 | config := new(lklConfig) 190 | 191 | // IPv4 address 192 | ifInfo, routeInfo, err := setupNetwork(spec) 193 | if err != nil { 194 | return nil, fmt.Errorf("failed to parse ipv4/ipv6 address: (%s)", err) 195 | } 196 | 197 | if ifInfo == nil { 198 | logrus.Warnf("no interface detected") 199 | *lklJsonOut = lklJson 200 | return config, nil 201 | } 202 | 203 | logrus.Debugf(ifInfo.ifAddrs[0].String()) 204 | logrus.Debugf("interface=>%+v", ifInfo) 205 | logrus.Debugf("route=>%+v", routeInfo) 206 | 207 | v4masklen, _ := ifInfo.ifAddrs[0].Mask.Size() 208 | v4addr := ifInfo.ifAddrs[0].IP 209 | v4addr = v4addr.To4() 210 | v4gw := ifInfo.v4Gw.To4() 211 | 212 | // read user-specified file 213 | if lklJson != "" { 214 | bytes, err := ioutil.ReadFile(lklJson) 215 | if err != nil { 216 | logrus.Errorf("failed to read JSON file: %s (%s)", 217 | lklJson, err) 218 | panic(err) 219 | } 220 | 221 | // decode json 222 | if err := json.Unmarshal(bytes, &config); err != nil { 223 | logrus.Errorf("failed to decode JSON file: %s (%s)", 224 | lklJson, err) 225 | panic(err) 226 | } 227 | 228 | // only replace IPv4 address when the address is written 229 | // with "AUTO": otherwise use user-specified address 230 | if len(config.Interfaces) > 0 { 231 | if config.Interfaces[0].V4Addr == "AUTO" { 232 | config.Interfaces[0].V4Addr = v4addr.String() 233 | config.Interfaces[0].V4MaskLen = strconv.Itoa(v4masklen) 234 | config.V4Gateway = v4gw.String() 235 | } 236 | } 237 | } else { 238 | config.Debug = "1" 239 | config.Interfaces = append(config.Interfaces, 240 | lklInterface{ 241 | V4Addr: v4addr.String(), 242 | V4MaskLen: strconv.Itoa(v4masklen), 243 | Name: ifInfo.ifName, 244 | Iftype: "rumpfd", 245 | }) 246 | config.V4Gateway = v4gw.String() 247 | for _, route := range routeInfo { 248 | config.Routes += route.Destination 249 | config.Routes += "=" 250 | config.Routes += route.Nexthop 251 | config.Routes += ";" 252 | } 253 | } 254 | 255 | outJson, err := json.MarshalIndent(config, "", " ") 256 | if err != nil { 257 | logrus.Errorf("failed to encode JSON file: %s (%s)", 258 | lklJson, err) 259 | panic(err) 260 | } 261 | 262 | ioutil.WriteFile(*lklJsonOut, outJson, os.ModePerm) 263 | 264 | return config, nil 265 | } 266 | 267 | func parseEnvs(spec *specs.Spec, context *cli.Context, rootfs string) ([]string, map[*os.File]bool) { 268 | specEnv := []string{} 269 | fds := map[*os.File]bool{} 270 | fdNum := 3 271 | hasRootFs := false 272 | lklJson := "" 273 | // check if -v is specified 274 | _, use9pFs := os.LookupEnv("LKL_USE_9PFS") 275 | 276 | for _, env := range spec.Process.Env { 277 | // look for LKL_ROOTFS env for .img/.iso files 278 | if strings.HasPrefix(env, "LKL_ROOTFS=") { 279 | lklRootfs := strings.TrimLeft(env, "LKL_ROOTFS=") 280 | 281 | // if a file exists in local/host, copy and use it 282 | if _, err := os.Stat(lklRootfs); err == nil { 283 | copyFile(lklRootfs, 284 | rootfs+"/"+filepath.Base(lklRootfs), 285 | 0644) 286 | lklRootfs = "/" + filepath.Base(lklRootfs) 287 | } 288 | 289 | fd, nonblock := openRootfsFd(rootfs + "/" + lklRootfs) 290 | fds[fd] = nonblock 291 | specEnv = append(specEnv, fdInfoEnvPrefixRoot+"="+strconv.Itoa(fdNum)) 292 | fdNum++ 293 | hasRootFs = true 294 | continue 295 | } 296 | // look for LKL_NET env for tap/macvtap devices 297 | if strings.HasPrefix(env, "LKL_NET=") { 298 | lklNet := strings.TrimLeft(env, "LKL_NET=") 299 | 300 | fd, nonblock := openNetFd(lklNet, spec.Process.Env) 301 | fds[fd] = nonblock 302 | specEnv = append(specEnv, fdInfoEnvPrefixNet+lklNet+"="+strconv.Itoa(fdNum)) 303 | fdNum++ 304 | continue 305 | } 306 | // look for LKL_CONFIG env for json file 307 | if strings.HasPrefix(env, "LKL_CONFIG=") { 308 | lklJson = strings.TrimLeft(env, "LKL_CONFIG=") 309 | copyFile(lklJson, rootfs+"/"+filepath.Base(lklJson), 0644) 310 | lklJson = rootfs + "/" + filepath.Base(lklJson) 311 | 312 | continue 313 | } 314 | 315 | // lookf for LKL_USE_9PFS 316 | if strings.HasPrefix(env, "LKL_USE_9PFS=") { 317 | use9pFs = true 318 | } 319 | 320 | // XXX: should exclude duplicated PATH variable in spec.Env since 321 | // it eliminates following values 322 | if !strings.HasPrefix(env, "PATH=") { 323 | specEnv = append(specEnv, env) 324 | } 325 | } 326 | 327 | // Set IPv4 addr/route from CNI info 328 | // and configure as lkl-$(container-id)-out.json file 329 | container := context.Args().First() 330 | clen := 10 331 | if len(container) < 10 { 332 | clen = len(container) 333 | } 334 | lklJsonOut := rootfs + "/" + "lkl-" + container[:clen] + "-out.json" 335 | 336 | jsonObj, err := generateLklJsonFile(lklJson, &lklJsonOut, spec) 337 | if err != nil { 338 | panic(err) 339 | } 340 | 341 | // XXX: eth0 should be somewhere else 342 | if len(jsonObj.Interfaces) > 0 { 343 | lklNet := "eth0" 344 | fd, nonblock := openNetFd(lklNet, spec.Process.Env) 345 | fds[fd] = nonblock 346 | specEnv = append(specEnv, fdInfoEnvPrefixNet+lklNet+"="+strconv.Itoa(fdNum)) 347 | fdNum++ 348 | } 349 | 350 | if lklJsonOut != "" { 351 | fd, nonblock := openJsonFd(lklJsonOut) 352 | fds[fd] = nonblock 353 | specEnv = append(specEnv, fdInfoConfigJson+"="+strconv.Itoa(fdNum)) 354 | fdNum++ 355 | } 356 | 357 | // start 9pfs server as a child process 358 | // if there is no rootfs disk image 359 | if !hasRootFs && use9pFs { 360 | childArgs := []string{"--9ps=" + rootfs + "/"} 361 | cmd := exec.Command(os.Args[0], childArgs[0:]...) 362 | cmd.Stderr = os.Stderr 363 | cmd.Stdout = os.Stdout 364 | if err := cmd.Start(); err != nil { 365 | panic(err) 366 | } 367 | // pid file for 9pfs server 368 | root := context.GlobalString("root") 369 | name := context.Args().Get(0) 370 | pidf := filepath.Join(root, name, pidFile9p) 371 | f, _ := os.OpenFile(pidf, 372 | os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666) 373 | _, _ = fmt.Fprintf(f, "%d", cmd.Process.Pid) 374 | f.Close() 375 | logrus.Debugf("Starting command %s, Env=%s, rootfs=%s\n", 376 | cmd.Args, cmd.Env, rootfs) 377 | 378 | time.Sleep(100 * time.Millisecond) 379 | fd, nonblock := connect9pfs() 380 | fds[fd] = nonblock 381 | specEnv = append(specEnv, "9PFS_FD="+strconv.Itoa(fdNum)) 382 | specEnv = append(specEnv, "9PFS_MNT=/") 383 | } 384 | 385 | return specEnv, fds 386 | } 387 | 388 | func prepareUkontainer(context *cli.Context) (*exec.Cmd, error) { 389 | container := context.Args().Get(0) 390 | spec, err := setupSpec(context) 391 | if err != nil { 392 | logrus.Warnf("setupSepc err %s\n", err) 393 | return nil, err 394 | } 395 | 396 | rootfs, _ := filepath.Abs(spec.Root.Path) 397 | // open fds to pass to main programs later 398 | specEnv, fds := parseEnvs(spec, context, rootfs) 399 | 400 | // fixup ldso to a pulled image 401 | err = changeLdso(spec, rootfs) 402 | if err != nil { 403 | logrus.Warnf("ldso fixup error. skipping (%s)", err) 404 | } 405 | 406 | for _, node := range DefaultDevices { 407 | createDeviceNode(rootfs, node) 408 | } 409 | 410 | // call rexec 411 | os.Setenv("PATH", rootfs+":"+rootfs+ 412 | "/sbin:"+rootfs+"/bin:/bin:/sbin:") 413 | 414 | cmd := exec.Command(spec.Process.Args[0], spec.Process.Args[1:]...) 415 | // XXX: need a better way to detect 416 | if isAlpineImage(rootfs) && goruntime.GOOS == "darwin" { 417 | logrus.Debugf("This is alpine linux image") 418 | cmd = exec.Command("lkick", spec.Process.Args[0:]...) 419 | } 420 | 421 | // do chroot(2) in rexec-ed processes 422 | cmd.SysProcAttr = &syscall.SysProcAttr{ 423 | Setpgid: false, 424 | } 425 | cmd.Dir = "/" 426 | 427 | // on Linux, libc.so is replaced in a chrooted directory 428 | if goruntime.GOOS == "linux" { 429 | cmd.SysProcAttr.Chroot = rootfs 430 | 431 | binpath, err := exec.LookPath(spec.Process.Args[0]) 432 | if err != nil { 433 | logrus.Errorf("cmd %s not found %s", 434 | spec.Process.Args[0], err) 435 | os.Setenv("PATH", "/bin:/sbin:/usr/bin:"+rootfs+":"+rootfs+ 436 | "/sbin:"+rootfs+"/bin") 437 | } 438 | 439 | if strings.HasPrefix(binpath, rootfs) { 440 | cmd.Path = strings.Split(binpath, rootfs)[1] 441 | logrus.Debugf("cmd %s found at %s=>%s", 442 | spec.Process.Args[0], binpath, cmd.Path) 443 | } 444 | } 445 | 446 | cmd.Env = append(specEnv, "PATH=/bin:/sbin:"+os.Getenv("PATH")) 447 | cmd.Stdin = os.Stdin 448 | cmd.Stdout = os.Stdout 449 | cmd.Stderr = os.Stderr 450 | 451 | // XXX: need to sort cmd.Extrafiles to sync with frankenlibc 452 | fdkeys := make([]*os.File, 0) 453 | for k := range fds { 454 | fdkeys = append(fdkeys, k) 455 | } 456 | sort.SliceStable(fdkeys, func(i, j int) bool { 457 | return fdkeys[i].Fd() < fdkeys[j].Fd() 458 | }) 459 | for k := range fdkeys { 460 | cmd.ExtraFiles = append(cmd.ExtraFiles, fdkeys[k]) 461 | } 462 | 463 | cwd, _ := os.Getwd() 464 | logrus.Debugf("Starting command %s, Env=%s, cwd=%s, chroot=%s", 465 | cmd.Args, cmd.Env, cwd, rootfs) 466 | if err := cmd.Start(); err != nil { 467 | logrus.Errorf("cmd error %s (cmd=%s)", err, cmd.Args) 468 | panic(err) 469 | } 470 | 471 | // pid file for forked process from `runu create` 472 | // it'll be used later by `runu start` 473 | root := context.GlobalString("root") 474 | pidf := filepath.Join(root, container, pidFilePriv) 475 | f, _ := os.OpenFile(pidf, 476 | os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666) 477 | 478 | _, _ = fmt.Fprintf(f, "%d", cmd.Process.Pid) 479 | f.Close() 480 | 481 | logrus.Debugf("PID=%d to pid file %s", 482 | cmd.Process.Pid, pidf) 483 | 484 | proc, err := os.FindProcess(cmd.Process.Pid) 485 | if err != nil { 486 | return nil, fmt.Errorf("couldn't find pid %d", cmd.Process.Pid) 487 | } 488 | proc.Signal(syscall.Signal(syscall.SIGSTOP)) 489 | 490 | // XXX: 491 | // os/exec.Start() close and open extrafiles thus strip O_NONBLOCK flag 492 | // thus re-enable it here 493 | for fd, nbFlag := range fds { 494 | if err := syscall.SetNonblock(int(fd.Fd()), nbFlag); err != nil { 495 | logrus.Errorf("setNonBlock %d error: %s\n", int(fd.Fd()), err) 496 | panic(err) 497 | } 498 | } 499 | 500 | return cmd, nil 501 | } 502 | -------------------------------------------------------------------------------- /cmd/containerd-shim-runu-v1/service.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | // +build darwin 3 | 4 | /* 5 | Copyright The containerd Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "context" 24 | "io/ioutil" 25 | "os" 26 | "os/exec" 27 | "path/filepath" 28 | "strings" 29 | "sync" 30 | "syscall" 31 | "time" 32 | 33 | eventstypes "github.com/containerd/containerd/api/events" 34 | "github.com/containerd/containerd/api/types/task" 35 | "github.com/containerd/containerd/errdefs" 36 | "github.com/containerd/containerd/log" 37 | "github.com/containerd/containerd/mount" 38 | "github.com/containerd/containerd/namespaces" 39 | "github.com/containerd/containerd/pkg/process" 40 | "github.com/containerd/containerd/pkg/stdio" 41 | "github.com/containerd/containerd/runtime" 42 | "github.com/containerd/containerd/runtime/v2/runc/options" 43 | "github.com/containerd/containerd/runtime/v2/shim" 44 | taskAPI "github.com/containerd/containerd/runtime/v2/task" 45 | runcC "github.com/containerd/go-runc" 46 | "github.com/containerd/typeurl" 47 | "github.com/gogo/protobuf/proto" 48 | ptypes "github.com/gogo/protobuf/types" 49 | "github.com/pkg/errors" 50 | "github.com/sirupsen/logrus" 51 | "golang.org/x/sys/unix" 52 | ) 53 | 54 | var ( 55 | _ = (taskAPI.TaskService)(&service{}) 56 | empty = &ptypes.Empty{} 57 | ) 58 | 59 | // group labels specifies how the shim groups services. 60 | // currently supports a runc.v2 specific .group label and the 61 | // standard k8s pod label. Order matters in this list 62 | var groupLabels = []string{ 63 | "io.containerd.runc.v2.group", 64 | "io.kubernetes.cri.sandbox-id", 65 | } 66 | 67 | type spec struct { 68 | Annotations map[string]string `json:"annotations,omitempty"` 69 | } 70 | 71 | // Container for operating on a runc container and its processes 72 | type darwinContainer struct { 73 | mu sync.Mutex 74 | 75 | // ID of the container 76 | ID string 77 | // Bundle path 78 | Bundle string 79 | 80 | process process.Process 81 | processes map[string]process.Process 82 | } 83 | 84 | func newInit(ctx context.Context, path, workDir, namespace string, platform stdio.Platform, 85 | r *process.CreateConfig, options *options.Options, rootfs string) (*process.Init, error) { 86 | runtime := process.NewRunc(options.Root, path, namespace, options.BinaryName, options.CriuPath, false) 87 | p := process.New(r.ID, runtime, stdio.Stdio{ 88 | Stdin: r.Stdin, 89 | Stdout: r.Stdout, 90 | Stderr: r.Stderr, 91 | Terminal: r.Terminal, 92 | }) 93 | p.Bundle = r.Bundle 94 | p.Platform = platform 95 | p.Rootfs = rootfs 96 | p.WorkDir = workDir 97 | p.IoUID = int(options.IoUid) 98 | p.IoGID = int(options.IoGid) 99 | p.NoPivotRoot = options.NoPivotRoot 100 | p.NoNewKeyring = options.NoNewKeyring 101 | p.CriuWorkPath = options.CriuWorkPath 102 | if p.CriuWorkPath == "" { 103 | // if criu work path not set, use container WorkDir 104 | p.CriuWorkPath = p.WorkDir 105 | } 106 | return p, nil 107 | } 108 | 109 | // NewContainer returns a new runc container 110 | func newContainer(ctx context.Context, platform stdio.Platform, r *taskAPI.CreateTaskRequest) (_ *darwinContainer, retErr error) { 111 | ns, err := namespaces.NamespaceRequired(ctx) 112 | if err != nil { 113 | return nil, errors.Wrap(err, "create namespace") 114 | } 115 | 116 | var opts options.Options 117 | if r.Options != nil && r.Options.GetTypeUrl() != "" { 118 | v, err := typeurl.UnmarshalAny(r.Options) 119 | if err != nil { 120 | return nil, err 121 | } 122 | opts = *v.(*options.Options) 123 | } 124 | 125 | var mounts []process.Mount 126 | for _, m := range r.Rootfs { 127 | mounts = append(mounts, process.Mount{ 128 | Type: m.Type, 129 | Source: m.Source, 130 | Target: m.Target, 131 | Options: m.Options, 132 | }) 133 | } 134 | 135 | rootfs := "" 136 | if len(mounts) > 0 { 137 | rootfs = filepath.Join(r.Bundle, "rootfs") 138 | if err := os.Mkdir(rootfs, 0711); err != nil && !os.IsExist(err) { 139 | return nil, err 140 | } 141 | } 142 | 143 | config := &process.CreateConfig{ 144 | ID: r.ID, 145 | Bundle: r.Bundle, 146 | Runtime: opts.BinaryName, 147 | Rootfs: mounts, 148 | Terminal: r.Terminal, 149 | Stdin: r.Stdin, 150 | Stdout: r.Stdout, 151 | Stderr: r.Stderr, 152 | Checkpoint: r.Checkpoint, 153 | ParentCheckpoint: r.ParentCheckpoint, 154 | Options: r.Options, 155 | } 156 | 157 | defer func() { 158 | if retErr != nil { 159 | if err := mount.UnmountAll(rootfs, 0); err != nil { 160 | logrus.WithError(err).Warn("failed to cleanup rootfs mount") 161 | } 162 | } 163 | }() 164 | for _, rm := range mounts { 165 | m := &mount.Mount{ 166 | Type: rm.Type, 167 | Source: rm.Source, 168 | Options: rm.Options, 169 | } 170 | if err := m.Mount(rootfs); err != nil { 171 | return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m) 172 | } 173 | } 174 | 175 | p, err := newInit( 176 | ctx, 177 | r.Bundle, 178 | filepath.Join(r.Bundle, "work"), 179 | ns, 180 | platform, 181 | config, 182 | &opts, 183 | rootfs, 184 | ) 185 | if err != nil { 186 | return nil, errdefs.ToGRPC(err) 187 | } 188 | if err := p.Create(ctx, config); err != nil { 189 | return nil, errdefs.ToGRPC(err) 190 | } 191 | container := &darwinContainer{ 192 | ID: r.ID, 193 | Bundle: r.Bundle, 194 | process: p, 195 | processes: make(map[string]process.Process), 196 | } 197 | return container, nil 198 | } 199 | 200 | // New returns a new shim service that can be used via GRPC 201 | func New(ctx context.Context, id string, publisher shim.Publisher, shutdown func()) (shim.Shim, error) { 202 | s := &service{ 203 | id: id, 204 | context: ctx, 205 | events: make(chan interface{}, 128), 206 | ec: Default.Subscribe(), 207 | cancel: shutdown, 208 | containers: make(map[string]*darwinContainer), 209 | } 210 | go s.processExits() 211 | 212 | // setup our own reaper (similar to runj/freebsd shim) 213 | SetupReaperSignals(ctx, log.G(ctx).WithField("id", id)) 214 | runcC.Monitor = Default 215 | if err := s.initPlatform(); err != nil { 216 | shutdown() 217 | return nil, errors.Wrap(err, "failed to initialized platform behavior") 218 | } 219 | go s.forward(ctx, publisher) 220 | return s, nil 221 | } 222 | 223 | // service is the shim implementation of a remote shim over GRPC 224 | type service struct { 225 | mu sync.Mutex 226 | eventSendMu sync.Mutex 227 | 228 | context context.Context 229 | events chan interface{} 230 | platform stdio.Platform 231 | ec chan runcC.Exit 232 | 233 | // id only used in cleanup case 234 | id string 235 | 236 | containers map[string]*darwinContainer 237 | 238 | cancel func() 239 | } 240 | 241 | func newCommand(ctx context.Context, id, containerdBinary, containerdAddress, containerdTTRPCAddress string) (*exec.Cmd, error) { 242 | ns, err := namespaces.NamespaceRequired(ctx) 243 | if err != nil { 244 | return nil, err 245 | } 246 | self, err := os.Executable() 247 | if err != nil { 248 | return nil, err 249 | } 250 | cwd, err := os.Getwd() 251 | if err != nil { 252 | return nil, err 253 | } 254 | args := []string{ 255 | "-namespace", ns, 256 | "-id", id, 257 | "-address", containerdAddress, 258 | } 259 | cmd := exec.Command(self, args...) 260 | cmd.Dir = cwd 261 | cmd.Env = append(os.Environ(), "GOMAXPROCS=4") 262 | cmd.SysProcAttr = &syscall.SysProcAttr{ 263 | Setpgid: true, 264 | } 265 | return cmd, nil 266 | } 267 | 268 | func getProcess(c *darwinContainer, id string) (process.Process, error) { 269 | c.mu.Lock() 270 | defer c.mu.Unlock() 271 | if id == "" { 272 | if c.process == nil { 273 | return nil, errors.Wrapf(errdefs.ErrFailedPrecondition, "container must be created") 274 | } 275 | return c.process, nil 276 | } 277 | p, ok := c.processes[id] 278 | if !ok { 279 | return nil, errors.Wrapf(errdefs.ErrNotFound, "process does not exist %s", id) 280 | } 281 | return p, nil 282 | } 283 | 284 | func (s *service) StartShim(ctx context.Context, opts shim.StartOpts) (_ string, retErr error) { 285 | cmd, err := newCommand(ctx, opts.ID, opts.ContainerdBinary, opts.Address, opts.TTRPCAddress) 286 | if err != nil { 287 | return "", err 288 | } 289 | grouping := opts.ID 290 | address, err := shim.SocketAddress(ctx, opts.Address, grouping) 291 | if err != nil { 292 | return "", err 293 | } 294 | 295 | os.Mkdir(filepath.Dir(address), 0711) 296 | unix.Unlink(address) 297 | 298 | socket, err := shim.NewSocket(address) 299 | if err != nil { 300 | if strings.Contains(err.Error(), "address already in use") { 301 | if err := shim.WriteAddress("address", address); err != nil { 302 | return "", err 303 | } 304 | return address, nil 305 | } 306 | return "", err 307 | } 308 | // XXX: defer socket.Close() and f.Close() may close sockets created 309 | // _immediately_ (on darwin?) thus, don't do for darwin 310 | f, err := socket.File() 311 | if err != nil { 312 | return "", err 313 | } 314 | 315 | cmd.ExtraFiles = append(cmd.ExtraFiles, f) 316 | if err := cmd.Start(); err != nil { 317 | return "", err 318 | } 319 | defer func() { 320 | if retErr != nil { 321 | cmd.Process.Kill() 322 | } 323 | }() 324 | // make sure to wait after start 325 | go cmd.Wait() 326 | if err := shim.WritePidFile("shim.pid", cmd.Process.Pid); err != nil { 327 | return "", err 328 | } 329 | if err := shim.WriteAddress("address", address); err != nil { 330 | return "", err 331 | } 332 | if data, err := ioutil.ReadAll(os.Stdin); err == nil { 333 | if len(data) > 0 { 334 | var any ptypes.Any 335 | if err := proto.Unmarshal(data, &any); err != nil { 336 | return "", err 337 | } 338 | _, err := typeurl.UnmarshalAny(&any) 339 | if err != nil { 340 | return "", err 341 | } 342 | } 343 | } 344 | 345 | // XXX: unix socket on darwin seems to take a bit moment 346 | // before listened socket will be ready, so sleep a little 347 | time.Sleep(time.Millisecond * 10) 348 | return address, nil 349 | } 350 | 351 | func (s *service) Cleanup(ctx context.Context) (*taskAPI.DeleteResponse, error) { 352 | cwd, err := os.Getwd() 353 | if err != nil { 354 | return nil, err 355 | } 356 | path := filepath.Join(filepath.Dir(cwd), s.id) 357 | 358 | if err := mount.UnmountAll(filepath.Join(path, "rootfs"), 0); err != nil { 359 | logrus.WithError(err).Warn("failed to cleanup rootfs mount") 360 | } 361 | return &taskAPI.DeleteResponse{ 362 | ExitedAt: time.Now(), 363 | ExitStatus: 128 + uint32(unix.SIGKILL), 364 | }, nil 365 | } 366 | 367 | // Create a new initial process and container with the underlying OCI runtime 368 | func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *taskAPI.CreateTaskResponse, err error) { 369 | opts := &options.Options{ 370 | BinaryName: "runu", 371 | Root: RunuRoot, 372 | } 373 | any, err := typeurl.MarshalAny(opts) 374 | if err != nil { 375 | return nil, err 376 | } 377 | r.Options = any 378 | 379 | container, err := newContainer(ctx, s.platform, r) 380 | if err != nil { 381 | return nil, err 382 | } 383 | 384 | s.containers[r.ID] = container 385 | 386 | s.send(&eventstypes.TaskCreate{ 387 | ContainerID: r.ID, 388 | Bundle: r.Bundle, 389 | Rootfs: r.Rootfs, 390 | IO: &eventstypes.TaskIO{ 391 | Stdin: r.Stdin, 392 | Stdout: r.Stdout, 393 | Stderr: r.Stderr, 394 | Terminal: r.Terminal, 395 | }, 396 | Checkpoint: r.Checkpoint, 397 | Pid: uint32(container.process.Pid()), 398 | }) 399 | 400 | return &taskAPI.CreateTaskResponse{ 401 | Pid: uint32(container.process.Pid()), 402 | }, nil 403 | } 404 | 405 | // Start a process 406 | func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (*taskAPI.StartResponse, error) { 407 | container, err := s.getContainer(r.ID) 408 | if err != nil { 409 | return nil, err 410 | } 411 | 412 | // hold the send lock so that the start events are sent before any exit events in the error case 413 | s.eventSendMu.Lock() 414 | p, err := getProcess(container, r.ExecID) 415 | if err != nil { 416 | s.eventSendMu.Unlock() 417 | return nil, errdefs.ToGRPC(err) 418 | } 419 | if err := p.Start(ctx); err != nil { 420 | s.eventSendMu.Unlock() 421 | return nil, errdefs.ToGRPC(err) 422 | } 423 | 424 | s.eventSendMu.Unlock() 425 | return &taskAPI.StartResponse{ 426 | Pid: uint32(p.Pid()), 427 | }, nil 428 | } 429 | 430 | // Delete the initial process and container 431 | func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (*taskAPI.DeleteResponse, error) { 432 | container, err := s.getContainer(r.ID) 433 | if err != nil { 434 | return nil, err 435 | } 436 | p, err := getProcess(container, r.ExecID) 437 | if err != nil { 438 | return nil, errdefs.ToGRPC(err) 439 | } 440 | if err := p.Delete(ctx); err != nil { 441 | return nil, errdefs.ToGRPC(err) 442 | } 443 | 444 | // if we deleted an init task, send the task delete event 445 | if r.ExecID == "" { 446 | s.mu.Lock() 447 | delete(s.containers, r.ID) 448 | s.mu.Unlock() 449 | s.send(&eventstypes.TaskDelete{ 450 | ContainerID: container.ID, 451 | Pid: uint32(p.Pid()), 452 | ExitStatus: uint32(p.ExitStatus()), 453 | ExitedAt: p.ExitedAt(), 454 | }) 455 | } else { 456 | container.mu.Lock() 457 | defer container.mu.Unlock() 458 | delete(container.processes, r.ExecID) 459 | } 460 | return &taskAPI.DeleteResponse{ 461 | ExitStatus: uint32(p.ExitStatus()), 462 | ExitedAt: p.ExitedAt(), 463 | Pid: uint32(p.Pid()), 464 | }, nil 465 | } 466 | 467 | // Exec an additional process inside the container 468 | // TODO: not implemented yet 469 | func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (*ptypes.Empty, error) { 470 | return empty, nil 471 | } 472 | 473 | // ResizePty of a process 474 | // TODO: not implemented yet 475 | func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (*ptypes.Empty, error) { 476 | return empty, nil 477 | } 478 | 479 | // State returns runtime state information for a process 480 | func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (*taskAPI.StateResponse, error) { 481 | container, err := s.getContainer(r.ID) 482 | if err != nil { 483 | return nil, err 484 | } 485 | p, err := getProcess(container, r.ExecID) 486 | if err != nil { 487 | logrus.WithError(err).Error(container.processes) 488 | return nil, err 489 | } 490 | st, err := p.Status(ctx) 491 | if err != nil { 492 | return nil, err 493 | } 494 | status := task.StatusUnknown 495 | switch st { 496 | case "created": 497 | status = task.StatusCreated 498 | case "running": 499 | status = task.StatusRunning 500 | case "stopped": 501 | status = task.StatusStopped 502 | case "paused": 503 | status = task.StatusPaused 504 | case "pausing": 505 | status = task.StatusPausing 506 | } 507 | sio := p.Stdio() 508 | return &taskAPI.StateResponse{ 509 | ID: p.ID(), 510 | Bundle: container.Bundle, 511 | Pid: uint32(p.Pid()), 512 | Status: status, 513 | Stdin: sio.Stdin, 514 | Stdout: sio.Stdout, 515 | Stderr: sio.Stderr, 516 | Terminal: sio.Terminal, 517 | ExitStatus: uint32(p.ExitStatus()), 518 | ExitedAt: p.ExitedAt(), 519 | }, nil 520 | } 521 | 522 | // Pause the container 523 | func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (*ptypes.Empty, error) { 524 | container, err := s.getContainer(r.ID) 525 | if err != nil { 526 | return nil, err 527 | } 528 | if err := container.process.(*process.Init).Pause(ctx); err != nil { 529 | return nil, errdefs.ToGRPC(err) 530 | } 531 | s.send(&eventstypes.TaskPaused{ 532 | ContainerID: container.ID, 533 | }) 534 | return empty, nil 535 | } 536 | 537 | // Resume the container 538 | func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (*ptypes.Empty, error) { 539 | container, err := s.getContainer(r.ID) 540 | if err != nil { 541 | return nil, err 542 | } 543 | if err := container.process.(*process.Init).Resume(ctx); err != nil { 544 | return nil, errdefs.ToGRPC(err) 545 | } 546 | s.send(&eventstypes.TaskResumed{ 547 | ContainerID: container.ID, 548 | }) 549 | return empty, nil 550 | } 551 | 552 | // Kill a process with the provided signal 553 | func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (*ptypes.Empty, error) { 554 | container, err := s.getContainer(r.ID) 555 | if err != nil { 556 | return nil, err 557 | } 558 | p, err := getProcess(container, r.ExecID) 559 | if err != nil { 560 | return nil, err 561 | } 562 | 563 | if err := p.Kill(ctx, r.Signal, r.All); err != nil { 564 | return nil, errdefs.ToGRPC(err) 565 | } 566 | return empty, nil 567 | } 568 | 569 | // Pids returns all pids inside the container 570 | func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (*taskAPI.PidsResponse, error) { 571 | container, err := s.getContainer(r.ID) 572 | if err != nil { 573 | return nil, err 574 | } 575 | pids, err := s.getContainerPids(ctx, r.ID) 576 | if err != nil { 577 | return nil, errdefs.ToGRPC(err) 578 | } 579 | var processes []*task.ProcessInfo 580 | for _, pid := range pids { 581 | pInfo := task.ProcessInfo{ 582 | Pid: pid, 583 | } 584 | for _, p := range container.processes { 585 | if p.Pid() == int(pid) { 586 | d := &options.ProcessDetails{ 587 | ExecID: p.ID(), 588 | } 589 | a, err := typeurl.MarshalAny(d) 590 | if err != nil { 591 | return nil, errors.Wrapf(err, "failed to marshal process %d info", pid) 592 | } 593 | pInfo.Info = a 594 | break 595 | } 596 | } 597 | processes = append(processes, &pInfo) 598 | } 599 | return &taskAPI.PidsResponse{ 600 | Processes: processes, 601 | }, nil 602 | } 603 | 604 | // CloseIO of a process 605 | func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (*ptypes.Empty, error) { 606 | container, err := s.getContainer(r.ID) 607 | if err != nil { 608 | return nil, err 609 | } 610 | p, err := getProcess(container, r.ExecID) 611 | if err != nil { 612 | return empty, err 613 | } 614 | if stdin := p.Stdin(); stdin != nil { 615 | if err := stdin.Close(); err != nil { 616 | return empty, errors.Wrap(err, "close stdin") 617 | } 618 | } 619 | return empty, nil 620 | } 621 | 622 | // Checkpoint the container 623 | // TODO: not implemented yet 624 | func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskRequest) (*ptypes.Empty, error) { 625 | return empty, nil 626 | } 627 | 628 | // Update a running container 629 | // TODO: not implemented yet 630 | func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (*ptypes.Empty, error) { 631 | return empty, nil 632 | } 633 | 634 | // Wait for a process to exit 635 | func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (*taskAPI.WaitResponse, error) { 636 | container, err := s.getContainer(r.ID) 637 | if err != nil { 638 | return nil, err 639 | } 640 | p, err := getProcess(container, r.ExecID) 641 | if err != nil { 642 | return nil, errdefs.ToGRPC(err) 643 | } 644 | p.Wait() 645 | 646 | return &taskAPI.WaitResponse{ 647 | ExitStatus: uint32(p.ExitStatus()), 648 | ExitedAt: p.ExitedAt(), 649 | }, nil 650 | } 651 | 652 | // Connect returns shim information such as the shim's pid 653 | func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (*taskAPI.ConnectResponse, error) { 654 | var pid int 655 | if container, err := s.getContainer(r.ID); err == nil { 656 | pid = container.process.Pid() 657 | } 658 | return &taskAPI.ConnectResponse{ 659 | ShimPid: uint32(os.Getpid()), 660 | TaskPid: uint32(pid), 661 | }, nil 662 | } 663 | 664 | func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (*ptypes.Empty, error) { 665 | s.mu.Lock() 666 | // return out if the shim is still servicing containers 667 | if len(s.containers) > 0 { 668 | s.mu.Unlock() 669 | return empty, nil 670 | } 671 | s.cancel() 672 | close(s.events) 673 | 674 | if s.platform != nil { 675 | s.platform.Close() 676 | } 677 | 678 | return empty, nil 679 | } 680 | 681 | func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (*taskAPI.StatsResponse, error) { 682 | return &taskAPI.StatsResponse{ 683 | Stats: nil, 684 | }, nil 685 | } 686 | 687 | func (s *service) processExits() { 688 | for e := range s.ec { 689 | s.checkProcesses(e) 690 | } 691 | } 692 | 693 | func (s *service) send(evt interface{}) { 694 | s.events <- evt 695 | } 696 | 697 | func (s *service) sendL(evt interface{}) { 698 | s.eventSendMu.Lock() 699 | s.events <- evt 700 | s.eventSendMu.Unlock() 701 | } 702 | 703 | func (s *service) checkProcesses(e runcC.Exit) { 704 | for _, container := range s.containers { 705 | o := []process.Process{} 706 | for _, p := range container.processes { 707 | o = append(o, p) 708 | } 709 | o = append(o, container.process) 710 | for _, p := range o { 711 | if p.Pid() != e.Pid { 712 | continue 713 | } 714 | 715 | p.SetExited(e.Status) 716 | s.sendL(&eventstypes.TaskExit{ 717 | ContainerID: container.ID, 718 | ID: p.ID(), 719 | Pid: uint32(e.Pid), 720 | ExitStatus: uint32(e.Status), 721 | ExitedAt: p.ExitedAt(), 722 | }) 723 | return 724 | } 725 | return 726 | } 727 | } 728 | 729 | func (s *service) getContainerPids(ctx context.Context, id string) ([]uint32, error) { 730 | container, err := s.getContainer(id) 731 | if err != nil { 732 | return nil, err 733 | } 734 | p, err := getProcess(container, "") 735 | if err != nil { 736 | return nil, errdefs.ToGRPC(err) 737 | } 738 | ps, err := p.(*process.Init).Runtime().Ps(ctx, id) 739 | if err != nil { 740 | return nil, err 741 | } 742 | pids := make([]uint32, 0, len(ps)) 743 | for _, pid := range ps { 744 | pids = append(pids, uint32(pid)) 745 | } 746 | return pids, nil 747 | } 748 | 749 | func getTopic(e interface{}) string { 750 | switch e.(type) { 751 | case *eventstypes.TaskCreate: 752 | return runtime.TaskCreateEventTopic 753 | case *eventstypes.TaskStart: 754 | return runtime.TaskStartEventTopic 755 | case *eventstypes.TaskOOM: 756 | return runtime.TaskOOMEventTopic 757 | case *eventstypes.TaskExit: 758 | return runtime.TaskExitEventTopic 759 | case *eventstypes.TaskDelete: 760 | return runtime.TaskDeleteEventTopic 761 | case *eventstypes.TaskExecAdded: 762 | return runtime.TaskExecAddedEventTopic 763 | case *eventstypes.TaskExecStarted: 764 | return runtime.TaskExecStartedEventTopic 765 | case *eventstypes.TaskPaused: 766 | return runtime.TaskPausedEventTopic 767 | case *eventstypes.TaskResumed: 768 | return runtime.TaskResumedEventTopic 769 | case *eventstypes.TaskCheckpointed: 770 | return runtime.TaskCheckpointedEventTopic 771 | default: 772 | logrus.Warnf("no topic for type %#v", e) 773 | } 774 | return runtime.TaskUnknownTopic 775 | } 776 | 777 | func (s *service) forward(ctx context.Context, publisher shim.Publisher) { 778 | ns, _ := namespaces.Namespace(ctx) 779 | ctx = namespaces.WithNamespace(context.Background(), ns) 780 | 781 | for e := range s.events { 782 | err := publisher.Publish(ctx, getTopic(e), e) 783 | if err != nil { 784 | logrus.WithError(err).Error("post event") 785 | } 786 | } 787 | publisher.Close() 788 | } 789 | 790 | func (s *service) getContainer(id string) (*darwinContainer, error) { 791 | s.mu.Lock() 792 | container := s.containers[id] 793 | s.mu.Unlock() 794 | if container == nil { 795 | return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "container not created") 796 | } 797 | return container, nil 798 | } 799 | --------------------------------------------------------------------------------