├── .dockerignore ├── .github └── workflows │ └── go.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── example_configs ├── alpine.json └── nginx.json ├── go.mod ├── go.sum ├── log └── log.go ├── main.go ├── rootfs ├── extract.go ├── image.go ├── pull.go └── pull_test.go ├── test ├── alpine.json ├── fas.json ├── integration.sh ├── junk.json ├── subuid.json └── timeout.json └── util ├── util.go └── util_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | .dockerignore 3 | rootfs_builder 4 | .git 5 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | if [ -f Gopkg.toml ]; then 29 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 30 | dep ensure 31 | fi 32 | 33 | - name: Build 34 | run: go build -ldflags "-linkmode external -extldflags -static" -tags="netgo osusergo" -o rootfs_builder -a main.go 35 | 36 | - name: Test 37 | run: docker run -i --privileged -v `pwd`:/rootfs_builder alpine:latest 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | rootfs_builder 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.14.3 2 | 3 | WORKDIR /rootfs_builder 4 | ADD . . 5 | # This automatically adds a subuid mapping 6 | RUN useradd fas 7 | RUN make local_build 8 | CMD make local_test 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 ForAllSecure, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: static dev build_in_container 2 | 3 | local_static: 4 | go build -ldflags "-linkmode external -extldflags -static" -tags="netgo osusergo" -o rootfs_builder -a main.go 5 | 6 | static: 7 | docker run --privileged -it -v `pwd`:/rootfs_builder golang:1.12 bash -c "cd /rootfs_builder && make local_static" 8 | 9 | dev: rootfs_image 10 | docker run -it --privileged -v `pwd`:/rootfs_builder rootfs_image bash -c "cd /rootfs_builder; bash" 11 | 12 | local_build: 13 | go build -o rootfs_builder main.go 14 | 15 | rootfs_image: 16 | docker build -t rootfs_image . 17 | 18 | local_test: local_build 19 | ./test/integration.sh 20 | 21 | test: rootfs_image 22 | docker run -it --privileged -v `pwd`:/rootfs_builder rootfs_image 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Go](https://github.com/ForAllSecure/rootfs_builder/workflows/Go/badge.svg?branch=master) 2 | 3 | Rootfs Builder 4 | ====== 5 | 6 | Rootfs builder pulls an image from a Docker registry and extracts the 7 | rootfs. This is equivalent to the command: 8 | 9 | `mkdir rootfs && docker export $(docker create busybox) | tar -C rootfs -xvf -` 10 | 11 | The rootfs generated is OCI compliant and can be run with RunC. The 12 | user can specify the user to chown the files to and whether or not to 13 | use a subuid mapping in case they want to unshare user namespaces. 14 | 15 | Installation 16 | ===== 17 | Install Go 1.12 18 | 19 | On debian:sid 20 | `apt-get install -y golang-1.12-go`. 21 | 22 | From source: 23 | ``` 24 | sudo apt-get update 25 | wget https://dl.google.com/go/go1.12.7.linux-amd64.tar.gz 26 | sudo tar -xvf go1.12.7.linux-amd64.tar.gz 27 | sudo mv go /usr/local 28 | sudo mv /usr/local/go/bin/go /bin 29 | ``` 30 | 31 | Rootfs builder can be statically built. This statically compiles 32 | rootfs builder in a container: 33 | 34 | `make static` 35 | 36 | Or if you want to develop Rootfs Builder in a container, run: 37 | `make dev` 38 | 39 | Usage 40 | ===== 41 | Rootfs builder can be run with: 42 | `./rootfs_builder ` 43 | 44 | An example config.json looks like: 45 | ``` 46 | { 47 | "Name": "debian:buster", 48 | "Cert": "/workdir/cert", 49 | "Retries": 3, 50 | "Spec": 51 | { 52 | "Dest": "/tmp/rootfs", 53 | "User": "fas", 54 | "UseSubuid": True 55 | } 56 | } 57 | ``` 58 | * **`Name`** (string, REQUIRED) Name of image to pull. 59 | * **`Cert`** (string, OPTIONAL) Path to cert to add to root CAs for the registry. 60 | * **`Retries`** (int, OPTIONAL) Number of attempts to connect to registry. 61 | * **`Spec`** (dict, OPTIONAL) Spec for the rootfs. 62 | * **`Dest`** (string, OPTIONAL) Destination to extract rootfs to. 63 | * **`User`** (string, OPTIONAL) User to chown files to. 64 | * **`UseSubuid`** (bool, OPTIONAL) Look up subuid mapping for giving user and chown to that uid. 65 | 66 | Tests 67 | ===== 68 | To run integration tests, run `make test`. 69 | 70 | Credits 71 | ===== 72 | 73 | This code is from [ForAllSecure](https://forallsecure.com) labs. It is 74 | not an official ForAllSecure maintained product or offering. 75 | 76 | Some code recycled from Google's Kaniko. 77 | -------------------------------------------------------------------------------- /example_configs/alpine.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "alpine:3.10", 3 | "Retries": 1, 4 | "HTTPS": false, 5 | "Spec": { 6 | "Dest": "/tmp/alpine", 7 | "User": "fas" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /example_configs/nginx.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "nginx:latest", 3 | "Retries": 1, 4 | "HTTPS": false, 5 | "Spec": { 6 | "Dest": "/tmp/alpine", 7 | "User": "root" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ForAllSecure/rootfs_builder 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/google/go-containerregistry v0.0.0-20190910142231-b02d448a3705 7 | github.com/pkg/errors v0.8.1 8 | github.com/stretchr/testify v1.4.0 9 | go.uber.org/atomic v1.4.0 // indirect 10 | go.uber.org/multierr v1.1.0 // indirect 11 | go.uber.org/zap v1.10.0 12 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= 2 | github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= 3 | github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= 4 | github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= 10 | github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= 11 | github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= 12 | github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 13 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 14 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 15 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 16 | github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= 17 | github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= 18 | github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= 19 | github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= 20 | github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 21 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 22 | github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 23 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 24 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 25 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 26 | github.com/google/go-containerregistry v0.0.0-20190820205713-31e00cede111 h1:5F39eE4QsUnAd6iGzt1/zBs3dhX877U2hJyOJHFmQF0= 27 | github.com/google/go-containerregistry v0.0.0-20190820205713-31e00cede111/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk= 28 | github.com/google/go-containerregistry v0.0.0-20190910142231-b02d448a3705 h1:rsBH4vQ2gLNUKf2+82LNQ45AsYnH12Q5ZnHiZXx9LZw= 29 | github.com/google/go-containerregistry v0.0.0-20190910142231-b02d448a3705/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk= 30 | github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= 31 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 32 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 33 | github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 34 | github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= 35 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 36 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 37 | github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 38 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 39 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 40 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 41 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 42 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 43 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 44 | github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 45 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 46 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 47 | github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 48 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 49 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 50 | github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 51 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 52 | github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 53 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 54 | github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 55 | github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 56 | github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 57 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 58 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 59 | github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 60 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 61 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 62 | github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 63 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 64 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 65 | github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 66 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 67 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 68 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 69 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 70 | go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= 71 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 72 | go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= 73 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 74 | go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= 75 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 76 | golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 77 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 78 | golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 79 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 80 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 81 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 82 | golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 83 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 84 | golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 85 | golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 86 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 87 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 88 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 89 | golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 90 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 91 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 92 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 93 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 94 | gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 95 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 96 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 97 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 98 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 99 | k8s.io/apimachinery v0.0.0-20190826114657-e31a5531b558 h1:89s1htYZqlP8Xpj+rGQ4ys/Xksmqrf4Xws0fZuWoWZg= 100 | k8s.io/apimachinery v0.0.0-20190826114657-e31a5531b558/go.mod h1:EZoIMuAgG/4v58YL+bz0kqnivqupk28fKYxFCa5e6X8= 101 | k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= 102 | k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= 103 | k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ= 104 | k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= 105 | k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= 106 | sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= 107 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 108 | -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "os" 5 | 6 | "go.uber.org/zap" 7 | "go.uber.org/zap/zapcore" 8 | ) 9 | 10 | // Fn typedefs a logging function so it can be passed around 11 | type Fn func(string, ...interface{}) 12 | 13 | var ( 14 | logger *zap.SugaredLogger 15 | ) 16 | 17 | func init() { 18 | // Define our level-handling logic. 19 | allPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { 20 | return true 21 | }) 22 | 23 | // Let's also log to stderr 24 | consoleLog := zapcore.Lock(os.Stderr) 25 | consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) 26 | 27 | // Create our custom loggers 28 | core := zapcore.NewTee( 29 | zapcore.NewCore(consoleEncoder, consoleLog, allPriority), 30 | ) 31 | log := zap.New(core) 32 | logger = log.Sugar() 33 | } 34 | 35 | func SetLogger(log *zap.SugaredLogger) { 36 | logger = log 37 | } 38 | 39 | func getLogger() *zap.SugaredLogger { 40 | return logger 41 | } 42 | 43 | func Debugf(format string, args ...interface{}) { 44 | getLogger().Debugf(format, args...) 45 | } 46 | 47 | func Infof(format string, args ...interface{}) { 48 | getLogger().Infof(format, args...) 49 | } 50 | 51 | func Warnf(format string, args ...interface{}) { 52 | getLogger().Warnf(format, args...) 53 | } 54 | 55 | func Errorf(format string, args ...interface{}) { 56 | getLogger().Errorf(format, args...) 57 | } 58 | 59 | func Panicf(format string, args ...interface{}) { 60 | getLogger().Panicf(format, args...) 61 | } 62 | 63 | func Fatalf(format string, args ...interface{}) { 64 | getLogger().Fatalf(format, args...) 65 | } 66 | 67 | func Debug(args ...interface{}) { 68 | getLogger().Debug(args...) 69 | } 70 | 71 | func Info(args ...interface{}) { 72 | getLogger().Info(args...) 73 | } 74 | 75 | func Warn(args ...interface{}) { 76 | getLogger().Warn(args...) 77 | } 78 | 79 | func Error(args ...interface{}) { 80 | getLogger().Error(args...) 81 | } 82 | 83 | func Panic(args ...interface{}) { 84 | getLogger().Panic(args...) 85 | } 86 | 87 | func Fatal(args ...interface{}) { 88 | getLogger().Fatal(args...) 89 | } 90 | 91 | func With(args ...interface{}) *zap.SugaredLogger { 92 | return getLogger().With(args...) 93 | } 94 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/ForAllSecure/rootfs_builder/log" 7 | "github.com/ForAllSecure/rootfs_builder/rootfs" 8 | ) 9 | 10 | func main() { 11 | if len(os.Args) > 3 || len(os.Args) < 2 { 12 | log.Fatal("Usage: rootfs_builder \n" + 13 | "\t\t\t\t\t--digest-only: only print the digest") 14 | } 15 | // Initialize pullable image from config 16 | pullableImage, err := rootfs.NewPullableImage(os.Args[1]) 17 | if err != nil { 18 | log.Errorf("Failed to initialize image from config: %+v", err) 19 | os.Exit(1) 20 | } 21 | pulledManifest, err := pullableImage.Pull() 22 | if err != nil { 23 | log.Errorf("Failed to pull image manifest: %+v", err) 24 | os.Exit(1) 25 | } 26 | 27 | // Extract rootfs 28 | if len(os.Args) == 2 { 29 | err = pulledManifest.Extract() 30 | if err != nil { 31 | log.Errorf("Failed to extract rootfs: %+v", err) 32 | os.Exit(1) 33 | } 34 | } else { 35 | // Digest only 36 | digest, err := pulledManifest.Digest() 37 | if err != nil { 38 | log.Errorf("Failed to get digest: %+v", err) 39 | os.Exit(1) 40 | } 41 | log.Info(digest) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /rootfs/extract.go: -------------------------------------------------------------------------------- 1 | // Parts of this file are modified from Kaniko, an Apache 2.0 licensed project, 2 | // and so this copyright applies. 3 | // 4 | // Copyright 2018 Google LLC 5 | // 6 | // https://github.com/GoogleContainerTools/kaniko/blob/master/pkg/util/fs_util.go 7 | // Commit # 3422d55 8 | 9 | package rootfs 10 | 11 | import ( 12 | "archive/tar" 13 | "fmt" 14 | "io" 15 | "io/ioutil" 16 | "os" 17 | "path/filepath" 18 | "strings" 19 | 20 | "github.com/ForAllSecure/rootfs_builder/log" 21 | v1 "github.com/google/go-containerregistry/pkg/v1" 22 | "github.com/google/go-containerregistry/pkg/v1/v1util" 23 | "github.com/pkg/errors" 24 | ) 25 | 26 | // extract a single file 27 | func extractFile(dest string, hdr *tar.Header, tr io.Reader, subuid int, subgid int) error { 28 | // Construct filepath from tar header 29 | path := filepath.Join(dest, filepath.Clean(hdr.Name)) 30 | dir := filepath.Dir(path) 31 | 32 | // Get metadata from tar header 33 | mode := hdr.FileInfo().Mode() 34 | uid := hdr.Uid + subuid 35 | gid := hdr.Gid + subgid 36 | 37 | switch hdr.Typeflag { 38 | case tar.TypeReg: 39 | // It's possible a file is in the tar before it's directory. 40 | if _, err := os.Stat(dir); os.IsNotExist(err) { 41 | if err := os.MkdirAll(dir, 0755); err != nil { 42 | return err 43 | } 44 | } 45 | // Check if something already exists at path (symlinks etc.) 46 | // If so, delete it 47 | if _, err := os.Lstat(path); !os.IsNotExist(err) { 48 | if err := os.RemoveAll(path); err != nil { 49 | return errors.Wrapf(err, "error removing %s to make way for new file.", path) 50 | } 51 | } 52 | currFile, err := os.Create(path) 53 | if err != nil { 54 | return err 55 | } 56 | // manually set permissions on file, since the default umask (022) will interfere 57 | if err = os.Chmod(path, mode); err != nil { 58 | return err 59 | } 60 | if _, err = io.Copy(currFile, tr); err != nil { 61 | return err 62 | } 63 | if err = currFile.Chown(uid, gid); err != nil { 64 | return err 65 | } 66 | currFile.Close() 67 | case tar.TypeDir: 68 | if err := os.MkdirAll(path, mode); err != nil { 69 | return err 70 | } 71 | // In some cases, MkdirAll doesn't change the permissions, so run Chmod 72 | if err := os.Chmod(path, mode); err != nil { 73 | return err 74 | } 75 | if err := os.Chown(path, uid, gid); err != nil { 76 | return err 77 | } 78 | 79 | // Hard link: Two files point to same data on disc. Assume OFS/Docker orders tarball such 80 | // that hard link comes after regular file that hard link points to. 81 | case tar.TypeLink: 82 | // The base directory for a link may not exist before it is created. 83 | if err := os.MkdirAll(dir, 0755); err != nil { 84 | return err 85 | } 86 | // Check if something already exists at path 87 | // If so, delete it 88 | if _, err := os.Lstat(path); !os.IsNotExist(err) { 89 | if err := os.RemoveAll(path); err != nil { 90 | return errors.Wrapf(err, "error removing %s to make way for new link", hdr.Name) 91 | } 92 | } 93 | // Link hard link to its target 94 | link := filepath.Clean(filepath.Join(dest, hdr.Linkname)) 95 | if err := os.Link(link, path); err != nil { 96 | return err 97 | } 98 | 99 | case tar.TypeSymlink: 100 | // The base directory for a symlink may not exist before it is created. 101 | if err := os.MkdirAll(dir, 0755); err != nil { 102 | return err 103 | } 104 | // Check if something already exists at path 105 | // If so, delete it 106 | if _, err := os.Lstat(path); !os.IsNotExist(err) { 107 | if err := os.RemoveAll(path); err != nil { 108 | return errors.Wrapf(err, "error removing %s to make way for new symlink", hdr.Name) 109 | } 110 | } 111 | if err := os.Symlink(hdr.Linkname, path); err != nil { 112 | return err 113 | } 114 | if err := os.Lchown(path, uid, gid); err != nil { 115 | return err 116 | } 117 | } 118 | return nil 119 | } 120 | 121 | // Whiteouts 122 | func whiteout(tr *tar.Reader, rootfs string) error { 123 | // Iterate through headers, removing whiteouts first 124 | for { 125 | hdr, err := tr.Next() 126 | // Done with this tar layer 127 | if err == io.EOF { 128 | break 129 | } 130 | // Something went wrong 131 | if err != nil { 132 | return err 133 | } 134 | path := filepath.Join(rootfs, filepath.Clean(hdr.Name)) 135 | base := filepath.Base(path) 136 | dir := filepath.Dir(path) 137 | // Opaque directory 138 | if strings.HasPrefix(base, ".wh..wh..opq") { 139 | if err := os.RemoveAll(dir); err != nil { 140 | return errors.Wrapf(err, "removing whiteout %s", hdr.Name) 141 | } 142 | } else if strings.HasPrefix(base, ".wh.") { 143 | name := strings.TrimPrefix(base, ".wh.") 144 | if err := os.RemoveAll(filepath.Join(dir, name)); err != nil { 145 | return errors.Wrapf(err, "removing whiteout %s", hdr.Name) 146 | } 147 | } else { 148 | continue 149 | } 150 | } 151 | return nil 152 | } 153 | 154 | // Handle regular files 155 | func handleFiles(tr *tar.Reader, rootfs string, subuid int, subgid int) error { 156 | // Iterate through the headers, extracting regular files 157 | for { 158 | hdr, err := tr.Next() 159 | // Done with this tar layer 160 | if err == io.EOF { 161 | break 162 | } 163 | // Something went wrong 164 | if err != nil { 165 | return err 166 | } 167 | path := filepath.Join(rootfs, filepath.Clean(hdr.Name)) 168 | base := filepath.Base(path) 169 | // This is a whiteout file/directory, skip! 170 | if strings.HasPrefix(base, ".wh.") { 171 | continue 172 | } 173 | if err := extractFile(rootfs, hdr, tr, subuid, subgid); err != nil { 174 | return err 175 | } 176 | } 177 | return nil 178 | } 179 | 180 | // Get a tar reader from a v1.Layer 181 | func tarReader(layer_file *os.File) (*tar.Reader, error) { 182 | r, err := v1util.GunzipReadCloser(layer_file) 183 | if err != nil { 184 | return nil, err 185 | } 186 | tr := tar.NewReader(r) 187 | return tr, nil 188 | } 189 | 190 | // saveLayer saves a Layer to disk 191 | func saveLayer(layer v1.Layer) (*os.File, error) { 192 | digest, err := layer.Digest() 193 | if err != nil { 194 | return nil, err 195 | } 196 | 197 | layer_file, err := ioutil.TempFile("", fmt.Sprintf("%s", digest)) 198 | if err != nil { 199 | return nil, errors.Wrapf(err, "generating tempfile") 200 | } 201 | 202 | rc, err := layer.Compressed() 203 | if err != nil { 204 | defer os.Remove(layer_file.Name()) 205 | return nil, err 206 | } 207 | 208 | _, err = io.Copy(layer_file, rc) 209 | if err != nil { 210 | defer os.Remove(layer_file.Name()) 211 | return nil, err 212 | } 213 | layer_file.Seek(0, 0) 214 | return layer_file, nil 215 | } 216 | 217 | // extractLayer accepts an open file descriptor to tarball and the destianation 218 | // to extract the rootfs to 219 | func extractLayer(layer v1.Layer, rootfs string, subuid int, subgid int) error { 220 | digest, err := layer.Digest() 221 | if err != nil { 222 | return err 223 | } 224 | size, err := layer.Size() 225 | if err != nil { 226 | return err 227 | } 228 | 229 | log.Debugf("Downloading layer %s, %d bytes", digest, size) 230 | layer_file, err := saveLayer(layer) 231 | if err != nil { 232 | return err 233 | } 234 | defer os.Remove(layer_file.Name()) 235 | 236 | tr, err := tarReader(layer_file) 237 | if err != nil { 238 | return err 239 | } 240 | log.Debugf("Whiting out layer %s", digest) 241 | err = whiteout(tr, rootfs) 242 | if err != nil { 243 | return err 244 | } 245 | 246 | layer_file.Seek(0, 0) 247 | tr, err = tarReader(layer_file) 248 | if err != nil { 249 | return err 250 | } 251 | 252 | log.Debugf("Extracting layer %s", digest) 253 | err = handleFiles(tr, rootfs, subuid, subgid) 254 | if err != nil { 255 | return err 256 | } 257 | return nil 258 | } 259 | -------------------------------------------------------------------------------- /rootfs/image.go: -------------------------------------------------------------------------------- 1 | package rootfs 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "os/user" 8 | "path/filepath" 9 | "strconv" 10 | 11 | "github.com/ForAllSecure/rootfs_builder/util" 12 | "github.com/google/go-containerregistry/pkg/name" 13 | v1 "github.com/google/go-containerregistry/pkg/v1" 14 | "github.com/google/go-containerregistry/pkg/v1/partial" 15 | "github.com/pkg/errors" 16 | ) 17 | 18 | // Spec for rootfs extraction 19 | type Spec struct { 20 | // Destination to extract to 21 | Dest string 22 | // User to chown files in rootfs to 23 | User string 24 | // Use the subuid associated with the given user for chowning 25 | UseSubuid bool 26 | subuid int 27 | subgid int 28 | } 29 | 30 | // PulledImage using provided PullableImage 31 | type PulledImage struct { 32 | // User specified requirements for rootfs 33 | spec Spec 34 | name string 35 | img v1.Image 36 | } 37 | 38 | // Digest from pulled image 39 | func (pulledImg *PulledImage) Digest() (string, error) { 40 | // Digest() fails silently on images older than June 2016 (i.e. returns a 41 | // random hash that changes), so check img age 42 | _, err := getConfig(pulledImg.img) 43 | if err != nil { 44 | return "", err 45 | } 46 | hash, err := pulledImg.img.Digest() 47 | if err != nil { 48 | return "", errors.WithStack(err) 49 | } 50 | ref, err := name.ParseReference(pulledImg.name, name.WeakValidation) 51 | if err != nil { 52 | return "", errors.WithStack(err) 53 | } 54 | buf := fmt.Sprintf("%s/%s@%s\n", ref.Context().RegistryStr(), ref.Context().RepositoryStr(), hash.String()) 55 | 56 | return buf, nil 57 | } 58 | 59 | // Extract rootfs 60 | func (pulledImg *PulledImage) Extract() error { 61 | // Ensure we have a valid location to extract to 62 | err := pulledImg.validateDest() 63 | if err != nil { 64 | return err 65 | } 66 | 67 | // Dump the config 68 | err = pulledImg.writeConfig() 69 | if err != nil { 70 | return err 71 | } 72 | 73 | // Get a list of layers 74 | layers, err := pulledImg.img.Layers() 75 | if err != nil { 76 | return err 77 | } 78 | 79 | rootfsPath := filepath.Join(pulledImg.spec.Dest, "rootfs") 80 | if err := os.MkdirAll(rootfsPath, 0755); err != nil { 81 | return err 82 | } 83 | 84 | if err := pulledImg.validateUser(); err != nil { 85 | return err 86 | } 87 | 88 | // Extract the layers 89 | for _, layer := range layers { 90 | err = extractLayer(layer, rootfsPath, pulledImg.spec.subuid, pulledImg.spec.subgid) 91 | if err != nil { 92 | return err 93 | } 94 | } 95 | 96 | if err := os.Chown(rootfsPath, pulledImg.spec.subuid, pulledImg.spec.subuid); err != nil { 97 | return err 98 | } 99 | 100 | return nil 101 | } 102 | 103 | // Confirm that the user exists, and look up the appropriate subuid/subgid 104 | func (pulledImg *PulledImage) validateUser() error { 105 | // Default to current user 106 | userObj, err := user.Current() 107 | 108 | // The config provided a user 109 | if pulledImg.spec.User != "" { 110 | userObj, err = user.Lookup(pulledImg.spec.User) 111 | } 112 | 113 | // Failed to find the user 114 | if err != nil { 115 | return err 116 | } 117 | 118 | // Get subuids for user namespace 119 | subuid, err := strconv.Atoi(userObj.Uid) 120 | if err != nil { 121 | return err 122 | } 123 | subgid, err := strconv.Atoi(userObj.Gid) 124 | if err != nil { 125 | return err 126 | } 127 | if pulledImg.spec.UseSubuid { 128 | subuid, subgid, err = util.GetSubid(userObj) 129 | if err != nil { 130 | return err 131 | } 132 | } 133 | 134 | pulledImg.spec.subuid = subuid 135 | pulledImg.spec.subgid = subgid 136 | 137 | return nil 138 | } 139 | 140 | // Validate the output location for the rootfs 141 | func (pulledImg *PulledImage) validateDest() error { 142 | if pulledImg.spec.Dest == "" { 143 | return errors.New("Specify output destination for rootfs") 144 | } 145 | // Create the directory if it doesn't exist 146 | if _, err := os.Stat(pulledImg.spec.Dest); os.IsNotExist(err) { 147 | _ = os.Mkdir(pulledImg.spec.Dest, 0755) 148 | } 149 | return nil 150 | } 151 | 152 | // extract config.json from image and write to image.Dest. 153 | // assumes image.Dest is valid. 154 | func (pulledImg *PulledImage) writeConfig() error { 155 | configFile, err := getConfig(pulledImg.img) 156 | if err != nil { 157 | return err 158 | } 159 | jdata, err := json.MarshalIndent(configFile, "", " ") 160 | if err != nil { 161 | return err 162 | } 163 | configPath := filepath.Join(pulledImg.spec.Dest, "config.json") 164 | jsonFile, err := os.Create(configPath) 165 | if err != nil { 166 | return err 167 | } 168 | _, err = jsonFile.Write(jdata) 169 | return err 170 | } 171 | 172 | // extract config.json from image and check for errors 173 | func getConfig(img partial.WithConfigFile) (*v1.ConfigFile, error) { 174 | configFile, err := img.ConfigFile() 175 | if err != nil { 176 | return nil, errors.Wrap(err, "could not retrieve config from image") 177 | } 178 | return configFile, nil 179 | } 180 | -------------------------------------------------------------------------------- /rootfs/pull.go: -------------------------------------------------------------------------------- 1 | package rootfs 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "io/ioutil" 7 | "math" 8 | "net" 9 | "net/http" 10 | "strings" 11 | "time" 12 | 13 | "github.com/ForAllSecure/rootfs_builder/log" 14 | "github.com/ForAllSecure/rootfs_builder/util" 15 | "github.com/google/go-containerregistry/pkg/authn" 16 | "github.com/google/go-containerregistry/pkg/name" 17 | v1 "github.com/google/go-containerregistry/pkg/v1" 18 | "github.com/google/go-containerregistry/pkg/v1/remote" 19 | "github.com/google/go-containerregistry/pkg/v1/remote/transport" 20 | "github.com/pkg/errors" 21 | ) 22 | 23 | // PullableImage contains metadata necessary for pulling images 24 | type PullableImage struct { 25 | // Name of image to pull 26 | Name string 27 | // Path to registry cert 28 | Cert *string 29 | // Number of attempts to retry pulling 30 | Retries int 31 | // Metadata for rootfs extraction 32 | Spec Spec 33 | https bool 34 | } 35 | 36 | // MaxBackoff is the maximum backoff time per retry in seconds 37 | var MaxBackoff float64 = 30 38 | 39 | // DefaultRetries is the default number of retries 40 | var DefaultRetries int = 3 41 | 42 | // NewPullableImage initializes a PullableImage spec from a user provided config 43 | func NewPullableImage(path string) (*PullableImage, error) { 44 | var pullableImage PullableImage 45 | err := util.UnmarshalFile(path, &pullableImage) 46 | if err != nil { 47 | return nil, err 48 | } 49 | if pullableImage.Retries <= 0 { 50 | pullableImage.Retries = DefaultRetries 51 | } 52 | pullableImage.https = true 53 | return &pullableImage, nil 54 | } 55 | 56 | // Pull a v1.Image and initialize a PulledImage struct to include the v1.img 57 | // and metadata for extracting to a rootfs 58 | func (pullable *PullableImage) Pull() (*PulledImage, error) { 59 | var err error 60 | var img v1.Image 61 | for i := 0; i < pullable.Retries; i++ { 62 | img, err = pullable.pull() 63 | if err == nil { 64 | break 65 | } 66 | if strings.Contains(err.Error(), "http: server gave HTTP response to HTTPS client") { 67 | log.Info("Retrying with HTTP") 68 | pullable.https = false 69 | } 70 | // This is a v1 schema, give up early 71 | if strings.Contains(err.Error(), "unsupported MediaType") { 72 | err = errors.WithMessage(err, "Image is v1 schema and too old to support") 73 | break 74 | } 75 | // Either we are unauthorized, or this is a bad registry/image name 76 | if strings.Contains(err.Error(), "UNAUTHORIZED: authentication required") { 77 | break 78 | } 79 | // If we get a i/o timeout, it's either intermittent network failure 80 | // or an incorrect ip address etc. This means we've already failed 5 81 | // retries internal to go-containerregistry, so fail 82 | if strings.Contains(err.Error(), "i/o timeout") { 83 | log.Warnf("Connection to server timed out %s", err) 84 | break 85 | } 86 | switch err := errors.Cause(err).(type) { 87 | case *transport.Error: 88 | break 89 | default: 90 | log.Warnf("Unrecognized error: %s Trying again", err) 91 | } 92 | 93 | backoff := math.Pow(2, float64(i)) 94 | backoff = math.Min(backoff, MaxBackoff) 95 | time.Sleep(time.Second * time.Duration(backoff)) 96 | } 97 | // Failed to pull, return an error 98 | if err != nil { 99 | return nil, err 100 | } 101 | // Initialize the image 102 | pulled := &PulledImage{ 103 | img: img, 104 | name: pullable.Name, 105 | spec: pullable.Spec, 106 | } 107 | return pulled, nil 108 | } 109 | 110 | // pull a v1.image 111 | func (pullable *PullableImage) pull() (v1.Image, error) { 112 | log.Debugf("Getting manifest for %s", pullable.Name) 113 | ref, err := name.ParseReference(pullable.Name, name.WeakValidation) 114 | if err != nil { 115 | return nil, errors.WithStack(err) 116 | } 117 | registryName := ref.Context().RegistryStr() 118 | 119 | var newReg name.Registry 120 | if pullable.https { 121 | newReg, err = name.NewRegistry(registryName, name.WeakValidation) 122 | } else { 123 | newReg, err = name.NewRegistry(registryName, name.Insecure) 124 | } 125 | 126 | if err != nil { 127 | return nil, errors.WithStack(err) 128 | } 129 | 130 | if tag, ok := ref.(name.Tag); ok { 131 | tag.Repository.Registry = newReg 132 | ref = tag 133 | } 134 | if digest, ok := ref.(name.Digest); ok { 135 | digest.Repository.Registry = newReg 136 | ref = digest 137 | } 138 | 139 | transport := http.DefaultTransport.(*http.Transport) 140 | transport.DialContext = (&net.Dialer{ 141 | Timeout: 10 * time.Second, 142 | KeepAlive: 10 * time.Second, 143 | DualStack: true, 144 | }).DialContext 145 | // A cert was provided 146 | if pullable.Cert != nil { 147 | rootCAs, err := x509.SystemCertPool() 148 | if err != nil { 149 | return nil, err 150 | } 151 | if rootCAs == nil { 152 | rootCAs = x509.NewCertPool() 153 | } 154 | // Read in the cert file 155 | certs, err := ioutil.ReadFile(*pullable.Cert) 156 | if err != nil { 157 | return nil, errors.Wrapf(err, "failed to read file %s to add to RootCAs", *pullable.Cert) 158 | } 159 | // Append our cert to the system pool 160 | if ok := rootCAs.AppendCertsFromPEM(certs); !ok { 161 | return nil, errors.Wrap(err, "Failed to append registry certificate") 162 | } 163 | 164 | // Trust the augmented cert pool in our client 165 | config := &tls.Config{ 166 | RootCAs: rootCAs, 167 | } 168 | 169 | transport.TLSClientConfig = config 170 | } 171 | transportOption := remote.WithTransport(transport) 172 | 173 | authnOption := remote.WithAuthFromKeychain(authn.NewMultiKeychain(authn.DefaultKeychain)) 174 | img, err := remote.Image(ref, transportOption, authnOption) 175 | if err != nil { 176 | return nil, errors.WithStack(err) 177 | } 178 | return img, nil 179 | } 180 | -------------------------------------------------------------------------------- /rootfs/pull_test.go: -------------------------------------------------------------------------------- 1 | package rootfs 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | "time" 7 | 8 | "github.com/ForAllSecure/rootfs_builder/rootfs" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestPull(t *testing.T) { 13 | pullable, err := rootfs.NewPullableImage("../test/alpine.json") 14 | require.NoError(t, err) 15 | _, err = pullable.Pull() 16 | require.NoError(t, err) 17 | } 18 | 19 | func TestPullTimeout(t *testing.T) { 20 | if testing.Short() { 21 | t.Skip("skipping timeout test in short mode") 22 | } 23 | 24 | pullable, err := rootfs.NewPullableImage("../test/timeout.json") 25 | require.NoError(t, err) 26 | 27 | start := time.Now() 28 | _, err = pullable.Pull() 29 | require.Less(t, int64(time.Since(start)/time.Second), int64(60)) 30 | require.Error(t, err) 31 | require.True(t, strings.Contains(err.Error(), "i/o timeout")) 32 | } 33 | -------------------------------------------------------------------------------- /test/alpine.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "alpine:3.10@sha256:c19173c5ada610a5989151111163d28a67368362762534d8a8121ce95cf2bd5a", 3 | "Retries": 3, 4 | "Spec": { 5 | "Dest": "/test", 6 | "User": "root" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /test/fas.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "alpine:3.10@sha256:c19173c5ada610a5989151111163d28a67368362762534d8a8121ce95cf2bd5a", 3 | "Retries": 3, 4 | "Spec": { 5 | "Dest": "/test", 6 | "User": "fas" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /test/integration.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Pull alpine:3.10, extract the rootfs, and verify its hash 4 | 5 | set -e 6 | set -x 7 | 8 | # set up 9 | printf "testing extracting alpine 3.10...\n" 10 | 11 | test_dir="/test/" 12 | if [ ! -d $test_dir ]; then 13 | mkdir $test_dir 14 | fi 15 | 16 | rm -rf $test_dir 17 | 18 | # run 19 | go run main.go test/alpine.json 20 | 21 | # check config hash 22 | config_md5=`md5sum $test_dir/config.json | head -n1 | awk '{print $1;}'` 23 | correct_config_md5="a7c6eead06dc2a2535d165d2db4d51f5" 24 | if [ "$config_md5" != "$correct_config_md5" ]; then 25 | echo "configs don't match" 26 | exit 1 27 | fi 28 | 29 | # check rootfs hash 30 | rootfs_md5=`find $test_dir/rootfs -type f -exec md5sum {} \; | sort -k 2 | md5sum | head -n1 | awk '{print $1;}'` 31 | correct_rootfs_md5="31ae55aacfa90c87e313a196617c5fe3" 32 | echo $rootfs_md5 33 | if [ "$rootfs_md5" != "$correct_rootfs_md5" ]; then 34 | echo "rootfs doesn't match" 35 | exit 1 36 | fi 37 | 38 | # tear down 39 | rm -rf $test_dir 40 | 41 | # set up 42 | printf "testing chowning to subuid...\n" 43 | test_dir="/test/" 44 | rm -rf $test_dir 45 | 46 | # run 47 | go run main.go test/subuid.json 48 | 49 | # Check that we chowned to the subuid mapping 100000 50 | uid=`ls -ld /test/rootfs/bin/cat | awk '{print $3}'` 51 | if [ "$uid" != "100000" ]; then 52 | echo "failed to chown to subuid 100000" 53 | exit 1 54 | fi 55 | 56 | # tear down 57 | rm -rf $test_dir 58 | 59 | # set up 60 | printf "testing chowning to a user besides root...\n" 61 | test_dir="/test/" 62 | rm -rf $test_dir 63 | 64 | # run 65 | go run main.go test/fas.json 66 | 67 | # Check that we chowned to the subuid mapping 100000 68 | uid=`ls -ld /test/rootfs/bin/cat | awk '{print $3}'` 69 | if [ "$uid" != "fas" ]; then 70 | echo "failed to chown to fas" 71 | exit 1 72 | fi 73 | 74 | # tear down 75 | rm -rf $test_dir 76 | -------------------------------------------------------------------------------- /test/junk.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "somejunk:latest", 3 | "Retries": 1 4 | } 5 | -------------------------------------------------------------------------------- /test/subuid.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "alpine:3.10@sha256:c19173c5ada610a5989151111163d28a67368362762534d8a8121ce95cf2bd5a", 3 | "Retries": 3, 4 | "Spec": { 5 | "Dest": "/test", 6 | "User": "fas", 7 | "UseSubuid": true 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /test/timeout.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "192.0.2.1:5000/does/not/exist", 3 | "Retries": 10, 4 | "Spec": { 5 | "Dest": "/test", 6 | "User": "root" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "os/user" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | // MapSize is the size of the subuid mapping 17 | const MapSize = 65536 18 | 19 | // PrettyPrintStruct prints a struct as info level by default 20 | // unless a different logging level function is passed in 21 | func PrettyPrintStruct(data interface{}) { 22 | jsonified, err := json.MarshalIndent(data, "", " ") 23 | if err != nil { 24 | err = fmt.Errorf("failed to pretty print json: %v", err) 25 | fmt.Println(err.Error()) 26 | return 27 | } 28 | fmt.Printf("%+v", (string(jsonified))) 29 | } 30 | 31 | // UnmarshalFile reads a json file into a struct 32 | func UnmarshalFile(filepath string, dst interface{}) error { 33 | data, err := ioutil.ReadFile(filepath) 34 | if err != nil { 35 | return errors.WithStack(err) 36 | } 37 | err = json.Unmarshal(data, dst) 38 | if err != nil { 39 | return errors.WithStack(err) 40 | } 41 | return nil 42 | } 43 | 44 | // GetSubid looks up the subuid and subgid for the given user 45 | func GetSubid(userObj *user.User) (int, int, error) { 46 | subuidFile, err := os.Open("/etc/subuid") 47 | if err != nil { 48 | return -1, -1, err 49 | } 50 | defer subuidFile.Close() 51 | subuid, err := parseSubidFile(subuidFile, userObj.Username, userObj.Uid) 52 | if err != nil { 53 | return -1, -1, err 54 | } 55 | 56 | subgidFile, err := os.Open("/etc/subgid") 57 | if err != nil { 58 | return -1, -1, err 59 | } 60 | defer subgidFile.Close() 61 | subgid, err := parseSubidFile(subgidFile, userObj.Username, userObj.Gid) 62 | if err != nil { 63 | return -1, -1, err 64 | } 65 | 66 | return subuid, subgid, nil 67 | } 68 | 69 | func parseSubidFile(subidFile *os.File, name string, id string) (int, error) { 70 | scanner := bufio.NewScanner(subidFile) 71 | for scanner.Scan() { 72 | err := scanner.Err() 73 | if err != nil { 74 | return -1, err 75 | } 76 | 77 | parts := strings.Split(scanner.Text(), ":") 78 | if len(parts) != 3 { 79 | return -1, fmt.Errorf("invalid /etc/sub[gu]id file") 80 | } 81 | if parts[0] == name || parts[0] == id { 82 | size, err := strconv.Atoi(parts[2]) 83 | if err != nil { 84 | return -1, err 85 | } 86 | if size >= MapSize { 87 | subid, err := strconv.Atoi(parts[1]) 88 | if err != nil { 89 | return -1, err 90 | } 91 | return subid, nil 92 | } 93 | } 94 | } 95 | return -1, fmt.Errorf("no matching sub[gu]id found for user %s", name) 96 | } 97 | -------------------------------------------------------------------------------- /util/util_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestUnmarshalFile(t *testing.T) { 13 | type Test struct { 14 | A string 15 | } 16 | test := Test{A: "hello"} 17 | data, err := json.MarshalIndent(test, "", " ") 18 | require.NoError(t, err) 19 | 20 | tmpfile, err := ioutil.TempFile("/tmp", "") 21 | require.NoError(t, err) 22 | defer os.Remove(tmpfile.Name()) 23 | 24 | _, err = tmpfile.Write(data) 25 | require.NoError(t, err) 26 | 27 | result := &Test{} 28 | err = UnmarshalFile(tmpfile.Name(), result) 29 | require.NoError(t, err) 30 | } 31 | 32 | // Test unmarshalling an empty file 33 | func TestUnmarshalFileEmpty(t *testing.T) { 34 | type Test struct { 35 | A string 36 | } 37 | test := &Test{} 38 | err := UnmarshalFile("foo.json", test) 39 | require.Error(t, err) 40 | } 41 | --------------------------------------------------------------------------------