├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── deploy.sh ├── drivers ├── goofys │ └── main.go └── pysssix │ └── main.go ├── github_deploy_key.enc └── helm-chart ├── chartpress.yaml └── s3-fuse-flex-volume ├── Chart.yaml ├── templates ├── NOTES.txt ├── daemonset.yaml └── fuse_install_deps.yaml └── values.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: python 3 | python: 4 | - '3.6' 5 | 6 | before_install: 7 | - openssl aes-256-cbc -K $encrypted_6f4fc646cfd4_key -iv $encrypted_6f4fc646cfd4_iv -in github_deploy_key.enc -out github_deploy_key -d 8 | 9 | install: 10 | - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash 11 | - pip install --default-timeout=100 git+https://github.com/jupyterhub/chartpress.git 12 | 13 | script: 14 | - git diff 15 | 16 | deploy: 17 | - provider: script 18 | skip_cleanup: true 19 | script: GIT_SSH_COMMAND="ssh -i ${PWD}/github_deploy_key" chartpress --tag $TRAVIS_TAG --publish-chart 20 | on: 21 | branch: master 22 | tags: true 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build pysssix driver 2 | FROM golang:1.9.2 3 | 4 | COPY drivers/pysssix/main.go /go 5 | RUN go build /go/main.go 6 | 7 | 8 | # Build goofys driver 9 | FROM golang:1.9.2 10 | 11 | COPY drivers/goofys/main.go /go 12 | RUN go build /go/main.go 13 | 14 | 15 | # Build deployment container 16 | FROM bash:4.4 17 | 18 | COPY deploy.sh /usr/local/bin 19 | COPY --from=0 /go/main /pysssix-flex-volume 20 | COPY --from=1 /go/main /goofys-flex-volume 21 | 22 | CMD /usr/local/bin/deploy.sh 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Yuvi Panda 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # S3 FUSE Flex Volume Drivers 2 | 3 | [![Docker Image](https://img.shields.io/badge/docker-ready-blue.svg)](https://hub.docker.com/r/informaticslab/s3-fuse-flex-volume/) [![Docker Layers](https://images.microbadger.com/badges/image/informaticslab/s3-fuse-flex-volume.svg)](https://microbadger.com/#/images/informaticslab/s3-fuse-flex-volume) 4 | 5 | This helm chart adds S3 FUSE flex volume drivers to your kubernetes cluster. 6 | 7 | The flex volume drivers require the `fuse` package and the S3 fuse libraries to be installed on the host nodes, the chart assumes the hosts are running ubuntu and uses a privileged container to install these. It then installs the flex volume drivers. 8 | 9 | This chart requires Kubernetes 1.8+ as previous versions require the `kubelet` to be restarted to pick up new flex volume drivers. 10 | 11 | Included S3 FUSE libraries: 12 | - [pysssix](https://github.com/met-office-lab/pysssix) 13 | - [goofys](https://github.com/kahing/goofys) 14 | 15 | ## Installation 16 | 17 | ``` 18 | cd helm-chart 19 | helm install --namespace kube-system --name s3-fuse-deployer s3-fuse-flex-volume 20 | ``` 21 | 22 | This helm chart will create a `DaemonSet` which uses privileged containers to install the fuse dependancies and the flex drivers on the kubernetes nodes. You are then able to use the drivers in your pod definitions. 23 | ## Usage examples 24 | 25 | ### pysssix 26 | 27 | Pysssix will mount "all" of S3 which is accessible to the authenticating user. A mount point is created which referrs to all of S3 and then you access objects at `///`. 28 | 29 | With this driver you are limited to read only. 30 | 31 | ```yaml 32 | volumes: 33 | - name: pysssix 34 | flexVolume: 35 | driver: "informaticslab/pysssix-flex-volume" 36 | options: 37 | # Optional 38 | subPath: "key/prefix" 39 | containers: 40 | - name: mycontainer 41 | ... 42 | volumeMounts: 43 | - name: pysssix 44 | mountPath: /s3 45 | ``` 46 | 47 | ### goofys 48 | 49 | Goofys will only mount a specific bucket so you must provide the `bucket` option. Make sure the instances running your kubernetes nodes have permission to write to the bucket (e.g on AWS console, select a node instance and make sure there is an IAM that has a S3 write policy attached) 50 | 51 | ```yaml 52 | volumes: 53 | - name: goofys-mybucket 54 | flexVolume: 55 | driver: "informaticslab/goofys-flex-volume" 56 | options: 57 | # Required 58 | bucket: "mybucket" 59 | # Optional 60 | dirMode: "0755" 61 | fileMode: "0644" 62 | uid: "501" 63 | gid: "20" 64 | subPath: "key/prefix" 65 | endpoint: "s3.not-aws.com" 66 | debug_s3: false 67 | region: "us-east-1" 68 | access-key: "XXXXXXXXXXXXXXXXXXXX" 69 | secret-key: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" 70 | containers: 71 | - name: mycontainer 72 | ... 73 | volumeMounts: 74 | - name: goofys-mybucket 75 | mountPath: /s3/mybucket 76 | ``` 77 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | 6 | VENDOR=informaticslab 7 | declare -a DRIVERS=("pysssix-flex-volume" "goofys-flex-volume") 8 | 9 | for DRIVER in "${DRIVERS[@]}" 10 | do 11 | echo "Installing $DRIVER" 12 | driver_dir=$VENDOR${VENDOR:+"~"}${DRIVER} 13 | if [ ! -d "/flexmnt/$driver_dir" ]; then 14 | mkdir "/flexmnt/$driver_dir" 15 | fi 16 | 17 | cp "/$DRIVER" "/flexmnt/$driver_dir/.$DRIVER" 18 | mv -f "/flexmnt/$driver_dir/.$DRIVER" "/flexmnt/$driver_dir/$DRIVER" 19 | done 20 | 21 | echo "Listing installed drivers:" 22 | ls -l /flexmnt 23 | -------------------------------------------------------------------------------- /drivers/goofys/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "path" 10 | "strconv" 11 | ) 12 | 13 | func makeResponse(status, message string) map[string]interface{} { 14 | return map[string]interface{}{ 15 | "status": status, 16 | "message": message, 17 | } 18 | } 19 | 20 | /// Return status 21 | func Init() interface{} { 22 | resp := makeResponse("Success", "No Initialization required") 23 | resp["capabilities"] = map[string]interface{}{ 24 | "attach": false, 25 | "selinuxRelabel": false, 26 | } 27 | return resp 28 | } 29 | 30 | func isMountPoint(path string) bool { 31 | cmd := exec.Command("mountpoint", path) 32 | err := cmd.Run() 33 | if err != nil { 34 | return false 35 | } 36 | return true 37 | } 38 | 39 | /// If goofys hasn't been mounted yet, mount! 40 | /// If mounted, bind mount to appropriate place. 41 | func Mount(target string, options map[string]string) interface{} { 42 | 43 | bucket := options["bucket"] 44 | subPath := options["subPath"] 45 | dirMode, ok := options["dirMode"] 46 | if !ok { 47 | dirMode = "0755" 48 | } 49 | fileMode, ok := options["fileMode"] 50 | if !ok { 51 | fileMode = "0644" 52 | } 53 | 54 | args := []string{ 55 | "-o", "allow_other", 56 | "--dir-mode", dirMode, 57 | "--file-mode", fileMode, 58 | } 59 | 60 | if endpoint, ok := options["endpoint"]; ok { 61 | args = append(args, "--endpoint", endpoint) 62 | } 63 | if region, ok := options["region"]; ok { 64 | args = append(args, "--region", region) 65 | } 66 | if uid, ok := options["uid"]; ok { 67 | args = append(args, "--uid", uid) 68 | } 69 | if gid, ok := options["gid"]; ok { 70 | args = append(args, "--gid", gid) 71 | } 72 | 73 | debug_s3, ok := options["debug_s3"] 74 | if ok && debug_s3 == "true" { 75 | args = append(args, "--debug_s3") 76 | } 77 | 78 | mountPath := path.Join("/mnt/goofys", bucket) 79 | 80 | args = append(args, bucket, mountPath) 81 | 82 | if !isMountPoint(mountPath) { 83 | exec.Command("umount", mountPath).Run() 84 | exec.Command("rm", "-rf", mountPath).Run() 85 | os.MkdirAll(mountPath, 0755) 86 | 87 | mountCmd := exec.Command("goofys", args...) 88 | mountCmd.Env = os.Environ() 89 | if accessKey, ok := options["access-key"]; ok { 90 | mountCmd.Env = append(mountCmd.Env, "AWS_ACCESS_KEY_ID=" + accessKey) 91 | } 92 | if secretKey, ok := options["secret-key"]; ok { 93 | mountCmd.Env = append(mountCmd.Env, "AWS_SECRET_ACCESS_KEY=" + secretKey) 94 | } 95 | var stderr bytes.Buffer 96 | mountCmd.Stderr = &stderr 97 | err := mountCmd.Run() 98 | if err != nil { 99 | errMsg := err.Error() + ": " + stderr.String() 100 | if debug_s3 == "true" { 101 | errMsg += fmt.Sprintf("; /var/log/syslog follows") 102 | grepCmd := exec.Command("sh", "-c", "grep goofys /var/log/syslog | tail") 103 | var stdout bytes.Buffer 104 | grepCmd.Stdout = &stdout 105 | grepCmd.Run() 106 | errMsg += stdout.String() 107 | } 108 | return makeResponse("Failure", errMsg) 109 | } 110 | } 111 | 112 | srcPath := path.Join(mountPath, subPath) 113 | 114 | // Create subpath if it does not exist 115 | intDirMode, _ := strconv.ParseUint(dirMode, 8, 32) 116 | os.MkdirAll(srcPath, os.FileMode(intDirMode)) 117 | 118 | // Now we rmdir the target, and then make a symlink to it! 119 | err := os.Remove(target) 120 | if err != nil { 121 | return makeResponse("Failure", err.Error()) 122 | } 123 | 124 | err = os.Symlink(srcPath, target) 125 | 126 | return makeResponse("Success", "Mount completed!") 127 | } 128 | 129 | func Unmount(target string) interface{} { 130 | err := os.Remove(target) 131 | if err != nil { 132 | return makeResponse("Failure", err.Error()) 133 | } 134 | return makeResponse("Success", "Successfully unmounted") 135 | } 136 | 137 | func printJSON(data interface{}) { 138 | jsonBytes, err := json.Marshal(data) 139 | if err != nil { 140 | panic(err) 141 | } 142 | fmt.Printf("%s", string(jsonBytes)) 143 | } 144 | 145 | func main() { 146 | switch action := os.Args[1]; action { 147 | case "init": 148 | printJSON(Init()) 149 | case "mount": 150 | optsString := os.Args[3] 151 | opts := make(map[string]string) 152 | json.Unmarshal([]byte(optsString), &opts) 153 | printJSON(Mount(os.Args[2], opts)) 154 | case "unmount": 155 | printJSON(Unmount(os.Args[2])) 156 | default: 157 | printJSON(makeResponse("Not supported", fmt.Sprintf("Operation %s is not supported", action))) 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /drivers/pysssix/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "path" 9 | ) 10 | 11 | func makeResponse(status, message string) map[string]interface{} { 12 | return map[string]interface{}{ 13 | "status": status, 14 | "message": message, 15 | } 16 | } 17 | 18 | /// Return status 19 | func Init() interface{} { 20 | resp := makeResponse("Success", "No Initialization required") 21 | resp["capabilities"] = map[string]interface{}{ 22 | "attach": false, 23 | } 24 | return resp 25 | } 26 | 27 | func isMountPoint(path string) bool { 28 | cmd := exec.Command("mountpoint", path) 29 | err := cmd.Run() 30 | if err != nil { 31 | return false 32 | } 33 | return true 34 | } 35 | 36 | /// If NFS hasn't been mounted yet, mount! 37 | /// If mounted, bind mount to appropriate place. 38 | func Mount(target string, options map[string]string) interface{} { 39 | 40 | subPath := options["subPath"] 41 | mountPath := "/mnt/pysssix" 42 | 43 | if !isMountPoint(mountPath) { 44 | os.MkdirAll(mountPath, 0755) 45 | mountCmd := exec.Command("pysssix", "-a", mountPath) 46 | mountCmd.Start() 47 | } 48 | 49 | srcPath := path.Join(mountPath, subPath) 50 | 51 | // Now we rmdir the target, and then make a symlink to it! 52 | err := os.Remove(target) 53 | if err != nil { 54 | return makeResponse("Failure", err.Error()) 55 | } 56 | 57 | err = os.Symlink(srcPath, target) 58 | 59 | return makeResponse("Success", "Mount completed!") 60 | } 61 | 62 | func Unmount(target string) interface{} { 63 | err := os.Remove(target) 64 | if err != nil { 65 | return makeResponse("Failure", err.Error()) 66 | } 67 | return makeResponse("Success", "Successfully unmounted") 68 | } 69 | 70 | func printJSON(data interface{}) { 71 | jsonBytes, err := json.Marshal(data) 72 | if err != nil { 73 | panic(err) 74 | } 75 | fmt.Printf("%s", string(jsonBytes)) 76 | } 77 | 78 | func main() { 79 | switch action := os.Args[1]; action { 80 | case "init": 81 | printJSON(Init()) 82 | case "mount": 83 | optsString := os.Args[3] 84 | opts := make(map[string]string) 85 | json.Unmarshal([]byte(optsString), &opts) 86 | printJSON(Mount(os.Args[2], opts)) 87 | case "unmount": 88 | printJSON(Unmount(os.Args[2])) 89 | default: 90 | printJSON(makeResponse("Not supported", fmt.Sprintf("Operation %s is not supported", action))) 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /github_deploy_key.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/informatics-lab/s3-fuse-flex-volume/db271276097c5c6331b872d2a6a1d2db0d776aac/github_deploy_key.enc -------------------------------------------------------------------------------- /helm-chart/chartpress.yaml: -------------------------------------------------------------------------------- 1 | charts: 2 | - name: s3-fuse-flex-volume 3 | repo: 4 | git: informatics-lab/helm-charts-repo 5 | published: http://charts.informaticslab.co.uk/ 6 | -------------------------------------------------------------------------------- /helm-chart/s3-fuse-flex-volume/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Helm chart for deploying s3-fuse-flex-volume 3 | name: s3-fuse-flex-volume 4 | version: 0.1.0 5 | -------------------------------------------------------------------------------- /helm-chart/s3-fuse-flex-volume/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | A flexVolume of type informaticslab.co.uk/s3-fuse-flex-volume has been registered 2 | to your kubernetes cluster! 3 | -------------------------------------------------------------------------------- /helm-chart/s3-fuse-flex-volume/templates/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: s3-fuse-flex-volume-deployer 5 | spec: 6 | updateStrategy: 7 | type: RollingUpdate 8 | template: 9 | metadata: 10 | name: s3-fuse-flex-volume-deployer 11 | labels: 12 | app: s3-fuse-flex-volume-deployer 13 | annotations: 14 | checksum/config-map: {{ include (print .Template.BasePath "/fuse_install_deps.yaml") . | sha256sum }} 15 | spec: 16 | initContainers: 17 | - name: install-host-deps 18 | image: busybox:latest 19 | imagePullPolicy: Always 20 | securityContext: 21 | privileged: true 22 | volumeMounts: 23 | - mountPath: /rootfs 24 | name: rootfs 25 | - mountPath: /script 26 | name: install-host-deps-script 27 | command: 28 | - sh 29 | - -c 30 | - cp /script/install_deps.sh /rootfs/tmp/ && chroot /rootfs sh /tmp/install_deps.sh 31 | - name: s3-flex-volume-drivers-deploy 32 | image: {{ .Values.image.repository }}:{{ .Values.image.tag }} 33 | imagePullPolicy: Always 34 | securityContext: 35 | privileged: true 36 | volumeMounts: 37 | - mountPath: /flexmnt 38 | name: flexvolume-plugindir 39 | - mountPath: /hostbin 40 | name: flexvolume-usrlocalbin 41 | containers: 42 | - name: pause 43 | image: gcr.io/google-containers/pause 44 | volumes: 45 | - name: flexvolume-plugindir 46 | hostPath: 47 | path: {{ .Values.flexVolume.pluginDir | quote }} 48 | - name: flexvolume-usrlocalbin 49 | hostPath: 50 | path: '/usr/local/bin' 51 | - name: rootfs 52 | hostPath: 53 | path: / 54 | type: Directory 55 | - name: install-host-deps-script 56 | configMap: 57 | name: fuse-install-deps-script 58 | -------------------------------------------------------------------------------- /helm-chart/s3-fuse-flex-volume/templates/fuse_install_deps.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: fuse-install-deps-script 5 | data: 6 | install_deps.sh: |- 7 | #!/usr/bin/env sh 8 | 9 | # See https://get.docker.com. 10 | get_distro() { 11 | distro="" 12 | # Every system that we officially support has /etc/os-release 13 | if [ -r /etc/os-release ]; then 14 | distro="$(. /etc/os-release && echo "$ID")" 15 | fi 16 | # Returning an empty string here should be alright since the 17 | # case statements don't act unless you provide an actual value 18 | echo "$distro" | tr '[:upper:]' '[:lower:]' 19 | } 20 | 21 | 22 | run_debian() { 23 | apt-get update 24 | apt-get install -y fuse python3-pip git 25 | pip3 install git+git://github.com/met-office-lab/pysssix.git@big_cache 26 | curl -L -o /usr/bin/goofys http://bit.ly/goofys-latest 27 | chmod +x /usr/bin/goofys 28 | } 29 | 30 | 31 | run_amazonLinux() { 32 | yum update -y 33 | yum install -y fuse-devel fuse-libs fuse python3-pip git 34 | pip3 install git+git://github.com/met-office-lab/pysssix.git@big_cache 35 | curl -L -o /usr/bin/goofys http://bit.ly/goofys-latest 36 | chmod +x /usr/bin/goofys 37 | } 38 | 39 | 40 | distro=$(get_distro) 41 | 42 | case $distro in 43 | 44 | amzn) 45 | run_amazonLinux 46 | ;; 47 | 48 | debian) 49 | run_debian 50 | ;; 51 | 52 | *) 53 | echo "Unsupported distro" 54 | exit 1 55 | ;; 56 | 57 | esac 58 | 59 | exit 0 60 | -------------------------------------------------------------------------------- /helm-chart/s3-fuse-flex-volume/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | repository: informaticslab/s3-fuse-flex-volume 3 | tag: latest 4 | pullPolicy: Always 5 | flexVolume: 6 | pluginDir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/ 7 | --------------------------------------------------------------------------------