├── VERSION ├── crd ├── VERSION ├── cmd │ └── example │ │ ├── example │ │ └── main.go ├── pkg │ ├── apis │ │ └── stable.nerdalize.com │ │ │ ├── register.go │ │ │ └── v1 │ │ │ ├── doc.go │ │ │ ├── types.go │ │ │ └── register.go │ ├── client │ │ ├── clientset │ │ │ └── versioned │ │ │ │ ├── doc.go │ │ │ │ ├── fake │ │ │ │ ├── doc.go │ │ │ │ └── register.go │ │ │ │ ├── scheme │ │ │ │ ├── doc.go │ │ │ │ └── register.go │ │ │ │ └── typed │ │ │ │ └── stable.nerdalize.com │ │ │ │ └── v1 │ │ │ │ ├── generated_expansion.go │ │ │ │ ├── doc.go │ │ │ │ ├── fake │ │ │ │ ├── doc.go │ │ │ │ └── fake_stable.nerdalize.com_client.go │ │ │ │ └── stable.nerdalize.com_client.go │ │ ├── listers │ │ │ └── stable.nerdalize.com │ │ │ │ └── v1 │ │ │ │ └── expansion_generated.go │ │ └── informers │ │ │ └── externalversions │ │ │ ├── internalinterfaces │ │ │ └── factory_interfaces.go │ │ │ ├── stable.nerdalize.com │ │ │ ├── v1 │ │ │ │ └── interface.go │ │ │ └── interface.go │ │ │ └── generic.go │ └── signals │ │ ├── signal_windows.go │ │ ├── signal_posix.go │ │ ├── signal.go │ │ └── BUILD ├── Dockerfile ├── artifacts │ ├── test.yaml │ └── datasets.yaml ├── hack │ ├── update-codegen.sh │ ├── custom-boilerplate.go.txt │ └── verify-codegen.sh ├── deployment.yml ├── main.go ├── README.md └── handler.go ├── .dockerignore ├── tool └── release │ ├── release │ └── main.go ├── installers └── msi │ ├── icon.ico │ ├── WixUIBannerBmp-32bit.bmp │ ├── WixUIDialogBmp-32bit.bmp │ ├── make.sh │ └── nerd.wixproject ├── examples ├── datasets │ ├── helloworld.zip │ └── upload.sh ├── docker-base-image │ ├── Dockerfile │ ├── entrypoint.sh │ └── README.md ├── quota │ └── q1.yml └── pods │ ├── non-existent-dataset.yaml │ ├── read-write-dataset.yaml │ └── input-and-output-dataset.yaml ├── pkg ├── transfer │ ├── doc.go │ ├── archiver │ │ ├── types.go │ │ ├── opts.go │ │ └── copy.go │ └── store │ │ └── opts.go ├── populator │ ├── errors.go │ ├── env.go │ └── endpoint.go ├── kubeconfig │ └── kubeconfig.go └── kubevisor │ └── types.go ├── nerd ├── service │ ├── datatransfer │ │ └── v1 │ │ │ ├── client │ │ │ ├── payload │ │ │ │ └── metadata.go │ │ │ ├── index_test.go │ │ │ ├── metadata.go │ │ │ ├── client.go │ │ │ └── index.go │ │ │ ├── pipe.go │ │ │ ├── upload.go │ │ │ ├── upload_process_test.go │ │ │ └── download.go │ └── working │ │ └── v1 │ │ └── utils.go ├── client │ ├── batch │ │ └── v1 │ │ │ ├── ping.go │ │ │ ├── payload │ │ │ ├── token.go │ │ │ ├── upload.go │ │ │ ├── worker.go │ │ │ ├── placement.go │ │ │ ├── error.go │ │ │ ├── dataset.go │ │ │ ├── run.go │ │ │ ├── task.go │ │ │ ├── plan.go │ │ │ ├── workload.go │ │ │ └── secret.go │ │ │ ├── error.go │ │ │ ├── token.go │ │ │ ├── worker.go │ │ │ ├── upload.go │ │ │ ├── dataset.go │ │ │ ├── placement.go │ │ │ ├── jwt_provider.go │ │ │ ├── plan.go │ │ │ ├── run.go │ │ │ └── workload.go │ ├── auth │ │ └── v1 │ │ │ ├── payload │ │ │ ├── error.go │ │ │ ├── jwt.go │ │ │ ├── project.go │ │ │ ├── oauth.go │ │ │ └── cluster.go │ │ │ ├── error.go │ │ │ └── oath_token_provider.go │ └── client.go ├── errors.go ├── conf │ └── conf_test.go ├── oauth │ ├── provider.go │ └── config_provider.go ├── utils │ └── test_utils.go ├── aws │ ├── credentials_provider_test.go │ ├── credentials_provider.go │ ├── queue_client.go │ └── data_client.go ├── version_test.go └── jwt │ ├── test_utils.go │ ├── provider.go │ ├── authapi_provider.go │ ├── config_provider.go │ ├── env_provider.go │ ├── provider_test.go │ └── jwt_test.go ├── .github ├── ISSUE_TEMPLATE.md └── CONTRIBUTING.md ├── flex.Dockerfile ├── .gitignore ├── Dockerfile ├── cmd ├── flex │ ├── dataset.yml │ ├── dataset-dev.yml │ └── install.sh ├── cluster.go ├── job.go ├── dataset.go ├── version.go ├── output.go ├── job_logs.go ├── dataset_list.go └── job_run_test.go ├── svc ├── kube_delete_job.go ├── kube.go ├── kube_update_secret_test.go ├── kube_delete_secret.go ├── kube_delete_dataset.go ├── errors.go ├── kube_get_secret.go ├── kube_update_secret.go ├── kube_create_dataset.go ├── kube_update_dataset.go ├── kube_get_dataset.go ├── kube_fetch_job_logs.go ├── kube_update_dataset_test.go ├── kube_create_secret_test.go ├── kube_list_datasets.go ├── kube_delete_dataset_test.go ├── kube_list_secret.go ├── kube_delete_job_test.go ├── kube_delete_secret_test.go ├── kube_list_quotas.go └── kube_get_dataset_test.go ├── main.go ├── glide.yaml └── README.md /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.2 2 | -------------------------------------------------------------------------------- /crd/VERSION: -------------------------------------------------------------------------------- 1 | 0.3 -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | *.Dockerfile 3 | Dockerfile 4 | -------------------------------------------------------------------------------- /tool/release/release: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdalize/nerd/HEAD/tool/release/release -------------------------------------------------------------------------------- /crd/cmd/example/example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdalize/nerd/HEAD/crd/cmd/example/example -------------------------------------------------------------------------------- /installers/msi/icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdalize/nerd/HEAD/installers/msi/icon.ico -------------------------------------------------------------------------------- /examples/datasets/helloworld.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdalize/nerd/HEAD/examples/datasets/helloworld.zip -------------------------------------------------------------------------------- /pkg/transfer/doc.go: -------------------------------------------------------------------------------- 1 | //Package transfer provides primitives for uploading and downloading datasets 2 | package transfer 3 | -------------------------------------------------------------------------------- /installers/msi/WixUIBannerBmp-32bit.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdalize/nerd/HEAD/installers/msi/WixUIBannerBmp-32bit.bmp -------------------------------------------------------------------------------- /installers/msi/WixUIDialogBmp-32bit.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdalize/nerd/HEAD/installers/msi/WixUIDialogBmp-32bit.bmp -------------------------------------------------------------------------------- /crd/pkg/apis/stable.nerdalize.com/register.go: -------------------------------------------------------------------------------- 1 | package nerdalizecom 2 | 3 | const ( 4 | GroupName = "stable.nerdalize.com" 5 | ) 6 | -------------------------------------------------------------------------------- /crd/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.1 2 | 3 | ADD controller /bin/ 4 | 5 | RUN apk update && apk add ca-certificates 6 | 7 | ENTRYPOINT ["/bin/controller"] 8 | 9 | -------------------------------------------------------------------------------- /examples/datasets/upload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | aws s3 cp helloworld.zip s3://nlz-datasets-dev/test-helloworld.zip --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers 3 | -------------------------------------------------------------------------------- /examples/docker-base-image/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nerdalize/nerd 2 | 3 | ADD entrypoint.sh /entrypoint.sh 4 | 5 | # Add your own Dockerfile commands 6 | 7 | ENTRYPOINT /entrypoint.sh 8 | -------------------------------------------------------------------------------- /crd/pkg/apis/stable.nerdalize.com/v1/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package,register 2 | 3 | // Package v1 is the v1 version of the API. 4 | // +groupName=nerdalize.com 5 | package v1 6 | -------------------------------------------------------------------------------- /pkg/populator/errors.go: -------------------------------------------------------------------------------- 1 | package populator 2 | 3 | // ErrNoSuchPopulator implements the error interface 4 | type ErrNoSuchPopulator string 5 | 6 | func (e ErrNoSuchPopulator) Error() string { return string(e) } 7 | -------------------------------------------------------------------------------- /crd/artifacts/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "stable.nerdalize.com/v1" 2 | kind: Dataset 3 | labels: 4 | nerd-app: cli 5 | metadata: 6 | name: my-1601-dataset-object 7 | spec: 8 | bucket: "a-s3-bucket-1501" 9 | key: my-awesome-dataset 10 | 11 | -------------------------------------------------------------------------------- /examples/docker-base-image/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Download dataset with nerd command 4 | nerd download $NERD_DATASET_INPUT /in 5 | 6 | 7 | #Start your own process here 8 | #For example: 9 | touch /out/test.txt 10 | 11 | 12 | #Upload /out folder 13 | nerd upload /out 14 | -------------------------------------------------------------------------------- /examples/quota/q1.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ResourceQuota 3 | metadata: 4 | name: compute-resources 5 | labels: 6 | flex-volume-size: "10000000000" 7 | spec: 8 | hard: 9 | requests.cpu: "1" 10 | requests.memory: 1Gi 11 | limits.cpu: "1" 12 | limits.memory: 1Gi 13 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/client/payload/metadata.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | import "time" 4 | 5 | //Metadata describes a dataset. 6 | type Metadata struct { 7 | Created time.Time `json:"created_at"` 8 | Updated time.Time `json:"updated_at"` 9 | Size int64 `json:"size"` 10 | } 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Expected Behavior 2 | 3 | ## Actual Behavior 4 | 5 | ## Steps to Reproduce the Problem 6 | 7 | 1. 8 | 1. 9 | 1. 10 | 11 | ## Specifications 12 | 13 | - Version (using `nerd version`): 14 | - Platform: 15 | - Subsystem: 16 | 17 | ## Anything else we need to know? 18 | -------------------------------------------------------------------------------- /flex.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1-stretch as build 2 | WORKDIR /go/src/github.com/nerdalize/nerd 3 | COPY . . 4 | RUN go build -o $GOPATH/bin/nerd-flex-volume cmd/flex/main.go 5 | 6 | FROM golang:1-alpine 7 | COPY --from=build $GOPATH/bin/nerd-flex-volume /dataset 8 | COPY cmd/flex/install.sh /run.sh 9 | RUN chmod +x /run.sh 10 | ENTRYPOINT ["/run.sh"] 11 | -------------------------------------------------------------------------------- /installers/msi/make.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | cp $GOPATH/bin/nerd ./nerd.exe 5 | VERSION=$(cat ../../VERSION | cut -f1 -d"-") 6 | 7 | sed -i "s/\sVersion=".*"/ Version=\"$VERSION\"/" Product.wxs 8 | 9 | $WINDIR/Microsoft.NET/Framework/v4.0.30319/MSBuild.exe nerd.wixproject 10 | 11 | mv "bin/Nerd Setup (x64).msi" "bin/Nerd.Windows.Installer.x64.msi" 12 | -------------------------------------------------------------------------------- /pkg/transfer/archiver/types.go: -------------------------------------------------------------------------------- 1 | package transferarchiver 2 | 3 | import "io" 4 | 5 | //Reporter describes how an archiver reports 6 | type Reporter interface { 7 | StartArchivingProgress(label string, total int64) func(int64) 8 | StopArchivingProgress() 9 | StartUnarchivingProgress(label string, total int64, rr io.Reader) io.Reader 10 | StopUnarchivingProgress() 11 | } 12 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/ping.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import "net/http" 4 | 5 | //ClientPingInterface is an interface so client ping calls can be mocked. 6 | type ClientPingInterface interface { 7 | Ping() error 8 | } 9 | 10 | //Ping will error if there are connection issues 11 | func (c *Client) Ping() error { 12 | return c.doRequest(http.MethodGet, "ping", nil, nil) 13 | } 14 | -------------------------------------------------------------------------------- /nerd/client/auth/v1/payload/error.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //Error is the error returned by the authentication server. 4 | type Error struct { 5 | Msg string `json:"error"` 6 | } 7 | 8 | //Error returns the error message. 9 | func (e Error) Error() string { 10 | return e.Msg 11 | } 12 | 13 | //Cause is implemented to be compatible with the pkg/errors package. 14 | func (e Error) Cause() error { 15 | return nil 16 | } 17 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/pipe.go: -------------------------------------------------------------------------------- 1 | package v1datatransfer 2 | 3 | import "io" 4 | 5 | type pipeErr struct { 6 | error 7 | } 8 | 9 | func newPipeErr(err error) *pipeErr { 10 | return &pipeErr{err} 11 | } 12 | 13 | func isPipeErr(err error) bool { 14 | _, ok := err.(*pipeErr) 15 | return ok 16 | } 17 | 18 | type pipe struct { 19 | r *io.PipeReader 20 | w *io.PipeWriter 21 | } 22 | 23 | func newPipe() *pipe { 24 | pr, pw := io.Pipe() 25 | return &pipe{ 26 | r: pr, 27 | w: pw, 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pkg/transfer/archiver/opts.go: -------------------------------------------------------------------------------- 1 | package transferarchiver 2 | 3 | //ArchiverType determines what type the object store will be 4 | type ArchiverType string 5 | 6 | const ( 7 | //ArchiverTypeTar uses the tar archiving format 8 | ArchiverTypeTar ArchiverType = "tar" 9 | ) 10 | 11 | //ArchiverOptions contain options for all stores 12 | type ArchiverOptions struct { 13 | Type ArchiverType `json:"type"` 14 | 15 | TarArchiverKeyPrefix string `json:"keyPrefix"` 16 | 17 | SizeLimit int64 `json:"sizeLimit"` 18 | } 19 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing guidelines 2 | 3 | ## Branch names 4 | Branch names should be in the following format: `{prefix}-{branch-name}`. 5 | `{prefix}` should be one of the following: 6 | 7 | | prefix | description | 8 | |---------|----------------------------------------| 9 | | docs | when the branch adds documentation | 10 | | feature | when the branch adds a feature | 11 | | bug | when the branch fixes a bug | 12 | | spike | when the branch adds experimental code | 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | secrets.env 26 | .DS_STORE 27 | bin 28 | vendor 29 | installers/msi/obj 30 | crd/controller 31 | *.env 32 | controller 33 | installers/msi/nerd 34 | installers/msi/nerd.* 35 | -------------------------------------------------------------------------------- /crd/hack/update-codegen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. 8 | CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)} 9 | 10 | vendor/k8s.io/code-generator/generate-groups.sh all \ 11 | github.com/nerdalize/nerd/crd/pkg/client github.com/nerdalize/nerd/crd/pkg/apis \ 12 | stable.nerdalize.com:v1 \ 13 | --go-header-file ${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1-alpine as build 2 | 3 | ENV NERD_PATH /go/src/github.com/nerdalize/nerd 4 | 5 | ADD . $NERD_PATH 6 | 7 | RUN mkdir /in; mkdir /out 8 | 9 | # Leave these go build flags, this way we can inject the nerd binary in other containers 10 | ENV CGO_ENABLED 0 11 | ENV GOOS linux 12 | ENV GOARCH amd64 13 | RUN cd $NERD_PATH; \ 14 | go build \ 15 | -ldflags "-X main.version=$(cat VERSION) -X main.commit=docker.build" \ 16 | -o /go/bin/nerd \ 17 | main.go 18 | 19 | FROM alpine:3.5 20 | COPY --from=build /go/bin/nerd /go/bin/nerd 21 | ENTRYPOINT ["/go/bin/nerd"] 22 | -------------------------------------------------------------------------------- /examples/pods/non-existent-dataset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: non-existent-dataset 5 | labels: 6 | app: non-existent-dataset 7 | spec: 8 | containers: 9 | - image: busybox 10 | command: [/bin/sh, -c, 'sleep 3600'] 11 | imagePullPolicy: IfNotPresent 12 | name: busybox 13 | volumeMounts: 14 | - name: dataset 15 | mountPath: /dataset 16 | restartPolicy: Always 17 | volumes: 18 | - name: dataset 19 | flexVolume: 20 | driver: "nerdalize.com/dataset" 21 | options: 22 | input/s3Bucket: nlz-datasets-dev 23 | input/s3Key: test-nonexistent.zip -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/token.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | import "time" 4 | 5 | //CreateTokenInput is input for token creation 6 | type CreateTokenInput struct { 7 | ProjectID string `json:"project_id" valid:"required"` 8 | } 9 | 10 | //CreateTokenOutput is output for token creation 11 | type CreateTokenOutput struct { 12 | AWSRegion string `json:"aws_region"` 13 | AWSAccessKeyID string `json:"aws_access_key_id"` 14 | AWSExpiration time.Time `json:"aws_expiration"` 15 | AWSSecretAccessKey string `json:"aws_secret_access_key"` 16 | AWSSessionToken string `json:"aws_session_token"` 17 | } 18 | -------------------------------------------------------------------------------- /crd/hack/custom-boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright YEAR Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /nerd/client/auth/v1/error.go: -------------------------------------------------------------------------------- 1 | package v1auth 2 | 3 | import ( 4 | v1payload "github.com/nerdalize/nerd/nerd/client/auth/v1/payload" 5 | ) 6 | 7 | //HTTPError is an error that is used when a server responded with a status code >= 400. 8 | //Based on the actual status code a custom error message will be generated. 9 | type HTTPError struct { 10 | StatusCode int 11 | Err *v1payload.Error 12 | } 13 | 14 | //Error returns the error message specific for the status code. 15 | func (e HTTPError) Error() string { 16 | return e.Err.Msg 17 | } 18 | 19 | //Cause is implemented to be compatible with the pkg/errors package. 20 | func (e HTTPError) Cause() error { 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/error.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 5 | ) 6 | 7 | //HTTPError is an error that is used when a server responded with a status code >= 400. 8 | //Based on the actual status code a custom error message will be generated. 9 | type HTTPError struct { 10 | StatusCode int 11 | Err *v1payload.Error 12 | } 13 | 14 | //Error returns the error message specific for the status code. 15 | func (e HTTPError) Error() string { 16 | return e.Err.Message 17 | } 18 | 19 | //Cause is implemented to be compatible with the pkg/errors package. 20 | func (e HTTPError) Cause() error { 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /pkg/transfer/store/opts.go: -------------------------------------------------------------------------------- 1 | package transferstore 2 | 3 | //StoreType determines what type the object store will be 4 | type StoreType string 5 | 6 | const ( 7 | //StoreTypeS3 uses a AWS S3 store 8 | StoreTypeS3 StoreType = "s3" 9 | ) 10 | 11 | //StoreOptions contain options for all stores 12 | type StoreOptions struct { 13 | Type StoreType `json:"type"` 14 | 15 | S3StoreBucket string `json:"s3StoreBucket"` 16 | S3StorePrefix string `json:"s3StorePrefix"` 17 | S3StoreAWSRegion string `json:"s3StoreAWSRegion"` 18 | S3StoreAccessKey string `json:"s3StoreAccessKey"` 19 | S3StoreSecretKey string `json:"s3StoreSecretKey"` 20 | S3SessionToken string `json:"s3SessionToken"` 21 | } 22 | -------------------------------------------------------------------------------- /examples/pods/read-write-dataset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: read-write-dataset 5 | labels: 6 | app: read-write-dataset 7 | spec: 8 | containers: 9 | - image: busybox 10 | command: [/bin/sh, -c, 'cat /dataset/test.txt; echo "foo" > /dataset/test.txt; cat /dataset/test.txt; sleep 3600'] 11 | imagePullPolicy: IfNotPresent 12 | name: busybox 13 | volumeMounts: 14 | - name: dataset 15 | mountPath: /dataset 16 | restartPolicy: Always 17 | volumes: 18 | - name: dataset 19 | flexVolume: 20 | driver: "nerdalize.com/dataset" 21 | options: 22 | input/s3Bucket: nlz-datasets-dev 23 | input/s3Key: test-helloworld.zip -------------------------------------------------------------------------------- /examples/docker-base-image/README.md: -------------------------------------------------------------------------------- 1 | # Using nerd docker container as base image 2 | 3 | When you want to run a container on the Nerdalize platform and want data management to be handled for you, the `nerdalize/nerd` container could be used as a convenient base container. 4 | Running the `upload` and `download` command is up to the user. This example contains a minimal Dockerfile that shows how to use the `upload` and `download` commands. 5 | 6 | Please note: 7 | * The `/in` and `/out` folders in the container are predefined locations to store input and output data. 8 | * The `nerdalize/nerd` container expects a config file with a valid nerd token to be provided as a volume (e.g. `docker run -v ~/.nerd:/root/.nerd [YOUR-CONTAINER]`) 9 | -------------------------------------------------------------------------------- /nerd/client/auth/v1/payload/jwt.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //GetJWTOutput is output when a JWT is requested 4 | type GetJWTOutput struct { 5 | Token string `json:"token"` 6 | } 7 | 8 | //GetWorkerJWTOutput is output when a worker JWT (JWT + RefreshToken) is requested 9 | type GetWorkerJWTOutput struct { 10 | Token string `json:"token"` 11 | Secret string `json:"secret"` 12 | } 13 | 14 | //RefreshWorkerJWTInput is input for refreshing a JWT 15 | type RefreshWorkerJWTInput struct { 16 | Token string `json:"jwt"` 17 | Secret string `json:"secret"` 18 | } 19 | 20 | //RefreshWorkerJWTOutput is output when a JWT refresh is requested 21 | type RefreshWorkerJWTOutput struct { 22 | Token string `json:"token"` 23 | } 24 | -------------------------------------------------------------------------------- /nerd/errors.go: -------------------------------------------------------------------------------- 1 | package nerd 2 | 3 | import "errors" 4 | 5 | var ( 6 | //ErrNotImplemented is returned when a function is not yet implemented 7 | ErrNotImplemented = errors.New("not yet implemented") 8 | 9 | //ErrTokenRevoked is returned when trying to refresh a revoked token 10 | ErrTokenRevoked = errors.New("ErrTokenRevoked") 11 | 12 | //ErrTokenUnset is returned when no oauth access token was found in the config file 13 | ErrTokenUnset = errors.New("You're not logged in. Please login with `nerd login`.") 14 | 15 | //ErrProjectIDNotSet is returned when no project id is set in the session 16 | ErrProjectIDNotSet = errors.New("No project ID specified, use `nerd project set` to configure a project to work on.") 17 | ) 18 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // This package has the automatically generated clientset. 17 | package versioned 18 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // This package has the automatically generated fake clientset. 17 | package fake 18 | -------------------------------------------------------------------------------- /cmd/flex/dataset.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | namespace: kube-system 5 | name: nlz-nerd-datasets 6 | spec: 7 | template: 8 | metadata: 9 | name: nlz-nerd-datasets 10 | labels: 11 | app: nlz-nerd-datasets 12 | spec: 13 | containers: 14 | - image: nerdalize/nerd-flex-volume:1.0.0-rc8 15 | name: nlz-nerd-datasets 16 | imagePullPolicy: Always 17 | securityContext: 18 | privileged: true 19 | volumeMounts: 20 | - mountPath: /flexmnt 21 | name: flexvolume-mount 22 | volumes: 23 | - name: flexvolume-mount 24 | hostPath: 25 | path: /var/lib/kubelet/volumeplugins/ 26 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/scheme/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // This package contains the scheme of the automatically generated clientset. 17 | package scheme 18 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/typed/stable.nerdalize.com/v1/generated_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package v1 17 | 18 | type DatasetExpansion interface{} 19 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/typed/stable.nerdalize.com/v1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // This package has the automatically generated typed clients. 17 | package v1 18 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/typed/stable.nerdalize.com/v1/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Package fake has the automatically generated clients. 17 | package fake 18 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/token.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | "net/http" 5 | 6 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 7 | ) 8 | 9 | //ClientTokenInterface is an interface so client token calls can be mocked. 10 | type ClientTokenInterface interface { 11 | CreateToken(projectID string) (output *v1payload.CreateTokenOutput, err error) 12 | } 13 | 14 | //CreateToken will create a token 15 | func (c *Client) CreateToken(projectID string) (output *v1payload.CreateTokenOutput, err error) { 16 | output = &v1payload.CreateTokenOutput{} 17 | input := &v1payload.CreateTokenInput{ 18 | ProjectID: projectID, 19 | } 20 | 21 | return output, c.doRequest(http.MethodPost, createPath(projectID, tokensEndpoint), input, output) 22 | } 23 | -------------------------------------------------------------------------------- /cmd/flex/dataset-dev.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | namespace: kube-system 5 | name: nlz-nerd-datasets-dev 6 | spec: 7 | template: 8 | metadata: 9 | name: nlz-nerd-datasets-dev 10 | labels: 11 | app: nlz-nerd-datasets-dev 12 | spec: 13 | containers: 14 | - image: nerdalize/nerd-flex-volume:dev 15 | name: nlz-nerd-datasets-dev 16 | imagePullPolicy: Always 17 | securityContext: 18 | privileged: true 19 | volumeMounts: 20 | - mountPath: /flexmnt 21 | name: flexvolume-mount 22 | volumes: 23 | - name: flexvolume-mount 24 | hostPath: 25 | path: /var/lib/kubelet/volumeplugins/ 26 | -------------------------------------------------------------------------------- /crd/pkg/signals/signal_windows.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package signals 18 | 19 | import ( 20 | "os" 21 | ) 22 | 23 | var shutdownSignals = []os.Signal{os.Interrupt} 24 | -------------------------------------------------------------------------------- /svc/kube_delete_job.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | ) 8 | 9 | //DeleteJobInput is the input to DeleteJob 10 | type DeleteJobInput struct { 11 | Name string `validate:"min=1,printascii"` 12 | } 13 | 14 | //DeleteJobOutput is the output to DeleteJob 15 | type DeleteJobOutput struct{} 16 | 17 | //DeleteJob will create a job on kubernetes 18 | func (k *Kube) DeleteJob(ctx context.Context, in *DeleteJobInput) (out *DeleteJobOutput, err error) { 19 | if err = k.checkInput(ctx, in); err != nil { 20 | return nil, err 21 | } 22 | 23 | err = k.visor.DeleteResource(ctx, kubevisor.ResourceTypeJobs, in.Name) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | return &DeleteJobOutput{}, nil 29 | } 30 | -------------------------------------------------------------------------------- /svc/kube.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | ) 8 | 9 | //Kube interacts with the kubernetes backend 10 | type Kube struct { 11 | visor *kubevisor.Visor 12 | val Validator 13 | logs Logger 14 | } 15 | 16 | //NewKube will setup the Kubernetes service 17 | func NewKube(di DI) (k *Kube) { 18 | k = &Kube{ 19 | visor: kubevisor.NewVisor(di.Namespace(), "", di.Kube(), di.Crd(), di.APIExt(), di.Logger()), 20 | val: di.Validator(), 21 | logs: di.Logger(), 22 | } 23 | 24 | return k 25 | } 26 | 27 | func (k *Kube) checkInput(ctx context.Context, in interface{}) (err error) { 28 | err = k.val.StructCtx(ctx, in) 29 | if err != nil { 30 | return errValidation{err} 31 | } 32 | 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /cmd/flex/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -o errexit 3 | set -o pipefail 4 | 5 | VENDOR=nerdalize.com 6 | DRIVER=dataset 7 | 8 | driver_dir=$VENDOR${VENDOR:+"~"}${DRIVER} 9 | if [ ! -d "/flexmnt/$driver_dir" ]; then 10 | mkdir "/flexmnt/$driver_dir" 11 | fi 12 | 13 | # atomically write (new) flex volume executable 14 | cp "/$DRIVER" "/flexmnt/$driver_dir/.$DRIVER" 15 | mv -f "/flexmnt/$driver_dir/.$DRIVER" "/flexmnt/$driver_dir/$DRIVER" 16 | 17 | # copy service account from pod to the host (for the flex volume) 18 | cp -R /var/run/secrets/kubernetes.io/serviceaccount /flexmnt/$driver_dir 19 | 20 | # write env information (for api host info and such) 21 | env > /flexmnt/$driver_dir/flex.env 22 | 23 | # block forever to keep the daemonset running 24 | while : ; do 25 | sleep 3600 26 | done 27 | -------------------------------------------------------------------------------- /examples/pods/input-and-output-dataset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: input-and-output-dataset 5 | labels: 6 | app: input-and-output-dataset 7 | spec: 8 | containers: 9 | - image: busybox 10 | command: [/bin/sh, -c, 'cat /dataset/test.txt | rev > /dataset/test_reversed.txt; sleep 3600'] 11 | imagePullPolicy: IfNotPresent 12 | name: busybox 13 | volumeMounts: 14 | - name: dataset 15 | mountPath: /dataset 16 | restartPolicy: Always 17 | volumes: 18 | - name: dataset 19 | flexVolume: 20 | driver: "nerdalize.com/dataset" 21 | options: 22 | input/s3Bucket: nlz-datasets-dev 23 | input/s3Key: test-helloworld.zip 24 | output/s3Bucket: nlz-datasets-dev 25 | output/s3Key: test-helloworld-reversed.zip -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/upload.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //SendUploadHeartbeatInput is input for dataset creation 4 | type SendUploadHeartbeatInput struct { 5 | ProjectID string `json:"project_id" valid:"required"` 6 | DatasetID string `json:"dataset_id" valid:"required"` 7 | } 8 | 9 | //SendUploadHeartbeatOutput is output for dataset creation 10 | type SendUploadHeartbeatOutput struct { 11 | HasExpired bool `json:"has_expired"` 12 | } 13 | 14 | //SendUploadSuccessInput is input for marking a run as failed 15 | type SendUploadSuccessInput struct { 16 | ProjectID string `json:"project_id" valid:"required"` 17 | DatasetID string `json:"dataset_id" valid:"required"` 18 | } 19 | 20 | //SendUploadSuccessOutput is output from marking a run as failed 21 | type SendUploadSuccessOutput struct{} 22 | -------------------------------------------------------------------------------- /svc/kube_update_secret_test.go: -------------------------------------------------------------------------------- 1 | package svc_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/nerdalize/nerd/svc" 9 | ) 10 | 11 | func TestUpdateSecret(t *testing.T) { 12 | di, clean := testDI(t) 13 | defer clean() 14 | 15 | ctx := context.Background() 16 | ctx, cancel := context.WithTimeout(ctx, time.Second*5) 17 | defer cancel() 18 | 19 | kube := svc.NewKube(di) 20 | out, err := kube.CreateSecret(ctx, &svc.CreateSecretInput{ 21 | Image: "smoketest", 22 | Project: "nerdalize", 23 | Registry: "quay.io", 24 | Username: "test", 25 | Password: "test", 26 | }) 27 | ok(t, err) 28 | 29 | _, err = kube.UpdateSecret(ctx, &svc.UpdateSecretInput{ 30 | Name: out.Name, 31 | Username: "newtest", 32 | Password: "newtest", 33 | }) 34 | ok(t, err) 35 | } 36 | -------------------------------------------------------------------------------- /pkg/kubeconfig/kubeconfig.go: -------------------------------------------------------------------------------- 1 | package kubeconfig 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | homedir "github.com/mitchellh/go-homedir" 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | // GetPath returns the expanded and normalized kube config path 11 | func GetPath(kubeConfig string) (string, error) { 12 | hdir, err := homedir.Dir() 13 | if err != nil { 14 | return "", err 15 | } 16 | if kubeConfig == "" { 17 | kubeConfig = filepath.Join(hdir, ".kube", "config") 18 | } 19 | kubeConfig, err = homedir.Expand(kubeConfig) 20 | if err != nil { 21 | return "", errors.Wrap(err, "failed to expand home directory in kube config file path") 22 | } 23 | //Normalize all slashes to native platform slashes (e.g. / to \ on Windows) 24 | kubeConfig = filepath.FromSlash(kubeConfig) 25 | return kubeConfig, nil 26 | } 27 | -------------------------------------------------------------------------------- /svc/kube_delete_secret.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | ) 8 | 9 | //DeleteSecretInput is the input to DeleteSecret 10 | type DeleteSecretInput struct { 11 | Name string `validate:"min=1,printascii"` 12 | } 13 | 14 | //DeleteSecretOutput is the output to DeleteSecret 15 | type DeleteSecretOutput struct{} 16 | 17 | //DeleteSecret will create a dataset on kubernetes 18 | func (k *Kube) DeleteSecret(ctx context.Context, in *DeleteSecretInput) (out *DeleteSecretOutput, err error) { 19 | if err = k.checkInput(ctx, in); err != nil { 20 | return nil, err 21 | } 22 | 23 | err = k.visor.DeleteResource(ctx, kubevisor.ResourceTypeSecrets, in.Name) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | return &DeleteSecretOutput{}, nil 29 | } 30 | -------------------------------------------------------------------------------- /svc/kube_delete_dataset.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | ) 8 | 9 | //DeleteDatasetInput is the input to DeleteDataset 10 | type DeleteDatasetInput struct { 11 | Name string `validate:"min=1,printascii"` 12 | } 13 | 14 | //DeleteDatasetOutput is the output to DeleteDataset 15 | type DeleteDatasetOutput struct{} 16 | 17 | //DeleteDataset will create a dataset on kubernetes 18 | func (k *Kube) DeleteDataset(ctx context.Context, in *DeleteDatasetInput) (out *DeleteDatasetOutput, err error) { 19 | if err = k.checkInput(ctx, in); err != nil { 20 | return nil, err 21 | } 22 | 23 | err = k.visor.DeleteResource(ctx, kubevisor.ResourceTypeDatasets, in.Name) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | return &DeleteDatasetOutput{}, nil 29 | } 30 | -------------------------------------------------------------------------------- /crd/pkg/signals/signal_posix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | /* 4 | Copyright 2017 The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package signals 20 | 21 | import ( 22 | "os" 23 | "syscall" 24 | ) 25 | 26 | var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} 27 | -------------------------------------------------------------------------------- /crd/hack/verify-codegen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/.. 8 | DIFFROOT="${SCRIPT_ROOT}/pkg" 9 | TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp/pkg" 10 | _tmp="${SCRIPT_ROOT}/_tmp" 11 | 12 | cleanup() { 13 | rm -rf "${_tmp}" 14 | } 15 | trap "cleanup" EXIT SIGINT 16 | 17 | cleanup 18 | 19 | mkdir -p "${TMP_DIFFROOT}" 20 | cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" 21 | 22 | "${SCRIPT_ROOT}/hack/update-codegen.sh" 23 | echo "diffing ${DIFFROOT} against freshly generated codegen" 24 | ret=0 25 | diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? 26 | cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" 27 | if [[ $ret -eq 0 ]] 28 | then 29 | echo "${DIFFROOT} up to date." 30 | else 31 | echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" 32 | exit 1 33 | fi 34 | -------------------------------------------------------------------------------- /nerd/conf/conf_test.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | ) 7 | 8 | const testConf = ` 9 | { 10 | "auth": { 11 | "public_key": "test_key", 12 | "api_endpoint": "test_url" 13 | } 14 | } 15 | ` 16 | 17 | func TestFromFile(t *testing.T) { 18 | temp, err := ioutil.TempFile("/tmp", "nerd_conf") 19 | if err != nil { 20 | t.Fatalf("Unexpected error for temp file: %v", err) 21 | } 22 | temp.WriteString(testConf) 23 | conf, err := Read(temp.Name()) 24 | if err != nil { 25 | t.Fatalf("Unexpected error: %v", err) 26 | } 27 | auth := conf.Auth 28 | if auth.APIEndpoint != "test_url" { 29 | t.Errorf("Expected api_endpoint %v but got %v", "test_url", auth.APIEndpoint) 30 | } 31 | if auth.PublicKey != "test_key" { 32 | t.Errorf("Expected api_endpoint %v but got %v", "test_key", auth.PublicKey) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tool/release/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | 9 | "github.com/google/go-github/github" 10 | "golang.org/x/oauth2" 11 | ) 12 | 13 | func main() { 14 | ctx := context.Background() 15 | ts := oauth2.StaticTokenSource( 16 | &oauth2.Token{AccessToken: os.Getenv("GITHUB_ACCESS_TOKEN")}, 17 | ) 18 | tc := oauth2.NewClient(ctx, ts) 19 | 20 | client := github.NewClient(tc) 21 | 22 | // list all releases for nerd 23 | releases, _, err := client.Repositories.ListReleases(ctx, "nerdalize", "nerd", nil) 24 | if err != nil { 25 | log.Fatal(err) 26 | } 27 | for _, release := range releases { 28 | fmt.Printf("Downloads for release [%s]:\n", *release.Name) 29 | for _, asset := range release.Assets { 30 | fmt.Printf("\t%s: %d times\n", *asset.Name, *asset.DownloadCount) 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /crd/deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: custom-dataset-controller 5 | namespace: kube-system 6 | labels: 7 | app: nerd-cli 8 | project: cli 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: nerd-cli 14 | project: cli 15 | template: 16 | metadata: 17 | name: custom-dataset-controller 18 | labels: 19 | app: nerd-cli 20 | project: cli 21 | spec: 22 | containers: 23 | - name: controller 24 | imagePullPolicy: Always 25 | image: nerdalize/custom-dataset-controller:0.3 26 | args: 27 | - -alsologtostderr 28 | # resources: 29 | # limits: 30 | # cpu: "1" 31 | # memory: "1Gi" 32 | # requests: 33 | # cpu: "500m" 34 | # memory: "512Mi" 35 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/worker.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | import "time" 4 | 5 | //WorkerCondition describes the worker status at a point in time 6 | type WorkerCondition struct { 7 | ProbeTime time.Time `json:"probe_time"` 8 | Type string `json:"type"` 9 | } 10 | 11 | //WorkerSummary is a small version 12 | type WorkerSummary struct { 13 | WorkerID string `json:"worker_id"` 14 | Status string `json:"status"` 15 | Conditions []*WorkerCondition `json:"conditions"` 16 | } 17 | 18 | //WorkerLogsInput is for fetching worker logs 19 | type WorkerLogsInput struct { 20 | ProjectID string `json:"project_id"` 21 | WorkloadID string `json:"workload_id"` 22 | WorkerID string `json:"worker_id"` 23 | } 24 | 25 | //WorkerLogsOutput contains raw log data from the cluster 26 | type WorkerLogsOutput struct { 27 | Data []byte `json:"data"` 28 | } 29 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/placement.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //PlaceProjectInput is input for placement creation 4 | type PlaceProjectInput struct { 5 | ProjectID string `json:"project_id" valid:"required"` 6 | Host string `json:"host" valid:"required"` 7 | Token string `json:"token"` 8 | CAPem string `json:"ca_pem"` 9 | Password string `json:"password"` 10 | Username string `json:"username"` 11 | Insecure bool `json:"insecure"` 12 | ComputeUnits string `json:"compute_units"` 13 | } 14 | 15 | //PlaceProjectOutput is output for placement creation 16 | type PlaceProjectOutput struct { 17 | } 18 | 19 | //ExpelProjectInput is input for placement creation 20 | type ExpelProjectInput struct { 21 | ProjectID string `json:"project_id" valid:"required"` 22 | } 23 | 24 | //ExpelProjectOutput is output for placement creation 25 | type ExpelProjectOutput struct{} 26 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/worker.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | "net/http" 5 | 6 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 7 | ) 8 | 9 | //ClientWorkerInterface is an interface so client workload calls can be mocked. 10 | type ClientWorkerInterface interface { 11 | WorkerLogs(projectID, workloadID, workerID string) (output *v1payload.WorkerLogsOutput, err error) 12 | } 13 | 14 | // WorkerLogs will return all workloads 15 | func (c *Client) WorkerLogs(projectID, workloadID, workerID string) (output *v1payload.WorkerLogsOutput, err error) { 16 | output = &v1payload.WorkerLogsOutput{} 17 | input := &v1payload.WorkerLogsInput{ 18 | ProjectID: projectID, 19 | WorkloadID: workloadID, 20 | WorkerID: workerID, 21 | } 22 | 23 | return output, c.doRequest(http.MethodGet, createPath(projectID, workloadsEndpoint, workloadID, "workers", workerID, "logs"), input, output) 24 | } 25 | -------------------------------------------------------------------------------- /crd/artifacts/datasets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1beta1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | # name must match the spec fields below, and be in the form: . 5 | name: datasets.stable.nerdalize.com 6 | spec: 7 | # group name to use for REST API: /apis// 8 | group: stable.nerdalize.com 9 | # version name to use for REST API: /apis// 10 | version: v1 11 | # either Namespaced or Cluster 12 | scope: Namespaced 13 | names: 14 | # plural name to be used in the URL: /apis/// 15 | plural: datasets 16 | # singular name to be used as an alias on the CLI and for display 17 | singular: dataset 18 | # kind is normally the CamelCased singular type. Your resource manifests use this. 19 | kind: Dataset 20 | # shortNames allow shorter string to match your resource on the CLI 21 | shortNames: 22 | - dts -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/error.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //Error struct is returned by the API if anything goes wrong 4 | type Error struct { 5 | 6 | //Retry indicates if the client can retry the request as is, this is mostly false on validation/encoding errors and true in other cases 7 | Retry bool `json:"retry"` 8 | 9 | //Message contains a overall message to the user, it should always be set to provide some feedback 10 | Message string `json:"message"` 11 | 12 | //Trace is set if the server is running in development mode, if it is empty it can be ignored 13 | Trace []string `json:"trace,omitempty"` 14 | 15 | //Fields can hold validation messages for individual fields, if empty the cause of the overal error is not due to specific field's input 16 | Fields map[string]string `json:"fields,omitempty"` 17 | } 18 | 19 | //Error returns the error message. 20 | func (e Error) Error() string { 21 | return e.Message 22 | } 23 | -------------------------------------------------------------------------------- /nerd/service/working/v1/utils.go: -------------------------------------------------------------------------------- 1 | package v1working 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | //RemoveContents removes all contents of a directory 10 | func RemoveContents(dir string) error { 11 | d, err := os.Open(dir) 12 | if err != nil { 13 | return err 14 | } 15 | defer d.Close() 16 | names, err := d.Readdirnames(-1) 17 | if err != nil { 18 | return err 19 | } 20 | for _, name := range names { 21 | err = os.RemoveAll(filepath.Join(dir, name)) 22 | if err != nil { 23 | return err 24 | } 25 | } 26 | return nil 27 | } 28 | 29 | //IsEmptyDir checks if a directory contains files 30 | func IsEmptyDir(name string) (bool, error) { 31 | f, err := os.Open(name) 32 | if err != nil { 33 | return false, err 34 | } 35 | defer f.Close() 36 | 37 | _, err = f.Readdirnames(1) // Or f.Readdir(1) 38 | if err == io.EOF { 39 | return true, nil 40 | } 41 | return false, err // Either not empty or error, suits both cases 42 | } 43 | -------------------------------------------------------------------------------- /cmd/cluster.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | flags "github.com/jessevdk/go-flags" 5 | "github.com/mitchellh/cli" 6 | ) 7 | 8 | //Cluster command 9 | type Cluster struct { 10 | *command 11 | } 12 | 13 | //ClusterFactory creates the command 14 | func ClusterFactory(ui cli.Ui) cli.CommandFactory { 15 | cmd := &Cluster{} 16 | cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, nil, flags.None, "nerd cluster") 17 | 18 | return func() (cli.Command, error) { 19 | return cmd, nil 20 | } 21 | } 22 | 23 | //Execute runs the command 24 | func (cmd *Cluster) Execute(args []string) (err error) { return errShowHelp("") } 25 | 26 | // Description returns long-form help text 27 | func (cmd *Cluster) Description() string { return cmd.Synopsis() } 28 | 29 | // Synopsis returns a one-line 30 | func (cmd *Cluster) Synopsis() string { 31 | return "Group of commands used to manage clusters." 32 | } 33 | 34 | // Usage shows usage 35 | func (cmd *Cluster) Usage() string { return "nerd cluster " } 36 | -------------------------------------------------------------------------------- /nerd/client/auth/v1/oath_token_provider.go: -------------------------------------------------------------------------------- 1 | package v1auth 2 | 3 | //OAuthTokenProvider is capable of providing a oauth access token. When IsExpired return false 4 | //the in-memory token will be used to prevent from calling Retrieve for each API call. 5 | type OAuthTokenProvider interface { 6 | IsExpired() bool 7 | Retrieve() (string, error) 8 | } 9 | 10 | //StaticOAuthTokenProvider is a simple oauth token provider that always returns the same token. 11 | type StaticOAuthTokenProvider struct { 12 | Token string 13 | } 14 | 15 | //NewStaticOAuthTokenProvider creates a new StaticOAuthTokenProvider for the given token. 16 | func NewStaticOAuthTokenProvider(token string) *StaticOAuthTokenProvider { 17 | return &StaticOAuthTokenProvider{token} 18 | } 19 | 20 | //IsExpired always returns false. 21 | func (s *StaticOAuthTokenProvider) IsExpired() bool { 22 | return false 23 | } 24 | 25 | //Retrieve always returns the given token. 26 | func (s *StaticOAuthTokenProvider) Retrieve() (string, error) { 27 | return s.Token, nil 28 | } 29 | -------------------------------------------------------------------------------- /crd/pkg/client/listers/stable.nerdalize.com/v1/expansion_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file was automatically generated by lister-gen 18 | 19 | package v1 20 | 21 | // DatasetListerExpansion allows custom methods to be added to 22 | // DatasetLister. 23 | type DatasetListerExpansion interface{} 24 | 25 | // DatasetNamespaceListerExpansion allows custom methods to be added to 26 | // DatasetNamespaceLister. 27 | type DatasetNamespaceListerExpansion interface{} 28 | -------------------------------------------------------------------------------- /nerd/oauth/provider.go: -------------------------------------------------------------------------------- 1 | package oauth 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | //DefaultExpireWindow is the default amount of seconds that a nerd token is assumed to be expired, before it's actually expired. 8 | //This will prevent the server from declining the token because it was just expired. 9 | const DefaultExpireWindow = 20 10 | 11 | //ProviderBasis is the basis for every provider. 12 | type ProviderBasis struct { 13 | expiration time.Time 14 | CurrentTime func() time.Time 15 | AlwaysValid bool 16 | 17 | ExpireWindow time.Duration 18 | } 19 | 20 | //IsExpired checks if the current token is expired. 21 | func (b *ProviderBasis) IsExpired() bool { 22 | if b.CurrentTime == nil { 23 | b.CurrentTime = time.Now 24 | } 25 | return !b.AlwaysValid && !b.CurrentTime().Before(b.expiration) 26 | } 27 | 28 | //SetExpiration sets the expiration field and takes the ExpireWindow into account. 29 | func (b *ProviderBasis) SetExpiration(expiration time.Time) { 30 | b.expiration = expiration 31 | if b.ExpireWindow > 0 { 32 | b.expiration = b.expiration.Add(-b.ExpireWindow) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /nerd/client/auth/v1/payload/project.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | import "time" 4 | 5 | //ListProjectsOutput contains a list of projects 6 | type ListProjectsOutput struct { 7 | Projects []*GetProjectOutput 8 | } 9 | 10 | //GetProjectOutput get some details of a specific project. Useful to setup kube config. 11 | type GetProjectOutput struct { 12 | ID int `json:"id"` 13 | URL string `json:"url"` 14 | Nk string `json:"nk"` 15 | Name string `json:"name"` 16 | Slug string `json:"slug"` 17 | Services struct { 18 | Cluster struct { 19 | URL string `json:"url"` 20 | ID int `json:"id"` 21 | Name string `json:"name"` 22 | Address string `json:"address"` 23 | Subaddress string `json:"subaddress"` 24 | B64CaData string `json:"b64_ca_data"` 25 | Default bool `json:"default"` 26 | } `json:"cluster"` 27 | } `json:"services"` 28 | CreatedAt time.Time `json:"created_at"` 29 | } 30 | 31 | //Project represents a project 32 | type Project struct { 33 | ID int `json:"id"` 34 | URL string `json:"url"` 35 | Slug string `json:"slug"` 36 | } 37 | -------------------------------------------------------------------------------- /nerd/utils/test_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "reflect" 7 | "runtime" 8 | "testing" 9 | ) 10 | 11 | //Assert fails the test if the condition is false. 12 | func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) { 13 | if !condition { 14 | _, file, line, _ := runtime.Caller(1) 15 | fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) 16 | tb.FailNow() 17 | } 18 | } 19 | 20 | //OK fails the test if an err is not nil. 21 | func OK(tb testing.TB, err error) { 22 | if err != nil { 23 | _, file, line, _ := runtime.Caller(1) 24 | fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) 25 | tb.FailNow() 26 | } 27 | } 28 | 29 | //Equals fails the test if exp is not equal to act. 30 | func Equals(tb testing.TB, exp, act interface{}) { 31 | if !reflect.DeepEqual(exp, act) { 32 | _, file, line, _ := runtime.Caller(1) 33 | fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) 34 | tb.FailNow() 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /svc/errors.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | type errValidation struct{ error } 4 | 5 | func (e errValidation) IsValidation() bool { return true } 6 | 7 | //IsValidationErr asserts for a validation error 8 | func IsValidationErr(err error) bool { 9 | type iface interface { 10 | IsValidation() bool 11 | } 12 | te, ok := err.(iface) 13 | return ok && te.IsValidation() 14 | } 15 | 16 | type errRaceCondition struct{ error } 17 | 18 | func (e errRaceCondition) IsRaceCondition() bool { return true } 19 | 20 | //IsRaceConditionErr is returned when we couldn't retrieve any logs for the job 21 | func IsRaceConditionErr(err error) bool { 22 | type iface interface { 23 | IsRaceCondition() bool 24 | } 25 | te, ok := err.(iface) 26 | return ok && te.IsRaceCondition() 27 | } 28 | 29 | type errDatasetSpec struct{ error } 30 | 31 | func (e errDatasetSpec) IsDatasetSpec() bool { return true } 32 | 33 | //IsDatasetSpecErr is returned when a invalid input/output spec was given 34 | func IsDatasetSpecErr(err error) bool { 35 | type iface interface { 36 | IsDatasetSpec() bool 37 | } 38 | te, ok := err.(iface) 39 | return ok && te.IsDatasetSpec() 40 | } 41 | -------------------------------------------------------------------------------- /cmd/job.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | flags "github.com/jessevdk/go-flags" 5 | "github.com/mitchellh/cli" 6 | ) 7 | 8 | //Job command 9 | type Job struct { 10 | *command 11 | } 12 | 13 | //JobFactory creates the command 14 | func JobFactory(ui cli.Ui) cli.CommandFactory { 15 | cmd := &Job{} 16 | cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, nil, flags.None, "nerd job") 17 | 18 | return func() (cli.Command, error) { 19 | return cmd, nil 20 | } 21 | } 22 | 23 | //Execute runs the command 24 | func (cmd *Job) Execute(args []string) (err error) { return errShowHelp("") } 25 | 26 | // Description returns long-form help text 27 | func (cmd *Job) Description() string { 28 | return "Group of commands used to manage the lifecycle of jobs. A job is a computation that takes some input data, runs an application to do operations on this data and stores the results." 29 | } 30 | 31 | // Synopsis returns a one-line 32 | func (cmd *Job) Synopsis() string { 33 | return "Group of commands used to manage the lifecycle of jobs." 34 | } 35 | 36 | // Usage shows usage 37 | func (cmd *Job) Usage() string { return "nerd job " } 38 | -------------------------------------------------------------------------------- /installers/msi/nerd.wixproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | $(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets 4 | Nerd Setup (x64) 5 | Package 6 | false 7 | bin\ 8 | 9 | 10 | 11 | 12 | x64 13 | 3.5 14 | {bf3f885c-7954-456b-8282-3171f571024d} 15 | 2.0 16 | 17 | obj\ 18 | False 19 | True 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /nerd/aws/credentials_provider_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 8 | ) 9 | 10 | type fakeClient struct { 11 | exp time.Time 12 | } 13 | 14 | func (f *fakeClient) CreateToken(projectID string) (output *v1payload.CreateTokenOutput, err error) { 15 | return &v1payload.CreateTokenOutput{ 16 | AWSExpiration: f.exp, 17 | }, nil 18 | } 19 | 20 | func TestExpired(t *testing.T) { 21 | cases := map[string]struct { 22 | expireAt time.Time 23 | expected bool 24 | }{ 25 | "expired": { 26 | expireAt: time.Now().Add(-time.Minute), 27 | expected: true, 28 | }, 29 | "notExpired": { 30 | expireAt: time.Now().Add(time.Minute), 31 | expected: false, 32 | }, 33 | } 34 | for name, tc := range cases { 35 | t.Run(name, func(t *testing.T) { 36 | cl := &fakeClient{tc.expireAt} 37 | prov := &Provider{ 38 | Client: cl, 39 | } 40 | _, err := prov.Retrieve() 41 | if err != nil { 42 | t.Fatalf("unexpected error: %v", err) 43 | } 44 | expired := prov.IsExpired() 45 | if expired != tc.expected { 46 | t.Errorf("expected %v but got %v", tc.expected, expired) 47 | } 48 | }) 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /crd/pkg/apis/stable.nerdalize.com/v1/types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/nerdalize/nerd/pkg/transfer/archiver" 5 | "github.com/nerdalize/nerd/pkg/transfer/store" 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | ) 8 | 9 | // +genclient 10 | // +genclient:noStatus 11 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 12 | 13 | // Dataset describes a nerd dataset. 14 | type Dataset struct { 15 | metav1.TypeMeta `json:",inline"` 16 | metav1.ObjectMeta `json:"metadata,omitempty"` 17 | 18 | Spec DatasetSpec `json:"spec"` 19 | } 20 | 21 | // DatasetSpec is the spec for a Dataset resource 22 | type DatasetSpec struct { 23 | StoreOptions transferstore.StoreOptions 24 | ArchiverOptions transferarchiver.ArchiverOptions 25 | 26 | Options map[string]string `json:"options"` 27 | Size uint64 `json:"size"` 28 | InputFor []string `json:"input"` 29 | OutputFrom []string `json:"output"` 30 | } 31 | 32 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 33 | 34 | // DatasetList is a list of Dataset resources 35 | type DatasetList struct { 36 | metav1.TypeMeta `json:",inline"` 37 | metav1.ListMeta `json:"metadata"` 38 | 39 | Items []Dataset `json:"items"` 40 | } 41 | -------------------------------------------------------------------------------- /svc/kube_get_secret.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | "path" 6 | "time" 7 | 8 | "github.com/nerdalize/nerd/pkg/kubevisor" 9 | "k8s.io/api/core/v1" 10 | ) 11 | 12 | //GetSecretInput is the input to GetSecret 13 | type GetSecretInput struct { 14 | Name string `validate:"printascii"` 15 | } 16 | 17 | //GetSecretOutput is the output to GetSecret 18 | type GetSecretOutput struct { 19 | Name string 20 | Size int 21 | Image string 22 | CreatedAt time.Time 23 | Type string 24 | } 25 | 26 | //GetSecret will retrieve the secret matching the provided name from kubernetes 27 | func (k *Kube) GetSecret(ctx context.Context, in *GetSecretInput) (out *GetSecretOutput, err error) { 28 | if err = k.checkInput(ctx, in); err != nil { 29 | return nil, err 30 | } 31 | 32 | secret := &v1.Secret{} 33 | err = k.visor.GetResource(ctx, kubevisor.ResourceTypeSecrets, secret, in.Name) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | return &GetSecretOutput{ 39 | Name: secret.Name, 40 | Type: string(secret.Type), 41 | Size: secret.Size(), 42 | CreatedAt: secret.CreationTimestamp.Local(), 43 | Image: path.Join(secret.Labels["registry"], secret.Labels["project"], secret.Labels["image"]), 44 | }, nil 45 | 46 | } 47 | -------------------------------------------------------------------------------- /cmd/dataset.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | flags "github.com/jessevdk/go-flags" 5 | "github.com/mitchellh/cli" 6 | ) 7 | 8 | //Dataset command 9 | type Dataset struct { 10 | *command 11 | } 12 | 13 | //DatasetFactory creates the command 14 | func DatasetFactory(ui cli.Ui) cli.CommandFactory { 15 | cmd := &Dataset{} 16 | cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, nil, flags.None, "nerd dataset") 17 | 18 | return func() (cli.Command, error) { 19 | return cmd, nil 20 | } 21 | } 22 | 23 | //Execute runs the command 24 | func (cmd *Dataset) Execute(args []string) (err error) { return errShowHelp("") } 25 | 26 | // Description returns long-form help text 27 | func (cmd *Dataset) Description() string { 28 | return "Group of commands used to manage datasets. A dataset is a collection of files, like a folder on a computer. They can be used as input for a job and, when an application creates output files, these can be automatically stored in a new dataset." 29 | } 30 | 31 | // Synopsis returns a one-line 32 | func (cmd *Dataset) Synopsis() string { 33 | return "Group of commands used to manage datasets (collection of files)." 34 | } 35 | 36 | // Usage shows usage 37 | func (cmd *Dataset) Usage() string { return "nerd dataset " } 38 | -------------------------------------------------------------------------------- /crd/cmd/example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | 7 | "github.com/golang/glog" 8 | 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/client-go/tools/clientcmd" 11 | 12 | nerdalizecs "github.com/nerdalize/nerd/crd/pkg/client/clientset/versioned" 13 | ) 14 | 15 | var ( 16 | kuberconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") 17 | master = flag.String("master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") 18 | ) 19 | 20 | func main() { 21 | flag.Parse() 22 | 23 | cfg, err := clientcmd.BuildConfigFromFlags(*master, *kuberconfig) 24 | if err != nil { 25 | glog.Fatalf("Error building kubeconfig: %v", err) 26 | } 27 | 28 | exampleClient, err := nerdalizecs.NewForConfig(cfg) 29 | if err != nil { 30 | glog.Fatalf("Error building example clientset: %v", err) 31 | } 32 | 33 | list, err := exampleClient.NerdalizeV1().Datasets("default").List(metav1.ListOptions{}) 34 | if err != nil { 35 | glog.Fatalf("Error listing all datasets: %v", err) 36 | } 37 | 38 | for _, db := range list.Items { 39 | fmt.Printf("datasets %s with key %q and bucket %q\n", db.Name, db.Spec.Key, db.Spec.Bucket) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /nerd/client/auth/v1/payload/oauth.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //GetOAuthCredentialsInput is input for getting oauth credentials 4 | type GetOAuthCredentialsInput struct { 5 | Code string `url:"code"` 6 | ClientID string `url:"client_id"` 7 | RedirectURI string `url:"redirect_uri"` 8 | GrantType string `url:"grant_type"` 9 | } 10 | 11 | //GetOAuthCredentialsOutput is output when getting oauth credentials 12 | type GetOAuthCredentialsOutput struct { 13 | OAuthCredentials 14 | } 15 | 16 | //RefreshOAuthCredentialsInput is input for refreshing oauth credentials 17 | type RefreshOAuthCredentialsInput struct { 18 | RefreshToken string `url:"refresh_token"` 19 | ClientID string `url:"client_id"` 20 | GrantType string `url:"grant_type"` 21 | } 22 | 23 | //RefreshOAuthCredentialsOutput is output when refreshing oauth credentials 24 | type RefreshOAuthCredentialsOutput struct { 25 | OAuthCredentials 26 | } 27 | 28 | // OAuthCredentials represents the OAuth access tokens 29 | type OAuthCredentials struct { 30 | AccessToken string `json:"access_token"` 31 | RefreshToken string `json:"refresh_token"` 32 | ExpiresIn int `json:"expires_in"` 33 | Scope string `json:"scope,omitempty"` 34 | TokenType string `json:"token_type,omitempty"` 35 | IDToken string `json:"id_token"` 36 | } 37 | -------------------------------------------------------------------------------- /svc/kube_update_secret.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | "k8s.io/api/core/v1" 8 | ) 9 | 10 | // UpdateSecretInput is the input for UpdateSecret 11 | type UpdateSecretInput struct { 12 | Name string `validate:"printascii"` 13 | Username string 14 | Password string 15 | } 16 | 17 | // UpdateSecretOutput is the output for UpdateSecret 18 | type UpdateSecretOutput struct { 19 | Name string 20 | } 21 | 22 | // UpdateSecret will update a secret resource. 23 | // Fields that can be updated: name, input, output and size. Input and output are the jobs the secret is used for or coming from. 24 | func (k *Kube) UpdateSecret(ctx context.Context, in *UpdateSecretInput) (out *UpdateSecretOutput, err error) { 25 | secret := &v1.Secret{} 26 | err = k.visor.GetResource(ctx, kubevisor.ResourceTypeSecrets, secret, in.Name) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | secret.Data[v1.DockerConfigJsonKey], err = transformCredentials(in.Username, in.Password, secret.Labels["registry"]) 32 | if err != nil { 33 | return nil, err 34 | } 35 | err = k.visor.UpdateResource(ctx, kubevisor.ResourceTypeSecrets, secret, in.Name) 36 | if err != nil { 37 | return nil, err 38 | } 39 | return &UpdateSecretOutput{ 40 | Name: secret.Name, 41 | }, nil 42 | } 43 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | flags "github.com/jessevdk/go-flags" 7 | "github.com/mitchellh/cli" 8 | "github.com/nerdalize/nerd/nerd" 9 | ) 10 | 11 | //Version command 12 | type Version struct { 13 | version string 14 | commit string 15 | *command 16 | } 17 | 18 | //VersionFactory returns a factory method for the join command 19 | func VersionFactory(version, commit string, ui cli.Ui) cli.CommandFactory { 20 | 21 | cmd := &Version{ 22 | version: version, 23 | commit: commit, 24 | } 25 | cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, nil, flags.None, "nerd version") 26 | return func() (cli.Command, error) { 27 | return cmd, nil 28 | } 29 | } 30 | 31 | //Execute is called by run and allows an error to be returned 32 | func (cmd *Version) Execute(args []string) (err error) { 33 | cmd.out.Info(fmt.Sprintf("%s (%s)", cmd.version, cmd.commit)) 34 | nerd.VersionMessage(cmd.version) 35 | return nil 36 | } 37 | 38 | // Description returns long-form help text 39 | func (cmd *Version) Description() string { return cmd.Synopsis() } 40 | 41 | // Synopsis returns a one-line 42 | func (cmd *Version) Synopsis() string { return "Check the current version of the CLI" } 43 | 44 | // Usage shows usage 45 | func (cmd *Version) Usage() string { return "nerd version" } 46 | -------------------------------------------------------------------------------- /nerd/version_test.go: -------------------------------------------------------------------------------- 1 | package nerd 2 | 3 | import "testing" 4 | 5 | func TestSemVer(t *testing.T) { 6 | testCases := []struct { 7 | big string 8 | small string 9 | }{ 10 | { 11 | big: "2.0.0", 12 | small: "1.0.0", 13 | }, 14 | { 15 | big: "2.0.0", 16 | small: "1.3.3", 17 | }, 18 | { 19 | big: "0.2.0", 20 | small: "0.1.0", 21 | }, 22 | { 23 | big: "0.2.0", 24 | small: "0.1.2", 25 | }, 26 | { 27 | big: "0.0.2", 28 | small: "0.0.1", 29 | }, 30 | { 31 | big: "0.1.0", 32 | small: "0.0.1", 33 | }, 34 | { 35 | big: "2.0.0", 36 | small: "1.20.30", 37 | }, 38 | { 39 | big: "20.0.0", 40 | small: "19.2.3", 41 | }, 42 | } 43 | for _, tc := range testCases { 44 | big, err := ParseSemVer(tc.big) 45 | if err != nil { 46 | t.Fatalf("Failed to parse semver %v (big)", tc.big) 47 | } 48 | small, err := ParseSemVer(tc.small) 49 | if err != nil { 50 | t.Fatalf("Failed to parse semver %v (small)", tc.small) 51 | } 52 | if !big.GreaterThan(small) { 53 | t.Errorf("Expected %v to be a larger semver than %v", tc.big, tc.small) 54 | } 55 | } 56 | } 57 | 58 | func TestSemVerFail(t *testing.T) { 59 | ver := "ver.ver.ver" 60 | _, err := ParseSemVer(ver) 61 | if err == nil { 62 | t.Errorf("Parsing %v should raise an error.", ver) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/typed/stable.nerdalize.com/v1/fake/fake_stable.nerdalize.com_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package fake 17 | 18 | import ( 19 | v1 "github.com/nerdalize/nerd/crd/pkg/client/clientset/versioned/typed/stable.nerdalize.com/v1" 20 | rest "k8s.io/client-go/rest" 21 | testing "k8s.io/client-go/testing" 22 | ) 23 | 24 | type FakeNerdalizeV1 struct { 25 | *testing.Fake 26 | } 27 | 28 | func (c *FakeNerdalizeV1) Datasets(namespace string) v1.DatasetInterface { 29 | return &FakeDatasets{c, namespace} 30 | } 31 | 32 | // RESTClient returns a RESTClient that is used to communicate 33 | // with API server by this client implementation. 34 | func (c *FakeNerdalizeV1) RESTClient() rest.Interface { 35 | var ret *rest.RESTClient 36 | return ret 37 | } 38 | -------------------------------------------------------------------------------- /crd/pkg/signals/signal.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package signals 18 | 19 | import ( 20 | "os" 21 | "os/signal" 22 | ) 23 | 24 | var onlyOneSignalHandler = make(chan struct{}) 25 | 26 | // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned 27 | // which is closed on one of these signals. If a second signal is caught, the program 28 | // is terminated with exit code 1. 29 | func SetupSignalHandler() (stopCh <-chan struct{}) { 30 | close(onlyOneSignalHandler) // panics when called twice 31 | 32 | stop := make(chan struct{}) 33 | c := make(chan os.Signal, 2) 34 | signal.Notify(c, shutdownSignals...) 35 | go func() { 36 | <-c 37 | close(stop) 38 | <-c 39 | os.Exit(1) // second signal. Exit directly. 40 | }() 41 | 42 | return stop 43 | } 44 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/client/index_test.go: -------------------------------------------------------------------------------- 1 | package v1data 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestIndexReader(t *testing.T) { 9 | keys := []Key{ 10 | Key{1, 2, 3}, 11 | Key{4, 5, 6}, 12 | Key{7, 8, 9}, 13 | } 14 | // create input stream 15 | buf := bytes.NewBufferString("") 16 | for _, key := range keys { 17 | buf.WriteString(key.ToString() + "\n") 18 | } 19 | 20 | // new index reader 21 | ir := NewIndexReader(buf) 22 | 23 | // compare input with reader result 24 | for _, key := range keys { 25 | read, err := ir.ReadKey() 26 | if err != nil { 27 | t.Fatalf("unexpected error: %v", err) 28 | } 29 | if key.ToString() != read.ToString() { 30 | t.Errorf("expected key %v but got %v", key, read) 31 | } 32 | } 33 | } 34 | 35 | func TestIndexWriter(t *testing.T) { 36 | keys := []Key{ 37 | Key{1, 2, 3}, 38 | Key{4, 5, 6}, 39 | Key{7, 8, 9}, 40 | } 41 | // create input stream 42 | expected := bytes.NewBufferString("") 43 | for _, key := range keys { 44 | expected.WriteString(key.ToString() + "\n") 45 | } 46 | result := bytes.NewBuffer(nil) 47 | iw := NewIndexWriter(result) 48 | for _, key := range keys { 49 | err := iw.WriteKey(key) 50 | if err != nil { 51 | t.Fatalf("unexpected error: %v", err) 52 | } 53 | } 54 | if expected.String() != result.String() { 55 | t.Errorf("expected %v but got %v", expected, result) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /nerd/client/auth/v1/payload/cluster.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //ListClustersOutput contains a list of projects 4 | type ListClustersOutput struct { 5 | Clusters []*GetClusterOutput 6 | } 7 | 8 | //GetClusterOutput get some details of a specific cluster. Useful to setup kube config. 9 | type GetClusterOutput struct { 10 | URL string `json:"url"` 11 | ShortName string `json:"short_name"` 12 | Name string `json:"name"` 13 | Namespaces []struct { 14 | Name string `json:"name"` 15 | } `json:"namespaces"` 16 | ServiceType string `json:"service_type"` 17 | ServiceURL string `json:"service_url"` 18 | CaCertificate string `json:"ca_certificate"` 19 | Capacity struct { 20 | CPU float64 `json:"cpu"` 21 | Memory float64 `json:"memory"` 22 | Pods int `json:"pods"` 23 | } `json:"capacity"` 24 | Usage struct { 25 | CPU float64 `json:"cpu"` 26 | Memory float64 `json:"memory"` 27 | Pods int `json:"pods"` 28 | } `json:"usage"` 29 | KubeConfigUser struct { 30 | Token string `json:"token"` 31 | AuthProvider struct { 32 | Config struct { 33 | IdpIssuerURL string `json:"idp-issuer-url"` 34 | ClientID string `json:"client-id"` 35 | RefreshToken string `json:"refresh-token"` 36 | IDToken string `json:"id-token"` 37 | } `json:"config"` 38 | Name string `json:"name"` 39 | } `json:"auth-provider"` 40 | } `json:"kubeconfig_user"` 41 | } 42 | -------------------------------------------------------------------------------- /cmd/output.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "text/tabwriter" 8 | 9 | "github.com/mitchellh/cli" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | //Output standardizes the way in which we would like to capture output 14 | //throughout the program 15 | type Output struct { 16 | uiw *cli.UiWriter 17 | cli.Ui 18 | } 19 | 20 | //NewOutput sets up our standardized program outputter 21 | func NewOutput(ui cli.Ui) *Output { 22 | return &Output{Ui: ui, uiw: &cli.UiWriter{Ui: ui}} 23 | } 24 | 25 | //Logger returns a logrus logger that writes to the UIs Stderr 26 | func (o *Output) Logger(level logrus.Level) *logrus.Logger { 27 | logs := logrus.New() 28 | logs.Out = o.uiw 29 | logs.Level = level 30 | return logs 31 | } 32 | 33 | //Errorf prints a formatted error to ErrorOutput 34 | func (o *Output) Errorf(format string, a ...interface{}) { 35 | o.Error(fmt.Sprintf(format, a...)) 36 | } 37 | 38 | //Infof prints a formatted message to 39 | func (o *Output) Infof(format string, a ...interface{}) { 40 | o.Info(fmt.Sprintf(format, a...)) 41 | } 42 | 43 | //Table will print a table 44 | func (o *Output) Table(header []string, rows [][]string) error { 45 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 46 | if len(header) > 0 { 47 | fmt.Fprintln(w, strings.Join(header, "\t")+"\t") 48 | } 49 | 50 | for _, r := range rows { 51 | fmt.Fprintln(w, strings.Join(r, "\t")+"\t") 52 | } 53 | 54 | return w.Flush() 55 | } 56 | -------------------------------------------------------------------------------- /pkg/transfer/archiver/copy.go: -------------------------------------------------------------------------------- 1 | package transferarchiver 2 | 3 | import ( 4 | "context" 5 | "io" 6 | ) 7 | 8 | // here is some syntaxic sugar inspired by the Tomas Senart's video, 9 | // it allows me to inline the Reader interface 10 | type readerFunc func(p []byte) (n int, err error) 11 | 12 | func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) } 13 | 14 | // Copy is taken from http://ixday.github.io/post/golang-cancel-copy/, slightly modified function signature: 15 | // - context has been added in order to propagate cancelation 16 | // - I do not return the number of bytes written, has it is not useful in my use case 17 | func Copy(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) { 18 | 19 | // Copy will call the Reader and Writer interface multiple time, in order 20 | // to copy by chunk (avoiding loading the whole file in memory). 21 | // I insert the ability to cancel before read time as it is the earliest 22 | // possible in the call process. 23 | n, err := io.Copy(dst, readerFunc(func(p []byte) (int, error) { 24 | 25 | // golang non-blocking channel: https://gobyexample.com/non-blocking-channel-operations 26 | select { 27 | 28 | // if context has been canceled 29 | case <-ctx.Done(): 30 | // stop process and propagate "context canceled" error 31 | return 0, ctx.Err() 32 | default: 33 | // otherwise just run default io.Reader implementation 34 | return src.Read(p) 35 | } 36 | })) 37 | return n, err 38 | } 39 | -------------------------------------------------------------------------------- /crd/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file was automatically generated by informer-gen 18 | 19 | package internalinterfaces 20 | 21 | import ( 22 | time "time" 23 | 24 | versioned "github.com/nerdalize/nerd/crd/pkg/client/clientset/versioned" 25 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | runtime "k8s.io/apimachinery/pkg/runtime" 27 | cache "k8s.io/client-go/tools/cache" 28 | ) 29 | 30 | type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer 31 | 32 | // SharedInformerFactory a small interface to allow for adding an informer without an import cycle 33 | type SharedInformerFactory interface { 34 | Start(stopCh <-chan struct{}) 35 | InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer 36 | } 37 | 38 | type TweakListOptionsFunc func(*v1.ListOptions) 39 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/upload.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | "net/http" 5 | 6 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 7 | ) 8 | 9 | //ClientUploadInterface is an interface so client task calls can be mocked. 10 | type ClientUploadInterface interface { 11 | SendUploadHeartbeat(projectID, datasetID string) (output *v1payload.SendUploadHeartbeatOutput, err error) 12 | SendUploadSuccess(projectID, datasetID string) (output *v1payload.SendUploadSuccessOutput, err error) 13 | } 14 | 15 | //SendUploadHeartbeat will send a heartbeat for a task run 16 | func (c *Client) SendUploadHeartbeat(projectID, datasetID string) (output *v1payload.SendUploadHeartbeatOutput, err error) { 17 | output = &v1payload.SendUploadHeartbeatOutput{} 18 | input := &v1payload.SendUploadHeartbeatInput{ 19 | ProjectID: projectID, 20 | DatasetID: datasetID, 21 | } 22 | 23 | return output, c.doRequest(http.MethodPost, createPath(projectID, datasetEndpoint, datasetID, "heartbeats"), input, output) 24 | } 25 | 26 | //SendUploadSuccess will send a successfully run for a task 27 | func (c *Client) SendUploadSuccess(projectID, datasetID string) (output *v1payload.SendUploadSuccessOutput, err error) { 28 | output = &v1payload.SendUploadSuccessOutput{} 29 | input := &v1payload.SendUploadSuccessInput{ 30 | ProjectID: projectID, 31 | DatasetID: datasetID, 32 | } 33 | 34 | return output, c.doRequest(http.MethodPost, createPath(projectID, datasetEndpoint, datasetID, "success"), input, output) 35 | } 36 | -------------------------------------------------------------------------------- /crd/pkg/apis/stable.nerdalize.com/v1/register.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | "k8s.io/apimachinery/pkg/runtime/schema" 7 | 8 | nerdalizecom "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com" 9 | ) 10 | 11 | // SchemeGroupVersion is group version used to register these objects 12 | var SchemeGroupVersion = schema.GroupVersion{Group: nerdalizecom.GroupName, Version: "v1"} 13 | 14 | // Resource takes an unqualified resource and returns a Group qualified GroupResource 15 | func Resource(resource string) schema.GroupResource { 16 | return SchemeGroupVersion.WithResource(resource).GroupResource() 17 | } 18 | 19 | var ( 20 | // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. 21 | SchemeBuilder runtime.SchemeBuilder 22 | localSchemeBuilder = &SchemeBuilder 23 | AddToScheme = localSchemeBuilder.AddToScheme 24 | ) 25 | 26 | func init() { 27 | // We only register manually written functions here. The registration of the 28 | // generated functions takes place in the generated files. The separation 29 | // makes the code compile even when the generated files are missing. 30 | localSchemeBuilder.Register(addKnownTypes) 31 | } 32 | 33 | // Adds the list of known types to api.Scheme. 34 | func addKnownTypes(scheme *runtime.Scheme) error { 35 | scheme.AddKnownTypes(SchemeGroupVersion, 36 | &Dataset{}, 37 | &DatasetList{}, 38 | ) 39 | metav1.AddToGroupVersion(scheme, SchemeGroupVersion) 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/dataset.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | "net/http" 5 | 6 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 7 | ) 8 | 9 | //ClientDatasetInterface is an interface so client dataset calls can be mocked. 10 | type ClientDatasetInterface interface { 11 | CreateDataset(projectID string) (output *v1payload.CreateDatasetOutput, err error) 12 | ListDatasets(projectID string) (output *v1payload.ListDatasetsOutput, err error) 13 | DescribeDataset(projectID, id string) (output *v1payload.DescribeDatasetOutput, err error) 14 | } 15 | 16 | //CreateDataset creates a new dataset. 17 | func (c *Client) CreateDataset(projectID string) (output *v1payload.CreateDatasetOutput, err error) { 18 | input := &v1payload.CreateDatasetInput{} 19 | output = &v1payload.CreateDatasetOutput{} 20 | return output, c.doRequest(http.MethodPost, createPath(projectID, datasetEndpoint), input, output) 21 | } 22 | 23 | //DescribeDataset gets a dataset by ID. 24 | func (c *Client) DescribeDataset(projectID, id string) (output *v1payload.DescribeDatasetOutput, err error) { 25 | output = &v1payload.DescribeDatasetOutput{} 26 | return output, c.doRequest(http.MethodGet, createPath(projectID, datasetEndpoint, id), nil, output) 27 | } 28 | 29 | //ListDatasets gets a dataset by ID. 30 | func (c *Client) ListDatasets(projectID string) (output *v1payload.ListDatasetsOutput, err error) { 31 | output = &v1payload.ListDatasetsOutput{} 32 | return output, c.doRequest(http.MethodGet, createPath(projectID, datasetEndpoint), nil, output) 33 | } 34 | -------------------------------------------------------------------------------- /svc/kube_create_dataset.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | 8 | datasetsv1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 9 | "github.com/nerdalize/nerd/pkg/transfer/archiver" 10 | "github.com/nerdalize/nerd/pkg/transfer/store" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | ) 13 | 14 | //CreateDatasetInput is the input to CreateDataset 15 | type CreateDatasetInput struct { 16 | Name string `validate:"printascii"` 17 | Size uint64 18 | 19 | StoreOptions transferstore.StoreOptions `validate:"required"` 20 | ArchiverOptions transferarchiver.ArchiverOptions `validate:"required"` 21 | } 22 | 23 | //CreateDatasetOutput is the output to CreateDataset 24 | type CreateDatasetOutput struct { 25 | Name string 26 | } 27 | 28 | //CreateDataset will create a dataset on kubernetes 29 | func (k *Kube) CreateDataset(ctx context.Context, in *CreateDatasetInput) (out *CreateDatasetOutput, err error) { 30 | if err = k.checkInput(ctx, in); err != nil { 31 | return nil, err 32 | } 33 | 34 | dataset := &datasetsv1.Dataset{ 35 | ObjectMeta: metav1.ObjectMeta{}, 36 | Spec: datasetsv1.DatasetSpec{ 37 | Size: in.Size, 38 | StoreOptions: in.StoreOptions, 39 | ArchiverOptions: in.ArchiverOptions, 40 | }, 41 | } 42 | 43 | err = k.visor.CreateResource(ctx, kubevisor.ResourceTypeDatasets, dataset, in.Name) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | return &CreateDatasetOutput{ 49 | Name: dataset.Name, 50 | }, nil 51 | } 52 | -------------------------------------------------------------------------------- /crd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "time" 6 | 7 | "github.com/golang/glog" 8 | "k8s.io/client-go/tools/clientcmd" 9 | 10 | clientset "github.com/nerdalize/nerd/crd/pkg/client/clientset/versioned" 11 | informers "github.com/nerdalize/nerd/crd/pkg/client/informers/externalversions" 12 | "github.com/nerdalize/nerd/crd/pkg/signals" 13 | ) 14 | 15 | var ( 16 | masterURL string 17 | kubeconfig string 18 | ) 19 | 20 | func main() { 21 | flag.Parse() 22 | 23 | // set up signals so we handle the first shutdown signal gracefully 24 | glog.Info("Setting up signal handler") 25 | stopCh := signals.SetupSignalHandler() 26 | 27 | cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) 28 | if err != nil { 29 | glog.Fatalf("Error building kubeconfig: %s", err.Error()) 30 | } 31 | 32 | datasetClient, err := clientset.NewForConfig(cfg) 33 | if err != nil { 34 | glog.Fatalf("Error building dataset clientset: %s", err.Error()) 35 | } 36 | 37 | datasetInformerFactory := informers.NewSharedInformerFactory(datasetClient, time.Second*30) 38 | eventHandler := new(S3AWS) 39 | 40 | controller := NewController(datasetClient, datasetInformerFactory, eventHandler) 41 | 42 | go datasetInformerFactory.Start(stopCh) 43 | 44 | controller.Run(stopCh) 45 | } 46 | 47 | func init() { 48 | flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") 49 | flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") 50 | } 51 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/placement.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | "net/http" 5 | 6 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 7 | ) 8 | 9 | //ClientPlacementInterface is an interface for placement of project 10 | type ClientPlacementInterface interface { 11 | PlaceProject(projectID, host, token, capem, username, password, computeUnits string, insecure bool) (output *v1payload.PlaceProjectOutput, err error) 12 | ExpelProject(projectID string) (output *v1payload.ExpelProjectOutput, err error) 13 | } 14 | 15 | //PlaceProject will create queue 16 | func (c *Client) PlaceProject(projectID, host, token, capem, username, password, computeUnits string, insecure bool) (output *v1payload.PlaceProjectOutput, err error) { 17 | output = &v1payload.PlaceProjectOutput{} 18 | input := &v1payload.PlaceProjectInput{ 19 | ProjectID: projectID, 20 | Host: host, 21 | Token: token, 22 | CAPem: capem, 23 | Username: username, 24 | Password: password, 25 | Insecure: insecure, 26 | ComputeUnits: computeUnits, 27 | } 28 | 29 | return output, c.doRequest(http.MethodPost, createPath(projectID, placementsEndpoint), input, output) 30 | } 31 | 32 | //ExpelProject will delete queue a queue with the provided id 33 | func (c *Client) ExpelProject(projectID string) (output *v1payload.ExpelProjectOutput, err error) { 34 | output = &v1payload.ExpelProjectOutput{} 35 | input := &v1payload.ExpelProjectInput{ 36 | ProjectID: projectID, 37 | } 38 | 39 | return output, c.doRequest(http.MethodDelete, createPath(projectID, placementsEndpoint), input, output) 40 | } 41 | -------------------------------------------------------------------------------- /nerd/jwt/test_utils.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "crypto/elliptic" 6 | "crypto/rand" 7 | "fmt" 8 | "testing" 9 | "time" 10 | 11 | jwt "github.com/dgrijalva/jwt-go" 12 | v1payload "github.com/nerdalize/nerd/nerd/client/auth/v1/payload" 13 | ) 14 | 15 | const ( 16 | minute = 60 17 | ) 18 | 19 | //testkey creates a new ecdsa keypair. 20 | func testkey(t *testing.T) *ecdsa.PrivateKey { 21 | k, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) 22 | if err != nil { 23 | t.Fatalf("failed to generate test key: %v", err) 24 | } 25 | return k 26 | } 27 | 28 | //tokenAndPub returns a token string and public key. 29 | func getToken(key *ecdsa.PrivateKey, claims *jwt.StandardClaims, t *testing.T) string { 30 | token := jwt.NewWithClaims(jwt.SigningMethodES384, claims) 31 | ss, err := token.SignedString(key) 32 | if err != nil { 33 | t.Fatalf("failed to sign claims: %v", err) 34 | } 35 | return ss 36 | } 37 | 38 | type tokenClient struct { 39 | token string 40 | } 41 | 42 | //RefreshJWT refreshes a JWT with a refresh token 43 | func (c *tokenClient) RefreshJWT(projectID, jwt, secret string) (output *v1payload.RefreshWorkerJWTOutput, err error) { 44 | output = &v1payload.RefreshWorkerJWTOutput{ 45 | Token: c.token, 46 | } 47 | return output, nil 48 | } 49 | 50 | //RevokeJWT revokes a JWT 51 | func (c *tokenClient) RevokeJWT(projectID, jwt, secret string) (output *v1payload.RefreshWorkerJWTOutput, err error) { 52 | return nil, fmt.Errorf("not implemented") 53 | } 54 | 55 | func timeFunc(t time.Time) func() time.Time { 56 | return func() time.Time { 57 | return t 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /nerd/jwt/provider.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "time" 6 | 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | //DefaultExpireWindow is the default amount of seconds that a nerd token is assumed to be expired, before it's actually expired. 11 | //This will prevent the server from declining the token because it was just expired. 12 | const DefaultExpireWindow = 20 13 | 14 | //ProviderBasis is the basis for every provider. 15 | type ProviderBasis struct { 16 | expiration time.Time 17 | CurrentTime func() time.Time 18 | AlwaysValid bool 19 | 20 | ExpireWindow time.Duration 21 | 22 | Pub *ecdsa.PublicKey 23 | } 24 | 25 | //IsExpired checks if the current token is expired. 26 | func (b *ProviderBasis) IsExpired() bool { 27 | if b.CurrentTime == nil { 28 | b.CurrentTime = time.Now 29 | } 30 | return !b.AlwaysValid && !b.CurrentTime().Before(b.expiration) 31 | } 32 | 33 | //SetExpiration sets the expiration field and takes the ExpireWindow into account. 34 | func (b *ProviderBasis) SetExpiration(expiration time.Time) { 35 | b.expiration = expiration 36 | if b.ExpireWindow > 0 { 37 | b.expiration = b.expiration.Add(-b.ExpireWindow) 38 | } 39 | } 40 | 41 | //SetExpirationFromJWT decodes the JWT and sets the provider expiration based on the JWT expiration field. 42 | func (b *ProviderBasis) SetExpirationFromJWT(jwt string) error { 43 | claims, err := DecodeTokenWithKey(jwt, b.Pub) 44 | if err != nil { 45 | return errors.Wrapf(err, "failed to decode jwt '%v'", jwt) 46 | } 47 | 48 | b.AlwaysValid = claims.ExpiresAt == 0 // if unset 49 | b.SetExpiration(time.Unix(claims.ExpiresAt, 0)) 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /svc/kube_update_dataset.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | 8 | datasetsv1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 9 | ) 10 | 11 | // UpdateDatasetInput is the input for UpdateDataset 12 | type UpdateDatasetInput struct { 13 | Name string `validate:"printascii"` 14 | NewName string `validate:"printascii"` 15 | Size *uint64 16 | InputFor string 17 | OutputFrom string 18 | } 19 | 20 | // UpdateDatasetOutput is the output for UpdateDataset 21 | type UpdateDatasetOutput struct { 22 | Name string 23 | } 24 | 25 | // UpdateDataset will update a dataset resource. 26 | // Fields that can be updated: name, input, output and size. Input and output are the jobs the dataset is used for or coming from. 27 | func (k *Kube) UpdateDataset(ctx context.Context, in *UpdateDatasetInput) (out *UpdateDatasetOutput, err error) { 28 | dataset := &datasetsv1.Dataset{} 29 | err = k.visor.GetResource(ctx, kubevisor.ResourceTypeDatasets, dataset, in.Name) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | if in.NewName != "" { 35 | dataset.SetName(in.NewName) 36 | } 37 | if in.Size != nil { 38 | dataset.Spec.Size = *in.Size 39 | } 40 | if in.InputFor != "" { 41 | dataset.Spec.InputFor = append(dataset.Spec.InputFor, in.InputFor) 42 | } 43 | if in.OutputFrom != "" { 44 | dataset.Spec.OutputFrom = append(dataset.Spec.OutputFrom, in.OutputFrom) 45 | } 46 | 47 | err = k.visor.UpdateResource(ctx, kubevisor.ResourceTypeDatasets, dataset, in.Name) 48 | if err != nil { 49 | return nil, err 50 | } 51 | return &UpdateDatasetOutput{ 52 | Name: dataset.Name, 53 | }, nil 54 | } 55 | -------------------------------------------------------------------------------- /nerd/jwt/authapi_provider.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | 6 | v1auth "github.com/nerdalize/nerd/nerd/client/auth/v1" 7 | "github.com/nerdalize/nerd/nerd/conf" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | const ( 12 | jwtScope = "nce.nerdalize.com" 13 | ) 14 | 15 | //AuthAPIProvider provides nerdalize credentials by making a request to the nerdalize auth server. 16 | //The UserPassProvider is used to retrieve the username and password required to authenticate with the auth server. 17 | type AuthAPIProvider struct { 18 | *ProviderBasis 19 | 20 | Client *v1auth.Client 21 | Session conf.SessionInterface 22 | } 23 | 24 | //NewAuthAPIProvider creates a new AuthAPIProvider provider. 25 | func NewAuthAPIProvider(pub *ecdsa.PublicKey, session conf.SessionInterface, c *v1auth.Client) *AuthAPIProvider { 26 | return &AuthAPIProvider{ 27 | ProviderBasis: &ProviderBasis{ 28 | ExpireWindow: DefaultExpireWindow, 29 | Pub: pub, 30 | }, 31 | Client: c, 32 | Session: session, 33 | } 34 | } 35 | 36 | //Retrieve retrieves the token from the authentication server. 37 | func (p *AuthAPIProvider) Retrieve() (string, error) { 38 | out, err := p.Client.GetJWT(jwtScope) 39 | if err != nil { 40 | return "", errors.Wrap(err, "failed to get nerd jwt") 41 | } 42 | err = p.SetExpirationFromJWT(out.Token) 43 | if err != nil { 44 | return "", errors.Wrap(err, "failed to set expiration") 45 | } 46 | err = isValid(out.Token, p.Pub) 47 | if err != nil { 48 | return "", err 49 | } 50 | err = p.Session.WriteJWT(out.Token, "") 51 | if err != nil { 52 | return "", errors.Wrap(err, "failed to write nerd jwt to config") 53 | } 54 | return out.Token, nil 55 | } 56 | -------------------------------------------------------------------------------- /svc/kube_get_dataset.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | 8 | datasetsv1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 9 | "github.com/nerdalize/nerd/pkg/transfer/archiver" 10 | "github.com/nerdalize/nerd/pkg/transfer/store" 11 | ) 12 | 13 | //GetDatasetInput is the input to GetDataset 14 | type GetDatasetInput struct { 15 | Name string `validate:"printascii"` 16 | } 17 | 18 | //GetDatasetOutput is the output to GetDataset 19 | type GetDatasetOutput struct { 20 | Name string 21 | Size uint64 22 | 23 | InputFor []string 24 | OutputFrom []string 25 | 26 | StoreOptions transferstore.StoreOptions 27 | ArchiverOptions transferarchiver.ArchiverOptions 28 | } 29 | 30 | //GetDataset will retrieve a dataset from kubernetes 31 | func (k *Kube) GetDataset(ctx context.Context, in *GetDatasetInput) (out *GetDatasetOutput, err error) { 32 | if err = k.checkInput(ctx, in); err != nil { 33 | return nil, err 34 | } 35 | 36 | dataset := &datasetsv1.Dataset{} 37 | err = k.visor.GetResource(ctx, kubevisor.ResourceTypeDatasets, dataset, in.Name) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | return GetDatasetOutputFromSpec(dataset), nil 43 | } 44 | 45 | //GetDatasetOutputFromSpec allows easy output creation from dataset 46 | func GetDatasetOutputFromSpec(dataset *datasetsv1.Dataset) *GetDatasetOutput { 47 | return &GetDatasetOutput{ 48 | Name: dataset.Name, 49 | Size: dataset.Spec.Size, 50 | InputFor: dataset.Spec.InputFor, 51 | OutputFrom: dataset.Spec.OutputFrom, 52 | StoreOptions: dataset.Spec.StoreOptions, 53 | ArchiverOptions: dataset.Spec.ArchiverOptions, 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/client/metadata.go: -------------------------------------------------------------------------------- 1 | package v1data 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "path" 8 | 9 | "github.com/nerdalize/nerd/nerd/client" 10 | v1payload "github.com/nerdalize/nerd/nerd/service/datatransfer/v1/client/payload" 11 | ) 12 | 13 | const ( 14 | //MetadataObjectKey is the key of the object that contains an a dataset's metadata. 15 | MetadataObjectKey = "metadata" 16 | ) 17 | 18 | //MetadataExists checks if the metadata object exists. 19 | func (c *Client) MetadataExists(ctx context.Context, bucket, root string) (bool, error) { 20 | return c.Exists(ctx, bucket, path.Join(root, MetadataObjectKey)) 21 | } 22 | 23 | //MetadataUpload uploads a dataset's metadata. 24 | func (c *Client) MetadataUpload(ctx context.Context, bucket, root string, m *v1payload.Metadata) error { 25 | dat, err := json.Marshal(m) 26 | if err != nil { 27 | return client.NewError("failed to encode metadata", err) 28 | } 29 | err = c.Upload(ctx, bucket, path.Join(root, MetadataObjectKey), bytes.NewReader(dat)) 30 | if err != nil { 31 | return client.NewError("failed to upload index file", err) 32 | } 33 | return nil 34 | } 35 | 36 | //MetadataDownload downloads a dataset's metadata. 37 | func (c *Client) MetadataDownload(ctx context.Context, bucket, root string) (*v1payload.Metadata, error) { 38 | r, err := c.Download(ctx, bucket, path.Join(root, MetadataObjectKey)) 39 | if err != nil { 40 | return nil, client.NewError("failed to download metadata", err) 41 | } 42 | defer r.Close() 43 | dec := json.NewDecoder(r) 44 | metadata := &v1payload.Metadata{} 45 | err = dec.Decode(metadata) 46 | if err != nil { 47 | return nil, client.NewError("failed to decode metadata", err) 48 | } 49 | return metadata, nil 50 | } 51 | -------------------------------------------------------------------------------- /crd/pkg/client/informers/externalversions/stable.nerdalize.com/v1/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file was automatically generated by informer-gen 18 | 19 | package v1 20 | 21 | import ( 22 | internalinterfaces "github.com/nerdalize/nerd/crd/pkg/client/informers/externalversions/internalinterfaces" 23 | ) 24 | 25 | // Interface provides access to all the informers in this group version. 26 | type Interface interface { 27 | // Datasets returns a DatasetInformer. 28 | Datasets() DatasetInformer 29 | } 30 | 31 | type version struct { 32 | factory internalinterfaces.SharedInformerFactory 33 | namespace string 34 | tweakListOptions internalinterfaces.TweakListOptionsFunc 35 | } 36 | 37 | // New returns a new Interface. 38 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 39 | return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 40 | } 41 | 42 | // Datasets returns a DatasetInformer. 43 | func (v *version) Datasets() DatasetInformer { 44 | return &datasetInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} 45 | } 46 | -------------------------------------------------------------------------------- /svc/kube_fetch_job_logs.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "sort" 7 | 8 | "github.com/nerdalize/nerd/pkg/kubevisor" 9 | batchv1 "k8s.io/api/batch/v1" 10 | ) 11 | 12 | //FetchJobLogsInput is the input to FetchJobLogs 13 | type FetchJobLogsInput struct { 14 | Tail int64 `validate:"min=0"` 15 | Name string `validate:"min=1,printascii"` 16 | } 17 | 18 | //FetchJobLogsOutput is the output to FetchJobLogs 19 | type FetchJobLogsOutput struct { 20 | Data []byte 21 | } 22 | 23 | //FetchJobLogs will create a job on kubernetes 24 | func (k *Kube) FetchJobLogs(ctx context.Context, in *FetchJobLogsInput) (out *FetchJobLogsOutput, err error) { 25 | if err = k.checkInput(ctx, in); err != nil { 26 | return nil, err 27 | } 28 | 29 | job := &batchv1.Job{} 30 | err = k.visor.GetResource(ctx, kubevisor.ResourceTypeJobs, job, in.Name) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | pods := &pods{} 36 | err = k.visor.ListResources(ctx, kubevisor.ResourceTypePods, pods, []string{"controller-uid=" + string(job.GetUID())}, []string{}) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | if len(pods.Items) < 1 { 42 | return &FetchJobLogsOutput{}, nil 43 | } 44 | 45 | //sort by latest created 46 | sort.Slice(pods.Items, func(i int, j int) bool { 47 | return pods.Items[i].CreationTimestamp.UnixNano() > pods.Items[j].CreationTimestamp.UnixNano() 48 | }) 49 | 50 | //loop over the pods, return output from the first pod that returns logs, at most 3 times 51 | buf := bytes.NewBuffer(nil) 52 | for i := 0; i < len(pods.Items) && i < 3; i++ { 53 | pod := pods.Items[i] 54 | _ = k.visor.FetchLogs(ctx, in.Tail, buf, "main", pod.Name) 55 | if buf.Len() > 0 { 56 | break 57 | } 58 | } 59 | 60 | return &FetchJobLogsOutput{Data: buf.Bytes()}, nil 61 | } 62 | -------------------------------------------------------------------------------- /crd/pkg/client/informers/externalversions/stable.nerdalize.com/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file was automatically generated by informer-gen 18 | 19 | package nerdalize 20 | 21 | import ( 22 | internalinterfaces "github.com/nerdalize/nerd/crd/pkg/client/informers/externalversions/internalinterfaces" 23 | v1 "github.com/nerdalize/nerd/crd/pkg/client/informers/externalversions/stable.nerdalize.com/v1" 24 | ) 25 | 26 | // Interface provides access to each of this group's versions. 27 | type Interface interface { 28 | // V1 provides access to shared informers for resources in V1. 29 | V1() v1.Interface 30 | } 31 | 32 | type group struct { 33 | factory internalinterfaces.SharedInformerFactory 34 | namespace string 35 | tweakListOptions internalinterfaces.TweakListOptionsFunc 36 | } 37 | 38 | // New returns a new Interface. 39 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 40 | return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 41 | } 42 | 43 | // V1 returns a new v1.Interface. 44 | func (g *group) V1() v1.Interface { 45 | return v1.New(g.factory, g.namespace, g.tweakListOptions) 46 | } 47 | -------------------------------------------------------------------------------- /crd/pkg/signals/BUILD: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "signal.go", 7 | ] + select({ 8 | "@io_bazel_rules_go//go/platform:android": [ 9 | "signal_posix.go", 10 | ], 11 | "@io_bazel_rules_go//go/platform:darwin": [ 12 | "signal_posix.go", 13 | ], 14 | "@io_bazel_rules_go//go/platform:dragonfly": [ 15 | "signal_posix.go", 16 | ], 17 | "@io_bazel_rules_go//go/platform:freebsd": [ 18 | "signal_posix.go", 19 | ], 20 | "@io_bazel_rules_go//go/platform:linux": [ 21 | "signal_posix.go", 22 | ], 23 | "@io_bazel_rules_go//go/platform:nacl": [ 24 | "signal_posix.go", 25 | ], 26 | "@io_bazel_rules_go//go/platform:netbsd": [ 27 | "signal_posix.go", 28 | ], 29 | "@io_bazel_rules_go//go/platform:openbsd": [ 30 | "signal_posix.go", 31 | ], 32 | "@io_bazel_rules_go//go/platform:plan9": [ 33 | "signal_posix.go", 34 | ], 35 | "@io_bazel_rules_go//go/platform:solaris": [ 36 | "signal_posix.go", 37 | ], 38 | "@io_bazel_rules_go//go/platform:windows": [ 39 | "signal_windows.go", 40 | ], 41 | "//conditions:default": [], 42 | }), 43 | importpath = "k8s.io/sample-controller/pkg/signals", 44 | visibility = ["//visibility:public"], 45 | ) 46 | 47 | filegroup( 48 | name = "package-srcs", 49 | srcs = glob(["**"]), 50 | tags = ["automanaged"], 51 | visibility = ["//visibility:private"], 52 | ) 53 | 54 | filegroup( 55 | name = "all-srcs", 56 | srcs = [":package-srcs"], 57 | tags = ["automanaged"], 58 | visibility = ["//visibility:public"], 59 | ) 60 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/dataset.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | import "time" 4 | 5 | const ( 6 | //DatasetUploadStatusCreated is the created upload status 7 | DatasetUploadStatusCreated = "CREATED" 8 | //DatasetUploadStatusUploading is the uploading upload status 9 | DatasetUploadStatusUploading = "UPLOADING" 10 | //DatasetUploadStatusSuccess is the success upload status 11 | DatasetUploadStatusSuccess = "SUCCESS" 12 | ) 13 | 14 | //CreateDatasetInput is used as input to dataset creation 15 | type CreateDatasetInput struct { 16 | ProjectID string `json:"project_id" valid:"required"` 17 | } 18 | 19 | //CreateDatasetOutput is returned from creating a dataset 20 | type CreateDatasetOutput struct { 21 | DatasetSummary 22 | HeartbeatInterval time.Duration `json:"heartbeat_interval"` 23 | } 24 | 25 | //DescribeDatasetInput is input for dataset creation 26 | type DescribeDatasetInput struct { 27 | ProjectID string `json:"project_id" valid:"required"` 28 | DatasetID string `json:"dataset_id" valid:"required"` 29 | } 30 | 31 | //DescribeDatasetOutput is output for dataset creation 32 | type DescribeDatasetOutput struct { 33 | DatasetSummary 34 | } 35 | 36 | //ListDatasetsInput is input for dataset creation 37 | type ListDatasetsInput struct { 38 | ProjectID string `json:"project_id" valid:"required"` 39 | } 40 | 41 | //DatasetSummary is a small version of 42 | type DatasetSummary struct { 43 | ProjectID string `json:"project_id"` 44 | DatasetID string `json:"dataset_id"` 45 | Bucket string `json:"bucket"` 46 | DatasetRoot string `json:"dataset_root"` 47 | ProjectRoot string `json:"project_root"` 48 | UploadExpire int64 `json:"upload_expire"` 49 | UploadStatus string `json:"upload_status"` 50 | CreatedAt int64 `json:"created_at"` 51 | } 52 | 53 | //ListDatasetsOutput is output for dataset creation 54 | type ListDatasetsOutput struct { 55 | Datasets []*DatasetSummary 56 | } 57 | -------------------------------------------------------------------------------- /svc/kube_update_dataset_test.go: -------------------------------------------------------------------------------- 1 | package svc_test 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "testing" 7 | "time" 8 | 9 | "github.com/nerdalize/nerd/pkg/transfer/archiver" 10 | "github.com/nerdalize/nerd/pkg/transfer/store" 11 | "github.com/nerdalize/nerd/svc" 12 | ) 13 | 14 | func TestUpdateDataset(t *testing.T) { 15 | di, clean := testDI(t) 16 | defer clean() 17 | 18 | ctx := context.Background() 19 | ctx, cancel := context.WithTimeout(ctx, time.Second*5) 20 | defer cancel() 21 | 22 | kube := svc.NewKube(di) 23 | out, err := kube.CreateDataset(ctx, &svc.CreateDatasetInput{ 24 | Name: "my-dataset", 25 | 26 | StoreOptions: transferstore.StoreOptions{Type: transferstore.StoreTypeS3}, ArchiverOptions: transferarchiver.ArchiverOptions{Type: transferarchiver.ArchiverTypeTar}, 27 | }) 28 | ok(t, err) 29 | 30 | newSize := uint64(1337) 31 | _, err = kube.UpdateDataset(ctx, &svc.UpdateDatasetInput{ 32 | Name: out.Name, 33 | InputFor: "j-123abc", 34 | OutputFrom: "j-456def", 35 | Size: &newSize, 36 | }) 37 | ok(t, err) 38 | 39 | o, err := kube.GetDataset(ctx, &svc.GetDatasetInput{Name: out.Name}) 40 | ok(t, err) 41 | assert(t, strings.Contains(strings.Join(o.InputFor, ""), "j-123abc"), "expected dataset to be up to date") 42 | assert(t, strings.Contains(strings.Join(o.OutputFrom, ""), "j-456def"), "expected dataset to be up to date and to contain job info for output section") 43 | assert(t, o.Size == 1337, "expected dataset to be up to date and contain new size") 44 | 45 | //Check if the output remains the same when not specifying any changes 46 | _, err = kube.UpdateDataset(ctx, &svc.UpdateDatasetInput{ 47 | Name: out.Name, 48 | }) 49 | ok(t, err) 50 | 51 | o2, err := kube.GetDataset(ctx, &svc.GetDatasetInput{Name: out.Name}) 52 | ok(t, err) 53 | equals(t, o.Size, o2.Size) 54 | equals(t, o.InputFor, o2.InputFor) 55 | equals(t, o.OutputFrom, o2.OutputFrom) 56 | } 57 | -------------------------------------------------------------------------------- /svc/kube_create_secret_test.go: -------------------------------------------------------------------------------- 1 | package svc_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | "runtime" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | "github.com/nerdalize/nerd/svc" 13 | ) 14 | 15 | func TestCreateSecret(t *testing.T) { 16 | for _, c := range []struct { 17 | Name string 18 | Timeout time.Duration 19 | Input *svc.CreateSecretInput 20 | IsOutput func(tb testing.TB, out *svc.CreateSecretOutput) 21 | IsErr func(error) bool 22 | }{ 23 | { 24 | Name: "when a zero value input is provided it should return a validation error", 25 | Timeout: time.Second * 5, 26 | Input: nil, 27 | IsErr: svc.IsValidationErr, 28 | IsOutput: func(t testing.TB, out *svc.CreateSecretOutput) { 29 | assert(t, out == nil, "output should be nil") 30 | }, 31 | }, 32 | { 33 | Name: "when a valid input is provided it should return a secret with a unique name", 34 | Timeout: time.Second * 5, 35 | Input: &svc.CreateSecretInput{Image: "smoketest", Project: "nerdalize", Registry: "quay.io", Username: "test", Password: "test"}, 36 | IsErr: nil, 37 | IsOutput: func(t testing.TB, out *svc.CreateSecretOutput) { 38 | assert(t, out != nil, "output should not be nil") 39 | assert(t, strings.Contains(out.Name, "s-"), "secret name should be generated and prefixed") 40 | }, 41 | }, 42 | } { 43 | t.Run(c.Name, func(t *testing.T) { 44 | di, clean := testDI(t) 45 | defer clean() 46 | 47 | ctx := context.Background() 48 | ctx, cancel := context.WithTimeout(ctx, c.Timeout) 49 | defer cancel() 50 | 51 | kube := svc.NewKube(di) 52 | out, err := kube.CreateSecret(ctx, c.Input) 53 | if c.IsErr != nil { 54 | assert(t, c.IsErr(err), fmt.Sprintf("unexpected '%#v' to match: %#v", err, runtime.FuncForPC(reflect.ValueOf(c.IsErr).Pointer()).Name())) 55 | } 56 | 57 | if c.IsOutput != nil { 58 | c.IsOutput(t, out) 59 | } 60 | }) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/nerdalize/nerd/cmd" 8 | "github.com/nerdalize/nerd/nerd" 9 | 10 | "github.com/mitchellh/cli" 11 | ) 12 | 13 | var ( 14 | name = "nerd" 15 | version = nerd.BuiltFromSourceVersion 16 | commit = "0000000" 17 | ) 18 | 19 | func create() *cli.CLI { 20 | args := os.Args[1:] 21 | for _, arg := range args { 22 | if arg == "-V" || arg == "-version" || arg == "--version" { 23 | newArgs := make([]string, len(args)+1) 24 | newArgs[0] = "version" 25 | copy(newArgs[1:], args) 26 | args = newArgs //overwrite args to use the version subcommand 27 | break 28 | } 29 | } 30 | 31 | ui := &cli.BasicUi{ 32 | Reader: os.Stdin, 33 | Writer: os.Stdout, 34 | ErrorWriter: os.Stderr, 35 | } 36 | 37 | c := &cli.CLI{ 38 | Name: name, 39 | Args: args, 40 | HiddenCommands: []string{}, 41 | Commands: map[string]cli.CommandFactory{ 42 | "version": cmd.VersionFactory(version, commit, ui), 43 | "login": cmd.LoginFactory(ui), 44 | "dataset": cmd.DatasetFactory(ui), 45 | "dataset upload": cmd.DatasetUploadFactory(ui), 46 | "dataset download": cmd.DatasetDownloadFactory(ui), 47 | "dataset list": cmd.DatasetListFactory(ui), 48 | "dataset delete": cmd.DatasetDeleteFactory(ui), 49 | "job": cmd.JobFactory(ui), 50 | "job run": cmd.JobRunFactory(ui), 51 | "job list": cmd.JobListFactory(ui), 52 | "job logs": cmd.JobLogsFactory(ui), 53 | "job delete": cmd.JobDeleteFactory(ui), 54 | "cluster": cmd.ClusterFactory(ui), 55 | "cluster list": cmd.ClusterListFactory(ui), 56 | "cluster use": cmd.ClusterUseFactory(ui), 57 | }, 58 | } 59 | 60 | return c 61 | } 62 | 63 | func main() { 64 | status, err := create().Run() 65 | if err != nil { 66 | fmt.Fprintf(os.Stderr, "%s: %s", name, err) 67 | } 68 | 69 | os.Exit(status) 70 | } 71 | -------------------------------------------------------------------------------- /crd/README.md: -------------------------------------------------------------------------------- 1 | # Custom Dataset controller 2 | 3 | This controller should be deployed on each cluster. 4 | It is used to delete S3 object when a Kubernetes Dataset is being deleted. 5 | 6 | ## Some resources regarding controllers 7 | 8 | - [Writing custom Kubernetes controllers](https://medium.com/@cloudark/kubernetes-custom-controllers-b6c7d0668fdf), it's a must read and there is a very good representation of how a custom controller works. 9 | 10 | - [Kubernetes sample controller](https://github.com/kubernetes/sample-controller) 11 | 12 | - [Kubewatch, an example of Kubernetes Custom Controller](https://engineering.bitnami.com/articles/kubewatch-an-example-of-kubernetes-custom-controller.html) 13 | 14 | ## How it works 15 | 16 | ![interaction between a custom controller and client-go](https://cdn-images-1.medium.com/max/800/1*dmvNSeSIORAMaTF2WdE9fg.jpeg) 17 | *Illustration from [Writing custom Kubernetes controllers](https://medium.com/@cloudark/kubernetes-custom-controllers-b6c7d0668fdf)* 18 | 19 | - The controller creates an Informer and an Indexer to list, watch and index a Kubernetes object, for our controller it's all about datasets 20 | 21 | - When a new object is being deleted, it calls an event handler to delete the s3 object. 22 | 23 | ## Running it locally 24 | 25 | ```bash 26 | $ go run *.go -kubeconfig= -alsologtostderr -v 4 27 | ``` 28 | 29 | ## Building the docker image 30 | 31 | ```bash 32 | $ CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o controller . 33 | $ docker build -t nerdalize/custom-dataset-controller: -f Dockerfile . 34 | $ docker push nerdalize/custom-dataset-controller 35 | ``` 36 | 37 | ## Deploying the controller on Kubernetes 38 | 39 | Update the docker image tag in the [deployment file](https://github.com/nerdalize/nerd/blob/master/crd/deployment.yml) and then apply it using kubectl: 40 | 41 | ```bash 42 | $ kubectl apply -f deployment.yml 43 | ``` 44 | 45 | The deployment is always done in the `kube-system` namespace. -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/fake/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package fake 17 | 18 | import ( 19 | nerdalizev1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 20 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | runtime "k8s.io/apimachinery/pkg/runtime" 22 | schema "k8s.io/apimachinery/pkg/runtime/schema" 23 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 24 | ) 25 | 26 | var scheme = runtime.NewScheme() 27 | var codecs = serializer.NewCodecFactory(scheme) 28 | var parameterCodec = runtime.NewParameterCodec(scheme) 29 | 30 | func init() { 31 | v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) 32 | AddToScheme(scheme) 33 | } 34 | 35 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 36 | // of clientsets, like in: 37 | // 38 | // import ( 39 | // "k8s.io/client-go/kubernetes" 40 | // clientsetscheme "k8s.io/client-go/kuberentes/scheme" 41 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 42 | // ) 43 | // 44 | // kclientset, _ := kubernetes.NewForConfig(c) 45 | // aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 46 | // 47 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 48 | // correctly. 49 | func AddToScheme(scheme *runtime.Scheme) { 50 | nerdalizev1.AddToScheme(scheme) 51 | 52 | } 53 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/scheme/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package scheme 17 | 18 | import ( 19 | nerdalizev1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 20 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | runtime "k8s.io/apimachinery/pkg/runtime" 22 | schema "k8s.io/apimachinery/pkg/runtime/schema" 23 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 24 | ) 25 | 26 | var Scheme = runtime.NewScheme() 27 | var Codecs = serializer.NewCodecFactory(Scheme) 28 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 29 | 30 | func init() { 31 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 32 | AddToScheme(Scheme) 33 | } 34 | 35 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 36 | // of clientsets, like in: 37 | // 38 | // import ( 39 | // "k8s.io/client-go/kubernetes" 40 | // clientsetscheme "k8s.io/client-go/kuberentes/scheme" 41 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 42 | // ) 43 | // 44 | // kclientset, _ := kubernetes.NewForConfig(c) 45 | // aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 46 | // 47 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 48 | // correctly. 49 | func AddToScheme(scheme *runtime.Scheme) { 50 | nerdalizev1.AddToScheme(scheme) 51 | 52 | } 53 | -------------------------------------------------------------------------------- /pkg/populator/env.go: -------------------------------------------------------------------------------- 1 | package populator 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "sync/atomic" 7 | 8 | "github.com/pkg/errors" 9 | "k8s.io/client-go/tools/clientcmd/api" 10 | ) 11 | 12 | //EnvPopulator is a populator implementation based on env variables. 13 | type EnvPopulator struct { 14 | // kubeConfigFile is the path where the kube config is stored 15 | // Only access this with atomic ops 16 | kubeConfigFile atomic.Value 17 | } 18 | 19 | func newEnv(kubeConfigFile string) *EnvPopulator { 20 | e := &EnvPopulator{} 21 | e.kubeConfigFile.Store(kubeConfigFile) 22 | return e 23 | } 24 | 25 | //GetKubeConfigFile returns the path where the kube config is stored. 26 | func (e *EnvPopulator) GetKubeConfigFile() string { 27 | return e.kubeConfigFile.Load().(string) 28 | } 29 | 30 | //RemoveConfig deletes the precised project context and cluster info. 31 | func (e *EnvPopulator) RemoveConfig(project string) error { 32 | return nil 33 | } 34 | 35 | // PopulateKubeConfig populates the kube config file with the info found in the environment. 36 | func (e *EnvPopulator) PopulateKubeConfig(project string) error { 37 | cluster := api.NewCluster() 38 | cluster.Server = os.Getenv("KUBE_CLUSTER_ADDR") 39 | 40 | // user 41 | user := api.NewAuthInfo() 42 | user.Token = os.Getenv("KUBE_TOKEN") 43 | 44 | // context 45 | context := api.NewContext() 46 | context.Cluster = project 47 | context.AuthInfo = project 48 | context.Namespace = os.Getenv("KUBE_NAMESPACE") 49 | clusterName := fmt.Sprintf("%s-%s", Prefix, project) 50 | 51 | // read existing config or create new if does not exist 52 | kubecfg, err := ReadConfigOrNew(e.GetKubeConfigFile()) 53 | if err != nil { 54 | return err 55 | } 56 | kubecfg.Clusters[project] = cluster 57 | kubecfg.AuthInfos[project] = user 58 | kubecfg.CurrentContext = clusterName 59 | kubecfg.Contexts[clusterName] = context 60 | 61 | // write back to disk 62 | if err := WriteConfig(kubecfg, e.GetKubeConfigFile()); err != nil { 63 | return errors.Wrap(err, "writing kubeconfig") 64 | } 65 | 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/run.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //An Run acts as an reference to a task instance 4 | type Run struct { 5 | ProjectID string `json:"project_id"` 6 | WorkloadID string `json:"workload_id" valid:"required"` 7 | TaskID int64 `json:"task_id"` 8 | Token string `json:"token"` 9 | 10 | Cmd []string `json:"cmd"` 11 | Env map[string]string `json:"env"` 12 | Stdin []byte `json:"stdin"` 13 | } 14 | 15 | //SendRunHeartbeatInput is input for run creation 16 | type SendRunHeartbeatInput struct { 17 | ProjectID string `json:"project_id" valid:"required"` 18 | WorkloadID string `json:"workload_id" valid:"required"` 19 | TaskID int64 `json:"task_id" valid:"required"` 20 | RunToken string `json:"run_token" valid:"required"` 21 | } 22 | 23 | //SendRunHeartbeatOutput is output for run creation 24 | type SendRunHeartbeatOutput struct { 25 | HasExpired bool `json:"has_expired"` 26 | } 27 | 28 | //SendRunSuccessInput is input for marking a run as failed 29 | type SendRunSuccessInput struct { 30 | ProjectID string `json:"project_id" valid:"required"` 31 | WorkloadID string `json:"workload_id" valid:"required"` 32 | TaskID int64 `json:"task_id" valid:"required"` 33 | RunToken string `json:"run_token" valid:"required"` 34 | 35 | Result string `json:"result"` 36 | OutputDatasetID string `json:"output_dataset_id"` 37 | } 38 | 39 | //SendRunSuccessOutput is output from marking a run as failed 40 | type SendRunSuccessOutput struct{} 41 | 42 | //SendRunFailureInput is input for marking a run as failed 43 | type SendRunFailureInput struct { 44 | ProjectID string `json:"project_id" valid:"required"` 45 | WorkloadID string `json:"workload_id" valid:"required"` 46 | TaskID int64 `json:"task_id" valid:"required"` 47 | RunToken string `json:"run_token" valid:"required"` 48 | 49 | ErrorCode string `json:"error_code"` 50 | ErrorMessage string `json:"error_message"` 51 | } 52 | 53 | //SendRunFailureOutput is output from marking a run as failed 54 | type SendRunFailureOutput struct{} 55 | -------------------------------------------------------------------------------- /nerd/aws/credentials_provider.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/aws/aws-sdk-go/aws/credentials" 7 | v1batch "github.com/nerdalize/nerd/nerd/client/batch/v1" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | // ProviderName is the name of the credentials provider. 12 | const ProviderName = `NerdalizeProvider` 13 | 14 | //DefaultExpireWindow is the default amount of seconds that the credentials are assumed to be expired, before they are actually expired. 15 | //This will prevent the server from rejecting the credentials because they were just expired. 16 | const DefaultExpireWindow = 20 17 | 18 | // Provider satisfies the credentials.Provider interface, and is a client to 19 | // retrieve credentials from the nerdalize api. 20 | type Provider struct { 21 | credentials.Expiry 22 | ExpiryWindow time.Duration 23 | Client v1batch.ClientTokenInterface 24 | NlzProjectID string 25 | } 26 | 27 | //NewNerdalizeCredentials creates a new credentials object with the NerdalizeProvider as provider. 28 | func NewNerdalizeCredentials(c v1batch.ClientTokenInterface, nlzProjectID string) *credentials.Credentials { 29 | return credentials.NewCredentials(&Provider{ 30 | Client: c, 31 | NlzProjectID: nlzProjectID, 32 | ExpiryWindow: DefaultExpireWindow, 33 | }) 34 | } 35 | 36 | //IsExpired checks if the AWS sessions is expired. 37 | func (p *Provider) IsExpired() bool { 38 | return p.Expiry.IsExpired() 39 | } 40 | 41 | // Retrieve will attempt to request the credentials from the nerdalize api. 42 | // And error will be returned if the retrieval fails. 43 | func (p *Provider) Retrieve() (credentials.Value, error) { 44 | token, err := p.Client.CreateToken(p.NlzProjectID) 45 | if err != nil { 46 | return credentials.Value{ProviderName: ProviderName}, errors.Wrap(err, "failed to get AWS credentials") 47 | } 48 | 49 | p.SetExpiration(token.AWSExpiration, p.ExpiryWindow) 50 | 51 | return credentials.Value{ 52 | AccessKeyID: token.AWSAccessKeyID, 53 | SecretAccessKey: token.AWSSecretAccessKey, 54 | SessionToken: token.AWSSessionToken, 55 | ProviderName: ProviderName, 56 | }, nil 57 | } 58 | -------------------------------------------------------------------------------- /pkg/populator/endpoint.go: -------------------------------------------------------------------------------- 1 | package populator 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "sync/atomic" 7 | 8 | "github.com/pkg/errors" 9 | "k8s.io/client-go/tools/clientcmd/api" 10 | ) 11 | 12 | //EndpointPopulator is an implementation of the P interface based on the retrieval of a conf file. 13 | type EndpointPopulator struct { 14 | // kubeConfigFile is the path where the kube config is stored 15 | // Only access this with atomic ops 16 | kubeConfigFile atomic.Value 17 | } 18 | 19 | func newEndpoint(kubeConfigFile string) *EndpointPopulator { 20 | e := &EndpointPopulator{} 21 | e.kubeConfigFile.Store(kubeConfigFile) 22 | return e 23 | } 24 | 25 | //GetKubeConfigFile returns the path where the kube config is stored 26 | func (e *EndpointPopulator) GetKubeConfigFile() string { 27 | return e.kubeConfigFile.Load().(string) 28 | } 29 | 30 | //RemoveConfig deletes the precised project context and cluster info. 31 | func (e *EndpointPopulator) RemoveConfig(project string) error { 32 | return nil 33 | } 34 | 35 | // PopulateKubeConfig populates an api.Config object and set the current context to the provided project. 36 | func (e *EndpointPopulator) PopulateKubeConfig(project string) error { 37 | cluster := api.NewCluster() 38 | cluster.Server = os.Getenv("KUBE_CLUSTER_ADDR") 39 | 40 | // user 41 | user := api.NewAuthInfo() 42 | user.Username = project 43 | user.Token = os.Getenv("KUBE_TOKEN") 44 | 45 | // context 46 | context := api.NewContext() 47 | context.Cluster = project 48 | context.AuthInfo = project 49 | context.Namespace = os.Getenv("KUBE_NAMESPACE") 50 | clusterName := fmt.Sprintf("%s-%s", Prefix, project) 51 | 52 | // read existing config or create new if does not exist 53 | kubecfg, err := ReadConfigOrNew(e.GetKubeConfigFile()) 54 | if err != nil { 55 | return err 56 | } 57 | kubecfg.Clusters[project] = cluster 58 | kubecfg.AuthInfos[project] = user 59 | kubecfg.CurrentContext = clusterName 60 | kubecfg.Contexts[clusterName] = context 61 | 62 | // write back to disk 63 | if err := WriteConfig(kubecfg, e.GetKubeConfigFile()); err != nil { 64 | return errors.Wrap(err, "writing kubeconfig") 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/upload.go: -------------------------------------------------------------------------------- 1 | package v1datatransfer 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | v1batch "github.com/nerdalize/nerd/nerd/client/batch/v1" 8 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 9 | v1data "github.com/nerdalize/nerd/nerd/service/datatransfer/v1/client" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | //UploadConfig is the config for Upload operations 14 | type UploadConfig struct { 15 | BatchClient *v1batch.Client 16 | DataOps v1data.DataOps 17 | LocalDir string 18 | ProjectID string 19 | Concurrency int 20 | ProgressCh chan<- int64 21 | } 22 | 23 | //Upload uploads a dataset 24 | func Upload(ctx context.Context, conf UploadConfig) (*v1payload.DatasetSummary, error) { 25 | if conf.ProgressCh != nil { 26 | defer close(conf.ProgressCh) 27 | } 28 | ds, err := conf.BatchClient.CreateDataset(conf.ProjectID) 29 | if err != nil { 30 | return nil, errors.Wrap(err, "failed to create dataset") 31 | } 32 | dataClient := v1data.NewClient(conf.DataOps) 33 | up := &uploadProcess{ 34 | batchClient: conf.BatchClient, 35 | dataClient: dataClient, 36 | dataset: ds.DatasetSummary, 37 | heartbeatInterval: ds.HeartbeatInterval, 38 | localDir: conf.LocalDir, 39 | concurrency: conf.Concurrency, 40 | progressCh: conf.ProgressCh, 41 | } 42 | return &ds.DatasetSummary, up.start(ctx) 43 | } 44 | 45 | //GetLocalDatasetSize calculates the total size in bytes of the archived version of a directory on disk 46 | func GetLocalDatasetSize(ctx context.Context, dataPath string) (int64, error) { 47 | type countResult struct { 48 | total int64 49 | err error 50 | } 51 | doneCh := make(chan countResult) 52 | pr, pw := io.Pipe() 53 | go func() { 54 | total, err := countBytes(ctx, pr) 55 | doneCh <- countResult{total, err} 56 | }() 57 | 58 | err := tardir(ctx, dataPath, pw) 59 | if err != nil { 60 | return 0, errors.Wrapf(err, "failed to tar '%s'", dataPath) 61 | } 62 | 63 | pw.Close() 64 | cr := <-doneCh 65 | if cr.err != nil { 66 | return 0, errors.Wrapf(err, "failed to count total disk size of '%v'", dataPath) 67 | } 68 | return cr.total, nil 69 | } 70 | -------------------------------------------------------------------------------- /nerd/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/http/httputil" 8 | 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | //Error is the default error returned by clients in the client package. 13 | //Error is compatible with the pkg/errors package. 14 | type Error struct { 15 | Msg string 16 | Underlying error 17 | } 18 | 19 | //NewError creates a new Error 20 | func NewError(msg string, underlying error) *Error { 21 | return &Error{ 22 | Msg: msg, 23 | Underlying: underlying, 24 | } 25 | } 26 | 27 | //Error returns the error message. 28 | func (e Error) Error() string { 29 | if e.Underlying != nil { 30 | return e.Msg + ": " + e.Underlying.Error() 31 | } 32 | return e.Msg 33 | } 34 | 35 | //Cause points to the underlying error. 36 | func (e Error) Cause() error { 37 | return e.Underlying 38 | } 39 | 40 | //Format implements different error formats. 41 | func (e Error) Format(s fmt.State, verb rune) { 42 | switch verb { 43 | case 'v': 44 | if s.Flag('+') { 45 | if e.Underlying != nil { 46 | fmt.Fprintf(s, "%+v\n", e.Underlying) 47 | } 48 | io.WriteString(s, e.Msg) 49 | return 50 | } 51 | fallthrough 52 | case 's', 'q': 53 | io.WriteString(s, e.Error()) 54 | } 55 | } 56 | 57 | //LogRequest is a util to log an HTTP request. 58 | func LogRequest(req *http.Request, logger *log.Logger) { 59 | txt, err := httputil.DumpRequest(req, true) 60 | // retry without printing the body 61 | if err != nil { 62 | txt, err = httputil.DumpRequest(req, false) 63 | } 64 | if err == nil { 65 | logger.Printf("[DEBUG] HTTP Request:\n%s\n", txt) 66 | } else { 67 | logger.Printf("[DEBUG] Failed to log HTTP request '%v'", err) 68 | } 69 | } 70 | 71 | //LogResponse is a util to log an HTTP response. 72 | func LogResponse(res *http.Response, logger *log.Logger) { 73 | txt, err := httputil.DumpResponse(res, true) 74 | // retry without printing the body 75 | if err != nil { 76 | txt, err = httputil.DumpResponse(res, false) 77 | } 78 | if err == nil { 79 | logger.Printf("[DEBUG] HTTP Response:\n%s\n", txt) 80 | } else { 81 | logger.Printf("[DEBUG] Failed to log HTTP response '%v'", err) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /nerd/jwt/config_provider.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | 6 | v1auth "github.com/nerdalize/nerd/nerd/client/auth/v1" 7 | "github.com/nerdalize/nerd/nerd/conf" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | //ConfigProvider provides a JWT from the config file. For the default file location please see TokenFilename(). 12 | type ConfigProvider struct { 13 | *ProviderBasis 14 | Session conf.SessionInterface 15 | Client v1auth.TokenClientInterface 16 | } 17 | 18 | //NewConfigProvider creates a new ConfigProvider provider. 19 | func NewConfigProvider(pub *ecdsa.PublicKey, session conf.SessionInterface, client v1auth.TokenClientInterface) *ConfigProvider { 20 | return &ConfigProvider{ 21 | ProviderBasis: &ProviderBasis{ 22 | ExpireWindow: DefaultExpireWindow, 23 | Pub: pub, 24 | }, 25 | Session: session, 26 | Client: client, 27 | } 28 | } 29 | 30 | //Retrieve retrieves the token from the nerd config file. 31 | func (e *ConfigProvider) Retrieve() (string, error) { 32 | ss, err := e.Session.Read() 33 | if err != nil { 34 | return "", errors.Wrap(err, "failed to read config") 35 | } 36 | jwt := ss.JWT.Token 37 | if jwt == "" { 38 | return "", errors.New(".jwt.token is not set in config") 39 | } 40 | err = e.SetExpirationFromJWT(jwt) 41 | if err != nil { 42 | return "", errors.Wrap(err, "failed to set expiration") 43 | } 44 | if ss.JWT.RefreshToken != "" && e.IsExpired() { 45 | jwt, err = e.refresh(jwt, ss.JWT.RefreshToken, ss.Project.Name) 46 | if err != nil { 47 | return "", errors.Wrap(err, "failed to refresh") 48 | } 49 | } 50 | err = isValid(jwt, e.Pub) 51 | if err != nil { 52 | return "", err 53 | } 54 | return jwt, nil 55 | } 56 | 57 | func (e *ConfigProvider) refresh(jwt, secret, projectID string) (string, error) { 58 | out, err := e.Client.RefreshJWT(projectID, jwt, secret) 59 | if err != nil { 60 | return "", errors.Wrap(err, "failed to refresh token") 61 | } 62 | err = e.SetExpirationFromJWT(out.Token) 63 | if err != nil { 64 | return "", errors.Wrap(err, "failed to set expiration") 65 | } 66 | err = e.Session.WriteJWT(out.Token, secret) 67 | if err != nil { 68 | return "", errors.Wrap(err, "failed to write jwt and secret to config") 69 | } 70 | return out.Token, nil 71 | } 72 | -------------------------------------------------------------------------------- /crd/pkg/client/informers/externalversions/generic.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file was automatically generated by informer-gen 18 | 19 | package externalversions 20 | 21 | import ( 22 | "fmt" 23 | 24 | v1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | cache "k8s.io/client-go/tools/cache" 27 | ) 28 | 29 | // GenericInformer is type of SharedIndexInformer which will locate and delegate to other 30 | // sharedInformers based on type 31 | type GenericInformer interface { 32 | Informer() cache.SharedIndexInformer 33 | Lister() cache.GenericLister 34 | } 35 | 36 | type genericInformer struct { 37 | informer cache.SharedIndexInformer 38 | resource schema.GroupResource 39 | } 40 | 41 | // Informer returns the SharedIndexInformer. 42 | func (f *genericInformer) Informer() cache.SharedIndexInformer { 43 | return f.informer 44 | } 45 | 46 | // Lister returns the GenericLister. 47 | func (f *genericInformer) Lister() cache.GenericLister { 48 | return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) 49 | } 50 | 51 | // ForResource gives generic access to a shared informer of the matching type 52 | // TODO extend this to unknown resources with a client pool 53 | func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { 54 | switch resource { 55 | // Group=nerdalize.com, Version=v1 56 | case v1.SchemeGroupVersion.WithResource("datasets"): 57 | return &genericInformer{resource: resource.GroupResource(), informer: f.Nerdalize().V1().Datasets().Informer()}, nil 58 | 59 | } 60 | 61 | return nil, fmt.Errorf("no informer found for %v", resource) 62 | } 63 | -------------------------------------------------------------------------------- /nerd/aws/queue_client.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/credentials" 8 | "github.com/aws/aws-sdk-go/aws/session" 9 | "github.com/aws/aws-sdk-go/service/sqs" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | //QueueClient is a client to AWS' SQS queueing service. The client implements the v1batch.QueueOps interface. 14 | type QueueClient struct { 15 | Service *sqs.SQS 16 | } 17 | 18 | //NewQueueClient creates a new QueueClient 19 | func NewQueueClient(c *credentials.Credentials, region string) (*QueueClient, error) { 20 | sess, err := session.NewSession(&aws.Config{ 21 | Credentials: c, 22 | Region: aws.String(region), 23 | }) 24 | if err != nil { 25 | return nil, errors.Wrap(err, "could not create AWS sessions") 26 | } 27 | return &QueueClient{ 28 | Service: sqs.New(sess), 29 | }, nil 30 | } 31 | 32 | //ReceiveMessages receives messages from the queue. 33 | func (c *QueueClient) ReceiveMessages(queueURL string, maxNoOfMessages, waitTimeSeconds int64) (messages []interface{}, err error) { 34 | out, err := c.Service.ReceiveMessage(&sqs.ReceiveMessageInput{ 35 | QueueUrl: aws.String(queueURL), 36 | MaxNumberOfMessages: aws.Int64(maxNoOfMessages), 37 | WaitTimeSeconds: aws.Int64(waitTimeSeconds), 38 | }) 39 | if err != nil { 40 | return nil, err 41 | } 42 | ret := make([]interface{}, len(out.Messages)) 43 | for i, msg := range out.Messages { 44 | ret[i] = msg 45 | } 46 | return ret, nil 47 | } 48 | 49 | //UnmarshalMessage decodes a message. 50 | func (c *QueueClient) UnmarshalMessage(message interface{}, v interface{}) error { 51 | msg, ok := message.(*sqs.Message) 52 | if !ok { 53 | return errors.New("message was not of type *sqs.Message") 54 | } 55 | return json.Unmarshal([]byte(aws.StringValue(msg.Body)), v) 56 | } 57 | 58 | //DeleteMessage deletes a message from the queue. 59 | func (c *QueueClient) DeleteMessage(queueURL string, message interface{}) error { 60 | msg, ok := message.(*sqs.Message) 61 | if !ok { 62 | return errors.New("message was not of type *sqs.Message") 63 | } 64 | _, err := c.Service.DeleteMessage(&sqs.DeleteMessageInput{ 65 | QueueUrl: aws.String(queueURL), 66 | ReceiptHandle: msg.ReceiptHandle, 67 | }) 68 | return err 69 | } 70 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/jwt_provider.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import "github.com/nerdalize/nerd/nerd/client" 4 | 5 | //JWTProvider is capable of providing a JWT. When IsExpired return false 6 | //the in-memory JWT will be used to prevent from calling Retrieve for each API call. 7 | type JWTProvider interface { 8 | IsExpired() bool 9 | Retrieve() (string, error) 10 | } 11 | 12 | //StaticJWTProvider is a simple JWT provider that always returns the same JWT. 13 | type StaticJWTProvider struct { 14 | JWT string 15 | } 16 | 17 | //NewStaticJWTProvider creates a new StaticJWTProvider for the given jwt. 18 | func NewStaticJWTProvider(jwt string) *StaticJWTProvider { 19 | return &StaticJWTProvider{jwt} 20 | } 21 | 22 | //IsExpired always returns false. 23 | func (s *StaticJWTProvider) IsExpired() bool { 24 | return false 25 | } 26 | 27 | //Retrieve always returns the given jwt. 28 | func (s *StaticJWTProvider) Retrieve() (string, error) { 29 | return s.JWT, nil 30 | } 31 | 32 | //ChainedJWTProvider provides a JWT based on multiple providers. The given providers are tried in sequential order. 33 | type ChainedJWTProvider struct { 34 | Providers []JWTProvider 35 | curr JWTProvider 36 | } 37 | 38 | //NewChainedJWTProvider creates a new chained jwt provider. 39 | func NewChainedJWTProvider(providers ...JWTProvider) *ChainedJWTProvider { 40 | return &ChainedJWTProvider{ 41 | Providers: providers, 42 | } 43 | } 44 | 45 | // Retrieve returns the jwt or error if no provider returned 46 | // without error. 47 | // 48 | // If a provider is found it will be cached and any calls to IsExpired() 49 | // will return the expired state of the cached provider. 50 | func (c *ChainedJWTProvider) Retrieve() (string, error) { 51 | var provErr error 52 | for _, p := range c.Providers { 53 | jwt, err := p.Retrieve() 54 | if err == nil { 55 | c.curr = p 56 | return jwt, nil 57 | } 58 | provErr = err 59 | } 60 | c.curr = nil 61 | 62 | return "", client.NewError("could not retrieve token from any provider", provErr) 63 | } 64 | 65 | // IsExpired will returned the expired state of the currently cached provider 66 | // if there is one. If there is no current provider, true will be returned. 67 | func (c *ChainedJWTProvider) IsExpired() bool { 68 | if c.curr != nil { 69 | return c.curr.IsExpired() 70 | } 71 | 72 | return true 73 | } 74 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/task.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //TaskSummary is a small version of 4 | type TaskSummary struct { 5 | TaskID int64 `json:"task_id"` 6 | TTL int64 `json:"ttl"` 7 | WorkloadID string `json:"workload_id"` 8 | Status string `json:"status,omitempty"` 9 | OutputDatasetID string `json:"output_dataset_id"` 10 | Cmd []string `json:"cmd"` 11 | } 12 | 13 | //StopTaskInput is input for task creation 14 | type StopTaskInput struct { 15 | ProjectID string `json:"project_id" valid:"required"` 16 | WorkloadID string `json:"workload_id" valid:"required"` 17 | TaskID int64 `json:"task_id" valid:"required"` 18 | } 19 | 20 | //StopTaskOutput is output for task creation 21 | type StopTaskOutput struct{} 22 | 23 | //StartTaskInput is input for task creation 24 | type StartTaskInput struct { 25 | ProjectID string `json:"project_id" valid:"required"` 26 | WorkloadID string `json:"workload_id" valid:"required"` 27 | 28 | Cmd []string `json:"cmd"` 29 | Env map[string]string `json:"env"` 30 | Stdin []byte `json:"stdin"` 31 | } 32 | 33 | //StartTaskOutput is output for task creation 34 | type StartTaskOutput struct { 35 | TaskSummary 36 | } 37 | 38 | //ListTasksInput is input for task creation 39 | type ListTasksInput struct { 40 | ProjectID string `json:"project_id" valid:"required"` 41 | WorkloadID string `json:"workload_id" valid:"required"` 42 | OnlySuccessTasks bool `json:"only_success_tasks" valid:"required"` 43 | } 44 | 45 | //ListTasksOutput is output for task creation 46 | type ListTasksOutput struct { 47 | Tasks []*TaskSummary 48 | } 49 | 50 | //DescribeTaskInput is input for task creation 51 | type DescribeTaskInput struct { 52 | ProjectID string `json:"project_id" valid:"required"` 53 | WorkloadID string `json:"workload_id" valid:"required"` 54 | TaskID int64 `json:"task_id" valid:"required"` 55 | } 56 | 57 | //DescribeTaskOutput is output for task creation 58 | type DescribeTaskOutput struct { 59 | TaskSummary 60 | ExecutionARN string `json:"execution_arn"` 61 | NumDispatches int64 `json:"num_dispatches"` 62 | Result string `json:"result,omitempty"` 63 | LastErrCode string `json:"last_err_code,omitempty"` 64 | LastErrMessage string `json:"last_err_message,omitempty"` 65 | } 66 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/client/client.go: -------------------------------------------------------------------------------- 1 | package v1data 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/nerdalize/nerd/nerd/client" 9 | ) 10 | 11 | const ( 12 | //LogGroup is the group for each log statement. 13 | LogGroup = "DataClient" 14 | //NoOfRetries is the amount of retries when uploading or downloading to S3. 15 | NoOfRetries = 2 16 | ) 17 | 18 | //Client holds a reference to an AWS session 19 | type Client struct { 20 | DataOps DataOps 21 | } 22 | 23 | //DataOps is an interface to a set of data operations. The interface can be implemented to store / retrieve data from different data backends. 24 | type DataOps interface { 25 | Upload(ctx context.Context, bucket, key string, body io.ReadSeeker) error 26 | Download(ctx context.Context, bucket, key string) (body io.ReadCloser, err error) 27 | Exists(ctx context.Context, bucket, key string) (exists bool, err error) 28 | } 29 | 30 | //NewClient creates a new data client that is capable of uploading and downloading (multiple) files. 31 | func NewClient(ops DataOps) *Client { 32 | return &Client{ops} 33 | } 34 | 35 | //Upload uploads a single object. 36 | func (c *Client) Upload(ctx context.Context, bucket, key string, body io.ReadSeeker) error { 37 | var err error 38 | for i := 0; i <= NoOfRetries; i++ { 39 | select { 40 | case <-ctx.Done(): 41 | return ctx.Err() 42 | default: 43 | err = c.DataOps.Upload(ctx, bucket, key, body) 44 | if err == nil { 45 | return nil 46 | } 47 | } 48 | } 49 | // TODO: Include logging. 50 | return client.NewError(fmt.Sprintf("failed to put '%v'", key), err) 51 | } 52 | 53 | //Download downloads a single object. 54 | func (c *Client) Download(ctx context.Context, bucket, key string) (io.ReadCloser, error) { 55 | var err error 56 | for i := 0; i <= NoOfRetries; i++ { 57 | select { 58 | case <-ctx.Done(): 59 | return nil, ctx.Err() 60 | default: 61 | var resp io.ReadCloser 62 | resp, err = c.DataOps.Download(ctx, bucket, key) 63 | if err == nil { 64 | return resp, nil 65 | } 66 | } 67 | } 68 | return nil, client.NewError(fmt.Sprintf("failed to download '%v'", key), err) 69 | } 70 | 71 | //Exists checks if a given object key exists. 72 | func (c *Client) Exists(ctx context.Context, bucket, key string) (has bool, err error) { 73 | return c.DataOps.Exists(ctx, bucket, key) 74 | } 75 | -------------------------------------------------------------------------------- /nerd/jwt/env_provider.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "os" 6 | 7 | v1auth "github.com/nerdalize/nerd/nerd/client/auth/v1" 8 | "github.com/nerdalize/nerd/nerd/conf" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | const ( 13 | //NerdTokenEnvVar is the environment variable used to set the JWT 14 | NerdTokenEnvVar = "NERD_JWT" 15 | //NerdSecretEnvVar is the environment variable used for the JWT refresh secret 16 | NerdSecretEnvVar = "NERD_JWT_REFRESH_TOKEN" 17 | ) 18 | 19 | //EnvProvider provides nerdalize credentials from the `credentials.NerdTokenEnvVar` environment variable. 20 | type EnvProvider struct { 21 | *ProviderBasis 22 | Client v1auth.TokenClientInterface 23 | Session conf.SessionInterface 24 | } 25 | 26 | //NewEnvProvider creates a new EnvProvider provider. 27 | func NewEnvProvider(pub *ecdsa.PublicKey, session conf.SessionInterface, client v1auth.TokenClientInterface) *EnvProvider { 28 | return &EnvProvider{ 29 | ProviderBasis: &ProviderBasis{ 30 | ExpireWindow: DefaultExpireWindow, 31 | Pub: pub, 32 | }, 33 | Session: session, 34 | Client: client, 35 | } 36 | } 37 | 38 | //Retrieve retrieves the jwt from the env variable. 39 | func (e *EnvProvider) Retrieve() (string, error) { 40 | jwt := os.Getenv(NerdTokenEnvVar) 41 | if jwt == "" { 42 | return "", errors.Errorf("environment variable %v is not set", NerdTokenEnvVar) 43 | } 44 | err := e.SetExpirationFromJWT(jwt) 45 | if err != nil { 46 | return "", errors.Wrap(err, "failed to set expiration") 47 | } 48 | 49 | jwtSecret := os.Getenv(NerdSecretEnvVar) 50 | if jwtSecret != "" && e.IsExpired() { 51 | jwt, err = e.refresh(jwt, jwtSecret) 52 | if err != nil { 53 | return "", errors.Wrap(err, "failed to refresh") 54 | } 55 | } 56 | err = isValid(jwt, e.Pub) 57 | if err != nil { 58 | return "", err 59 | } 60 | return jwt, nil 61 | } 62 | 63 | func (e *EnvProvider) refresh(jwt, secret string) (string, error) { 64 | ss, err := e.Session.Read() 65 | if err != nil { 66 | return "", errors.Wrap(err, "failed to read config") 67 | } 68 | out, err := e.Client.RefreshJWT(ss.Project.Name, jwt, secret) 69 | if err != nil { 70 | return "", errors.Wrap(err, "failed to refresh token") 71 | } 72 | err = e.SetExpirationFromJWT(out.Token) 73 | if err != nil { 74 | return "", errors.Wrap(err, "failed to set expiration") 75 | } 76 | return out.Token, nil 77 | } 78 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/client/index.go: -------------------------------------------------------------------------------- 1 | package v1data 2 | 3 | import ( 4 | "bufio" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "fmt" 8 | "io" 9 | 10 | "github.com/nerdalize/nerd/nerd/client" 11 | ) 12 | 13 | const ( 14 | //IndexObjectKey is the key of the object that contains an index of all the chunks of a dataset. 15 | IndexObjectKey = "index" 16 | //UploadPolynomal is the polynomal that is used for chunked uploading. 17 | UploadPolynomal = 0x3DA3358B4DC173 18 | ) 19 | 20 | //Key is the identifier of a chunk of data. 21 | type Key [sha256.Size]byte 22 | 23 | //ToString returns the string representation of a key. 24 | func (k Key) ToString() string { 25 | return fmt.Sprintf("%x", k) 26 | } 27 | 28 | //ZeroKey is an empty key. 29 | var ZeroKey = Key{} 30 | 31 | //KeyReader can be implemented by objects capable of reading Keys. 32 | type KeyReader interface { 33 | ReadKey() (Key, error) 34 | } 35 | 36 | //KeyWriter can be implemented by objects capable of writing Keys. 37 | type KeyWriter interface { 38 | WriteKey(Key) error 39 | } 40 | 41 | //IndexReader can be used to read keys from the "index" object. 42 | type IndexReader struct { 43 | s *bufio.Scanner 44 | } 45 | 46 | //NewIndexReader creates a new IndexReader. 47 | func NewIndexReader(r io.Reader) *IndexReader { 48 | return &IndexReader{ 49 | s: bufio.NewScanner(r), 50 | } 51 | } 52 | 53 | //ReadKey reads Keys from the provided io.Reader. 54 | func (r *IndexReader) ReadKey() (Key, error) { 55 | if !r.s.Scan() { 56 | return ZeroKey, io.EOF 57 | } 58 | line := r.s.Text() 59 | bytes, err := hex.DecodeString(line) 60 | if err != nil { 61 | return ZeroKey, client.NewError(fmt.Sprintf("could not decode key string '%v'", line), err) 62 | } 63 | var k Key 64 | copy(k[:], bytes) 65 | return k, nil 66 | } 67 | 68 | //IndexWriter can be used to write keys to the "index" object. 69 | type IndexWriter struct { 70 | w io.Writer 71 | } 72 | 73 | //NewIndexWriter creates a new IndexWriter. 74 | func NewIndexWriter(w io.Writer) *IndexWriter { 75 | return &IndexWriter{ 76 | w: w, 77 | } 78 | } 79 | 80 | //WriteKey writes a Key to the io.WriteCloser. 81 | func (w *IndexWriter) WriteKey(k Key) error { 82 | _, err := w.w.Write([]byte(fmt.Sprintf("%v\n", k.ToString()))) 83 | if err != nil { 84 | return client.NewError(fmt.Sprintf("failed to write key '%v' to writer", k.ToString()), err) 85 | } 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /cmd/job_logs.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "strings" 8 | 9 | flags "github.com/jessevdk/go-flags" 10 | "github.com/mitchellh/cli" 11 | "github.com/nerdalize/nerd/pkg/kubevisor" 12 | "github.com/nerdalize/nerd/svc" 13 | ) 14 | 15 | //JobLogs command 16 | type JobLogs struct { 17 | Tail int64 `long:"tail" short:"t" description:"only return the oldest N lines of the process logs"` 18 | 19 | *command 20 | } 21 | 22 | //JobLogsFactory creates the command 23 | func JobLogsFactory(ui cli.Ui) cli.CommandFactory { 24 | cmd := &JobLogs{} 25 | cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, nil, flags.None, "nerd job logs") 26 | return func() (cli.Command, error) { 27 | return cmd, nil 28 | } 29 | } 30 | 31 | //Execute runs the command 32 | func (cmd *JobLogs) Execute(args []string) (err error) { 33 | if len(args) < 1 { 34 | return errShowUsage(fmt.Sprintf(MessageNotEnoughArguments, 1, "")) 35 | } else if len(args) > 1 { 36 | return errShowUsage(fmt.Sprintf(MessageTooManyArguments, 1, "")) 37 | } 38 | 39 | kopts := cmd.globalOpts.KubeOpts 40 | deps, err := NewDeps(cmd.Logger(), kopts) 41 | if err != nil { 42 | return renderConfigError(err, "failed to configure") 43 | } 44 | 45 | ctx := context.Background() 46 | ctx, cancel := context.WithTimeout(ctx, kopts.Timeout) 47 | defer cancel() 48 | 49 | in := &svc.FetchJobLogsInput{ 50 | Name: args[0], 51 | Tail: cmd.Tail, 52 | } 53 | 54 | kube := svc.NewKube(deps) 55 | out, err := kube.FetchJobLogs(ctx, in) 56 | if err != nil { 57 | return renderServiceError(err, "failed to fetch job logs") 58 | } 59 | 60 | lines := string(bytes.TrimSpace(out.Data)) 61 | if len(lines) < 1 { 62 | cmd.out.Info("-- no visible logs returned --") 63 | return nil 64 | } 65 | 66 | cmd.out.Output(strings.TrimSpace(string(out.Data))) //trim trailing newline, which is re-added by the output function 67 | if int64(len(out.Data)) == kubevisor.MaxLogBytes { 68 | cmd.out.Info("-- logs are trimmed after this point --") 69 | } 70 | 71 | return nil 72 | } 73 | 74 | // Description returns long-form help text 75 | func (cmd *JobLogs) Description() string { return cmd.Synopsis() } 76 | 77 | // Synopsis returns a one-line 78 | func (cmd *JobLogs) Synopsis() string { return "Return logs for a running job." } 79 | 80 | // Usage shows usage 81 | func (cmd *JobLogs) Usage() string { return "nerd job logs [OPTIONS] JOB" } 82 | -------------------------------------------------------------------------------- /cmd/dataset_list.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | "strings" 7 | 8 | humanize "github.com/dustin/go-humanize" 9 | flags "github.com/jessevdk/go-flags" 10 | "github.com/mitchellh/cli" 11 | "github.com/nerdalize/nerd/svc" 12 | ) 13 | 14 | //DatasetList command 15 | type DatasetList struct { 16 | *command 17 | } 18 | 19 | //DatasetListFactory creates the command 20 | func DatasetListFactory(ui cli.Ui) cli.CommandFactory { 21 | cmd := &DatasetList{} 22 | cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, nil, flags.None, "nerd dataset list") 23 | return func() (cli.Command, error) { 24 | return cmd, nil 25 | } 26 | } 27 | 28 | //Execute runs the command 29 | func (cmd *DatasetList) Execute(args []string) (err error) { 30 | if len(args) > 0 { 31 | return errShowUsage(MessageNoArgumentRequired) 32 | } 33 | kopts := cmd.globalOpts.KubeOpts 34 | deps, err := NewDeps(cmd.Logger(), kopts) 35 | if err != nil { 36 | return renderConfigError(err, "failed to configure") 37 | } 38 | 39 | ctx := context.Background() 40 | ctx, cancel := context.WithTimeout(ctx, kopts.Timeout) 41 | defer cancel() 42 | 43 | in := &svc.ListDatasetsInput{} 44 | kube := svc.NewKube(deps) 45 | out, err := kube.ListDatasets(ctx, in) 46 | if err != nil { 47 | return renderServiceError(err, "failed to list datasets") 48 | } 49 | 50 | if len(out.Items) == 0 { 51 | cmd.out.Infof("No dataset found.") 52 | return nil 53 | } 54 | 55 | sort.Slice(out.Items, func(i int, j int) bool { 56 | return out.Items[i].Details.CreatedAt.After(out.Items[j].Details.CreatedAt) 57 | }) 58 | 59 | hdr := []string{"DATASET", "CREATED AT", "SIZE", "INPUT FOR", "OUTPUT FROM"} 60 | rows := [][]string{} 61 | for _, item := range out.Items { 62 | rows = append(rows, []string{ 63 | item.Name, 64 | humanize.Time(item.Details.CreatedAt), 65 | humanize.Bytes(item.Details.Size), 66 | strings.Join(item.Details.InputFor, ","), 67 | strings.Join(item.Details.OutputFrom, ","), 68 | }) 69 | } 70 | 71 | return cmd.out.Table(hdr, rows) 72 | } 73 | 74 | // Description returns long-form help text 75 | func (cmd *DatasetList) Description() string { return cmd.Synopsis() } 76 | 77 | // Synopsis returns a one-line 78 | func (cmd *DatasetList) Synopsis() string { return "Return datasets that are managed by the cluster." } 79 | 80 | // Usage shows usage 81 | func (cmd *DatasetList) Usage() string { return "nerd dataset list [OPTIONS]" } 82 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/nerdalize/nerd 2 | ignore: 3 | - github.com/go-ini/ini 4 | import: 5 | - package: github.com/mitchellh/cli 6 | version: 65fcae5817c8600da98ada9d7edf26dd1a84837b 7 | - package: github.com/jessevdk/go-flags 8 | version: 4e64e4a4e2552194cf594243e23aa9baf3b4297e 9 | - package: github.com/aws/aws-sdk-go 10 | version: ^v1.7.0 11 | - package: github.com/dghubble/sling 12 | version: v1.1.0 13 | - package: github.com/pkg/errors 14 | version: v0.8.0 15 | - package: github.com/mitchellh/go-homedir 16 | version: b8bc1bf767474819792c23f32d8286a45736f1c6 17 | - package: github.com/dgrijalva/jwt-go 18 | version: 2268707a8f0843315e2004ee4f1d021dc08baedf 19 | - package: github.com/olekukonko/tablewriter 20 | version: febf2d34b54a69ce7530036c7503b1c9fbfdf0bb 21 | - package: github.com/dustin/go-humanize 22 | version: 259d2a102b871d17f30e3cd9881a642961a1e486 23 | - package: github.com/restic/chunker 24 | version: v0.1.0 25 | - package: gopkg.in/cheggaaa/pb.v1 26 | version: v1.0.11 27 | - package: github.com/sirupsen/logrus 28 | version: v1.0.3 29 | - package: github.com/dchest/safefile 30 | version: 855e8d98f1852d48dde521e0522408d1fe7e836a 31 | - package: github.com/google/go-querystring 32 | version: 53e6ce116135b80d037921a7fdd5138cf32d7a8a 33 | - package: github.com/skratchdot/open-golang 34 | version: 75fb7ed4208cf72d323d7d02fd1a5964a7a9073c 35 | - package: github.com/hashicorp/logutils 36 | version: 0dc08b1671f34c4250ce212759ebd880f743d883 37 | - package: github.com/gorilla/schema 38 | version: e6c82218a8b3ed3cbeb5407429849c0b0b597d40 39 | - package: github.com/go-playground/validator 40 | version: ~v9.9.0 41 | - package: k8s.io/client-go 42 | - package: github.com/satori/go.uuid 43 | version: 36e9d2ebbde5e3f13ab2e25625fd453271d6522e 44 | subpackages: 45 | - kubernetes 46 | - tools/clientcmd 47 | - package: golang.org/x/sys 48 | - package: k8s.io/code-generator 49 | - package: k8s.io/api 50 | version: 184e700b32b7f1b532b9fce8dd8c1f412d297c4b 51 | subpackages: 52 | - apps/v1 53 | - package: k8s.io/apimachinery 54 | version: 084add7e300b3c88d77dded09a20d5063d632b5b 55 | subpackages: 56 | - pkg/runtime 57 | - package: github.com/google/go-github 58 | subpackages: 59 | - github 60 | - package: github.com/joho/godotenv 61 | version: 1.2.0 62 | - package: github.com/cheggaaa/pb 63 | version: ~v1.0.22 64 | - package: k8s.io/apiextensions-apiserver 65 | subpackages: 66 | - pkg/apis/apiextensions/v1beta1 67 | -------------------------------------------------------------------------------- /svc/kube_list_datasets.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | datasetsv1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 8 | "github.com/nerdalize/nerd/pkg/kubevisor" 9 | 10 | "k8s.io/apimachinery/pkg/types" 11 | ) 12 | 13 | //DatasetDetails tells us more about the dataset by looking at underlying resources 14 | type DatasetDetails struct { 15 | CreatedAt time.Time 16 | Size uint64 17 | InputFor []string 18 | OutputFrom []string 19 | } 20 | 21 | //ListDatasetItem is a dataset listing item 22 | type ListDatasetItem struct { 23 | Name string 24 | Details DatasetDetails 25 | } 26 | 27 | //ListDatasetsInput is the input to ListDatasets 28 | type ListDatasetsInput struct{} 29 | 30 | //ListDatasetsOutput is the output to ListDatasets 31 | type ListDatasetsOutput struct { 32 | Items []*ListDatasetItem 33 | } 34 | 35 | //ListDatasets will create a dataset on kubernetes 36 | func (k *Kube) ListDatasets(ctx context.Context, in *ListDatasetsInput) (out *ListDatasetsOutput, err error) { 37 | if err = k.checkInput(ctx, in); err != nil { 38 | return nil, err 39 | } 40 | 41 | //Step 0: Get all the datasets under nerd-app=cli 42 | datasets := &datasets{} 43 | err = k.visor.ListResources(ctx, kubevisor.ResourceTypeDatasets, datasets, nil, nil) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | //Step 1: Analyse dataset structure and formulate our output items 49 | out = &ListDatasetsOutput{} 50 | mapping := map[types.UID]*ListDatasetItem{} 51 | for _, dataset := range datasets.Items { 52 | item := &ListDatasetItem{ 53 | Name: dataset.GetName(), 54 | Details: DatasetDetails{ 55 | Size: dataset.Spec.Size, 56 | InputFor: dataset.Spec.InputFor, 57 | OutputFrom: dataset.Spec.OutputFrom, 58 | CreatedAt: dataset.CreationTimestamp.Local(), 59 | }, 60 | } 61 | 62 | mapping[dataset.UID] = item 63 | out.Items = append(out.Items, item) 64 | } 65 | 66 | return out, nil 67 | } 68 | 69 | //datasets implements the list transformer interface to allow the kubevisor to manage names for us 70 | type datasets struct{ *datasetsv1.DatasetList } 71 | 72 | func (datasets *datasets) Transform(fn func(in kubevisor.ManagedNames) (out kubevisor.ManagedNames)) { 73 | for i, d1 := range datasets.DatasetList.Items { 74 | datasets.Items[i] = *(fn(&d1).(*datasetsv1.Dataset)) 75 | } 76 | } 77 | 78 | func (datasets *datasets) Len() int { 79 | return len(datasets.DatasetList.Items) 80 | } 81 | -------------------------------------------------------------------------------- /cmd/job_run_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/mitchellh/cli" 9 | 10 | "github.com/nerdalize/nerd/cmd" 11 | ) 12 | 13 | func TestJobExecute(t *testing.T) { 14 | ui := &cli.BasicUi{ 15 | Reader: os.Stdin, 16 | Writer: os.Stdout, 17 | ErrorWriter: os.Stderr, 18 | } 19 | 20 | jobRun, err := cmd.JobRunFactory(ui)() 21 | if err != nil { 22 | t.Fatal("expected job run factory to return an instance, but got ", err) 23 | } 24 | command := jobRun.(*cmd.JobRun) 25 | 26 | err = command.Execute([]string{}) 27 | if err == nil { 28 | t.Error("expected job run without arguments to return error, but got ", err) 29 | } 30 | } 31 | 32 | func TestParseInputSpecification(t *testing.T) { 33 | var pathTests = []struct { 34 | input string 35 | parts []string 36 | err bool 37 | }{ 38 | // Generic 39 | {"somedataset:/input", []string{"somedataset", "/input"}, false}, 40 | {"some/relative/directory/:/input", []string{"some/relative/directory/", "/input"}, false}, 41 | {"./dot/relative/path:/input", []string{"./dot/relative/path", "/input"}, false}, 42 | {"./data:/~/valid/abs/path", []string{"./data", "/~/valid/abs/path"}, false}, 43 | {"C:/input", []string{"C", "/input"}, false}, // Can we detect this "mistake"? 44 | 45 | // Failure cases 46 | {"", nil, true}, 47 | {"nocolons", nil, true}, 48 | {"/too:/many:/colons:/here", nil, true}, 49 | {"./data:", nil, true}, 50 | {":/input", nil, true}, 51 | {" :/input", nil, true}, 52 | {"./data: ", nil, true}, 53 | 54 | // Windows 55 | {"C:/some/dir:/input", []string{"C:/some/dir", "/input"}, false}, 56 | {"//some/dir:/input", []string{"//some/dir", "/input"}, false}, 57 | {"C:\\some\\dir:/input", []string{"C:\\some\\dir", "/input"}, false}, 58 | 59 | // Linux 60 | {"/some/abs/path:/input", []string{"/some/abs/path", "/input"}, false}, 61 | } 62 | 63 | for _, testCase := range pathTests { 64 | parts, err := cmd.ParseInputSpecification(testCase.input) 65 | 66 | if testCase.err && err == nil { 67 | t.Errorf("expected error for input %s, but got no error and output %v", testCase.input, parts) 68 | } else if !testCase.err && err != nil { 69 | t.Errorf("expected no error for input %s, but got %s", testCase.input, err) 70 | } 71 | 72 | if !reflect.DeepEqual(parts, testCase.parts) { 73 | t.Errorf("expected %s to be parsed into %v, but got %v", testCase.input, testCase.parts, parts) 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/upload_process_test.go: -------------------------------------------------------------------------------- 1 | package v1datatransfer 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "runtime" 9 | "testing" 10 | "time" 11 | 12 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 13 | v1data "github.com/nerdalize/nerd/nerd/service/datatransfer/v1/client" 14 | ) 15 | 16 | func testfile(dir string, name string, size, seed int64, t interface { 17 | Fatalf(format string, args ...interface{}) 18 | }) (os.FileInfo, []byte) { 19 | path := filepath.Join(dir, name) 20 | err := os.MkdirAll(filepath.Dir(path), 0777) 21 | if err != nil { 22 | t.Fatalf("failed to create file dir for '%v': %v", path, err) 23 | } 24 | 25 | data := randb(size, seed) 26 | err = ioutil.WriteFile(path, data, 0666) 27 | if err != nil { 28 | t.Fatalf("failed to write file '%v': %v", path, err) 29 | } 30 | 31 | fi, err := os.Stat(path) 32 | if err != nil { 33 | t.Fatalf("failed to stat test file '%v': %v", path, err) 34 | } 35 | 36 | return fi, data 37 | } 38 | 39 | type clientUpload struct{} 40 | 41 | func (c *clientUpload) SendUploadHeartbeat(projectID, datasetID string) (output *v1payload.SendUploadHeartbeatOutput, err error) { 42 | return output, err 43 | } 44 | func (c *clientUpload) SendUploadSuccess(projectID, datasetID string) (output *v1payload.SendUploadSuccessOutput, err error) { 45 | return output, err 46 | } 47 | 48 | func TestUploadContext(t *testing.T) { 49 | baseNum := runtime.NumGoroutine() 50 | dir, err := ioutil.TempDir("", "test_upload_context") 51 | if err != nil { 52 | t.Fatalf("failed to setup tempdir: %v", err) 53 | } 54 | name := "test" 55 | testfile(dir, name, 10*MiB, 42, t) 56 | 57 | dp := &uploadProcess{ 58 | batchClient: &clientUpload{}, 59 | dataClient: v1data.NewClient(&blockingOps{}), 60 | dataset: v1payload.DatasetSummary{}, 61 | heartbeatInterval: time.Second * 10, 62 | localDir: dir, 63 | concurrency: 5, 64 | progressCh: nil, 65 | } 66 | ctx, cancel := context.WithCancel(context.Background()) 67 | go func() { 68 | dp.start(ctx) 69 | }() 70 | time.Sleep(time.Second * 5) 71 | expected := baseNum + 13 72 | if runtime.NumGoroutine() != expected { 73 | t.Fatalf("expected %v goroutines, got: %v", expected, runtime.NumGoroutine()) 74 | } 75 | cancel() 76 | time.Sleep(time.Second * 5) 77 | if runtime.NumGoroutine() != baseNum { 78 | t.Fatalf("expected %v goroutines, got: %v", baseNum, runtime.NumGoroutine()) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /nerd/service/datatransfer/v1/download.go: -------------------------------------------------------------------------------- 1 | package v1datatransfer 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | v1batch "github.com/nerdalize/nerd/nerd/client/batch/v1" 8 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 9 | v1data "github.com/nerdalize/nerd/nerd/service/datatransfer/v1/client" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | //DownloadConfig is the config for Download operations 14 | type DownloadConfig struct { 15 | BatchClient v1batch.ClientInterface 16 | DataOps v1data.DataOps 17 | LocalDir string 18 | ProjectID string 19 | DatasetID string 20 | Concurrency int 21 | ProgressCh chan<- int64 22 | } 23 | 24 | //Download downloads a dataset or fails if it is still being uploaded 25 | func Download(ctx context.Context, conf DownloadConfig) error { 26 | if conf.ProgressCh != nil { 27 | defer close(conf.ProgressCh) 28 | } 29 | var ds v1payload.DatasetSummary 30 | for { 31 | out, err := conf.BatchClient.DescribeDataset(conf.ProjectID, conf.DatasetID) 32 | if err != nil { 33 | return errors.Wrap(err, "failed to get dataset") 34 | } 35 | ds = out.DatasetSummary 36 | if ds.UploadStatus == v1payload.DatasetUploadStatusSuccess { 37 | break 38 | } 39 | if ds.UploadStatus == v1payload.DatasetUploadStatusUploading && ds.UploadExpire < time.Now().Unix() { 40 | return errors.Errorf("cannot start download, because the upload timed out") 41 | } 42 | wait := ds.UploadExpire - time.Now().Unix() 43 | select { 44 | case <-ctx.Done(): 45 | return ctx.Err() 46 | case <-time.After(time.Second * time.Duration(wait)): 47 | } 48 | } 49 | dataClient := v1data.NewClient(conf.DataOps) 50 | down := &downloadProcess{ 51 | dataClient: dataClient, 52 | dataset: ds, 53 | localDir: conf.LocalDir, 54 | concurrency: conf.Concurrency, 55 | progressCh: conf.ProgressCh, 56 | } 57 | return down.start(ctx) 58 | } 59 | 60 | //GetRemoteDatasetSize gets the size of a dataset from the metadata object 61 | func GetRemoteDatasetSize(ctx context.Context, batchClient *v1batch.Client, dataOps v1data.DataOps, projectID, datasetID string) (int64, error) { 62 | dataClient := v1data.NewClient(dataOps) 63 | ds, err := batchClient.DescribeDataset(projectID, datasetID) 64 | if err != nil { 65 | return 0, errors.Wrap(err, "failed to get dataset") 66 | } 67 | metadata, err := dataClient.MetadataDownload(ctx, ds.Bucket, ds.DatasetRoot) 68 | if err != nil { 69 | return 0, errors.Wrap(err, "failed to download metadata") 70 | } 71 | return metadata.Size, nil 72 | } 73 | -------------------------------------------------------------------------------- /svc/kube_delete_dataset_test.go: -------------------------------------------------------------------------------- 1 | package svc_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | "runtime" 8 | "testing" 9 | "time" 10 | 11 | "github.com/nerdalize/nerd/pkg/transfer/archiver" 12 | 13 | "github.com/nerdalize/nerd/pkg/kubevisor" 14 | "github.com/nerdalize/nerd/pkg/transfer/store" 15 | "github.com/nerdalize/nerd/svc" 16 | ) 17 | 18 | func TestDeleteDataset(t *testing.T) { 19 | for _, c := range []struct { 20 | Name string 21 | Timeout time.Duration 22 | Datasets []*svc.CreateDatasetInput 23 | Input *svc.DeleteDatasetInput 24 | Output *svc.DeleteDatasetOutput 25 | Listing *svc.ListDatasetsOutput 26 | IsOutput func(tb testing.TB, out *svc.DeleteDatasetOutput, l *svc.ListDatasetsOutput) 27 | IsErr func(error) bool 28 | }{ 29 | { 30 | Name: "when no name is provided it should provide a validation error", 31 | Timeout: time.Second * 5, 32 | Input: &svc.DeleteDatasetInput{}, 33 | Output: &svc.DeleteDatasetOutput{}, 34 | IsErr: svc.IsValidationErr, 35 | }, 36 | { 37 | Name: "when a non-existing dataset is deleted it should return NotExists error", 38 | Timeout: time.Second * 5, 39 | Input: &svc.DeleteDatasetInput{Name: "foo"}, 40 | Output: &svc.DeleteDatasetOutput{}, 41 | IsErr: kubevisor.IsNotExistsErr, 42 | }, 43 | { 44 | Name: "when a valid dataset is deleted it should return no error", 45 | Timeout: time.Second * 5, 46 | Datasets: []*svc.CreateDatasetInput{{Name: "test", StoreOptions: transferstore.StoreOptions{}, ArchiverOptions: transferarchiver.ArchiverOptions{}}}, 47 | Input: &svc.DeleteDatasetInput{Name: "test"}, 48 | Output: &svc.DeleteDatasetOutput{}, 49 | IsErr: nil, 50 | }, 51 | } { 52 | t.Run(c.Name, func(t *testing.T) { 53 | di, clean := testDI(t) 54 | defer clean() 55 | 56 | ctx := context.Background() 57 | ctx, cancel := context.WithTimeout(ctx, c.Timeout) 58 | defer cancel() 59 | 60 | kube := svc.NewKube(di) 61 | for _, dataset := range c.Datasets { 62 | _, err := kube.CreateDataset(ctx, dataset) 63 | ok(t, err) 64 | } 65 | 66 | out, err := kube.DeleteDataset(ctx, c.Input) 67 | if c.IsErr != nil { 68 | assert(t, c.IsErr(err), fmt.Sprintf("unexpected '%#v' to match: %#v", err, runtime.FuncForPC(reflect.ValueOf(c.IsErr).Pointer()).Name())) 69 | } 70 | 71 | list, err := kube.ListDatasets(ctx, &svc.ListDatasetsInput{}) 72 | ok(t, err) 73 | 74 | if c.IsOutput != nil { 75 | c.IsOutput(t, out, list) 76 | } 77 | }) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /svc/kube_list_secret.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | "path" 6 | "time" 7 | 8 | "github.com/nerdalize/nerd/pkg/kubevisor" 9 | 10 | "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/types" 12 | ) 13 | 14 | //SecretDetails tells us more about the secret by looking at underlying resources 15 | type SecretDetails struct { 16 | CreatedAt time.Time 17 | Size int 18 | Type string 19 | Image string 20 | } 21 | 22 | //ListSecretItem is a secret listing item 23 | type ListSecretItem struct { 24 | Name string 25 | Details SecretDetails 26 | } 27 | 28 | //ListSecretsInput is the input to ListSecrets 29 | type ListSecretsInput struct { 30 | Labels []string 31 | } 32 | 33 | //ListSecretsOutput is the output to ListSecrets 34 | type ListSecretsOutput struct { 35 | Items []*ListSecretItem 36 | } 37 | 38 | //ListSecrets will create a secret on kubernetes 39 | func (k *Kube) ListSecrets(ctx context.Context, in *ListSecretsInput) (out *ListSecretsOutput, err error) { 40 | if err = k.checkInput(ctx, in); err != nil { 41 | return nil, err 42 | } 43 | 44 | //Step 0: Get all the secrets under nerd-app=cli 45 | secrets := &secrets{} 46 | err = k.visor.ListResources(ctx, kubevisor.ResourceTypeSecrets, secrets, in.Labels, nil) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | //Step 1: Analyse secret structure and formulate our output items 52 | out = &ListSecretsOutput{} 53 | mapping := map[types.UID]*ListSecretItem{} 54 | for _, secret := range secrets.Items { 55 | if secret.Labels["registry"] == "index.docker.io" { 56 | secret.Labels["registry"] = "" 57 | } 58 | item := &ListSecretItem{ 59 | Name: secret.GetName(), 60 | Details: SecretDetails{ 61 | Type: string(secret.Type), 62 | Size: secret.Size(), 63 | CreatedAt: secret.CreationTimestamp.Local(), 64 | Image: path.Join(secret.Labels["registry"], secret.Labels["project"], secret.Labels["image"]), 65 | }, 66 | } 67 | 68 | mapping[secret.UID] = item 69 | out.Items = append(out.Items, item) 70 | } 71 | 72 | return out, nil 73 | } 74 | 75 | //secrets implements the list transformer interface to allow the kubevisor to manage names for us 76 | type secrets struct{ *v1.SecretList } 77 | 78 | func (secrets *secrets) Transform(fn func(in kubevisor.ManagedNames) (out kubevisor.ManagedNames)) { 79 | for i, d1 := range secrets.SecretList.Items { 80 | secrets.Items[i] = *(fn(&d1).(*v1.Secret)) 81 | } 82 | } 83 | 84 | func (secrets *secrets) Len() int { 85 | return len(secrets.SecretList.Items) 86 | } 87 | -------------------------------------------------------------------------------- /crd/handler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golang/glog" 7 | datasetsv1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 8 | transferv2 "github.com/nerdalize/nerd/pkg/transfer" 9 | ) 10 | 11 | type glogReporter struct{} 12 | 13 | func (r *glogReporter) HandledKey(key string) { 14 | glog.Infof("handled dataset key '%s'", key) 15 | } 16 | 17 | // Handler is implemented by any handler. 18 | // The Handle method is used to process event 19 | type Handler interface { 20 | ObjectCreated(obj interface{}) 21 | ObjectDeleted(obj interface{}, key string) 22 | ObjectUpdated(oldObj, newObj interface{}) 23 | } 24 | 25 | // S3AWS handler implements Handler interface 26 | type S3AWS struct{} 27 | 28 | // ObjectCreated will be called each time an object is created 29 | func (s *S3AWS) ObjectCreated(obj interface{}) { 30 | if dataset, ok := obj.(*datasetsv1.Dataset); ok { 31 | glog.Infof("New dataset created %s from namespace %s", dataset.Name, dataset.Namespace) 32 | } 33 | } 34 | 35 | // ObjectDeleted will be called each time an object is deleted 36 | // If the object is a dataset, the corresponding dataset will be removed from s3 37 | func (s *S3AWS) ObjectDeleted(obj interface{}, key string) { 38 | if dataset, ok := obj.(*datasetsv1.Dataset); ok { 39 | store, err := transferv2.CreateStore(dataset.Spec.StoreOptions) 40 | if err != nil { 41 | glog.Errorf("failed to create store with options '%#v': %v", dataset.Spec.StoreOptions, err) 42 | return 43 | } 44 | 45 | archiver, err := transferv2.CreateArchiver(dataset.Spec.ArchiverOptions) 46 | if err != nil { 47 | glog.Errorf("failed to create archiver with options '%#v': %v", dataset.Spec.ArchiverOptions, err) 48 | return 49 | } 50 | 51 | h, err := transferv2.CreateStdHandle(dataset.GetName(), store, archiver, nil) 52 | if err != nil { 53 | glog.Errorf("failed to create standard handle: %v", err) 54 | return 55 | } 56 | 57 | //@TODO decide on the timeout of the dataset clear 58 | err = h.Clear(context.TODO(), &glogReporter{}) 59 | if err != nil { 60 | glog.Errorf("failed to clear the dataset: %v", err) 61 | return 62 | } 63 | 64 | glog.Infof("Dataset deleted %s from namespace %s", dataset.Name, dataset.Namespace) 65 | } 66 | } 67 | 68 | // ObjectUpdated will be called each time an object is updated 69 | func (s *S3AWS) ObjectUpdated(oldObj, newObj interface{}) { 70 | if dataset, ok := newObj.(*datasetsv1.Dataset); ok { 71 | glog.Infof("New dataset updated %s from namespace %s", dataset.Name, dataset.Namespace) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /svc/kube_delete_job_test.go: -------------------------------------------------------------------------------- 1 | package svc_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | "runtime" 8 | "testing" 9 | "time" 10 | 11 | "github.com/nerdalize/nerd/pkg/kubevisor" 12 | "github.com/nerdalize/nerd/svc" 13 | ) 14 | 15 | func TestDeleteJob(t *testing.T) { 16 | for _, c := range []struct { 17 | Name string 18 | Timeout time.Duration 19 | Jobs []*svc.RunJobInput 20 | Input *svc.DeleteJobInput 21 | Output *svc.DeleteJobOutput 22 | Listing *svc.ListJobsOutput 23 | IsOutput func(tb testing.TB, out *svc.DeleteJobOutput, list *svc.ListJobsOutput) 24 | IsErr func(error) bool 25 | }{ 26 | { 27 | Name: "when no name is provided it should provide a validation error", 28 | Timeout: time.Second * 5, 29 | Input: &svc.DeleteJobInput{}, 30 | Output: &svc.DeleteJobOutput{}, 31 | IsErr: svc.IsValidationErr, 32 | }, 33 | { 34 | Name: "when an existing job is delete it should be marked for garbage collection", 35 | Timeout: time.Second * 5, 36 | Jobs: []*svc.RunJobInput{{Image: "nginx", Name: "my-job"}}, 37 | Input: &svc.DeleteJobInput{Name: "my-job"}, 38 | Output: &svc.DeleteJobOutput{}, 39 | IsErr: isNilErr, 40 | IsOutput: func(t testing.TB, out *svc.DeleteJobOutput, list *svc.ListJobsOutput) { 41 | assert(t, len(list.Items) == 1, "job should still be there") 42 | assert(t, !list.Items[0].DeletedAt.IsZero(), "delete at should not be zero") 43 | }, 44 | }, 45 | { 46 | Name: "when a non-existing job is delete it should return NotExists error", 47 | Timeout: time.Second * 5, 48 | Jobs: []*svc.RunJobInput{{Image: "nginx", Name: "my-job"}}, 49 | Input: &svc.DeleteJobInput{Name: "foo"}, 50 | Output: &svc.DeleteJobOutput{}, 51 | IsErr: kubevisor.IsNotExistsErr, 52 | }, 53 | } { 54 | t.Run(c.Name, func(t *testing.T) { 55 | di, clean := testDI(t) 56 | defer clean() 57 | 58 | ctx := context.Background() 59 | ctx, cancel := context.WithTimeout(ctx, c.Timeout) 60 | defer cancel() 61 | 62 | kube := svc.NewKube(di) 63 | for _, job := range c.Jobs { 64 | _, err := kube.RunJob(ctx, job) 65 | ok(t, err) 66 | } 67 | 68 | out, err := kube.DeleteJob(ctx, c.Input) 69 | if c.IsErr != nil { 70 | assert(t, c.IsErr(err), fmt.Sprintf("unexpected '%#v' to match: %#v", err, runtime.FuncForPC(reflect.ValueOf(c.IsErr).Pointer()).Name())) 71 | } 72 | 73 | list, err := kube.ListJobs(ctx, &svc.ListJobsInput{}) 74 | ok(t, err) 75 | 76 | if c.IsOutput != nil { 77 | c.IsOutput(t, out, list) 78 | } 79 | }) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /nerd/jwt/provider_test.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "testing" 6 | "time" 7 | 8 | jwt "github.com/dgrijalva/jwt-go" 9 | ) 10 | 11 | func TestExpiration(t *testing.T) { 12 | cases := []struct { 13 | alwaysValid bool 14 | window time.Duration 15 | expireAt time.Duration 16 | expected bool 17 | }{ 18 | {false, 0, 0, true}, 19 | {false, 0, +1 * time.Second, false}, 20 | {false, 0, +10 * time.Second, false}, 21 | {false, 0, -1 * time.Second, true}, 22 | {false, 0, -10 * time.Second, true}, 23 | {false, 10 * time.Second, 0, true}, 24 | {false, 10 * time.Second, +1 * time.Second, true}, 25 | {false, 10 * time.Second, +10 * time.Second, true}, 26 | {false, 10 * time.Second, +11 * time.Second, false}, 27 | {false, 10 * time.Second, -1 * time.Second, true}, 28 | {false, 10 * time.Second, -10 * time.Second, true}, 29 | 30 | {true, 0, 0, false}, 31 | {true, 0, +1 * time.Second, false}, 32 | {true, 0, +10 * time.Second, false}, 33 | {true, 0, -1 * time.Second, false}, 34 | {true, 0, -10 * time.Second, false}, 35 | {true, 10 * time.Second, 0, false}, 36 | {true, 10 * time.Second, +1 * time.Second, false}, 37 | {true, 10 * time.Second, +10 * time.Second, false}, 38 | {true, 10 * time.Second, +11 * time.Second, false}, 39 | {true, 10 * time.Second, -1 * time.Second, false}, 40 | {true, 10 * time.Second, -10 * time.Second, false}, 41 | } 42 | for _, c := range cases { 43 | now := time.Now() 44 | basis := &ProviderBasis{ 45 | CurrentTime: func() time.Time { 46 | return now 47 | }, 48 | AlwaysValid: c.alwaysValid, 49 | ExpireWindow: c.window, 50 | } 51 | basis.SetExpiration(now.Add(c.expireAt)) 52 | if basis.IsExpired() != c.expected { 53 | t.Errorf("Expected %v but got %v for case %v", c.expected, basis.IsExpired(), c) 54 | } 55 | } 56 | } 57 | 58 | func TestExpirationFromJWT(t *testing.T) { 59 | key := testkey(t) 60 | exp := time.Unix(time.Now().Add(time.Minute).Unix(), 0) // we need a little hack to make sure we round to seconds 61 | token := jwt.NewWithClaims(jwt.SigningMethodES384, &jwt.StandardClaims{ 62 | ExpiresAt: exp.Unix(), 63 | }) 64 | ss, err := token.SignedString(key) 65 | if err != nil { 66 | t.Fatalf("failed to sign test token: %v", err) 67 | } 68 | pub, _ := key.Public().(*ecdsa.PublicKey) 69 | basis := &ProviderBasis{ 70 | Pub: pub, 71 | } 72 | err = basis.SetExpirationFromJWT(ss) 73 | if err != nil { 74 | t.Fatalf("unexpected error: %v", err) 75 | } 76 | if basis.expiration != exp { 77 | t.Errorf("expected expiration %v but got %v", exp, basis.expiration) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /nerd/aws/data_client.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/awserr" 9 | "github.com/aws/aws-sdk-go/aws/credentials" 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | "github.com/aws/aws-sdk-go/service/s3" 12 | "github.com/aws/aws-sdk-go/service/sns" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | const ( 17 | //StatusCodeForbidden is returned by AWS when a user does not have access to perform a given operation 18 | StatusCodeForbidden = "Forbidden" 19 | ) 20 | 21 | //DataClient is a client to AWS' S3 service. The client implements the v1data.DataOps interface. 22 | type DataClient struct { 23 | Service *s3.S3 24 | } 25 | 26 | //NewDataClient creates a new DataClient. 27 | func NewDataClient(c *credentials.Credentials, region string) (*DataClient, error) { 28 | sess, err := session.NewSession(&aws.Config{ 29 | Credentials: c, 30 | Region: aws.String(region), 31 | }) 32 | if err != nil { 33 | return nil, errors.Wrap(err, "could not create AWS sessions") 34 | } 35 | return &DataClient{ 36 | Service: s3.New(sess), 37 | }, nil 38 | } 39 | 40 | //Upload uploads an object to S3. 41 | func (c *DataClient) Upload(ctx context.Context, bucket, key string, body io.ReadSeeker) error { 42 | input := &s3.PutObjectInput{ 43 | Bucket: aws.String(bucket), // Required 44 | Key: aws.String(key), // Required 45 | Body: body, 46 | } 47 | _, err := c.Service.PutObjectWithContext(ctx, input) 48 | return err 49 | } 50 | 51 | //Download downloads an object to S3. 52 | func (c *DataClient) Download(ctx context.Context, bucket, key string) (body io.ReadCloser, err error) { 53 | input := &s3.GetObjectInput{ 54 | Bucket: aws.String(bucket), // Required 55 | Key: aws.String(key), // Required 56 | } 57 | resp, err := c.Service.GetObjectWithContext(ctx, input) 58 | 59 | if err != nil { 60 | return nil, err 61 | } 62 | return resp.Body, nil 63 | } 64 | 65 | //Exists checks whether an object exists on S3. 66 | func (c *DataClient) Exists(ctx context.Context, bucket, key string) (exists bool, err error) { 67 | input := &s3.HeadObjectInput{ 68 | Bucket: aws.String(bucket), // Required 69 | Key: aws.String(key), 70 | } 71 | _, err = c.Service.HeadObjectWithContext(ctx, input) 72 | if err != nil { 73 | if aerr, ok := err.(awserr.Error); ok && (aerr.Code() == s3.ErrCodeNoSuchKey || aerr.Code() == sns.ErrCodeNotFoundException || aerr.Code() == StatusCodeForbidden) { 74 | return false, nil 75 | } 76 | return false, errors.Wrapf(err, "failed to check if key %v exists", key) 77 | } 78 | return true, nil 79 | } 80 | -------------------------------------------------------------------------------- /nerd/oauth/config_provider.go: -------------------------------------------------------------------------------- 1 | package oauth 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/nerdalize/nerd/nerd" 8 | v1auth "github.com/nerdalize/nerd/nerd/client/auth/v1" 9 | "github.com/nerdalize/nerd/nerd/conf" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | //ConfigProvider provides a oauth access token from the config file. For the default file location please see TokenFilename(). 14 | type ConfigProvider struct { 15 | *ProviderBasis 16 | Client v1auth.OpsClientInterface 17 | OAuthClientID string 18 | OAuthClientSecret string 19 | Session conf.SessionInterface 20 | } 21 | 22 | //NewConfigProvider creates a new ConfigProvider provider. 23 | func NewConfigProvider(client v1auth.OpsClientInterface, oauthClientID, oauthClientSecret string, session conf.SessionInterface) *ConfigProvider { 24 | return &ConfigProvider{ 25 | ProviderBasis: &ProviderBasis{ 26 | ExpireWindow: DefaultExpireWindow, 27 | }, 28 | Client: client, 29 | OAuthClientID: oauthClientID, 30 | OAuthClientSecret: oauthClientSecret, 31 | Session: session, 32 | } 33 | } 34 | 35 | //Retrieve retrieves the token from the nerd config file. 36 | func (e *ConfigProvider) Retrieve() (string, error) { 37 | ss, err := e.Session.Read() 38 | if err != nil { 39 | return "", errors.Wrap(err, "failed to read config") 40 | } 41 | if ss.OAuth.AccessToken == "" { 42 | return "", nerd.ErrTokenUnset 43 | } 44 | e.SetExpiration(ss.OAuth.Expiration) 45 | if e.IsExpired() { 46 | token, err := e.refresh(ss.OAuth.RefreshToken, e.OAuthClientID, e.OAuthClientSecret) 47 | if err != nil { 48 | return "", errors.Wrap(err, "failed to refresh oauth access token") 49 | } 50 | return token, nil 51 | } 52 | return ss.OAuth.AccessToken, nil 53 | } 54 | 55 | //refresh refreshes the oath token with the refresh token 56 | func (e *ConfigProvider) refresh(refreshToken, clientID, clientSecret string) (string, error) { 57 | out, err := e.Client.RefreshOAuthCredentials(refreshToken, clientID, clientSecret) 58 | if err != nil { 59 | if herr, ok := err.(*v1auth.HTTPError); ok && herr.StatusCode == http.StatusUnauthorized { 60 | return "", nerd.ErrTokenRevoked 61 | } 62 | return "", errors.Wrap(err, "failed to get oauth credentials") 63 | } 64 | expiration := time.Unix(e.CurrentTime().Unix()+int64(out.ExpiresIn), 0) 65 | e.SetExpiration(expiration) 66 | err = e.Session.WriteOAuth(out.AccessToken, out.RefreshToken, "", expiration, out.Scope, out.TokenType) 67 | if err != nil { 68 | return "", errors.Wrap(err, "failed to write oauth tokens to config") 69 | } 70 | return out.AccessToken, nil 71 | } 72 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/plan.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | // CreatePlanInput is the input for assigning a plan to a project. 4 | // This results in the creation of a quota in the right namespace. 5 | type CreatePlanInput struct { 6 | PlanID string `json:"billing_package_id"` 7 | ComputeUnits string `json:"compute_units" valid:"required"` 8 | } 9 | 10 | // CreatePlanOutput is the output from assigning a plan to a project. 11 | type CreatePlanOutput struct { 12 | ProjectID string `json:"project_id" valid:"required"` 13 | PlanID string `json:"billing_package_id" valid:"required"` 14 | ComputeUnits string `json:"compute_units" valid:"required"` 15 | } 16 | 17 | // UpdatePlanInput is the input for updating the plan capacity 18 | type UpdatePlanInput struct { 19 | OnDemand bool `json:"on_demand"` 20 | ComputeUnits string `json:"compute_units" valid:"required"` 21 | } 22 | 23 | // UpdatePlanOutput is the output for updating the plan capacity 24 | type UpdatePlanOutput struct { 25 | ProjectID string `json:"project_id" valid:"required"` 26 | PlanID string `json:"billing_package_id" valid:"required"` 27 | ComputeUnits string `json:"compute_units" valid:"required"` 28 | } 29 | 30 | // RemovePlanInput is the input for removing a plan from a project 31 | type RemovePlanInput struct { 32 | } 33 | 34 | // RemovePlanOutput is the output from removing a plan from a project 35 | type RemovePlanOutput struct { 36 | } 37 | 38 | // DeletePlanInput is the input for deleting a plan 39 | type DeletePlanInput struct { 40 | } 41 | 42 | // DeletePlanOutput is the output from deleting a plan 43 | type DeletePlanOutput struct { 44 | } 45 | 46 | //PlanSummary is summary of a plan 47 | type PlanSummary struct { 48 | ComputeUnits string `json:"compute_units" valid:"required"` 49 | PlanID string `json:"billing_package_id" valid:"required"` 50 | } 51 | 52 | // ListPlansInput is the input for listing plans. 53 | type ListPlansInput struct { 54 | } 55 | 56 | // ListPlansOutput is the output from listing plans of a project 57 | type ListPlansOutput struct { 58 | ProjectID string `json:"project_id" valid:"required"` 59 | Plans []*PlanSummary `json:"billing_packages" valid:"required"` 60 | Total *Resource 61 | Used *Resource 62 | } 63 | 64 | // Resource is a general struct that will be used in our list payloads. 65 | type Resource struct { 66 | RequestsCPU string `json:"requests_cpu" valid:"required"` 67 | RequestsMemory string `json:"requests_memory" valid:"required"` 68 | LimitsCPU string `json:"limits_cpu" valid:"required"` 69 | LimitsMemory string `json:"limits_memory" valid:"required"` 70 | } 71 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/workload.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | //WorkloadSummary is a smaller representation of a workload 4 | type WorkloadSummary struct { 5 | ProjectID string `json:"project_id"` 6 | WorkloadID string `json:"workload_id"` 7 | QueueURL string `json:"queue_url"` 8 | Image string `json:"image"` 9 | NrOfWorkers int `json:"nr_of_workers"` 10 | InputDatasetID string `json:"input_dataset_id"` 11 | CreatedAt int64 `json:"created_at"` 12 | ComputeUnits string `json:"compute_units"` 13 | Workers []*WorkerSummary `json:"workers"` 14 | } 15 | 16 | //ListWorkloadsInput is input for workload listing 17 | type ListWorkloadsInput struct { 18 | ProjectID string `json:"project_id" valid:"required"` 19 | } 20 | 21 | //ListWorkloadsOutput is output for workload listing 22 | type ListWorkloadsOutput struct { 23 | Workloads []*WorkloadSummary 24 | } 25 | 26 | //DescribeWorkloadInput is input for getting workload information 27 | type DescribeWorkloadInput struct { 28 | ProjectID string `json:"project_id" valid:"required"` 29 | WorkloadID string `json:"workload_id"` 30 | } 31 | 32 | //TaskStatus represents the status of a task 33 | type TaskStatus string 34 | 35 | //DescribeWorkloadOutput is output for getting workload information 36 | type DescribeWorkloadOutput struct { 37 | WorkloadSummary 38 | TaskCount map[TaskStatus]int `json:"task_count"` 39 | Env map[string]string `json:"env"` 40 | PullSecret string `json:"pull_secret"` 41 | } 42 | 43 | //CreateWorkloadInput is input for workload creation 44 | type CreateWorkloadInput struct { 45 | ProjectID string `json:"project_id" valid:"required"` 46 | Image string `json:"image" valid:"required"` 47 | NrOfWorkers int `json:"nr_of_workers" valid:"required"` 48 | InputDatasetID string `json:"input_dataset_id"` 49 | UseCuteur bool `json:"use_cuteur"` 50 | Env map[string]string `json:"env"` 51 | PullSecret string `json:"pull_secret"` 52 | ComputeUnits uint64 `json:"compute_units"` 53 | } 54 | 55 | //CreateWorkloadOutput is output for workload creation 56 | type CreateWorkloadOutput struct { 57 | WorkloadSummary 58 | } 59 | 60 | //StopWorkloadInput is input for workload deletion 61 | type StopWorkloadInput struct { 62 | ProjectID string `json:"project_id" valid:"required"` 63 | WorkloadID string `json:"workload_id" valid:"required"` 64 | } 65 | 66 | //StopWorkloadOutput is output for workload deletion 67 | type StopWorkloadOutput struct{} 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nerd - Nerdalize Command Line Interface 2 | Your personal `nerd` that takes care of running compute jobs on the [Nerdalize cloud](https://www.nerdalize.com/). 3 | 4 | --- 5 | 6 | 7 | 8 | [Nerdalize](https://www.nerdalize.com/) is building a different cloud. Instead of constructing huge datacenters, we're distributing our servers over homes. Homeowners use the residual heat for hot showers and to warm their house, and we don't need to build new infrastructure. 9 | 10 | In order to make our cloud resources accessible and easy to use, we've developed a CLI that fits your workflow. Whether you’re a researcher, engineer or developer, it allows you to easily run your computations, simulations and analyses on our cloud infrastructure. 11 | 12 | __Features__: 13 | - Moving __datasets__ from you workstation to the cloud and back is included right into the workflow 14 | - Nerd ensures efficient and quick datatransfers through a __deduplication__ algorithm 15 | - Send in __thousands of jobs__, Nerd makes sure your resources are used as efficiently as possible 16 | - Package your software using industry-standard __Docker__ containers 17 | - Follows basic CLI conventions to provide a __scriptable__ interface your daily dose of automation goodness 18 | 19 | ## Documentation 20 | To start running your compute on the Nerdalize cloud you'll need to set up an account and download the Nerd CLI itself. 21 | 22 | - [Quickstarts](https://www.nerdalize.com/docs/) - To quickly get you up and running. 23 | - [Ready-to-use Software ](https://www.nerdalize.com/applications/) - We have application-specific guides for Python or FFmpeg for you to get started. 24 | - [CLI Reference](https://www.nerdalize.com/docs/reference/cli/) - For a reference of all available commands 25 | 26 | --- 27 | ## Building from Source 28 | If you would like to contribute to the project it is possible to build the Nerd from source: 29 | 30 | 1. The CLI is written in Go. Make sure you've installed the language SDK as documented [here](https://golang.org/dl/) 31 | 2. Checkout the repository in your `GOPATH`: 32 | ``` 33 | git clone git@github.com:nerdalize/nerd.git $GOPATH/src/github.com/nerdalize/nerd 34 | ``` 35 | 3. Go to the checked out repository and build the binary using the included bash script: 36 | ``` 37 | cd $GOPATH/src/github.com/nerdalize/nerd 38 | ./make.sh build 39 | ``` 40 | 4. The Nerd CLI is now ready to be used in the `$GOPATH/bin` directory: 41 | ``` 42 | $GOPATH/bin/nerd 43 | Usage: nerd [--version] [--help] [] 44 | 45 | Available commands are: 46 | ... 47 | ``` 48 | -------------------------------------------------------------------------------- /nerd/jwt/jwt_test.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | const key = ` 9 | -----BEGIN PUBLIC KEY----- 10 | MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEAkYbLnam4wo+heLlTZEeh1ZWsfruz9nk 11 | kyvc4LwKZ8pez5KYY76H1ox+AfUlWOEq+bExypcFfEIrJkf/JXa7jpzkOWBDF9Sa 12 | OWbQHMK+vvUXieCJvCc9Vj084ABwLBgX 13 | -----END PUBLIC KEY----- 14 | ` 15 | 16 | func TestParseECDSAPublicKeyFromPemBytes(t *testing.T) { 17 | _, err := ParseECDSAPublicKeyFromPemBytes([]byte(key)) 18 | if err != nil { 19 | t.Errorf("Failed to parse valid public key. Error message: %v", err) 20 | } 21 | } 22 | 23 | func TestDecodeToken(t *testing.T) { 24 | testCases := map[string]struct { 25 | token string 26 | success bool 27 | errorMsg string 28 | expiresAt int64 29 | audience string 30 | }{ 31 | "expired valid": { 32 | token: "eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJhdXRoLm5lcmRhbGl6ZS5jb20iLCJleHAiOjE0ODgxODk1MzcsImF1ZCI6Im5jZS5uZXJkYWxpemUuY29tIiwibmJmIjoxNDg4MTg5MTc3LCJhY2Nlc3MiOlt7InNlcnZpY2UiOiJuY2UubmVyZGFsaXplLmNvbSIsInJlc291cmNlX3R5cGUiOiJjbHVzdGVyIiwicmVzb3VyY2VfaWRlbnRpZmllciI6ImNsdXN0ZXIxLm5lcmQubmV0L2UxNWxqMTJhYWZzZCIsImFjdGlvbnMiOiJDUlVEIn0seyJzZXJ2aWNlIjoibmNlLm5lcmRhbGl6ZS5jb20iLCJyZXNvdXJjZV90eXBlIjoib2JqZWN0X3N0b3JlIiwicmVzb3VyY2VfaWRlbnRpZmllciI6Im5lcmRzLnMzLmFtYXphbmF3cy5jb20vMjQ1bGtqMjM0NSIsImFjdGlvbnMiOiJSIn1dLCJzdWIiOiI0IiwiaWF0IjoxNDg4MTg5MjM3fQ.pk7yAGW8L80uEKAFUctupj4PO8UHIGmpikEi-ERkwZao73dEx5GlAnVmNTnXOO-xxjT8BomQtqL6Od15d7K6c4fY5YU8s64di4HA1SJuqIK0u0Mk8N6oVS216Y3FJkkD", 33 | success: true, 34 | errorMsg: "", 35 | expiresAt: 1488189537, 36 | audience: "nce.nerdalize.com", 37 | }, 38 | "json parse error": { 39 | token: "jwt.jwt.jwt", 40 | success: false, 41 | errorMsg: "failed to parse nerd token", 42 | expiresAt: 111, 43 | audience: "nlz.com", 44 | }, 45 | } 46 | 47 | for name, tc := range testCases { 48 | claims, err := DecodeTokenWithPEM(tc.token, key) 49 | if tc.success { 50 | if err != nil { 51 | t.Errorf("%v: expected success but got error '%v'", name, err) 52 | continue 53 | } 54 | if claims.Audience != tc.audience { 55 | t.Errorf("%v: expected audience to be '%v' but was '%v'", name, tc.audience, claims.Audience) 56 | } 57 | if claims.ExpiresAt != tc.expiresAt { 58 | t.Errorf("%v: expected expiresAt to be '%v' but was '%v'", name, tc.expiresAt, claims.ExpiresAt) 59 | } 60 | } else { 61 | if err == nil { 62 | t.Errorf("%v: expected failure but got success", name) 63 | continue 64 | } 65 | if !strings.Contains(err.Error(), tc.errorMsg) { 66 | t.Errorf("%v: expected error message to contain '%v' but error message was '%v'", name, tc.errorMsg, err.Error()) 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/plan.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | "net/http" 5 | 6 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 7 | ) 8 | 9 | // ClientPlanInterface is an interface so client plan calls can be mocked. 10 | type ClientPlanInterface interface { 11 | CreatePlan(projectID, PlanID, computeUnits string) (output *v1payload.CreatePlanOutput, err error) 12 | RemovePlan(projectID, PlanID string) (output *v1payload.RemovePlanOutput, err error) 13 | DeletePlan(PlanID string) (output *v1payload.DeletePlanOutput, err error) 14 | ListPlans(projectID string) (output *v1payload.ListPlansOutput, err error) 15 | UpdatePlan(projectID, PlanID, computeUnits string) (output *v1payload.UpdatePlanOutput, err error) 16 | } 17 | 18 | // CreatePlan will create a plan for the precised project. 19 | func (c *Client) CreatePlan(projectID, PlanID, computeUnits string) (output *v1payload.CreatePlanOutput, err error) { 20 | output = &v1payload.CreatePlanOutput{} 21 | input := &v1payload.CreatePlanInput{ 22 | PlanID: PlanID, 23 | ComputeUnits: computeUnits, 24 | } 25 | 26 | return output, c.doRequest(http.MethodPost, createPath(projectID, plansEndpoint), input, output) 27 | } 28 | 29 | // RemovePlan will delete a plan from the precised project. 30 | func (c *Client) RemovePlan(projectID, PlanID string) (output *v1payload.RemovePlanOutput, err error) { 31 | output = &v1payload.RemovePlanOutput{} 32 | input := &v1payload.RemovePlanInput{} 33 | 34 | return output, c.doRequest(http.MethodDelete, createPath(projectID, plansEndpoint, PlanID), input, output) 35 | } 36 | 37 | // DeletePlan will delete a plan with the provided . 38 | func (c *Client) DeletePlan(PlanID string) (output *v1payload.DeletePlanOutput, err error) { 39 | output = &v1payload.DeletePlanOutput{} 40 | input := &v1payload.DeletePlanInput{} 41 | 42 | return output, c.doRequest(http.MethodDelete, createPath("", plansEndpoint, PlanID), input, output) 43 | } 44 | 45 | // ListPlans will return all Plans for a particular project if precised. 46 | func (c *Client) ListPlans(projectID string) (output *v1payload.ListPlansOutput, err error) { 47 | output = &v1payload.ListPlansOutput{} 48 | input := &v1payload.ListPlansInput{} 49 | 50 | return output, c.doRequest(http.MethodGet, createPath(projectID, plansEndpoint), input, output) 51 | } 52 | 53 | // UpdatePlan returns a plan with an updated cpu request. 54 | func (c *Client) UpdatePlan(projectID, PlanID, computeUnits string) (output *v1payload.UpdatePlanOutput, err error) { 55 | output = &v1payload.UpdatePlanOutput{} 56 | input := &v1payload.UpdatePlanInput{ 57 | ComputeUnits: computeUnits, 58 | } 59 | 60 | return output, c.doRequest(http.MethodPut, createPath(projectID, plansEndpoint, PlanID), input, output) 61 | } 62 | -------------------------------------------------------------------------------- /svc/kube_delete_secret_test.go: -------------------------------------------------------------------------------- 1 | package svc_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | "runtime" 8 | "testing" 9 | "time" 10 | 11 | "github.com/nerdalize/nerd/pkg/kubevisor" 12 | "github.com/nerdalize/nerd/svc" 13 | ) 14 | 15 | func TestDeleteSecret(t *testing.T) { 16 | for _, c := range []struct { 17 | Name string 18 | Timeout time.Duration 19 | Secrets []*svc.CreateSecretInput 20 | Input *svc.DeleteSecretInput 21 | Output *svc.DeleteSecretOutput 22 | Listing *svc.ListSecretsOutput 23 | IsOutput func(tb testing.TB, out *svc.DeleteSecretOutput, l *svc.ListSecretsOutput) 24 | IsErr func(error) bool 25 | }{ 26 | { 27 | Name: "when no name is provided it should provide a validation error", 28 | Timeout: time.Second * 5, 29 | Input: &svc.DeleteSecretInput{}, 30 | Output: &svc.DeleteSecretOutput{}, 31 | IsErr: svc.IsValidationErr, 32 | }, 33 | { 34 | Name: "when a non-existing secret is deleted it should return NotExists error", 35 | Timeout: time.Second * 5, 36 | Input: &svc.DeleteSecretInput{Name: "foo"}, 37 | Output: &svc.DeleteSecretOutput{}, 38 | IsErr: kubevisor.IsNotExistsErr, 39 | }, 40 | } { 41 | t.Run(c.Name, func(t *testing.T) { 42 | di, clean := testDI(t) 43 | defer clean() 44 | 45 | ctx := context.Background() 46 | ctx, cancel := context.WithTimeout(ctx, c.Timeout) 47 | defer cancel() 48 | 49 | kube := svc.NewKube(di) 50 | for _, secret := range c.Secrets { 51 | _, err := kube.CreateSecret(ctx, secret) 52 | ok(t, err) 53 | } 54 | 55 | out, err := kube.DeleteSecret(ctx, c.Input) 56 | if c.IsErr != nil { 57 | assert(t, c.IsErr(err), fmt.Sprintf("unexpected '%#v' to match: %#v", err, runtime.FuncForPC(reflect.ValueOf(c.IsErr).Pointer()).Name())) 58 | } 59 | 60 | list, err := kube.ListSecrets(ctx, &svc.ListSecretsInput{}) 61 | ok(t, err) 62 | 63 | if c.IsOutput != nil { 64 | c.IsOutput(t, out, list) 65 | } 66 | }) 67 | } 68 | } 69 | 70 | func TestDeleteSpecificSecret(t *testing.T) { 71 | timeout := time.Minute 72 | 73 | if testing.Short() { 74 | t.Skipf("skipping long test with contex timeout: %s", timeout) 75 | } 76 | 77 | di, clean := testDI(t) 78 | defer clean() 79 | 80 | ctx := context.Background() 81 | ctx, cancel := context.WithTimeout(ctx, timeout) 82 | defer cancel() 83 | 84 | kube := svc.NewKube(di) 85 | secret, err := kube.CreateSecret(ctx, &svc.CreateSecretInput{Image: "smoketest", Project: "nerdalize", Registry: "quay.io", Username: "test", Password: "test"}) 86 | ok(t, err) 87 | 88 | out, err := kube.DeleteSecret(ctx, &svc.DeleteSecretInput{Name: secret.Name}) 89 | ok(t, err) 90 | assert(t, out != nil, "expected to find a DeleteSecretOutput") 91 | } 92 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/run.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 8 | ) 9 | 10 | //ClientRunInterface is an interface so client task calls can be mocked. 11 | type ClientRunInterface interface { 12 | SendRunHeartbeat(projectID, workloadID string, taskID int64, runToken string) (output *v1payload.SendRunHeartbeatOutput, err error) 13 | SendRunSuccess(projectID, workloadID string, taskID int64, runToken, result, outputDatasetID string) (output *v1payload.SendRunSuccessOutput, err error) 14 | SendRunFailure(projectID, workloadID string, taskID int64, runToken, errCode, errMessage string) (output *v1payload.SendRunFailureOutput, err error) 15 | } 16 | 17 | //SendRunHeartbeat will send a heartbeat for a task run 18 | func (c *Client) SendRunHeartbeat(projectID, workloadID string, taskID int64, runToken string) (output *v1payload.SendRunHeartbeatOutput, err error) { 19 | output = &v1payload.SendRunHeartbeatOutput{} 20 | input := &v1payload.SendRunHeartbeatInput{ 21 | TaskID: taskID, 22 | ProjectID: projectID, 23 | WorkloadID: workloadID, 24 | RunToken: runToken, 25 | } 26 | 27 | return output, c.doRequest(http.MethodPost, createPath(projectID, workloadsEndpoint, workloadID, "tasks", strconv.FormatInt(taskID, 10), "heartbeats"), input, output) 28 | } 29 | 30 | //SendRunSuccess will send a successfully run for a task 31 | func (c *Client) SendRunSuccess(projectID, workloadID string, taskID int64, runToken, result, outputDatasetID string) (output *v1payload.SendRunSuccessOutput, err error) { 32 | output = &v1payload.SendRunSuccessOutput{} 33 | input := &v1payload.SendRunSuccessInput{ 34 | TaskID: taskID, 35 | ProjectID: projectID, 36 | WorkloadID: workloadID, 37 | RunToken: runToken, 38 | Result: result, 39 | OutputDatasetID: outputDatasetID, 40 | } 41 | 42 | return output, c.doRequest(http.MethodPost, createPath(projectID, workloadsEndpoint, workloadID, "tasks", strconv.FormatInt(taskID, 10), "success"), input, output) 43 | } 44 | 45 | //SendRunFailure will send a failure for a run 46 | func (c *Client) SendRunFailure(projectID, workloadID string, taskID int64, runToken, errCode, errMessage string) (output *v1payload.SendRunFailureOutput, err error) { 47 | output = &v1payload.SendRunFailureOutput{} 48 | input := &v1payload.SendRunFailureInput{ 49 | TaskID: taskID, 50 | ProjectID: projectID, 51 | WorkloadID: workloadID, 52 | RunToken: runToken, 53 | ErrorCode: errCode, 54 | ErrorMessage: errMessage, 55 | } 56 | 57 | return output, c.doRequest(http.MethodPost, createPath(projectID, workloadsEndpoint, workloadID, "tasks", strconv.FormatInt(taskID, 10), "failure"), input, output) 58 | } 59 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/payload/secret.go: -------------------------------------------------------------------------------- 1 | package v1payload 2 | 3 | const ( 4 | //SecretTypeRegistry is the type for registring images 5 | SecretTypeRegistry string = "registry" 6 | 7 | //SecretTypeOpaque is the type that allows secrets that are opaque 8 | SecretTypeOpaque string = "opaque" 9 | ) 10 | 11 | // CreateSecretInput is the input for creating a secret 12 | type CreateSecretInput struct { 13 | ProjectID string `json:"project_id" valid:"required"` 14 | Name string `json:"name" valid:"required"` 15 | Key string `json:"key"` 16 | Value string `json:"value"` 17 | DockerUsername string `json:"dockerUsername"` 18 | DockerPassword string `json:"dockerPassword"` 19 | Type string `json:"type" valid:"required,in(opaque|registry)"` 20 | } 21 | 22 | // CreateSecretOutput is the output from creating a secret 23 | type CreateSecretOutput struct { 24 | ProjectID string `json:"project_id" valid:"required"` 25 | Name string `json:"name" valid:"required"` 26 | Type string `json:"type" valid:"required,in(opaque|registry)"` 27 | } 28 | 29 | // DescribeSecretInput is the input for describing a secret 30 | type DescribeSecretInput struct { 31 | ProjectID string `json:"project_id" valid:"required"` 32 | Name string `json:"name" valid:"required"` 33 | } 34 | 35 | // DescribeSecretOutput is the output from describing a secret 36 | type DescribeSecretOutput struct { 37 | ProjectID string `json:"project_id" valid:"required"` 38 | Name string `json:"name" valid:"required"` 39 | Key string `json:"key"` 40 | Value string `json:"value"` 41 | DockerUsername string `json:"dockerUsername"` 42 | DockerPassword string `json:"dockerPassword"` 43 | Type string `json:"type" valid:"required,in(opaque|registry)"` 44 | } 45 | 46 | // DeleteSecretInput is the input for deleting a secret 47 | type DeleteSecretInput struct { 48 | ProjectID string `json:"project_id" valid:"required"` 49 | Name string `json:"name" valid:"required"` 50 | } 51 | 52 | // DeleteSecretOutput is the output from deleting a secret 53 | type DeleteSecretOutput struct { 54 | } 55 | 56 | //SecretSummary is summary of a secret 57 | type SecretSummary struct { 58 | ProjectID string `json:"project_id" valid:"required"` 59 | Name string `json:"name" valid:"required"` 60 | Type string `json:"type" valid:"required,in(opaque|registry)"` 61 | } 62 | 63 | // ListSecretsInput is the input for listing secrets 64 | type ListSecretsInput struct { 65 | ProjectID string `json:"project_id" valid:"required"` 66 | } 67 | 68 | // ListSecretsOutput is the output from listing secrets 69 | type ListSecretsOutput struct { 70 | ProjectID string `json:"project_id" valid:"required"` 71 | Secrets []*SecretSummary `json:"secrets" valid:"required"` 72 | } 73 | -------------------------------------------------------------------------------- /svc/kube_list_quotas.go: -------------------------------------------------------------------------------- 1 | package svc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/nerdalize/nerd/pkg/kubevisor" 7 | 8 | corev1 "k8s.io/api/core/v1" 9 | ) 10 | 11 | //ListQuotaItem describes a namespace quota 12 | type ListQuotaItem struct { 13 | RequestCPU int64 14 | LimitCPU int64 15 | LimitMemory int64 16 | RequestMemory int64 17 | 18 | UseRequestCPU int64 19 | UseLimitCPU int64 20 | UseLimitMemory int64 21 | UseRequestMemory int64 22 | 23 | Labels map[string]string 24 | } 25 | 26 | //NodeLimitedQuota is used when no quota is configured 27 | var NodeLimitedQuota = ListQuotaItem{} 28 | 29 | //ListQuotasInput is the input to ListQuotas 30 | type ListQuotasInput struct{} 31 | 32 | //ListQuotasOutput is the output to ListQuotas 33 | type ListQuotasOutput struct { 34 | Items []*ListQuotaItem 35 | } 36 | 37 | //ListQuotas will list quotas on kubernetes 38 | func (k *Kube) ListQuotas(ctx context.Context, in *ListQuotasInput) (out *ListQuotasOutput, err error) { 39 | if err = k.checkInput(ctx, in); err != nil { 40 | return nil, err 41 | } 42 | 43 | quotas := "as{} 44 | err = k.visor.ListResources(ctx, kubevisor.ResourceTypeQuota, quotas, nil, nil) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | out = &ListQuotasOutput{} 50 | for _, q := range quotas.Items { 51 | reqCPU, _ := q.Status.Hard[corev1.ResourceRequestsCPU] 52 | reqMem, _ := q.Status.Hard[corev1.ResourceRequestsMemory] 53 | limCPU, _ := q.Status.Hard[corev1.ResourceLimitsCPU] 54 | limMem, _ := q.Status.Hard[corev1.ResourceLimitsMemory] 55 | useReqCPU, _ := q.Status.Used[corev1.ResourceRequestsCPU] 56 | useReqMem, _ := q.Status.Used[corev1.ResourceRequestsMemory] 57 | useLimCPU, _ := q.Status.Used[corev1.ResourceLimitsCPU] 58 | useLimMem, _ := q.Status.Used[corev1.ResourceLimitsMemory] 59 | 60 | out.Items = append(out.Items, &ListQuotaItem{ 61 | RequestCPU: reqCPU.MilliValue(), 62 | RequestMemory: reqMem.MilliValue(), 63 | LimitCPU: limCPU.MilliValue(), 64 | LimitMemory: limMem.MilliValue(), 65 | 66 | UseLimitCPU: useLimCPU.MilliValue(), 67 | UseRequestCPU: useReqCPU.MilliValue(), 68 | UseLimitMemory: useLimMem.MilliValue(), 69 | UseRequestMemory: useReqMem.MilliValue(), 70 | 71 | Labels: q.Labels, 72 | }) 73 | } 74 | 75 | return out, nil 76 | } 77 | 78 | //quotas implements the list transformer interface to allow the kubevisor the manage names for us 79 | type quotas struct{ *corev1.ResourceQuotaList } 80 | 81 | func (quotas *quotas) Transform(fn func(in kubevisor.ManagedNames) (out kubevisor.ManagedNames)) { 82 | for i, j1 := range quotas.ResourceQuotaList.Items { 83 | quotas.Items[i] = *(fn(&j1).(*corev1.ResourceQuota)) 84 | } 85 | } 86 | 87 | func (quotas *quotas) Len() int { 88 | return len(quotas.ResourceQuotaList.Items) 89 | } 90 | -------------------------------------------------------------------------------- /svc/kube_get_dataset_test.go: -------------------------------------------------------------------------------- 1 | package svc_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | "runtime" 8 | "testing" 9 | "time" 10 | 11 | "github.com/nerdalize/nerd/pkg/kubevisor" 12 | "github.com/nerdalize/nerd/pkg/transfer/archiver" 13 | "github.com/nerdalize/nerd/pkg/transfer/store" 14 | "github.com/nerdalize/nerd/svc" 15 | ) 16 | 17 | func TestGetDataset(t *testing.T) { 18 | for _, c := range []struct { 19 | Name string 20 | Timeout time.Duration 21 | Datasets []*svc.CreateDatasetInput 22 | Input *svc.GetDatasetInput 23 | IsOutput func(tb testing.TB, out *svc.GetDatasetOutput) bool 24 | IsErr func(error) bool 25 | }{ 26 | { 27 | Name: "when a zero value input is provided it should return a validation error", 28 | Timeout: time.Second * 5, 29 | Datasets: nil, 30 | Input: nil, 31 | IsErr: svc.IsValidationErr, 32 | IsOutput: func(t testing.TB, out *svc.GetDatasetOutput) bool { 33 | return true 34 | }, 35 | }, 36 | { 37 | Name: "when dataset doesnt exist it should return an error", 38 | Timeout: time.Second * 5, 39 | Input: &svc.GetDatasetInput{Name: "my-dataset"}, 40 | IsErr: kubevisor.IsNotExistsErr, 41 | IsOutput: func(t testing.TB, out *svc.GetDatasetOutput) bool { 42 | return true 43 | }, 44 | }, 45 | { 46 | Name: "when one dataset has been uploaded it should be available for download", 47 | Timeout: time.Minute, 48 | Datasets: []*svc.CreateDatasetInput{{Name: "my-dataset", StoreOptions: transferstore.StoreOptions{Type: transferstore.StoreTypeS3}, ArchiverOptions: transferarchiver.ArchiverOptions{Type: transferarchiver.ArchiverTypeTar}}}, 49 | Input: &svc.GetDatasetInput{Name: "my-datasets"}, 50 | IsErr: nil, 51 | IsOutput: func(t testing.TB, out *svc.GetDatasetOutput) bool { 52 | if out == nil { 53 | return false 54 | } 55 | return true 56 | }, 57 | }, 58 | } { 59 | t.Run(c.Name, func(t *testing.T) { 60 | if c.Timeout > time.Second*5 && testing.Short() { 61 | t.Skipf("skipping long test with contex timeout: %s", c.Timeout) 62 | } 63 | 64 | di, clean := testDI(t) 65 | defer clean() 66 | 67 | ctx := context.Background() 68 | ctx, cancel := context.WithTimeout(ctx, c.Timeout) 69 | defer cancel() 70 | 71 | kube := svc.NewKube(di) 72 | for _, datasets := range c.Datasets { 73 | _, err := kube.CreateDataset(ctx, datasets) 74 | ok(t, err) 75 | } 76 | 77 | out, err := kube.GetDataset(ctx, c.Input) 78 | if c.IsErr != nil { //if c.IsErr is nil we dont care about errors 79 | assert(t, c.IsErr(err), fmt.Sprintf("unexpected '%#v' to match: %#v", err, runtime.FuncForPC(reflect.ValueOf(c.IsErr).Pointer()).Name())) 80 | } 81 | 82 | if c.IsOutput != nil { 83 | c.IsOutput(t, out) 84 | } 85 | }) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /pkg/kubevisor/types.go: -------------------------------------------------------------------------------- 1 | package kubevisor 2 | 3 | import ( 4 | crd "github.com/nerdalize/nerd/crd/pkg/client/clientset/versioned" 5 | apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 6 | "k8s.io/client-go/kubernetes" 7 | ) 8 | 9 | //ResourceType is a type of Kubernetes resource 10 | type ResourceType string 11 | 12 | var ( 13 | //ResourceTypeJobs is used for job management 14 | ResourceTypeJobs = ResourceType("jobs") 15 | 16 | //ResourceTypePods is used for pod inspection 17 | ResourceTypePods = ResourceType("pods") 18 | 19 | //ResourceTypeDatasets is used for dataset management 20 | ResourceTypeDatasets = ResourceType("datasets") 21 | 22 | //ResourceTypeEvents is the resource type for event fetching 23 | ResourceTypeEvents = ResourceType("events") 24 | 25 | //ResourceTypeQuota can be used to retrieve quota information 26 | ResourceTypeQuota = ResourceType("resourcequotas") 27 | 28 | //ResourceTypeSecrets can be used to get secret information 29 | ResourceTypeSecrets = ResourceType("secrets") 30 | 31 | //ResourceTypeDeployments is used for deployment management 32 | ResourceTypeDeployments = ResourceType("deployments") 33 | 34 | //ResourceTypeRoles is used for role management 35 | ResourceTypeRoles = ResourceType("roles") 36 | 37 | //ResourceTypeRoleBindings is used for role bindings management 38 | ResourceTypeRoleBindings = ResourceType("rolebindings") 39 | 40 | //ResourceTypeClusterRoles is used for cluster roles management 41 | ResourceTypeClusterRoles = ResourceType("clusterroles") 42 | 43 | //ResourceTypeClusterRoleBindings is used for cluster role bindings management 44 | ResourceTypeClusterRoleBindings = ResourceType("clusterrolebindings") 45 | 46 | //ResourceTypeDaemonsets is used for daemonset management 47 | ResourceTypeDaemonsets = ResourceType("daemonsets") 48 | 49 | //ResourceTypeCustomResourceDefinition is used for crd management 50 | ResourceTypeCustomResourceDefinition = ResourceType("customresourcedefinitions") 51 | ) 52 | 53 | //ManagedNames allows for Nerd to transparently manage resources based on names and there prefixes 54 | type ManagedNames interface { 55 | GetName() string 56 | GetLabels() map[string]string 57 | SetLabels(map[string]string) 58 | SetName(name string) 59 | SetGenerateName(name string) 60 | } 61 | 62 | //ListTranformer must be implemented to allow Nerd to transparently manage resource names 63 | type ListTranformer interface { 64 | Transform(fn func(in ManagedNames) (out ManagedNames)) 65 | Len() int 66 | } 67 | 68 | //Visor provides access to Kubernetes resources while transparently filtering, naming and labeling 69 | //resources that are managed by the CLI. 70 | type Visor struct { 71 | prefix string 72 | ns string 73 | api kubernetes.Interface 74 | crd crd.Interface 75 | apiext apiext.Interface 76 | logs Logger 77 | } 78 | -------------------------------------------------------------------------------- /nerd/client/batch/v1/workload.go: -------------------------------------------------------------------------------- 1 | package v1batch 2 | 3 | import ( 4 | "net/http" 5 | 6 | v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload" 7 | ) 8 | 9 | //ClientWorkloadInterface is an interface so client workload calls can be mocked. 10 | type ClientWorkloadInterface interface { 11 | CreateWorkload(projectID, image, inputDatasetID, pullSecret string, computeUnits uint64, env map[string]string, instances int, useCuteur bool) (output *v1payload.CreateWorkloadOutput, err error) 12 | StopWorkload(projectID, workloadID string) (output *v1payload.StopWorkloadOutput, err error) 13 | ListWorkloads(projectID string) (output *v1payload.ListWorkloadsOutput, err error) 14 | DescribeWorkload(projectID, workloadID string) (output *v1payload.DescribeWorkloadOutput, err error) 15 | } 16 | 17 | //CreateWorkload will start a workload 18 | func (c *Client) CreateWorkload(projectID, image, inputDatasetID, pullSecret string, computeUnits uint64, env map[string]string, nrOfWorkers int, useCuteur bool) (output *v1payload.CreateWorkloadOutput, err error) { 19 | output = &v1payload.CreateWorkloadOutput{} 20 | input := &v1payload.CreateWorkloadInput{ 21 | ProjectID: projectID, 22 | Image: image, 23 | InputDatasetID: inputDatasetID, 24 | Env: env, 25 | NrOfWorkers: nrOfWorkers, 26 | UseCuteur: useCuteur, 27 | PullSecret: pullSecret, 28 | ComputeUnits: computeUnits, 29 | } 30 | 31 | return output, c.doRequest(http.MethodPost, createPath(projectID, workloadsEndpoint), input, output) 32 | } 33 | 34 | //StopWorkload will stop a workload 35 | func (c *Client) StopWorkload(projectID, workloadID string) (output *v1payload.StopWorkloadOutput, err error) { 36 | output = &v1payload.StopWorkloadOutput{} 37 | input := &v1payload.StopWorkloadInput{ 38 | ProjectID: projectID, 39 | WorkloadID: workloadID, 40 | } 41 | 42 | return output, c.doRequest(http.MethodDelete, createPath(projectID, workloadsEndpoint, workloadID), input, output) 43 | } 44 | 45 | // ListWorkloads will return all workloads 46 | func (c *Client) ListWorkloads(projectID string) (output *v1payload.ListWorkloadsOutput, err error) { 47 | output = &v1payload.ListWorkloadsOutput{} 48 | input := &v1payload.ListWorkloadsInput{ 49 | ProjectID: projectID, 50 | } 51 | 52 | return output, c.doRequest(http.MethodGet, createPath(projectID, workloadsEndpoint), input, output) 53 | } 54 | 55 | //DescribeWorkload returns detailed information of a workload 56 | func (c *Client) DescribeWorkload(projectID, workloadID string) (output *v1payload.DescribeWorkloadOutput, err error) { 57 | output = &v1payload.DescribeWorkloadOutput{} 58 | input := &v1payload.DescribeWorkloadInput{ 59 | ProjectID: projectID, 60 | WorkloadID: workloadID, 61 | } 62 | 63 | return output, c.doRequest(http.MethodGet, createPath(projectID, workloadsEndpoint, workloadID), input, output) 64 | } 65 | -------------------------------------------------------------------------------- /crd/pkg/client/clientset/versioned/typed/stable.nerdalize.com/v1/stable.nerdalize.com_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Nerdalize 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package v1 17 | 18 | import ( 19 | v1 "github.com/nerdalize/nerd/crd/pkg/apis/stable.nerdalize.com/v1" 20 | "github.com/nerdalize/nerd/crd/pkg/client/clientset/versioned/scheme" 21 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 22 | rest "k8s.io/client-go/rest" 23 | ) 24 | 25 | type NerdalizeV1Interface interface { 26 | RESTClient() rest.Interface 27 | DatasetsGetter 28 | } 29 | 30 | // NerdalizeV1Client is used to interact with features provided by the nerdalize.com group. 31 | type NerdalizeV1Client struct { 32 | restClient rest.Interface 33 | } 34 | 35 | func (c *NerdalizeV1Client) Datasets(namespace string) DatasetInterface { 36 | return newDatasets(c, namespace) 37 | } 38 | 39 | // NewForConfig creates a new NerdalizeV1Client for the given config. 40 | func NewForConfig(c *rest.Config) (*NerdalizeV1Client, error) { 41 | config := *c 42 | if err := setConfigDefaults(&config); err != nil { 43 | return nil, err 44 | } 45 | client, err := rest.RESTClientFor(&config) 46 | if err != nil { 47 | return nil, err 48 | } 49 | return &NerdalizeV1Client{client}, nil 50 | } 51 | 52 | // NewForConfigOrDie creates a new NerdalizeV1Client for the given config and 53 | // panics if there is an error in the config. 54 | func NewForConfigOrDie(c *rest.Config) *NerdalizeV1Client { 55 | client, err := NewForConfig(c) 56 | if err != nil { 57 | panic(err) 58 | } 59 | return client 60 | } 61 | 62 | // New creates a new NerdalizeV1Client for the given RESTClient. 63 | func New(c rest.Interface) *NerdalizeV1Client { 64 | return &NerdalizeV1Client{c} 65 | } 66 | 67 | func setConfigDefaults(config *rest.Config) error { 68 | gv := v1.SchemeGroupVersion 69 | config.GroupVersion = &gv 70 | config.APIPath = "/apis" 71 | config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} 72 | 73 | if config.UserAgent == "" { 74 | config.UserAgent = rest.DefaultKubernetesUserAgent() 75 | } 76 | 77 | return nil 78 | } 79 | 80 | // RESTClient returns a RESTClient that is used to communicate 81 | // with API server by this client implementation. 82 | func (c *NerdalizeV1Client) RESTClient() rest.Interface { 83 | if c == nil { 84 | return nil 85 | } 86 | return c.restClient 87 | } 88 | --------------------------------------------------------------------------------