├── .circleci ├── config.yml └── images │ └── primary │ └── Dockerfile ├── .gitignore ├── .goreleaser.yml ├── LICENSE ├── Makefile ├── README.rst ├── cmd └── gaia │ └── main.go ├── docker ├── Dockerfile ├── Dockerfile.cpp ├── Dockerfile.golang ├── Dockerfile.java ├── Dockerfile.nodejs ├── Dockerfile.python ├── Dockerfile.ruby ├── docker-entrypoint.sh └── settings-docker.xml ├── docs ├── docs.go ├── swagger.json └── swagger.yaml ├── frontend ├── .browserslistrc ├── .editorconfig ├── .eslintrc.js ├── babel.config.js ├── package-lock.json ├── package.json ├── postcss.config.js ├── public │ ├── favicon.ico │ └── index.html ├── src │ ├── App.vue │ ├── assets │ │ ├── fonts │ │ │ └── Lobster-Regular.ttf │ │ └── images │ │ │ ├── cpp.png │ │ │ ├── fail.png │ │ │ ├── golang.png │ │ │ ├── inprogress.png │ │ │ ├── java.png │ │ │ ├── logo.png │ │ │ ├── nodejs.png │ │ │ ├── python.png │ │ │ ├── questionmark.png │ │ │ ├── ruby.png │ │ │ └── success.png │ ├── auth │ │ └── index.js │ ├── components │ │ └── layout │ │ │ ├── AppMain.vue │ │ │ ├── Headerbar.vue │ │ │ ├── Login.vue │ │ │ ├── Navbar.vue │ │ │ ├── Sidebar.vue │ │ │ └── index.js │ ├── filters │ │ └── index.js │ ├── helper │ │ └── index.js │ ├── main.js │ ├── router │ │ └── index.js │ ├── store │ │ ├── actions.js │ │ ├── getters.js │ │ ├── index.js │ │ ├── modules │ │ │ ├── app.js │ │ │ └── menu │ │ │ │ ├── index.js │ │ │ │ └── lazyLoading.js │ │ ├── mutation-types.js │ │ └── mutations.js │ └── views │ │ ├── overview │ │ └── index.vue │ │ ├── pipeline │ │ ├── create.vue │ │ ├── detail.vue │ │ ├── log.vue │ │ └── params.vue │ │ ├── settings │ │ ├── index.vue │ │ ├── permissions │ │ │ └── manage-permissions.vue │ │ ├── settings │ │ │ └── manage-settings.vue │ │ └── worker │ │ │ └── manage-worker.vue │ │ └── vault │ │ └── index.vue └── vue.config.js ├── gaia.go ├── go.mod ├── go.sum ├── handlers ├── auth.go ├── auth_test.go ├── errors │ └── handler_errors.go ├── handler.go ├── handler_test.go ├── permission.go ├── permission_test.go ├── service.go ├── settings.go ├── settings_test.go ├── vault.go └── vault_test.go ├── helm ├── .helmignore ├── Chart.yaml ├── _system │ ├── default-http-backend │ │ ├── deployment.yaml │ │ └── service.yaml │ └── nginx-ingress │ │ ├── configmap-lb.yaml │ │ ├── configmap-tcp.yaml │ │ └── daemonset.yaml ├── templates │ ├── deployment.yaml │ ├── ingress.yaml │ └── service.yaml └── values.yaml ├── helper ├── assethelper │ └── helper.go ├── filehelper │ ├── filehelper.go │ └── filehelper_test.go ├── pipelinehelper │ ├── pipelinehelper.go │ └── pipelinehelper_test.go ├── rolehelper │ ├── role.go │ └── role_test.go └── stringhelper │ ├── stringhelper.go │ └── stringhelper_test.go ├── plugin ├── grpc.go ├── plugin.go └── plugin_test.go ├── providers ├── pipelines │ ├── fixtures │ │ └── hook_basic_push_payload.json │ ├── hook.go │ ├── pipeline.go │ ├── pipeline_provider.go │ ├── pipeline_run.go │ ├── pipeline_test.go │ └── settings.go ├── provider.go ├── rbac │ ├── provider.go │ └── provider_test.go ├── user │ ├── provider.go │ └── provider_test.go └── workers │ ├── worker_provider.go │ ├── workers.go │ └── workers_test.go ├── screenshots ├── create-pipeline.png ├── detail-pipeline.png ├── login.png ├── logs-pipeline.png ├── overview.png ├── settings.png └── vault.png ├── security ├── README.md ├── ca.go ├── ca_test.go ├── legacy_vault.go ├── rbac │ ├── api_mappings_test.go │ ├── endpoint_enforcer.go │ ├── endpoint_enforcer_test.go │ ├── noop.go │ ├── noop_test.go │ ├── service.go │ └── service_test.go ├── secret_generator.go ├── secret_generator_test.go ├── testdata │ ├── ca.key │ └── gaia_vault ├── vault.go └── vault_test.go ├── server └── server.go ├── services ├── service_provider.go └── service_provider_test.go ├── static ├── rbac-api-mappings.yml ├── rbac-model.conf └── rbac-policy.csv ├── store ├── memdb │ ├── memdb.go │ ├── memdb_schema.go │ └── memdb_test.go ├── permission.go ├── pipeline.go ├── settings.go ├── settings_test.go ├── sha_pair.go ├── sha_pair_test.go ├── store.go ├── store_test.go ├── upgrade.go ├── user.go ├── worker.go └── worker_test.go └── workers ├── agent ├── agent.go ├── agent_test.go ├── api │ ├── api.go │ └── api_test.go ├── fixtures │ ├── caCert.pem │ ├── cert.pem │ └── key.pem ├── tags.go └── tags_test.go ├── docker └── docker.go ├── pipeline ├── build_cpp.go ├── build_cpp_test.go ├── build_golang.go ├── build_golang_test.go ├── build_java.go ├── build_java_test.go ├── build_nodejs.go ├── build_nodejs_test.go ├── build_python.go ├── build_python_test.go ├── build_ruby.go ├── build_ruby_test.go ├── create_pipeline.go ├── create_pipeline_test.go ├── git.go ├── git_test.go ├── pipeline.go ├── pipeline_test.go ├── service.go ├── test_helper_test.go ├── testacc │ ├── build_pipeline_test.go │ └── test.pem ├── ticker.go ├── ticker_test.go ├── update_pipeline.go └── update_pipeline_test.go ├── proto ├── README.md ├── worker.pb.go └── worker.proto ├── scheduler ├── gaiascheduler │ ├── create_cmd.go │ ├── create_cmd_test.go │ ├── scheduler.go │ ├── scheduler_test.go │ ├── workload.go │ └── workload_test.go └── service │ └── scheduler.go └── server ├── server.go ├── server_test.go ├── worker.go └── worker_test.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | linter: 5 | working_directory: /go/src/github.com/gaia-pipeline/gaia 6 | docker: 7 | - image: circleci/golang:1.17 8 | environment: 9 | GO111MODULE: "on" 10 | steps: 11 | - checkout 12 | - run: 13 | name: Install linter 14 | command: | 15 | go get -u golang.org/x/lint/golint 16 | - run: 17 | name: Run linter 18 | command: | 19 | make lint 20 | test_and_coverage: 21 | working_directory: /go/src/github.com/gaia-pipeline/gaia 22 | docker: 23 | - image: circleci/golang:1.17 24 | environment: 25 | GO111MODULE: "on" 26 | steps: 27 | - checkout 28 | - run: 29 | name: Run unit tests 30 | command: | 31 | set -e 32 | echo "" > coverage.txt 33 | 34 | for d in $(go list ./... | grep -v vendor | grep -v /testacc); do 35 | go test -v -timeout 50s -race -coverprofile=profile.out -covermode=atomic $d 36 | if [ -f profile.out ]; then 37 | cat profile.out >> coverage.txt 38 | rm profile.out 39 | fi 40 | done 41 | - run: 42 | name: Upload test report to codecov.io 43 | command: bash <(curl -s https://codecov.io/bash) 44 | - run: 45 | name: Build binary without frontend 46 | command: | 47 | make download 48 | make compile_backend 49 | ./gaia-linux-amd64 --version 50 | acceptance_tests: 51 | working_directory: /go/src/github.com/gaia-pipeline/gaia 52 | docker: 53 | - image: gaiapipeline/circleci:0.0.7 54 | environment: 55 | GO111MODULE: "on" 56 | steps: 57 | - checkout 58 | - run: 59 | name: Run acceptance tests 60 | command: | 61 | set -e 62 | make download 63 | make test-acc 64 | compile: 65 | working_directory: /go/src/github.com/gaia-pipeline/gaia 66 | docker: 67 | - image: circleci/golang:1.17 68 | environment: 69 | GO111MODULE: "on" 70 | steps: 71 | - checkout 72 | - run: 73 | name: Install nvm, node, and npm 74 | command: | 75 | wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash 76 | echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV 77 | touch $HOME/.nvmrc 78 | echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV 79 | - run: 80 | name: Compile frontend and final binary 81 | command: | 82 | cd frontend 83 | nvm install v12.6.0 84 | npm cache clean --force 85 | cd .. 86 | make download 87 | make release 88 | - store_artifacts: 89 | path: gaia-linux-amd64 90 | 91 | workflows: 92 | version: 2 93 | test_and_compile: 94 | jobs: 95 | - linter 96 | - test_and_coverage 97 | - acceptance_tests 98 | - compile 99 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | *.db 7 | rice-box.go 8 | gaia-linux-amd64 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Visual studio 17 | .vscode 18 | debug 19 | 20 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 21 | .glide/ 22 | 23 | # Node/Javascript 24 | .DS_Store 25 | node_modules/ 26 | dist/ 27 | .idea 28 | npm-debug.log 29 | yarn-error.log 30 | selenium-debug.log 31 | test/unit/coverage 32 | test/e2e/reports 33 | 34 | # home folder generated by gaia during local test 35 | tmp/ 36 | 37 | # Ignore the vault file. 38 | .gaia_vault -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - make compile_frontend static_assets 4 | builds: 5 | - main: ./cmd/gaia/main.go 6 | binary: gaia-{{ .Os }}-{{ .Arch }} 7 | goos: 8 | - linux 9 | - freebsd 10 | goarch: 11 | - amd64 12 | checksum: 13 | name_template: 'checksums.txt' 14 | release: 15 | prerelease: true 16 | changelog: 17 | sort: asc 18 | filters: 19 | exclude: 20 | - '^docs:' 21 | - '^test:' 22 | dockers: 23 | - image_templates: 24 | - "gaiapipeline/gaia:latest" 25 | - "gaiapipeline/gaia:{{ .Tag }}" 26 | binaries: 27 | - gaia-linux-amd64 28 | dockerfile: docker/Dockerfile 29 | extra_files: 30 | - docker/docker-entrypoint.sh 31 | - docker/settings-docker.xml 32 | - image_templates: 33 | - "gaiapipeline/gaia:latest-go" 34 | - "gaiapipeline/gaia:{{ .Tag }}-go" 35 | binaries: 36 | - gaia-linux-amd64 37 | dockerfile: docker/Dockerfile.golang 38 | extra_files: 39 | - docker/docker-entrypoint.sh 40 | - image_templates: 41 | - "gaiapipeline/gaia:latest-java" 42 | - "gaiapipeline/gaia:{{ .Tag }}-java" 43 | binaries: 44 | - gaia-linux-amd64 45 | dockerfile: docker/Dockerfile.java 46 | extra_files: 47 | - docker/docker-entrypoint.sh 48 | - docker/settings-docker.xml 49 | - image_templates: 50 | - "gaiapipeline/gaia:latest-python" 51 | - "gaiapipeline/gaia:{{ .Tag }}-python" 52 | binaries: 53 | - gaia-linux-amd64 54 | dockerfile: docker/Dockerfile.python 55 | extra_files: 56 | - docker/docker-entrypoint.sh 57 | - image_templates: 58 | - "gaiapipeline/gaia:latest-cpp" 59 | - "gaiapipeline/gaia:{{ .Tag }}-cpp" 60 | binaries: 61 | - gaia-linux-amd64 62 | dockerfile: docker/Dockerfile.cpp 63 | extra_files: 64 | - docker/docker-entrypoint.sh 65 | - image_templates: 66 | - "gaiapipeline/gaia:latest-ruby" 67 | - "gaiapipeline/gaia:{{ .Tag }}-ruby" 68 | binaries: 69 | - gaia-linux-amd64 70 | dockerfile: docker/Dockerfile.ruby 71 | extra_files: 72 | - docker/docker-entrypoint.sh 73 | - image_templates: 74 | - "gaiapipeline/gaia:latest-nodejs" 75 | - "gaiapipeline/gaia:{{ .Tag }}-nodejs" 76 | binaries: 77 | - gaia-linux-amd64 78 | dockerfile: docker/Dockerfile.nodejs 79 | extra_files: 80 | - docker/docker-entrypoint.sh 81 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME=gaia 2 | GO_LDFLAGS_STATIC="-s -w -extldflags -static" 3 | NAMESPACE=${NAME} 4 | RELEASE_NAME=${NAME} 5 | HELM_DIR=$(shell pwd)/helm 6 | TEST=$$(go list ./... | grep -v /vendor/ | grep /testacc) 7 | TEST_TIMEOUT_ACC?=20m 8 | TEST_TIMEOUT?=50s 9 | # Set the build dir, where built cross-compiled binaries will be output 10 | BUILDDIR := bin 11 | BINARIES="linux/amd64 linux/arm darwin/amd64 windows/amd64" 12 | 13 | default: dev 14 | 15 | dev: 16 | go run ./cmd/gaia/main.go -home-path=${PWD}/tmp -dev=true 17 | 18 | compile_frontend: 19 | cd ./frontend && \ 20 | rm -rf dist && \ 21 | npm install && \ 22 | npm run build 23 | 24 | static_assets: 25 | go get github.com/GeertJohan/go.rice && \ 26 | go get github.com/GeertJohan/go.rice/rice && \ 27 | cd ./handlers && \ 28 | rm -f rice-box.go && \ 29 | rice embed-go && \ 30 | cd ../helper/assethelper && \ 31 | rm -f rice-box.go && \ 32 | rice embed-go 33 | 34 | compile_backend: 35 | env GOOS=linux GOARCH=amd64 go build -ldflags $(GO_LDFLAGS_STATIC) -o $(NAME)-linux-amd64 ./cmd/gaia/main.go 36 | 37 | binaries: 38 | CGO_ENABLED=0 gox \ 39 | -osarch=${BINARIES} \ 40 | -ldflags=${GO_LDFLAGS_STATIC} \ 41 | -output="$(BUILDDIR)/{{.OS}}/{{.Arch}}/$(NAME)" \ 42 | -tags="netgo" \ 43 | ./cmd/gaia/. 44 | 45 | download: 46 | go mod download 47 | 48 | get: 49 | go get ./... 50 | 51 | test: 52 | go test -v -race -timeout=$(TEST_TIMEOUT) ./... 53 | 54 | test-cover: 55 | go test -v -timeout=$(TEST_TIMEOUT) ./... --coverprofile=cover.out 56 | 57 | test-acc: 58 | GAIA_RUN_ACC=true GAIA_DEV=true go test -v $(TEST) -timeout=$(TEST_TIMEOUT_ACC) 59 | 60 | release: compile_frontend static_assets compile_backend 61 | 62 | deploy-kube: 63 | helm upgrade --install ${RELEASE_NAME} ${HELM_DIR} --namespace ${NAMESPACE} 64 | 65 | kube-ingress-lb: 66 | kubectl apply -R -f ${HELM_DIR}/_system 67 | 68 | lint: 69 | golint -set_exit_status ./... 70 | -------------------------------------------------------------------------------- /cmd/gaia/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | _ "github.com/gaia-pipeline/gaia/docs" 7 | "github.com/gaia-pipeline/gaia/server" 8 | ) 9 | 10 | // @title Gaia API 11 | // @version 1.0 12 | // @description This is the API that the Gaia Admin UI uses. 13 | // @termsOfService https://github.com/gaia-pipeline/gaia/blob/master/LICENSE 14 | 15 | // @contact.name API Support 16 | // @contact.url https://github.com/gaia-pipeline/gaia 17 | 18 | // @license.name Apache 2.0 19 | // @license.url http://www.apache.org/licenses/LICENSE-2.0.html 20 | 21 | // @securityDefinitions.apiKey ApiKeyAuth 22 | // @in header 23 | // @name Authorization 24 | 25 | // @BasePath /api/v1 26 | func main() { 27 | // Start the server. 28 | if err := server.Start(); err != nil { 29 | os.Exit(1) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /docker/Dockerfile.cpp: -------------------------------------------------------------------------------- 1 | FROM debian:buster 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | build-essential autoconf git pkg-config \ 5 | automake libtool curl make g++ unzip \ 6 | && apt-get clean 7 | 8 | # install protobuf first, then grpc 9 | ENV GRPC_RELEASE_TAG v1.29.x 10 | RUN git clone -b ${GRPC_RELEASE_TAG} https://github.com/grpc/grpc /var/local/git/grpc && \ 11 | cd /var/local/git/grpc && \ 12 | git submodule update --init && \ 13 | echo "--- installing protobuf ---" && \ 14 | cd third_party/protobuf && \ 15 | ./autogen.sh && ./configure --enable-shared && \ 16 | make -j$(nproc) && make install && make clean && ldconfig && \ 17 | echo "--- installing grpc ---" && \ 18 | cd /var/local/git/grpc && \ 19 | make -j$(nproc) && make install && make clean && ldconfig 20 | 21 | # Gaia internal port and data path. 22 | ENV GAIA_PORT=8080 \ 23 | GAIA_HOME_PATH=/data 24 | 25 | # Directory for the binary 26 | WORKDIR /app 27 | 28 | # Copy gaia binary into docker image 29 | COPY gaia-linux-amd64 /app 30 | 31 | # Fix permissions & setup known hosts file for ssh agent. 32 | RUN chmod +x ./gaia-linux-amd64 \ 33 | && mkdir -p /root/.ssh \ 34 | && touch /root/.ssh/known_hosts \ 35 | && chmod 600 /root/.ssh 36 | 37 | # Set homepath as volume 38 | VOLUME [ "${GAIA_HOME_PATH}" ] 39 | 40 | # Expose port 41 | EXPOSE ${GAIA_PORT} 42 | 43 | # Copy entry point script 44 | COPY docker/docker-entrypoint.sh /usr/local/bin/ 45 | 46 | # Start gaia 47 | ENTRYPOINT [ "docker-entrypoint.sh" ] 48 | -------------------------------------------------------------------------------- /docker/Dockerfile.golang: -------------------------------------------------------------------------------- 1 | FROM golang:1.17-alpine 2 | 3 | # Version and other variables which can be changed. 4 | ENV GAIA_PORT=8080 \ 5 | GAIA_HOME_PATH=/data 6 | 7 | # Directory for the binary 8 | WORKDIR /app 9 | 10 | # Copy gaia binary into docker image 11 | COPY gaia-linux-amd64 /app 12 | 13 | # Fix permissions & setup known hosts file for ssh agent. 14 | # Install git. 15 | RUN chmod +x ./gaia-linux-amd64 \ 16 | && apk add --no-cache git \ 17 | && mkdir -p /root/.ssh \ 18 | && touch /root/.ssh/known_hosts \ 19 | && chmod 600 /root/.ssh 20 | 21 | # Set homepath as volume 22 | VOLUME [ "${GAIA_HOME_PATH}" ] 23 | 24 | # Expose port 25 | EXPOSE ${GAIA_PORT} 26 | 27 | # Copy entry point script 28 | COPY docker/docker-entrypoint.sh /usr/local/bin/ 29 | 30 | # Start gaia 31 | ENTRYPOINT [ "docker-entrypoint.sh" ] 32 | -------------------------------------------------------------------------------- /docker/Dockerfile.java: -------------------------------------------------------------------------------- 1 | FROM maven:3.6.3-openjdk-11 2 | 3 | # Version and other variables which can be changed. 4 | ENV GAIA_PORT=8080 \ 5 | GAIA_HOME_PATH=/data 6 | 7 | # Directory for the binary 8 | WORKDIR /app 9 | 10 | # Copy gaia binary into docker image 11 | COPY gaia-linux-amd64 /app 12 | 13 | # Fix permissions & setup known hosts file for ssh agent. 14 | RUN chmod +x ./gaia-linux-amd64 \ 15 | && mkdir -p /root/.ssh \ 16 | && touch /root/.ssh/known_hosts \ 17 | && chmod 600 /root/.ssh 18 | 19 | # Set homepath as volume 20 | VOLUME [ "${GAIA_HOME_PATH}" ] 21 | 22 | # Expose port 23 | EXPOSE ${GAIA_PORT} 24 | 25 | # Copy entry point script 26 | COPY docker/docker-entrypoint.sh /usr/local/bin/ 27 | 28 | # Start gaia 29 | ENTRYPOINT [ "docker-entrypoint.sh" ] 30 | -------------------------------------------------------------------------------- /docker/Dockerfile.nodejs: -------------------------------------------------------------------------------- 1 | FROM node:12.17.0-stretch 2 | 3 | # Version and other variables which can be changed. 4 | ENV GAIA_PORT=8080 \ 5 | GAIA_HOME_PATH=/data 6 | 7 | # Directory for the binary 8 | WORKDIR /app 9 | 10 | # Copy gaia binary into docker image 11 | COPY gaia-linux-amd64 /app 12 | 13 | # Fix permissions & setup known hosts file for ssh agent. 14 | RUN chmod +x ./gaia-linux-amd64 \ 15 | && mkdir -p /root/.ssh \ 16 | && touch /root/.ssh/known_hosts \ 17 | && chmod 600 /root/.ssh 18 | 19 | # Set homepath as volume 20 | VOLUME [ "${GAIA_HOME_PATH}" ] 21 | 22 | # Expose port 23 | EXPOSE ${GAIA_PORT} 24 | 25 | # Copy entry point script 26 | COPY docker/docker-entrypoint.sh /usr/local/bin/ 27 | 28 | # Start gaia 29 | ENTRYPOINT [ "docker-entrypoint.sh" ] 30 | -------------------------------------------------------------------------------- /docker/Dockerfile.python: -------------------------------------------------------------------------------- 1 | FROM python:3.8-alpine3.12 2 | 3 | # Version and other variables which can be changed. 4 | ENV GAIA_PORT=8080 \ 5 | GAIA_HOME_PATH=/data 6 | 7 | # install additional deps 8 | RUN set -ex; \ 9 | apk add --no-cache build-base python3-dev linux-headers \ 10 | && pip install virtualenv grpcio 11 | 12 | # Directory for the binary 13 | WORKDIR /app 14 | 15 | # Copy gaia binary into docker image 16 | COPY gaia-linux-amd64 /app 17 | 18 | # Fix permissions & setup known hosts file for ssh agent. 19 | RUN chmod +x ./gaia-linux-amd64 \ 20 | && mkdir -p /root/.ssh \ 21 | && touch /root/.ssh/known_hosts \ 22 | && chmod 600 /root/.ssh 23 | 24 | # Set homepath as volume 25 | VOLUME [ "${GAIA_HOME_PATH}" ] 26 | 27 | # Expose port 28 | EXPOSE ${GAIA_PORT} 29 | 30 | # Copy entry point script 31 | COPY docker/docker-entrypoint.sh /usr/local/bin/ 32 | 33 | # Start gaia 34 | ENTRYPOINT [ "docker-entrypoint.sh" ] 35 | -------------------------------------------------------------------------------- /docker/Dockerfile.ruby: -------------------------------------------------------------------------------- 1 | FROM ruby:2.7-buster 2 | 3 | # Version and other variables which can be changed. 4 | ENV GAIA_PORT=8080 \ 5 | GAIA_HOME_PATH=/data 6 | 7 | # Directory for the binary 8 | WORKDIR /app 9 | 10 | # Copy gaia binary into docker image 11 | COPY gaia-linux-amd64 /app 12 | 13 | # Fix permissions & setup known hosts file for ssh agent. 14 | RUN chmod +x ./gaia-linux-amd64 \ 15 | && mkdir -p /root/.ssh \ 16 | && touch /root/.ssh/known_hosts \ 17 | && chmod 600 /root/.ssh 18 | 19 | # Set homepath as volume 20 | VOLUME [ "${GAIA_HOME_PATH}" ] 21 | 22 | # Expose port 23 | EXPOSE ${GAIA_PORT} 24 | 25 | # Copy entry point script 26 | COPY docker/docker-entrypoint.sh /usr/local/bin/ 27 | 28 | # Start gaia 29 | ENTRYPOINT [ "docker-entrypoint.sh" ] 30 | -------------------------------------------------------------------------------- /docker/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Start gaia 4 | exec /app/gaia-linux-amd64 5 | -------------------------------------------------------------------------------- /docker/settings-docker.xml: -------------------------------------------------------------------------------- 1 | 5 | /usr/share/maven/ref/repository 6 | -------------------------------------------------------------------------------- /frontend/.browserslistrc: -------------------------------------------------------------------------------- 1 | > 1% 2 | last 2 versions 3 | -------------------------------------------------------------------------------- /frontend/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{js,jsx,ts,tsx,vue}] 2 | indent_style = space 3 | indent_size = 2 4 | trim_trailing_whitespace = true 5 | insert_final_newline = true 6 | ij_javascript_force_semicolon_style = false 7 | -------------------------------------------------------------------------------- /frontend/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { 4 | node: true 5 | }, 6 | 'extends': [ 7 | 'plugin:vue/essential', 8 | '@vue/standard' 9 | ], 10 | rules: { 11 | 'no-console': process.env.NODE_ENV === 'production' ? 'error' : 'off', 12 | 'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off' 13 | }, 14 | parserOptions: { 15 | parser: 'babel-eslint' 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /frontend/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | '@vue/app' 4 | ], 5 | plugins: [ 6 | '@babel/plugin-proposal-export-default-from' 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gaia", 3 | "version": "0.2.5", 4 | "description": "Build powerful pipelines in any programming language.", 5 | "repository": "gaia-pipeline/gaia", 6 | "homepage": "https://github.com/gaia-pipeline/gaia", 7 | "license": "Apache-2.0", 8 | "author": { 9 | "name": "Michel Vocks", 10 | "email": "michelvocks@gmail.com", 11 | "url": "https://github.com/michelvocks" 12 | }, 13 | "keywords": [ 14 | "automation", 15 | "pipeline", 16 | "go", 17 | "build", 18 | "deployment", 19 | "java", 20 | "nodejs", 21 | "cplusplus", 22 | "python", 23 | "ruby", 24 | "nodejs" 25 | ], 26 | "engines": { 27 | "node": ">=4", 28 | "npm": ">=3" 29 | }, 30 | "scripts": { 31 | "serve": "vue-cli-service serve", 32 | "build": "vue-cli-service build", 33 | "lint": "vue-cli-service lint" 34 | }, 35 | "dependencies": { 36 | "@johmun/vue-tags-input": "^2.1.0", 37 | "core-js": "^2.6.5", 38 | "acorn": "^6.0.5", 39 | "animate.css": "3.7.2", 40 | "axios": "^0.21.2", 41 | "bulma": "^0.7.5", 42 | "font-awesome": "^4.7.0", 43 | "jdenticon": "^2.1.1", 44 | "lodash": "^4.17.21", 45 | "mdi": "^2.2.43", 46 | "moment": "^2.24.0", 47 | "npm": "^6.9.2", 48 | "plotly.js": "^1.48.3", 49 | "shelljs": "^0.8.3", 50 | "vis": "^4.21.0", 51 | "vue": "^2.6.10", 52 | "vue-bulma-collapse-fixed": "^1.0.4", 53 | "vue-bulma-datepicker": "^1.3.6", 54 | "vue-bulma-expanding": "^0.0.1", 55 | "vue-bulma-message-html": "^1.1.2", 56 | "vue-bulma-modal": "1.0.1", 57 | "vue-bulma-notification-fixed": "^1.1.0", 58 | "vue-bulma-progress-bar": "^1.0.2", 59 | "vue-bulma-tabs": "^1.1.3", 60 | "vue-good-table": "^2.16.5", 61 | "vue-js-toggle-button": "^1.3.2", 62 | "vue-lodash": "^2.0.2", 63 | "vue-nprogress": "0.1.5", 64 | "vue-router": "^3.0.6", 65 | "vue-tippy": "^2.1.3", 66 | "vuex": "^3.1.1", 67 | "vuex-router-sync": "^5.0.0", 68 | "wysiwyg.css": "0.0.3" 69 | }, 70 | "devDependencies": { 71 | "@babel/plugin-proposal-export-default-from": "^7.5.2", 72 | "@vue/cli-plugin-babel": "^3.9.0", 73 | "@vue/cli-plugin-eslint": "^3.9.0", 74 | "@vue/cli-service": "^3.9.0", 75 | "@vue/eslint-config-standard": "^4.0.0", 76 | "babel-eslint": "^10.0.1", 77 | "eslint": "^5.16.0", 78 | "eslint-plugin-vue": "^5.0.0", 79 | "node-sass": "^4.13.1", 80 | "sass-loader": "^7.1.0", 81 | "vue-runtime-helpers": "^1.0.1", 82 | "vue-template-compiler": "^2.6.10" 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /frontend/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | autoprefixer: {} 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/public/favicon.ico -------------------------------------------------------------------------------- /frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Gaia - Build powerful pipelines in any programming language. 9 | 10 | 11 | 14 |
15 | 16 | 17 | -------------------------------------------------------------------------------- /frontend/src/assets/fonts/Lobster-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/fonts/Lobster-Regular.ttf -------------------------------------------------------------------------------- /frontend/src/assets/images/cpp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/cpp.png -------------------------------------------------------------------------------- /frontend/src/assets/images/fail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/fail.png -------------------------------------------------------------------------------- /frontend/src/assets/images/golang.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/golang.png -------------------------------------------------------------------------------- /frontend/src/assets/images/inprogress.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/inprogress.png -------------------------------------------------------------------------------- /frontend/src/assets/images/java.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/java.png -------------------------------------------------------------------------------- /frontend/src/assets/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/logo.png -------------------------------------------------------------------------------- /frontend/src/assets/images/nodejs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/nodejs.png -------------------------------------------------------------------------------- /frontend/src/assets/images/python.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/python.png -------------------------------------------------------------------------------- /frontend/src/assets/images/questionmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/questionmark.png -------------------------------------------------------------------------------- /frontend/src/assets/images/ruby.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/ruby.png -------------------------------------------------------------------------------- /frontend/src/assets/images/success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/assets/images/success.png -------------------------------------------------------------------------------- /frontend/src/auth/index.js: -------------------------------------------------------------------------------- 1 | export default { 2 | 3 | login (context, creds) { 4 | return context.$http.post('/api/v1/login', creds) 5 | .then((response) => { 6 | var newSession = { 7 | 'token': response.data.tokenstring, 8 | 'display_name': response.data.display_name, 9 | 'username': response.data.username, 10 | 'jwtexpiry': response.data.jwtexpiry 11 | } 12 | window.localStorage.setItem('session', JSON.stringify(newSession)) 13 | context.$store.commit('setSession', newSession) 14 | 15 | // set success to true 16 | return true 17 | }) 18 | .catch((error) => { 19 | if (error) { 20 | return false 21 | } 22 | }) 23 | }, 24 | 25 | logout (context) { 26 | window.localStorage.removeItem('session') 27 | context.$store.commit('clearSession') 28 | }, 29 | 30 | getSession () { 31 | let session = JSON.parse(window.localStorage.getItem('session')) 32 | if (!session) { 33 | return '' 34 | } 35 | return session 36 | }, 37 | 38 | getToken () { 39 | let session = this.getSession() 40 | if (!session) { 41 | return '' 42 | } 43 | return session['token'] 44 | }, 45 | 46 | getAuthHeader () { 47 | return { 48 | 'Authorization': 'Bearer ' + this.getToken() 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /frontend/src/components/layout/AppMain.vue: -------------------------------------------------------------------------------- 1 | 15 | 16 | 35 | 36 | 55 | -------------------------------------------------------------------------------- /frontend/src/components/layout/Headerbar.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 22 | 23 | 31 | -------------------------------------------------------------------------------- /frontend/src/components/layout/Login.vue: -------------------------------------------------------------------------------- 1 | 31 | 32 | 83 | 84 | 122 | -------------------------------------------------------------------------------- /frontend/src/components/layout/Sidebar.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 33 | 34 | 127 | -------------------------------------------------------------------------------- /frontend/src/components/layout/index.js: -------------------------------------------------------------------------------- 1 | export Navbar from './Navbar' 2 | 3 | export Sidebar from './Sidebar' 4 | 5 | export Login from './Login' 6 | 7 | export AppMain from './AppMain' 8 | -------------------------------------------------------------------------------- /frontend/src/filters/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/frontend/src/filters/index.js -------------------------------------------------------------------------------- /frontend/src/helper/index.js: -------------------------------------------------------------------------------- 1 | export default { 2 | 3 | StartPipelineWithArgsCheck (context, pipeline) { 4 | // check if this pipeline has args 5 | if (pipeline.jobs) { 6 | for (let pipelineCurr = 0; pipelineCurr < pipeline.jobs.length; pipelineCurr++) { 7 | if (pipeline.jobs[pipelineCurr].args) { 8 | for (let argsCurr = 0; argsCurr < pipeline.jobs[pipelineCurr].args.length; argsCurr++) { 9 | if (pipeline.jobs[pipelineCurr].args[argsCurr].type !== 'vault') { 10 | // we found args. Redirect user to params view. 11 | context.$router.push({ path: '/pipeline/params', query: { pipelineid: pipeline.id, docker: pipeline.docker } }) 12 | return 13 | } 14 | } 15 | } 16 | } 17 | } 18 | 19 | // Start the pipeline directly. 20 | this.StartPipeline(context, pipeline) 21 | }, 22 | 23 | StartPipeline (context, pipeline) { 24 | // Send start request 25 | context.$http 26 | .post('/api/v1/pipeline/' + pipeline.id + '/start', [{ key: 'docker', value: this.docker ? '1' : '0' }]) 27 | .then(response => { 28 | if (response.data) { 29 | context.$router.push({ path: '/pipeline/detail', query: { pipelineid: pipeline.id, runid: response.data.id } }) 30 | } 31 | }) 32 | .catch((error) => { 33 | context.$store.commit('clearIntervals') 34 | context.$onError(error) 35 | }) 36 | }, 37 | 38 | PullPipeline (context, pipeline) { 39 | // Send pull request 40 | context.$http 41 | .post('/api/v1/pipeline/' + pipeline.id + '/pull', { docker: pipeline.docker }) 42 | .then(response => { 43 | context.$notify({ 44 | title: 'Successfully pulled new code', 45 | message: `Pipeline "${pipeline.name}" has been updated successfully.`, 46 | type: 'success' 47 | }) 48 | }) 49 | .catch((error) => { 50 | context.$store.commit('clearIntervals') 51 | context.$onError(error) 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /frontend/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import axios from 'axios' 3 | import NProgress from 'vue-nprogress' 4 | import { sync } from 'vuex-router-sync' 5 | import App from './App.vue' 6 | import router from './router' 7 | import store from './store' 8 | import * as filters from './filters' 9 | import Notification from 'vue-bulma-notification-fixed' 10 | import auth from './auth' 11 | import lodash from 'lodash' 12 | import VueLodash from 'vue-lodash' 13 | 14 | const axiosInstance = axios.create() 15 | 16 | Vue.config.productionTip = false 17 | Vue.prototype.$http = axiosInstance 18 | Vue.axios = axiosInstance 19 | Vue.router = router 20 | Vue.use(NProgress, { 21 | http: false, 22 | router: false 23 | }) 24 | Vue.use(VueLodash, lodash) 25 | 26 | // Auth interceptors 27 | axiosInstance.interceptors.request.use(function (request) { 28 | request.headers['Authorization'] = 'Bearer ' + auth.getToken() 29 | return request 30 | }) 31 | 32 | // Enable devtools 33 | Vue.config.devtools = true 34 | sync(store, router) 35 | 36 | const nprogress = new NProgress({ parent: '.nprogress-container' }) 37 | axiosInstance.interceptors.request.use(function (config) { 38 | if (!config.params || config.params.hideProgressBar !== true) { 39 | nprogress.start() 40 | } 41 | return config 42 | }) 43 | axiosInstance.interceptors.response.use(function (response) { 44 | nprogress.done() 45 | return response 46 | }) 47 | 48 | Vue.directive('focus', { 49 | // When the bound element is inserted into the DOM... 50 | inserted: function (el) { 51 | // Focus the element 52 | el.focus() 53 | } 54 | }) 55 | 56 | const NotificationComponent = Vue.extend(Notification) 57 | const openNotification = (propsData = { 58 | title: '', 59 | message: '', 60 | type: '', 61 | direction: '', 62 | duration: 4500, 63 | container: '.notifications' 64 | }) => { 65 | return new NotificationComponent({ 66 | el: document.createElement('div'), 67 | propsData 68 | }) 69 | } 70 | Vue.prototype.$notify = openNotification 71 | 72 | function handleError (error) { 73 | // if the server gave a response message, print that 74 | if (error.response) { 75 | // duration should be proportional to the error message length 76 | openNotification({ 77 | title: 'Error: ' + error.response.status, 78 | message: error.response.data, 79 | type: 'danger' 80 | }) 81 | } else if (error.request) { 82 | openNotification({ 83 | title: 'Error: No response received!', 84 | message: 'Please verify if the backend is running!', 85 | type: 'danger' 86 | }) 87 | } else { 88 | openNotification({ 89 | title: 'Error: Cannot setup request!', 90 | message: error.message, 91 | type: 'danger' 92 | }) 93 | } 94 | 95 | // Finish progress bar 96 | nprogress.done() 97 | } 98 | 99 | Vue.prototype.$onError = handleError 100 | 101 | Vue.prototype.$onSuccess = (title, message) => { 102 | openNotification({ 103 | title: title, 104 | message: message, 105 | type: 'success', 106 | duration: message > 60 ? 20000 : 4500 107 | }) 108 | } 109 | 110 | Vue.prototype.$prettifyTags = (tags) => { 111 | let prettyTags = '' 112 | for (let i = 0; i < tags.length; i++) { 113 | if (i === (tags.length - 1)) { 114 | prettyTags += tags[i] 115 | } else { 116 | prettyTags += tags[i] + ', ' 117 | } 118 | } 119 | return prettyTags 120 | } 121 | 122 | Object.keys(filters).forEach(key => { 123 | Vue.filter(key, filters[key]) 124 | }) 125 | 126 | const app = new Vue({ 127 | router, 128 | store, 129 | nprogress, 130 | ...App 131 | }).$mount('#app') 132 | 133 | // A simple event bus 134 | export const EventBus = new Vue() 135 | 136 | export { app, router, store } 137 | -------------------------------------------------------------------------------- /frontend/src/router/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import Router from 'vue-router' 3 | import menuModule from '../store/modules/menu' 4 | import lazyLoading from '../store/modules/menu/lazyLoading' 5 | Vue.use(Router) 6 | 7 | export default new Router({ 8 | mode: 'hash', // Demo is living in GitHub.io, so required! 9 | linkActiveClass: 'is-active', 10 | scrollBehavior: () => ({ y: 0 }), 11 | routes: [ 12 | ...generateRoutesFromMenu(menuModule.state.items), 13 | { 14 | path: '*', 15 | redirect: '/overview' 16 | }, 17 | { 18 | name: 'Pipeline Detail', 19 | path: '/pipeline/detail', 20 | component: lazyLoading('pipeline/detail') 21 | }, 22 | { 23 | name: 'Pipeline Logs', 24 | path: '/pipeline/log', 25 | component: lazyLoading('pipeline/log') 26 | }, 27 | { 28 | name: 'Pipeline Parameters', 29 | path: '/pipeline/params', 30 | component: lazyLoading('pipeline/params') 31 | } 32 | ] 33 | }) 34 | 35 | // Menu should have 1 level. 36 | function generateRoutesFromMenu (menu = [], routes = []) { 37 | for (let i = 0, l = menu.length; i < l; i++) { 38 | routes.push(menu[i]) 39 | } 40 | return routes 41 | } 42 | -------------------------------------------------------------------------------- /frontend/src/store/actions.js: -------------------------------------------------------------------------------- 1 | import * as types from './mutation-types' 2 | 3 | export const toggleSidebar = ({ commit }, config) => { 4 | if (config instanceof Object) { 5 | commit(types.TOGGLE_SIDEBAR, config) 6 | } 7 | } 8 | 9 | export const toggleDevice = ({ commit }, device) => commit(types.TOGGLE_DEVICE, device) 10 | -------------------------------------------------------------------------------- /frontend/src/store/getters.js: -------------------------------------------------------------------------------- 1 | const pkg = state => state.pkg 2 | const app = state => state.app 3 | const device = state => state.app.device 4 | const sidebar = state => state.app.sidebar 5 | const effect = state => state.app.effect 6 | const menuitems = state => state.menu.items 7 | const session = state => state.session 8 | const intervals = state => state.intervals 9 | 10 | export { 11 | pkg, 12 | app, 13 | device, 14 | sidebar, 15 | effect, 16 | menuitems, 17 | session, 18 | intervals 19 | } 20 | -------------------------------------------------------------------------------- /frontend/src/store/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import Vuex from 'vuex' 3 | import * as actions from './actions' 4 | import * as getters from './getters' 5 | import * as mutations from './mutations' 6 | 7 | import app from './modules/app' 8 | import menu from './modules/menu' 9 | 10 | Vue.use(Vuex) 11 | 12 | const store = new Vuex.Store({ 13 | strict: true, // process.env.NODE_ENV !== 'production', 14 | actions, 15 | getters, 16 | modules: { 17 | app, 18 | menu 19 | }, 20 | state: { 21 | session: null 22 | }, 23 | mutations 24 | }) 25 | 26 | export default store 27 | -------------------------------------------------------------------------------- /frontend/src/store/modules/app.js: -------------------------------------------------------------------------------- 1 | import * as types from '../mutation-types' 2 | 3 | const state = { 4 | device: { 5 | isMobile: false, 6 | isTablet: false 7 | }, 8 | sidebar: { 9 | opened: false, 10 | hidden: false 11 | }, 12 | effect: { 13 | translate3d: true 14 | } 15 | } 16 | 17 | const mutations = { 18 | [types.TOGGLE_DEVICE] (state, device) { 19 | state.device.isMobile = device === 'mobile' 20 | state.device.isTablet = device === 'tablet' 21 | }, 22 | 23 | [types.TOGGLE_SIDEBAR] (state, config) { 24 | if (state.device.isMobile && config.hasOwnProperty('opened')) { 25 | state.sidebar.opened = config.opened 26 | } else { 27 | state.sidebar.opened = true 28 | } 29 | 30 | if (config.hasOwnProperty('hidden')) { 31 | state.sidebar.hidden = config.hidden 32 | } 33 | }, 34 | 35 | [types.SWITCH_EFFECT] (state, effectItem) { 36 | for (let name in effectItem) { 37 | state.effect[name] = effectItem[name] 38 | } 39 | } 40 | } 41 | 42 | export default { 43 | state, 44 | mutations 45 | } 46 | -------------------------------------------------------------------------------- /frontend/src/store/modules/menu/index.js: -------------------------------------------------------------------------------- 1 | import lazyLoading from './lazyLoading' 2 | 3 | // show: meta.label -> name 4 | // name: component name 5 | // meta.label: display label 6 | 7 | const state = { 8 | items: [ 9 | { 10 | name: 'Overview', 11 | path: '/overview', 12 | meta: { 13 | icon: 'fa-th' 14 | }, 15 | component: lazyLoading('overview', true) 16 | }, 17 | { 18 | name: 'Create Pipeline', 19 | path: '/pipeline/create', 20 | meta: { 21 | icon: 'fa-plus' 22 | }, 23 | component: lazyLoading('pipeline/create') 24 | }, 25 | { 26 | name: 'Vault', 27 | path: '/vault', 28 | meta: { 29 | icon: 'fa-lock' 30 | }, 31 | component: lazyLoading('vault', true) 32 | }, 33 | { 34 | name: 'Settings', 35 | path: '/settings', 36 | meta: { 37 | icon: 'fa-cogs' 38 | }, 39 | component: lazyLoading('settings', true) 40 | } 41 | ] 42 | } 43 | 44 | export default { 45 | state 46 | } 47 | -------------------------------------------------------------------------------- /frontend/src/store/modules/menu/lazyLoading.js: -------------------------------------------------------------------------------- 1 | // lazy loading Components 2 | // https://github.com/vuejs/vue-router/blob/dev/examples/lazy-loading/app.js#L8 3 | export default (name, index = false) => () => import(`../../../views/${name}${index ? '/index' : ''}.vue`) 4 | -------------------------------------------------------------------------------- /frontend/src/store/mutation-types.js: -------------------------------------------------------------------------------- 1 | export const TOGGLE_DEVICE = 'TOGGLE_DEVICE' 2 | 3 | export const TOGGLE_SIDEBAR = 'TOGGLE_SIDEBAR' 4 | 5 | export const SWITCH_EFFECT = 'SWITCH_EFFECT' 6 | -------------------------------------------------------------------------------- /frontend/src/store/mutations.js: -------------------------------------------------------------------------------- 1 | export const setSession = (state, session) => { 2 | state.session = session 3 | } 4 | 5 | export const clearSession = (state) => { 6 | state.session = null 7 | } 8 | 9 | export const appendInterval = (state, interval) => { 10 | if (!state.intervals) { 11 | state.intervals = [] 12 | } 13 | 14 | state.intervals.push(interval) 15 | } 16 | 17 | export const clearIntervals = (state) => { 18 | if (state.intervals) { 19 | for (let i = 0, l = state.intervals.length; i < l; i++) { 20 | clearInterval(state.intervals[i]) 21 | } 22 | state.intervals = null 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /frontend/src/views/pipeline/log.vue: -------------------------------------------------------------------------------- 1 | 15 | 16 | 96 | 97 | 129 | -------------------------------------------------------------------------------- /frontend/src/views/pipeline/params.vue: -------------------------------------------------------------------------------- 1 | 32 | 33 | 117 | 118 | 130 | -------------------------------------------------------------------------------- /frontend/src/views/settings/settings/manage-settings.vue: -------------------------------------------------------------------------------- 1 | 44 | 45 | 137 | 138 | 158 | -------------------------------------------------------------------------------- /frontend/vue.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | devServer: { 3 | port: 8081, 4 | proxy: { 5 | '^/api/v1': { 6 | target: 'http://localhost:8080', 7 | changeOrigin: false 8 | } 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gaia-pipeline/gaia 2 | 3 | require ( 4 | github.com/GeertJohan/go.rice v1.0.2 5 | github.com/Pallinder/go-randomdata v1.2.0 6 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 7 | github.com/casbin/casbin/v2 v2.37.0 8 | github.com/docker/docker v20.10.8+incompatible 9 | github.com/gaia-pipeline/flag v1.7.4-pre 10 | github.com/gaia-pipeline/protobuf v0.0.0-20180812091451-7be8a901b55a 11 | github.com/gofrs/uuid v4.0.0+incompatible 12 | github.com/golang-jwt/jwt v3.2.2+incompatible 13 | github.com/golang/protobuf v1.5.2 14 | github.com/google/go-github v17.0.0+incompatible 15 | github.com/hashicorp/go-hclog v0.16.2 16 | github.com/hashicorp/go-memdb v1.3.2 17 | github.com/hashicorp/go-plugin v1.4.3 18 | github.com/labstack/echo/v4 v4.5.0 19 | github.com/pkg/errors v0.9.1 20 | github.com/robfig/cron v1.2.0 21 | github.com/speza/casbin-bolt-adapter v0.0.0-20200919192425-e2008c12e733 22 | github.com/stretchr/testify v1.6.1 23 | github.com/swaggo/echo-swagger v1.1.3 24 | github.com/swaggo/swag v1.7.1 25 | go.etcd.io/bbolt v1.3.6 26 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 27 | golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f 28 | google.golang.org/grpc v1.40.0 29 | gopkg.in/src-d/go-git.v4 v4.13.1 30 | gopkg.in/yaml.v2 v2.4.0 31 | ) 32 | 33 | require ( 34 | github.com/GeertJohan/go.incremental v1.0.0 // indirect 35 | github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect 36 | github.com/KyleBanks/depth v1.2.1 // indirect 37 | github.com/Microsoft/go-winio v0.5.0 // indirect 38 | github.com/PuerkitoBio/purell v1.1.1 // indirect 39 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect 40 | github.com/akavel/rsrc v0.8.0 // indirect 41 | github.com/containerd/containerd v1.5.5 // indirect 42 | github.com/daaku/go.zipexe v1.0.1 // indirect 43 | github.com/davecgh/go-spew v1.1.1 // indirect 44 | github.com/docker/distribution v2.7.1+incompatible // indirect 45 | github.com/docker/go-connections v0.4.0 // indirect 46 | github.com/docker/go-units v0.4.0 // indirect 47 | github.com/emirpasic/gods v1.12.0 // indirect 48 | github.com/fatih/color v1.12.0 // indirect 49 | github.com/go-openapi/jsonpointer v0.19.5 // indirect 50 | github.com/go-openapi/jsonreference v0.19.6 // indirect 51 | github.com/go-openapi/spec v0.20.3 // indirect 52 | github.com/go-openapi/swag v0.19.15 // indirect 53 | github.com/gogo/protobuf v1.3.2 // indirect 54 | github.com/google/go-cmp v0.5.6 // indirect 55 | github.com/google/go-querystring v1.1.0 // indirect 56 | github.com/gorilla/mux v1.7.3 // indirect 57 | github.com/hashicorp/go-immutable-radix v1.3.1 // indirect 58 | github.com/hashicorp/golang-lru v0.5.4 // indirect 59 | github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 // indirect 60 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 61 | github.com/jessevdk/go-flags v1.4.0 // indirect 62 | github.com/josharian/intern v1.0.0 // indirect 63 | github.com/kevinburke/ssh_config v1.1.0 // indirect 64 | github.com/labstack/gommon v0.3.0 // indirect 65 | github.com/mailru/easyjson v0.7.7 // indirect 66 | github.com/mattn/go-colorable v0.1.8 // indirect 67 | github.com/mattn/go-isatty v0.0.14 // indirect 68 | github.com/mitchellh/go-homedir v1.1.0 // indirect 69 | github.com/mitchellh/go-testing-interface v1.14.1 // indirect 70 | github.com/morikuni/aec v1.0.0 // indirect 71 | github.com/nkovacs/streamquote v1.0.0 // indirect 72 | github.com/oklog/run v1.1.0 // indirect 73 | github.com/opencontainers/go-digest v1.0.0 // indirect 74 | github.com/opencontainers/image-spec v1.0.1 // indirect 75 | github.com/pmezard/go-difflib v1.0.0 // indirect 76 | github.com/sergi/go-diff v1.2.0 // indirect 77 | github.com/sirupsen/logrus v1.8.1 // indirect 78 | github.com/src-d/gcfg v1.4.0 // indirect 79 | github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 // indirect 80 | github.com/valyala/bytebufferpool v1.0.0 // indirect 81 | github.com/valyala/fasttemplate v1.2.1 // indirect 82 | github.com/xanzy/ssh-agent v0.3.1 // indirect 83 | golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect 84 | golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect 85 | golang.org/x/text v0.3.7 // indirect 86 | golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect 87 | golang.org/x/tools v0.1.5 // indirect 88 | google.golang.org/appengine v1.6.7 // indirect 89 | google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af // indirect 90 | google.golang.org/protobuf v1.27.1 // indirect 91 | gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect 92 | gopkg.in/warnings.v0 v0.1.2 // indirect 93 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect 94 | ) 95 | 96 | go 1.17 97 | 98 | replace github.com/swaggo/swag => github.com/swaggo/swag v1.6.10-0.20201104153820-3f47d68f8872 99 | 100 | replace github.com/ugorji/go/codec => github.com/ugorji/go/codec v1.2.0 101 | -------------------------------------------------------------------------------- /handlers/auth.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "crypto/rsa" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | 10 | "github.com/golang-jwt/jwt" 11 | "github.com/labstack/echo/v4" 12 | 13 | "github.com/gaia-pipeline/gaia" 14 | "github.com/gaia-pipeline/gaia/helper/rolehelper" 15 | "github.com/gaia-pipeline/gaia/security/rbac" 16 | ) 17 | 18 | var ( 19 | // errNotAuthorized is thrown when user wants to access resource which is protected 20 | errNotAuthorized = errors.New("no or invalid jwt token provided. You are not authorized") 21 | ) 22 | 23 | func authMiddleware(authCfg *AuthConfig) echo.MiddlewareFunc { 24 | return func(next echo.HandlerFunc) echo.HandlerFunc { 25 | return func(c echo.Context) error { 26 | token, err := getToken(c) 27 | if err != nil { 28 | return c.String(http.StatusUnauthorized, err.Error()) 29 | } 30 | 31 | // Validate token 32 | if claims, okClaims := token.Claims.(jwt.MapClaims); okClaims && token.Valid { 33 | // All ok, continue 34 | username, hasUsername := claims["username"] 35 | roles, hasRoles := claims["roles"] 36 | if hasUsername && hasRoles && roles != nil { 37 | // Look through the perms until we find that the user has this permission 38 | if err := authCfg.checkRole(roles, c.Request().Method, c.Path()); err != nil { 39 | return c.String(http.StatusForbidden, "Permission denied for user.") 40 | } 41 | 42 | username, okUsername := username.(string) 43 | if !okUsername { 44 | gaia.Cfg.Logger.Error("username is not type string") 45 | return c.String(http.StatusInternalServerError, "Unknown error has occurred.") 46 | } 47 | 48 | // Currently this lives inside the existing auth middleware. Ideally we would have independent 49 | // middleware for enforcing RBAC. For now I will leave this here so we avoid parsing the token 50 | // and claims multiple times. 51 | params := map[string]string{} 52 | for i, n := range c.ParamNames() { 53 | params[n] = c.ParamValues()[i] 54 | } 55 | err := authCfg.rbacEnforcer.Enforce(username, c.Request().Method, c.Path(), params) 56 | if err != nil { 57 | if _, permDenied := err.(*rbac.ErrPermissionDenied); permDenied { 58 | return c.String(http.StatusForbidden, err.Error()) 59 | } 60 | gaia.Cfg.Logger.Error("rbacEnforcer error", "error", err.Error()) 61 | return c.String(http.StatusInternalServerError, "Unknown error has occurred while validating permissions.") 62 | } 63 | } 64 | return next(c) 65 | } 66 | return c.String(http.StatusUnauthorized, errNotAuthorized.Error()) 67 | } 68 | } 69 | } 70 | 71 | // AuthConfig is a simple config struct to be passed into AuthMiddleware. Currently allows the ability to specify 72 | // the permission roles required for each echo endpoint. 73 | type AuthConfig struct { 74 | RoleCategories []*gaia.UserRoleCategory 75 | rbacEnforcer rbac.EndpointEnforcer 76 | } 77 | 78 | // Finds the required role for the metho & path specified. If it exists we validate that the provided user roles have 79 | // the permission role. If not, error specifying the required role. 80 | func (ra *AuthConfig) checkRole(userRoles interface{}, method, path string) error { 81 | perm := ra.getRequiredRole(method, path) 82 | if perm == "" { 83 | return nil 84 | } 85 | for _, role := range userRoles.([]interface{}) { 86 | if role.(string) == perm { 87 | return nil 88 | } 89 | } 90 | return fmt.Errorf("required permission role %s", perm) 91 | } 92 | 93 | // Iterate over each category to find a permission (if existing) for this API endpoint. 94 | func (ra *AuthConfig) getRequiredRole(method, path string) string { 95 | for _, category := range ra.RoleCategories { 96 | for _, role := range category.Roles { 97 | for _, endpoint := range role.APIEndpoint { 98 | // If the http method & path match then return the role required for this endpoint 99 | if method == endpoint.Method && path == endpoint.Path { 100 | return rolehelper.FullUserRoleName(category, role) 101 | } 102 | } 103 | } 104 | } 105 | return "" 106 | } 107 | 108 | // Get the JWT token from the echo context 109 | func getToken(c echo.Context) (*jwt.Token, error) { 110 | // Get the token 111 | jwtRaw := c.Request().Header.Get("Authorization") 112 | split := strings.Split(jwtRaw, " ") 113 | if len(split) != 2 { 114 | return nil, errNotAuthorized 115 | } 116 | jwtString := split[1] 117 | 118 | // Parse token 119 | token, err := jwt.Parse(jwtString, func(token *jwt.Token) (interface{}, error) { 120 | signingMethodError := fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) 121 | switch token.Method.(type) { 122 | case *jwt.SigningMethodHMAC: 123 | if _, ok := gaia.Cfg.JWTKey.([]byte); !ok { 124 | return nil, signingMethodError 125 | } 126 | return gaia.Cfg.JWTKey, nil 127 | case *jwt.SigningMethodRSA: 128 | if _, ok := gaia.Cfg.JWTKey.(*rsa.PrivateKey); !ok { 129 | return nil, signingMethodError 130 | } 131 | return gaia.Cfg.JWTKey.(*rsa.PrivateKey).Public(), nil 132 | default: 133 | return nil, signingMethodError 134 | } 135 | }) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | return token, nil 141 | } 142 | -------------------------------------------------------------------------------- /handlers/errors/handler_errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | -------------------------------------------------------------------------------- /handlers/handler_test.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | rbacProvider "github.com/gaia-pipeline/gaia/providers/rbac" 10 | userProvider "github.com/gaia-pipeline/gaia/providers/user" 11 | "github.com/gaia-pipeline/gaia/security/rbac" 12 | 13 | "github.com/gaia-pipeline/gaia/providers/pipelines" 14 | "github.com/gaia-pipeline/gaia/providers/workers" 15 | "github.com/gaia-pipeline/gaia/security" 16 | "github.com/gaia-pipeline/gaia/store" 17 | 18 | "github.com/gaia-pipeline/gaia" 19 | "github.com/gaia-pipeline/gaia/workers/pipeline" 20 | "github.com/hashicorp/go-hclog" 21 | "github.com/labstack/echo/v4" 22 | ) 23 | 24 | type mockStorageService struct { 25 | store.GaiaStore 26 | mockPipeline *gaia.Pipeline 27 | } 28 | 29 | func (s *mockStorageService) PipelineGetRunByPipelineIDAndID(pipelineid int, runid int) (*gaia.PipelineRun, error) { 30 | return generateTestData(), nil 31 | } 32 | func (s *mockStorageService) PipelinePutRun(r *gaia.PipelineRun) error { return nil } 33 | func (s *mockStorageService) PipelineGet(id int) (pipeline *gaia.Pipeline, err error) { 34 | return s.mockPipeline, nil 35 | } 36 | func (s *mockStorageService) PipelineGetRunByID(runID string) (*gaia.PipelineRun, error) { 37 | return &gaia.PipelineRun{}, nil 38 | } 39 | 40 | func TestInitHandler(t *testing.T) { 41 | dataDir, err := ioutil.TempDir("", "TestInitHandler") 42 | if err != nil { 43 | t.Fatalf("error creating data dir %v", err.Error()) 44 | } 45 | 46 | defer func() { 47 | gaia.Cfg = nil 48 | _ = os.RemoveAll(dataDir) 49 | }() 50 | gaia.Cfg = &gaia.Config{ 51 | Logger: hclog.NewNullLogger(), 52 | DataPath: dataDir, 53 | CAPath: dataDir, 54 | VaultPath: dataDir, 55 | HomePath: dataDir, 56 | Mode: gaia.ModeServer, 57 | DevMode: true, 58 | } 59 | e := echo.New() 60 | 61 | // Initialize global active pipelines 62 | ap := pipeline.NewActivePipelines() 63 | pipeline.GlobalActivePipelines = ap 64 | 65 | p := gaia.Pipeline{ 66 | ID: 1, 67 | Name: "Pipeline A", 68 | Type: gaia.PTypeGolang, 69 | Created: time.Now(), 70 | Repo: &gaia.GitRepo{ 71 | URL: "https://github.com/Codertocat/Hello-World", 72 | }, 73 | } 74 | 75 | ap.Append(p) 76 | ms := &mockScheduleService{} 77 | // Initialize handlers 78 | pipelineService := pipeline.NewGaiaPipelineService(pipeline.Dependencies{ 79 | Scheduler: ms, 80 | }) 81 | pp := pipelines.NewPipelineProvider(pipelines.Dependencies{ 82 | Scheduler: ms, 83 | PipelineService: pipelineService, 84 | }) 85 | ca, err := security.InitCA() 86 | if err != nil { 87 | gaia.Cfg.Logger.Error("cannot create CA", "error", err.Error()) 88 | return 89 | } 90 | wp := workers.NewWorkerProvider(workers.Dependencies{ 91 | Certificate: ca, 92 | Scheduler: ms, 93 | }) 94 | mStore := &mockStorageService{mockPipeline: &p} 95 | rbacService := rbac.NewNoOpService() 96 | rbacPrv := rbacProvider.NewProvider(rbacService) 97 | userPrv := userProvider.NewProvider(mStore, rbacService) 98 | handlerService := NewGaiaHandler(Dependencies{ 99 | Scheduler: ms, 100 | PipelineService: pipelineService, 101 | PipelineProvider: pp, 102 | Certificate: ca, 103 | WorkerProvider: wp, 104 | Store: mStore, 105 | UserProvider: userPrv, 106 | RBACProvider: rbacPrv, 107 | }) 108 | if err := handlerService.InitHandlers(e); err != nil { 109 | t.Fatal(err) 110 | } 111 | } 112 | 113 | func generateTestData() *gaia.PipelineRun { 114 | return &gaia.PipelineRun{ 115 | UniqueID: "first-pipeline-run", 116 | ID: 1, 117 | PipelineID: 1, 118 | Jobs: []*gaia.Job{ 119 | { 120 | ID: 1, 121 | Title: "first-job", 122 | Status: gaia.JobWaitingExec, 123 | }, 124 | }, 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /handlers/permission.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gaia-pipeline/gaia/helper/rolehelper" 7 | "github.com/labstack/echo/v4" 8 | ) 9 | 10 | // PermissionGetAll simply returns a list of all the roles available. 11 | // @Summary Returns a list of default roles. 12 | // @Description Returns a list of all the roles available. 13 | // @Tags rbac 14 | // @Security ApiKeyAuth 15 | // @Success 200 {array} gaia.UserRoleCategory 16 | // @Router /permission [get] 17 | func PermissionGetAll(c echo.Context) error { 18 | return c.JSON(http.StatusOK, rolehelper.DefaultUserRoles) 19 | } 20 | -------------------------------------------------------------------------------- /handlers/permission_test.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | "github.com/labstack/echo/v4" 9 | ) 10 | 11 | func TestPermissionGetAll(t *testing.T) { 12 | e := echo.New() 13 | req := httptest.NewRequest(http.MethodGet, "/", nil) 14 | rec := httptest.NewRecorder() 15 | c := e.NewContext(req, rec) 16 | c.SetPath("/api/v1/permission") 17 | err := PermissionGetAll(c) 18 | 19 | if err != nil { 20 | t.Fatal("should not error") 21 | } 22 | if rec.Code != http.StatusOK { 23 | t.Fatal("code should be 200") 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /handlers/service.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/gaia-pipeline/gaia/providers" 5 | "github.com/gaia-pipeline/gaia/providers/pipelines" 6 | "github.com/gaia-pipeline/gaia/providers/workers" 7 | "github.com/gaia-pipeline/gaia/security" 8 | "github.com/gaia-pipeline/gaia/security/rbac" 9 | "github.com/gaia-pipeline/gaia/store" 10 | "github.com/gaia-pipeline/gaia/workers/pipeline" 11 | "github.com/gaia-pipeline/gaia/workers/scheduler/service" 12 | ) 13 | 14 | // Dependencies define dependencies for this service. 15 | type Dependencies struct { 16 | Scheduler service.GaiaScheduler 17 | PipelineService pipeline.Servicer 18 | PipelineProvider pipelines.PipelineProviderer 19 | UserProvider providers.UserProvider 20 | RBACProvider providers.RBACProvider 21 | WorkerProvider workers.WorkerProviderer 22 | Certificate security.CAAPI 23 | RBACService rbac.Service 24 | Store store.GaiaStore 25 | } 26 | 27 | // GaiaHandler defines handler functions throughout Gaia. 28 | type GaiaHandler struct { 29 | deps Dependencies 30 | } 31 | 32 | // NewGaiaHandler creates a new handler service with the required dependencies. 33 | func NewGaiaHandler(deps Dependencies) *GaiaHandler { 34 | return &GaiaHandler{deps: deps} 35 | } 36 | -------------------------------------------------------------------------------- /handlers/settings.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/labstack/echo/v4" 7 | 8 | "github.com/gaia-pipeline/gaia" 9 | "github.com/gaia-pipeline/gaia/store" 10 | ) 11 | 12 | const msgSomethingWentWrong = "Something went wrong while retrieving settings information." 13 | 14 | type settingsHandler struct { 15 | store store.SettingsStore 16 | } 17 | 18 | func newSettingsHandler(store store.SettingsStore) *settingsHandler { 19 | return &settingsHandler{store: store} 20 | } 21 | 22 | type rbacPutRequest struct { 23 | Enabled bool `json:"enabled"` 24 | } 25 | 26 | // @Summary Put RBAC settings 27 | // @Description Save the given RBAC settings. 28 | // @Tags settings 29 | // @Accept json 30 | // @Produce plain 31 | // @Security ApiKeyAuth 32 | // @Param RbacPutRequest body rbacPutRequest true "RBAC setting details." 33 | // @Success 200 {string} string "Settings have been updated." 34 | // @Failure 400 {string} string "Invalid body." 35 | // @Failure 500 {string} string "Something went wrong while saving or retrieving rbac settings." 36 | // @Router /settings/rbac [put] 37 | func (h *settingsHandler) rbacPut(c echo.Context) error { 38 | var request rbacPutRequest 39 | if err := c.Bind(&request); err != nil { 40 | gaia.Cfg.Logger.Error("failed to bind body", "error", err.Error()) 41 | return c.String(http.StatusBadRequest, "Invalid body provided.") 42 | } 43 | 44 | settings, err := h.store.SettingsGet() 45 | if err != nil { 46 | gaia.Cfg.Logger.Error("failed to get store settings", "error", err.Error()) 47 | return c.String(http.StatusInternalServerError, msgSomethingWentWrong) 48 | } 49 | 50 | settings.RBACEnabled = request.Enabled 51 | 52 | if err := h.store.SettingsPut(settings); err != nil { 53 | gaia.Cfg.Logger.Error("failed to put store settings", "error", err.Error()) 54 | return c.String(http.StatusInternalServerError, "An error occurred while saving the settings.") 55 | } 56 | 57 | return c.String(http.StatusOK, "Settings have been updated.") 58 | } 59 | 60 | type rbacGetResponse struct { 61 | Enabled bool `json:"enabled"` 62 | } 63 | 64 | // @Summary Get RBAC settings 65 | // @Description Get the given RBAC settings. 66 | // @Tags settings 67 | // @Produce json 68 | // @Security ApiKeyAuth 69 | // @Success 200 {object} rbacGetResponse 70 | // @Failure 500 {string} string "Something went wrong while saving or retrieving rbac settings." 71 | // @Router /settings/rbac [get] 72 | func (h *settingsHandler) rbacGet(c echo.Context) error { 73 | settings, err := h.store.SettingsGet() 74 | if err != nil { 75 | gaia.Cfg.Logger.Error("failed to get store settings", "error", err.Error()) 76 | return c.String(http.StatusInternalServerError, msgSomethingWentWrong) 77 | } 78 | 79 | response := rbacGetResponse{} 80 | // If RBAC is applied via config it takes priority. 81 | if gaia.Cfg.RBACEnabled { 82 | response.Enabled = true 83 | } else { 84 | response.Enabled = settings.RBACEnabled 85 | } 86 | 87 | return c.JSON(http.StatusOK, rbacGetResponse{Enabled: settings.RBACEnabled}) 88 | } 89 | -------------------------------------------------------------------------------- /handlers/vault_test.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | 11 | "github.com/hashicorp/go-hclog" 12 | "github.com/labstack/echo/v4" 13 | 14 | "github.com/gaia-pipeline/gaia" 15 | ) 16 | 17 | func TestVaultWorkflowAddListDelete(t *testing.T) { 18 | dataDir, _ := ioutil.TempDir("", "TestVaultWorkflowAddListDelete") 19 | 20 | defer func() { 21 | gaia.Cfg = nil 22 | }() 23 | 24 | gaia.Cfg = &gaia.Config{ 25 | Logger: hclog.NewNullLogger(), 26 | DataPath: dataDir, 27 | CAPath: dataDir, 28 | VaultPath: dataDir, 29 | } 30 | 31 | e := echo.New() 32 | t.Run("can add secret", func(t *testing.T) { 33 | body := map[string]string{ 34 | "Key": "Key", 35 | "Value": "Value", 36 | } 37 | bodyBytes, _ := json.Marshal(body) 38 | req := httptest.NewRequest(echo.POST, "/api/"+gaia.APIVersion+"/secret", bytes.NewBuffer(bodyBytes)) 39 | req.Header.Set("Content-Type", "application/json") 40 | rec := httptest.NewRecorder() 41 | c := e.NewContext(req, rec) 42 | 43 | _ = CreateSecret(c) 44 | 45 | if rec.Code != http.StatusCreated { 46 | t.Fatalf("expected response code %v got %v", http.StatusCreated, rec.Code) 47 | } 48 | }) 49 | 50 | t.Run("can update secret", func(t *testing.T) { 51 | body := map[string]string{ 52 | "Key": "Key", 53 | "Value": "Value", 54 | } 55 | bodyBytes, _ := json.Marshal(body) 56 | req := httptest.NewRequest(echo.PUT, "/api/"+gaia.APIVersion+"/secret", bytes.NewBuffer(bodyBytes)) 57 | req.Header.Set("Content-Type", "application/json") 58 | rec := httptest.NewRecorder() 59 | c := e.NewContext(req, rec) 60 | 61 | _ = UpdateSecret(c) 62 | 63 | if rec.Code != http.StatusCreated { 64 | t.Fatalf("expected response code %v got %v", http.StatusCreated, rec.Code) 65 | } 66 | }) 67 | 68 | t.Run("can list secrets", func(t *testing.T) { 69 | req := httptest.NewRequest(echo.GET, "/api/"+gaia.APIVersion+"/secrets", nil) 70 | req.Header.Set("Content-Type", "application/json") 71 | rec := httptest.NewRecorder() 72 | c := e.NewContext(req, rec) 73 | 74 | _ = ListSecrets(c) 75 | 76 | if rec.Code != http.StatusOK { 77 | t.Fatalf("expected response code %v got %v", http.StatusCreated, rec.Code) 78 | } 79 | body, _ := ioutil.ReadAll(rec.Body) 80 | expectedBody := "[{\"key\":\"Key\",\"value\":\"**********\"}]\n" 81 | if string(body) != expectedBody { 82 | t.Fatalf("response body did not equal expected body: expected: %s, actual: %s", expectedBody, string(body)) 83 | } 84 | }) 85 | 86 | t.Run("can delete secrets", func(t *testing.T) { 87 | req := httptest.NewRequest(echo.DELETE, "/api/"+gaia.APIVersion+"/secret/:key", nil) 88 | req.Header.Set("Content-Type", "application/json") 89 | rec := httptest.NewRecorder() 90 | c := e.NewContext(req, rec) 91 | c.SetParamNames("key") 92 | c.SetParamValues("Key") 93 | 94 | _ = RemoveSecret(c) 95 | 96 | if rec.Code != http.StatusOK { 97 | t.Fatalf("expected response code %v got %v", http.StatusCreated, rec.Code) 98 | } 99 | }) 100 | 101 | t.Run("can delete fails if no secret is provided", func(t *testing.T) { 102 | req := httptest.NewRequest(echo.DELETE, "/api/"+gaia.APIVersion+"/secret/:key", nil) 103 | req.Header.Set("Content-Type", "application/json") 104 | rec := httptest.NewRecorder() 105 | c := e.NewContext(req, rec) 106 | 107 | _ = RemoveSecret(c) 108 | 109 | if rec.Code != http.StatusBadRequest { 110 | t.Fatalf("expected response code %v got %v", http.StatusCreated, rec.Code) 111 | } 112 | }) 113 | } 114 | -------------------------------------------------------------------------------- /helm/.helmignore: -------------------------------------------------------------------------------- 1 | _system/ 2 | -------------------------------------------------------------------------------- /helm/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | name: gaiapipeline/gaia 3 | version: latest 4 | description: Build powerful pipelines in any programming language. 5 | -------------------------------------------------------------------------------- /helm/_system/default-http-backend/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: default-http-backend 5 | namespace: kube-system 6 | labels: 7 | k8s-app: default-http-backend 8 | spec: 9 | replicas: 2 10 | selector: 11 | matchLabels: 12 | k8s-app: default-http-backend 13 | template: 14 | metadata: 15 | labels: 16 | k8s-app: default-http-backend 17 | annotations: 18 | scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated", "value":"master"}]' 19 | spec: 20 | terminationGracePeriodSeconds: 60 21 | containers: 22 | - name: default-http-backend 23 | # Any image is permissable as long as: 24 | # 1. It serves a 404 page at / 25 | # 2. It serves 200 on a /healthz endpoint 26 | image: gcr.io/google_containers/defaultbackend:1.2 27 | livenessProbe: 28 | httpGet: 29 | path: /healthz 30 | port: 8080 31 | scheme: HTTP 32 | initialDelaySeconds: 30 33 | timeoutSeconds: 5 34 | ports: 35 | - containerPort: 8080 36 | resources: 37 | limits: 38 | cpu: 10m 39 | memory: 20Mi 40 | requests: 41 | cpu: 10m 42 | memory: 20Mi 43 | -------------------------------------------------------------------------------- /helm/_system/default-http-backend/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | k8s-app: default-http-backend 6 | name: default-http-backend 7 | namespace: kube-system 8 | spec: 9 | ports: 10 | - name: http 11 | port: 80 12 | targetPort: 8080 13 | protocol: TCP 14 | selector: 15 | k8s-app: default-http-backend 16 | sessionAffinity: None 17 | type: ClusterIP 18 | -------------------------------------------------------------------------------- /helm/_system/nginx-ingress/configmap-lb.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: nginx-ingress-lb 5 | namespace: kube-system 6 | creationTimestamp: null 7 | data: 8 | enable-vts-status: "true" 9 | worker-processes: "4" 10 | -------------------------------------------------------------------------------- /helm/_system/nginx-ingress/configmap-tcp.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: nginx-ingress-tcp 5 | namespace: kube-system 6 | data: 7 | # 53: "kube-system/kube-dns:53" 8 | -------------------------------------------------------------------------------- /helm/_system/nginx-ingress/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | labels: 5 | name: nginx-ingress-lb 6 | name: nginx-ingress-lb 7 | namespace: kube-system 8 | spec: 9 | selector: 10 | matchLabels: 11 | name: nginx-ingress-lb 12 | template: 13 | metadata: 14 | labels: 15 | name: nginx-ingress-lb 16 | annotations: 17 | pod.beta.kubernetes.io/init-containers: '[ 18 | { 19 | "name": "sysctl", 20 | "image": "busybox", 21 | "imagePullPolicy": "IfNotPresent", 22 | "command": ["sh", "-c", "sysctl -w net.core.somaxconn=32768; sysctl -w net.ipv4.ip_local_port_range=\"1024 65535\""], 23 | "securityContext": { 24 | "privileged": true 25 | } 26 | } 27 | ]' 28 | spec: 29 | hostNetwork: true 30 | containers: 31 | - args: 32 | - /nginx-ingress-controller 33 | - --ingress-class=service 34 | - --default-backend-service=$(POD_NAMESPACE)/default-http-backend 35 | - --configmap=$(POD_NAMESPACE)/nginx-ingress-lb 36 | - --tcp-services-configmap=$(POD_NAMESPACE)/nginx-ingress-tcp 37 | env: 38 | - name: POD_NAME 39 | valueFrom: 40 | fieldRef: 41 | apiVersion: v1 42 | fieldPath: metadata.name 43 | - name: POD_NAMESPACE 44 | valueFrom: 45 | fieldRef: 46 | apiVersion: v1 47 | fieldPath: metadata.namespace 48 | image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 49 | imagePullPolicy: Always 50 | name: nginx-ingress-lb 51 | ports: 52 | - name: web 53 | containerPort: 80 54 | protocol: TCP 55 | readinessProbe: 56 | failureThreshold: 3 57 | httpGet: 58 | path: /healthz 59 | port: 10254 60 | scheme: HTTP 61 | periodSeconds: 10 62 | successThreshold: 1 63 | timeoutSeconds: 1 64 | 65 | -------------------------------------------------------------------------------- /helm/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Release.Name }} 5 | labels: 6 | service: {{ .Release.Name }} 7 | app: service 8 | spec: 9 | replicas: {{ .Values.replicas }} 10 | selector: 11 | matchLabels: 12 | app: service 13 | strategy: 14 | type: RollingUpdate 15 | rollingUpdate: 16 | maxSurge: {{ .Values.rollingUpdate.maxSurge }} 17 | maxUnavailable: {{ .Values.rollingUpdate.maxUnavailable }} 18 | minReadySeconds: {{ .Values.minReadySeconds }} 19 | revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} 20 | template: 21 | metadata: 22 | labels: 23 | service: {{ .Release.Name }} 24 | app: service 25 | spec: 26 | containers: 27 | - name: service 28 | image: gaiapipeline/gaia:{{ .Values.imageTag }} 29 | imagePullPolicy: {{ .Values.imagePullPolicy }} 30 | lifecycle: 31 | # @see https://github.com/kubernetes/contrib/issues/1140 32 | preStop: 33 | exec: 34 | command: ["sleep", "{{ .Values.preStop.sleep }}"] 35 | env: 36 | - name: GAIA_PORT 37 | value: "8080" 38 | - name: GAIA_HOST_NAME 39 | value: http:0.0.0.0 40 | ports: 41 | - containerPort: 8080 42 | resources: 43 | {{ toYaml .Values.resources | indent 10 }} 44 | readinessProbe: 45 | httpGet: 46 | path: / 47 | port: 8080 48 | initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} 49 | timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} 50 | successThreshold: {{ .Values.readinessProbe.successThreshold }} 51 | failureThreshold: {{ .Values.readinessProbe.failureThreshold }} 52 | -------------------------------------------------------------------------------- /helm/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ingress.gaia.enabled }} 2 | apiVersion: extensions/v1beta1 3 | kind: Ingress 4 | metadata: 5 | name: {{ .Release.Name }} 6 | labels: 7 | service: {{ .Release.Name }} 8 | app: service 9 | annotations: 10 | kubernetes.io/ingress.class: service 11 | nginx.ingress.kubernetes.io/rewrite-target: / 12 | nginx.ingress.kubernetes.io/ssl-redirect: "false" 13 | nginx.ingress.kubernetes.io/force-ssl-redirect: "false" 14 | spec: 15 | rules: 16 | - http: 17 | paths: 18 | - backend: 19 | serviceName: {{ .Release.Name }} 20 | servicePort: 8080 21 | path: / 22 | host: {{ .Values.ingress.gaia.host }} 23 | {{ end }} 24 | -------------------------------------------------------------------------------- /helm/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ .Release.Name }} 5 | labels: 6 | service: {{ .Release.Name }} 7 | app: service 8 | spec: 9 | ports: 10 | - port: 8080 11 | targetPort: 8080 12 | selector: 13 | service: {{ .Release.Name }} 14 | app: service 15 | -------------------------------------------------------------------------------- /helm/values.yaml: -------------------------------------------------------------------------------- 1 | replicas: 1 2 | rollingUpdate: 3 | maxSurge: 50% 4 | maxUnavailable: 20% 5 | minReadySeconds: 3 6 | revisionHistoryLimit: 3 7 | imageTag: latest 8 | imagePullPolicy: Always 9 | preStop: 10 | sleep: 3 11 | 12 | resources: 13 | requests: 14 | cpu: 50m 15 | memory: 50Mi 16 | limits: 17 | cpu: 500m 18 | memory: 500Mi 19 | 20 | readinessProbe: 21 | initialDelaySeconds: 3 22 | timeoutSeconds: 3 23 | successThreshold: 1 24 | failureThreshold: 3 25 | 26 | ingress: 27 | gaia: 28 | enabled: true 29 | host: gaia.k8s.dev 30 | -------------------------------------------------------------------------------- /helper/assethelper/helper.go: -------------------------------------------------------------------------------- 1 | package assethelper 2 | 3 | import ( 4 | rice "github.com/GeertJohan/go.rice" 5 | ) 6 | 7 | const ( 8 | rbacBuiltinPolicy = "rbac-policy.csv" 9 | rbacModel = "rbac-model.conf" 10 | rbacAPIMappings = "rbac-api-mappings.yml" 11 | ) 12 | 13 | func loadStaticFile(filename string) (string, error) { 14 | box, err := rice.FindBox("../../static") 15 | if err != nil { 16 | return "", err 17 | } 18 | filestr, err := box.String(filename) 19 | if err != nil { 20 | return "", err 21 | } 22 | return filestr, nil 23 | } 24 | 25 | // LoadRBACBuiltinPolicy loads the builtin rbac-policy.csv 26 | func LoadRBACBuiltinPolicy() (string, error) { 27 | return loadStaticFile(rbacBuiltinPolicy) 28 | } 29 | 30 | // LoadRBACAPIMappings loads the rbac-api-mappings.yml 31 | func LoadRBACAPIMappings() (string, error) { 32 | return loadStaticFile(rbacAPIMappings) 33 | } 34 | 35 | // LoadRBACModel loads the rbac-model.conf 36 | func LoadRBACModel() (string, error) { 37 | return loadStaticFile(rbacModel) 38 | } 39 | -------------------------------------------------------------------------------- /helper/filehelper/filehelper.go: -------------------------------------------------------------------------------- 1 | package filehelper 2 | 3 | import ( 4 | "crypto/sha256" 5 | "io" 6 | "os" 7 | ) 8 | 9 | // GetSHA256Sum accepts a path to a file. 10 | // It load's the file and calculates a SHA256 Checksum and returns it. 11 | func GetSHA256Sum(path string) ([]byte, error) { 12 | // Open file 13 | f, err := os.Open(path) 14 | if err != nil { 15 | return nil, err 16 | } 17 | defer f.Close() 18 | 19 | // Create sha256 obj and insert bytes 20 | h := sha256.New() 21 | if _, err := io.Copy(h, f); err != nil { 22 | return nil, err 23 | } 24 | 25 | // return sha256 checksum 26 | return h.Sum(nil), nil 27 | } 28 | 29 | // CopyFileContents copies the content from source to destination. 30 | func CopyFileContents(src, dst string) (err error) { 31 | in, err := os.Open(src) 32 | if err != nil { 33 | return 34 | } 35 | defer in.Close() 36 | out, err := os.Create(dst) 37 | if err != nil { 38 | return 39 | } 40 | defer func() { 41 | cerr := out.Close() 42 | if err == nil { 43 | err = cerr 44 | } 45 | }() 46 | if _, err = io.Copy(out, in); err != nil { 47 | return 48 | } 49 | err = out.Sync() 50 | return 51 | } 52 | -------------------------------------------------------------------------------- /helper/filehelper/filehelper_test.go: -------------------------------------------------------------------------------- 1 | package filehelper 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | ) 11 | 12 | func TestGetSHA256Sum(t *testing.T) { 13 | tmp, _ := ioutil.TempDir("", "TestGetSHA256Sum") 14 | sumText := []byte("hello world\n") 15 | filePath := filepath.Join(tmp, "test.file") 16 | err := ioutil.WriteFile(filePath, sumText, 0777) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | defer os.RemoveAll(tmp) 21 | calcSha, err := GetSHA256Sum(filePath) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | h := sha256.New() 26 | if _, err := h.Write(sumText); err != nil { 27 | t.Fatal(err) 28 | } 29 | if !bytes.Equal(h.Sum(nil), calcSha) { 30 | t.Fatal("bytes are not identical") 31 | } 32 | } 33 | 34 | func TestCopyFileContents(t *testing.T) { 35 | tmp, _ := ioutil.TempDir("", "TestCopyFileContents") 36 | sumText := []byte("hello world\n") 37 | filePath := filepath.Join(tmp, "test.file") 38 | err := ioutil.WriteFile(filePath, sumText, 0777) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | defer os.RemoveAll(tmp) 43 | calcSha, err := GetSHA256Sum(filePath) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | output := filepath.Join(tmp, "copy.file") 48 | if err := CopyFileContents(filePath, output); err != nil { 49 | t.Fatal(err) 50 | } 51 | copySha, err := GetSHA256Sum(output) 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | if !bytes.Equal(copySha, calcSha) { 56 | t.Fatal("bytes are not identical") 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /helper/pipelinehelper/pipelinehelper.go: -------------------------------------------------------------------------------- 1 | package pipelinehelper 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/gaia-pipeline/gaia" 10 | ) 11 | 12 | const ( 13 | typeDelimiter = "_" 14 | ) 15 | 16 | // GetRealPipelineName removes the suffix from the pipeline name. 17 | func GetRealPipelineName(name string, pType gaia.PipelineType) string { 18 | return strings.TrimSuffix(name, typeDelimiter+pType.String()) 19 | } 20 | 21 | // AppendTypeToName appends the type to the output binary name. 22 | // This allows later to define the pipeline type by the pipeline binary name. 23 | func AppendTypeToName(n string, pType gaia.PipelineType) string { 24 | return fmt.Sprintf("%s%s%s", n, typeDelimiter, pType.String()) 25 | } 26 | 27 | // GetLocalDestinationForPipeline computes the local location of a pipeline on disk based on the pipeline's 28 | // type and configuration of Gaia such as, temp folder and data folder. 29 | func GetLocalDestinationForPipeline(p gaia.Pipeline) (string, error) { 30 | tmpFolder, err := tmpFolderFromPipelineType(p) 31 | if err != nil { 32 | gaia.Cfg.Logger.Error("Pipeline type invalid", "type", p.Type) 33 | return "", err 34 | } 35 | return filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, tmpFolder, gaia.SrcFolder, p.UUID), nil 36 | } 37 | 38 | // tmpFolderFromPipelineType returns the gaia specific tmp folder for a pipeline 39 | // based on the type of the pipeline. 40 | func tmpFolderFromPipelineType(foundPipeline gaia.Pipeline) (string, error) { 41 | switch foundPipeline.Type { 42 | case gaia.PTypeCpp: 43 | return gaia.TmpCppFolder, nil 44 | case gaia.PTypeGolang: 45 | return gaia.TmpGoFolder, nil 46 | case gaia.PTypeNodeJS: 47 | return gaia.TmpNodeJSFolder, nil 48 | case gaia.PTypePython: 49 | return gaia.TmpPythonFolder, nil 50 | case gaia.PTypeRuby: 51 | return gaia.TmpRubyFolder, nil 52 | case gaia.PTypeJava: 53 | return gaia.TmpJavaFolder, nil 54 | } 55 | return "", errors.New("invalid pipeline type") 56 | } 57 | -------------------------------------------------------------------------------- /helper/pipelinehelper/pipelinehelper_test.go: -------------------------------------------------------------------------------- 1 | package pipelinehelper 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gaia-pipeline/gaia/helper/stringhelper" 7 | 8 | "github.com/gaia-pipeline/gaia" 9 | ) 10 | 11 | func TestGetRealPipelineName(t *testing.T) { 12 | pipeGo := "my_pipeline_golang" 13 | if GetRealPipelineName(pipeGo, gaia.PTypeGolang) != "my_pipeline" { 14 | t.Fatalf("output should be my_pipeline but is %s", GetRealPipelineName(pipeGo, gaia.PTypeGolang)) 15 | } 16 | } 17 | 18 | func TestAppendTypeToName(t *testing.T) { 19 | expected := []string{"my_pipeline_golang", "my_pipeline2_java", "my_pipeline_python"} 20 | input := []struct { 21 | name string 22 | pType gaia.PipelineType 23 | }{ 24 | { 25 | "my_pipeline", 26 | gaia.PTypeGolang, 27 | }, 28 | { 29 | "my_pipeline2", 30 | gaia.PTypeJava, 31 | }, 32 | { 33 | "my_pipeline", 34 | gaia.PTypePython, 35 | }, 36 | } 37 | 38 | for _, inp := range input { 39 | got := AppendTypeToName(inp.name, inp.pType) 40 | if !stringhelper.IsContainedInSlice(expected, got, false) { 41 | t.Fatalf("expected name not contained: %s", got) 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /helper/rolehelper/role_test.go: -------------------------------------------------------------------------------- 1 | package rolehelper 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gaia-pipeline/gaia" 7 | ) 8 | 9 | var mockData = []*gaia.UserRoleCategory{ 10 | { 11 | Name: "CategoryA", 12 | Roles: []*gaia.UserRole{ 13 | { 14 | Name: "RoleA", 15 | }, 16 | { 17 | Name: "RoleB", 18 | }, 19 | }, 20 | }, 21 | { 22 | Name: "CategoryB", 23 | Roles: []*gaia.UserRole{ 24 | { 25 | Name: "RoleA", 26 | }, 27 | { 28 | Name: "RoleB", 29 | }, 30 | }, 31 | }, 32 | } 33 | 34 | func TestFlatRoleName(t *testing.T) { 35 | value := FullUserRoleName(mockData[0], mockData[0].Roles[0]) 36 | expect := "CategoryARoleA" 37 | if value != expect { 38 | t.Fatalf("value %s should equal: %s", value, expect) 39 | } 40 | } 41 | 42 | func TestFlattenUserCategoryRoles(t *testing.T) { 43 | value := FlattenUserCategoryRoles(mockData) 44 | expect := []string{"CategoryARoleA", "CategoryARoleB", "CategoryBRoleA", "CategoryBRoleB"} 45 | 46 | for i := range expect { 47 | if expect[i] != value[i] { 48 | t.Fatalf("value %s should exist: %s", expect[i], value[i]) 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /helper/stringhelper/stringhelper.go: -------------------------------------------------------------------------------- 1 | package stringhelper 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | ) 7 | 8 | // IsContainedInSlice checks if the given string is contained in the slice. 9 | // If case insensitive is enabled, checks are case insensitive. 10 | func IsContainedInSlice(s []string, c string, caseIns bool) bool { 11 | for _, curr := range s { 12 | if caseIns && strings.EqualFold(curr, c) { 13 | return true 14 | } else if curr == c { 15 | return true 16 | } 17 | } 18 | return false 19 | } 20 | 21 | // DiffSlices returns A - B set difference of the two given slices. 22 | // It also sorts the returned slice. 23 | func DiffSlices(a []string, b []string, caseIns bool) []string { 24 | m := map[string]bool{} 25 | for _, aItem := range a { 26 | if caseIns { 27 | aItem = strings.ToLower(aItem) 28 | } 29 | 30 | m[aItem] = true 31 | } 32 | 33 | // Check if equal 34 | for _, bItem := range b { 35 | if _, ok := m[bItem]; ok { 36 | m[bItem] = false 37 | } 38 | } 39 | 40 | var items []string 41 | for item, exists := range m { 42 | if exists { 43 | items = append(items, item) 44 | } 45 | } 46 | sort.Strings(items) 47 | return items 48 | } -------------------------------------------------------------------------------- /helper/stringhelper/stringhelper_test.go: -------------------------------------------------------------------------------- 1 | package stringhelper 2 | 3 | import "testing" 4 | 5 | func TestIsContainedInSlice(t *testing.T) { 6 | slice := []string{"ITEM1", "item1", "ITEM2"} 7 | 8 | // Simple contains check 9 | if !IsContainedInSlice(slice, "item1", false) { 10 | t.Fatal("item1 not contained in slice") 11 | } 12 | 13 | // Check without case insensitive 14 | if IsContainedInSlice(slice, "Item2", false) { 15 | t.Fatal("Item2 is contained but should not") 16 | } 17 | 18 | // Check with case insensitive 19 | if !IsContainedInSlice(slice, "item2", true) { 20 | t.Fatal("item2 case insensitive check failed") 21 | } 22 | 23 | // Should also work for duplicates 24 | if !IsContainedInSlice(slice, "item1", true) { 25 | t.Fatal("item1 not contained") 26 | } 27 | } 28 | 29 | func TestDiffSlices(t *testing.T) { 30 | a := []string{"ITEM1", "item1", "item3", "item2"} 31 | b := []string{"item1"} 32 | 33 | // Simple case insensitive check 34 | out := DiffSlices(a, b, true) 35 | for _, item := range DiffSlices(a, b, true) { 36 | if item == "ITEM1" || item == "item1" { 37 | t.Fatalf("item1 should be non-existend: %s", item) 38 | } 39 | } 40 | 41 | // Check if it is sorted 42 | if len(out) != 2 { 43 | t.Fatalf("expected 2 but got %d", len(out)) 44 | } 45 | if out[1] != "item3" { 46 | t.Fatalf("expected '%s' but got '%s'", "item3", out[1]) 47 | } 48 | 49 | 50 | // Multiple different values 51 | b = append(b, "nonexistend") 52 | b = append(b, "item2") 53 | out = DiffSlices(a, b, false) 54 | 55 | // Check 56 | if len(out) != 2 { 57 | t.Fatalf("expected 2 but got %d", len(out)) 58 | } 59 | if out[0] != "ITEM1" { 60 | t.Fatalf("expected '%s' but got '%s'", "ITEM1", out[0]) 61 | } 62 | if out[1] != "item3" { 63 | t.Fatalf("expected '%s' but got '%s'", "item3", out[1]) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /plugin/grpc.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "context" 5 | 6 | proto "github.com/gaia-pipeline/protobuf" 7 | plugin "github.com/hashicorp/go-plugin" 8 | "google.golang.org/grpc" 9 | ) 10 | 11 | // GaiaPlugin is the Gaia plugin interface used for communication 12 | // with the plugin. 13 | type GaiaPlugin interface { 14 | GetJobs() (proto.Plugin_GetJobsClient, error) 15 | ExecuteJob(job *proto.Job) (*proto.JobResult, error) 16 | } 17 | 18 | // GaiaPluginClient represents gRPC client 19 | type GaiaPluginClient struct { 20 | client proto.PluginClient 21 | } 22 | 23 | // GaiaPluginImpl represents the plugin implementation on client side. 24 | type GaiaPluginImpl struct { 25 | Impl GaiaPlugin 26 | 27 | plugin.NetRPCUnsupportedPlugin 28 | } 29 | 30 | // GRPCServer is needed here to implement hashicorp 31 | // plugin.Plugin interface. Real implementation is 32 | // in the plugin(s). 33 | func (p *GaiaPluginImpl) GRPCServer(b *plugin.GRPCBroker, s *grpc.Server) error { 34 | // Real implementation defined in plugin 35 | return nil 36 | } 37 | 38 | // GRPCClient is the passing method for the gRPC client. 39 | func (p *GaiaPluginImpl) GRPCClient(context context.Context, b *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { 40 | return &GaiaPluginClient{client: proto.NewPluginClient(c)}, nil 41 | } 42 | 43 | // GetJobs requests all jobs from the plugin. 44 | // We get a stream of proto.Job back. 45 | func (m *GaiaPluginClient) GetJobs() (proto.Plugin_GetJobsClient, error) { 46 | return m.client.GetJobs(context.Background(), &proto.Empty{}) 47 | } 48 | 49 | // ExecuteJob triggers the execution of the given job in the plugin. 50 | func (m *GaiaPluginClient) ExecuteJob(job *proto.Job) (*proto.JobResult, error) { 51 | return m.client.ExecuteJob(context.Background(), job) 52 | } 53 | -------------------------------------------------------------------------------- /providers/pipelines/pipeline_provider.go: -------------------------------------------------------------------------------- 1 | package pipelines 2 | 3 | import ( 4 | "github.com/labstack/echo/v4" 5 | 6 | "github.com/gaia-pipeline/gaia/store" 7 | "github.com/gaia-pipeline/gaia/workers/pipeline" 8 | "github.com/gaia-pipeline/gaia/workers/scheduler/service" 9 | ) 10 | 11 | // Dependencies define providers and services which this service needs. 12 | type Dependencies struct { 13 | Scheduler service.GaiaScheduler 14 | PipelineService pipeline.Servicer 15 | SettingsStore store.SettingsStore 16 | } 17 | 18 | // PipelineProvider is a provider for all pipeline related operations. 19 | type PipelineProvider struct { 20 | deps Dependencies 21 | } 22 | 23 | // PipelineProviderer defines functionality which this provider provides. 24 | // These are used by the handler service. 25 | type PipelineProviderer interface { 26 | PipelineGitLSRemote(c echo.Context) error 27 | CreatePipeline(c echo.Context) error 28 | CreatePipelineGetAll(c echo.Context) error 29 | PipelineNameAvailable(c echo.Context) error 30 | PipelineGet(c echo.Context) error 31 | PipelineGetAll(c echo.Context) error 32 | PipelineUpdate(c echo.Context) error 33 | PipelinePull(c echo.Context) error 34 | PipelineDelete(c echo.Context) error 35 | PipelineTrigger(c echo.Context) error 36 | PipelineResetToken(c echo.Context) error 37 | PipelineTriggerAuth(c echo.Context) error 38 | PipelineStart(c echo.Context) error 39 | PipelineGetAllWithLatestRun(c echo.Context) error 40 | PipelineCheckPeriodicSchedules(c echo.Context) error 41 | PipelineStop(c echo.Context) error 42 | PipelineRunGet(c echo.Context) error 43 | PipelineGetAllRuns(c echo.Context) error 44 | PipelineGetLatestRun(c echo.Context) error 45 | GetJobLogs(c echo.Context) error 46 | GitWebHook(c echo.Context) error 47 | SettingsPollOn(c echo.Context) error 48 | SettingsPollOff(c echo.Context) error 49 | SettingsPollGet(c echo.Context) error 50 | } 51 | 52 | // NewPipelineProvider creates a new provider with the needed dependencies. 53 | func NewPipelineProvider(deps Dependencies) *PipelineProvider { 54 | return &PipelineProvider{deps: deps} 55 | } 56 | -------------------------------------------------------------------------------- /providers/pipelines/settings.go: -------------------------------------------------------------------------------- 1 | package pipelines 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/labstack/echo/v4" 7 | 8 | "github.com/gaia-pipeline/gaia" 9 | ) 10 | 11 | // SettingsPollOn turn on polling functionality 12 | // @Summary Turn on polling functionality. 13 | // @Description Turns on the polling functionality for Gaia which periodically checks if there is new code to deploy for all pipelines. 14 | // @Tags settings 15 | // @Produce plain 16 | // @Security ApiKeyAuth 17 | // @Success 200 {string} string "Polling is turned on." 18 | // @Failure 400 {string} string "Error while toggling poll setting." 19 | // @Failure 500 {string} string "Internal server error while getting setting." 20 | // @Router /settings/poll/on [post] 21 | func (pp *PipelineProvider) SettingsPollOn(c echo.Context) error { 22 | settingsStore := pp.deps.SettingsStore 23 | 24 | configStore, err := settingsStore.SettingsGet() 25 | if err != nil { 26 | return c.String(http.StatusInternalServerError, "Something went wrong while retrieving settings information.") 27 | } 28 | if configStore == nil { 29 | configStore = &gaia.StoreConfig{} 30 | } 31 | 32 | gaia.Cfg.Poll = true 33 | err = pp.deps.PipelineService.StartPoller() 34 | if err != nil { 35 | return c.String(http.StatusBadRequest, err.Error()) 36 | } 37 | 38 | configStore.Poll = true 39 | err = settingsStore.SettingsPut(configStore) 40 | if err != nil { 41 | return c.String(http.StatusBadRequest, err.Error()) 42 | } 43 | return c.String(http.StatusOK, "Polling is turned on.") 44 | } 45 | 46 | // SettingsPollOff turn off polling functionality. 47 | // @Summary Turn off polling functionality. 48 | // @Description Turns off the polling functionality for Gaia which periodically checks if there is new code to deploy for all pipelines. 49 | // @Tags settings 50 | // @Produce plain 51 | // @Security ApiKeyAuth 52 | // @Success 200 {string} string "Polling is turned off." 53 | // @Failure 400 {string} string "Error while toggling poll setting." 54 | // @Failure 500 {string} string "Internal server error while getting setting." 55 | // @Router /settings/poll/off [post] 56 | func (pp *PipelineProvider) SettingsPollOff(c echo.Context) error { 57 | settingsStore := pp.deps.SettingsStore 58 | 59 | configStore, err := settingsStore.SettingsGet() 60 | if err != nil { 61 | return c.String(http.StatusInternalServerError, "Something went wrong while retrieving settings information.") 62 | } 63 | if configStore == nil { 64 | configStore = &gaia.StoreConfig{} 65 | } 66 | gaia.Cfg.Poll = false 67 | err = pp.deps.PipelineService.StopPoller() 68 | if err != nil { 69 | return c.String(http.StatusBadRequest, err.Error()) 70 | } 71 | configStore.Poll = true 72 | err = settingsStore.SettingsPut(configStore) 73 | if err != nil { 74 | return c.String(http.StatusBadRequest, err.Error()) 75 | } 76 | return c.String(http.StatusOK, "Polling is turned off.") 77 | } 78 | 79 | type pollStatus struct { 80 | Status bool 81 | } 82 | 83 | // SettingsPollGet get status of polling functionality. 84 | // @Summary Get the status of the poll setting. 85 | // @Description Gets the status of the poll setting. 86 | // @Tags settings 87 | // @Produce json 88 | // @Security ApiKeyAuth 89 | // @Success 200 {object} pollStatus "Poll status" 90 | // @Failure 500 {string} string "Internal server error while getting setting." 91 | // @Router /settings/poll [get] 92 | func (pp *PipelineProvider) SettingsPollGet(c echo.Context) error { 93 | settingsStore := pp.deps.SettingsStore 94 | 95 | configStore, err := settingsStore.SettingsGet() 96 | if err != nil { 97 | return c.String(http.StatusInternalServerError, "Something went wrong while retrieving settings information.") 98 | } 99 | var ps pollStatus 100 | if configStore == nil { 101 | ps.Status = gaia.Cfg.Poll 102 | } else { 103 | ps.Status = configStore.Poll 104 | } 105 | return c.JSON(http.StatusOK, ps) 106 | } 107 | -------------------------------------------------------------------------------- /providers/provider.go: -------------------------------------------------------------------------------- 1 | package providers 2 | 3 | import ( 4 | "github.com/labstack/echo/v4" 5 | ) 6 | 7 | // RBACProvider provides all the handler endpoints for RBAC actions. 8 | type RBACProvider interface { 9 | AddRole(c echo.Context) error 10 | DeleteRole(c echo.Context) error 11 | GetAllRoles(c echo.Context) error 12 | GetUserAttachedRoles(c echo.Context) error 13 | GetRoleAttachedUsers(c echo.Context) error 14 | AttachRole(c echo.Context) error 15 | DetachRole(c echo.Context) error 16 | } 17 | 18 | // UserProvider provides all the handler endpoints for User actions. 19 | type UserProvider interface { 20 | UserLogin(c echo.Context) error 21 | UserGetAll(c echo.Context) error 22 | UserChangePassword(c echo.Context) error 23 | UserResetTriggerToken(c echo.Context) error 24 | UserDelete(c echo.Context) error 25 | UserAdd(c echo.Context) error 26 | UserGetPermissions(c echo.Context) error 27 | UserPutPermissions(c echo.Context) error 28 | } 29 | -------------------------------------------------------------------------------- /providers/workers/worker_provider.go: -------------------------------------------------------------------------------- 1 | package workers 2 | 3 | import ( 4 | "github.com/labstack/echo/v4" 5 | 6 | "github.com/gaia-pipeline/gaia/security" 7 | "github.com/gaia-pipeline/gaia/workers/scheduler/service" 8 | ) 9 | 10 | // Dependencies define dependencies which this service needs. 11 | type Dependencies struct { 12 | Scheduler service.GaiaScheduler 13 | Certificate security.CAAPI 14 | } 15 | 16 | // WorkerProvider has all the operations for a worker. 17 | type WorkerProvider struct { 18 | deps Dependencies 19 | } 20 | 21 | // WorkerProviderer defines functionality which this provider provides. 22 | type WorkerProviderer interface { 23 | RegisterWorker(c echo.Context) error 24 | DeregisterWorker(c echo.Context) error 25 | GetWorkerRegisterSecret(c echo.Context) error 26 | GetWorkerStatusOverview(c echo.Context) error 27 | ResetWorkerRegisterSecret(c echo.Context) error 28 | GetWorker(c echo.Context) error 29 | } 30 | 31 | // NewWorkerProvider creates a provider which provides worker related functionality. 32 | func NewWorkerProvider(deps Dependencies) *WorkerProvider { 33 | return &WorkerProvider{ 34 | deps: deps, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /screenshots/create-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/screenshots/create-pipeline.png -------------------------------------------------------------------------------- /screenshots/detail-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/screenshots/detail-pipeline.png -------------------------------------------------------------------------------- /screenshots/login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/screenshots/login.png -------------------------------------------------------------------------------- /screenshots/logs-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/screenshots/logs-pipeline.png -------------------------------------------------------------------------------- /screenshots/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/screenshots/overview.png -------------------------------------------------------------------------------- /screenshots/settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/screenshots/settings.png -------------------------------------------------------------------------------- /screenshots/vault.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaia-pipeline/gaia/79cb63136ac0a70a13fee5054f4dd8c71001a35a/screenshots/vault.png -------------------------------------------------------------------------------- /security/README.md: -------------------------------------------------------------------------------- 1 | # Security 2 | 3 | ## Certificates 4 | 5 | Gaia, when first started will create a signed certificate in a location 6 | defined by the user under `gaia.Cfg.CAPath` which can be set by the runtime flag 7 | `-capath=/etc/gaia/cert` for example. It is recommended that the certificate 8 | is kept separate from the main Gaia work folder and in a secure location. 9 | 10 | This certificate is used in two places. First, in the communication between the 11 | admin portal and the back-end. Second, by the Vault. 12 | 13 | ## The Vault 14 | 15 | The Vault is a secure storage for secret values like, password, tokens and other 16 | things that the user would like to pass securly into a Pipeline. The Vault is 17 | encrypted using AES cipher technology where the key is derived from the above 18 | certificate and the IV is included in the encrypted content. 19 | 20 | The Vault file's location can be configured through the runtime variable called 21 | `VaultPath`. For maximum security it is recommended that this file is kept on an 22 | encrypted, mounted drive. In case there is a breach the drive can be quickly removed 23 | and the file deleted, thus rotating all of the secrets at once, under Gaia. 24 | 25 | To create an encrypted MacOSX image follow this guide: [Encrypted Secure Disk Image on Mac](https://www.howtogeek.com/183826/how-to-create-an-encrypted-file-container-disk-image-on-a-mac/). 26 | 27 | To create an encrypted disk on Linux follow this guide: [Encrypted Disk Image on Linux](http://freesoftwaremagazine.com/articles/create_encrypted_disk_image_gnulinux/). 28 | 29 | The admin will never see the secure values, not when editing, not when adding and not 30 | when looking at the list of secrets. Only the Key names are displayed at all times. 31 | 32 | It's possible to Add, Delete, Update and List secrets in the system. 33 | -------------------------------------------------------------------------------- /security/ca_test.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "io/ioutil" 7 | "testing" 8 | 9 | "github.com/gaia-pipeline/gaia" 10 | ) 11 | 12 | func TestInitCA(t *testing.T) { 13 | gaia.Cfg = &gaia.Config{} 14 | tmp, _ := ioutil.TempDir("", "TestInitCA") 15 | gaia.Cfg.DataPath = tmp 16 | 17 | c, err := InitCA() 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | 22 | // Get root CA cert path 23 | caCertPath, caKeyPath := c.GetCACertPath() 24 | 25 | // Load CA plain 26 | caPlain, err := tls.LoadX509KeyPair(caCertPath, caKeyPath) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | // Parse certificate 32 | ca, err := x509.ParseCertificate(caPlain.Certificate[0]) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | // Create cert pool and load ca root 38 | certPool := x509.NewCertPool() 39 | rootCA, err := ioutil.ReadFile(caCertPath) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | ok := certPool.AppendCertsFromPEM(rootCA) 45 | if !ok { 46 | t.Fatalf("Cannot append root cert to cert pool!\n") 47 | } 48 | 49 | _, err = ca.Verify(x509.VerifyOptions{ 50 | Roots: certPool, 51 | DNSName: orgDNS, 52 | }) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | 57 | err = c.CleanupCerts(caCertPath, caKeyPath) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | } 62 | 63 | func TestCreateSignedCert(t *testing.T) { 64 | gaia.Cfg = &gaia.Config{} 65 | tmp, _ := ioutil.TempDir("", "TestCreateSignedCert") 66 | gaia.Cfg.DataPath = tmp 67 | 68 | c, err := InitCA() 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | 73 | // Get root ca cert path 74 | caCertPath, caKeyPath := c.GetCACertPath() 75 | 76 | certPath, keyPath, err := c.CreateSignedCert() 77 | if err != nil { 78 | t.Fatal(err) 79 | } 80 | 81 | // Load CA plain 82 | caPlain, err := tls.LoadX509KeyPair(certPath, keyPath) 83 | if err != nil { 84 | t.Fatal(err) 85 | } 86 | 87 | // Parse certificate 88 | ca, err := x509.ParseCertificate(caPlain.Certificate[0]) 89 | if err != nil { 90 | t.Fatal(err) 91 | } 92 | 93 | // Create cert pool and load ca root 94 | certPool := x509.NewCertPool() 95 | rootCA, err := ioutil.ReadFile(caCertPath) 96 | if err != nil { 97 | t.Fatal(err) 98 | } 99 | 100 | ok := certPool.AppendCertsFromPEM(rootCA) 101 | if !ok { 102 | t.Fatalf("Cannot append root cert to cert pool!\n") 103 | } 104 | 105 | _, err = ca.Verify(x509.VerifyOptions{ 106 | Roots: certPool, 107 | DNSName: orgDNS, 108 | }) 109 | if err != nil { 110 | t.Fatal(err) 111 | } 112 | 113 | err = c.CleanupCerts(caCertPath, caKeyPath) 114 | if err != nil { 115 | t.Fatal(err) 116 | } 117 | err = c.CleanupCerts(certPath, keyPath) 118 | if err != nil { 119 | t.Fatal(err) 120 | } 121 | } 122 | 123 | func TestGenerateTLSConfig(t *testing.T) { 124 | gaia.Cfg = &gaia.Config{} 125 | tmp, _ := ioutil.TempDir("", "TestGenerateTLSConfig") 126 | gaia.Cfg.DataPath = tmp 127 | 128 | c, err := InitCA() 129 | if err != nil { 130 | t.Fatal(err) 131 | } 132 | 133 | // Get root ca cert path 134 | caCertPath, caKeyPath := c.GetCACertPath() 135 | 136 | certPath, keyPath, err := c.CreateSignedCert() 137 | if err != nil { 138 | t.Fatal(err) 139 | } 140 | 141 | // Generate TLS Config 142 | _, err = c.GenerateTLSConfig(certPath, keyPath) 143 | if err != nil { 144 | t.Fatal(err) 145 | } 146 | 147 | err = c.CleanupCerts(caCertPath, caKeyPath) 148 | if err != nil { 149 | t.Fatal(err) 150 | } 151 | err = c.CleanupCerts(certPath, keyPath) 152 | if err != nil { 153 | t.Fatal(err) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /security/legacy_vault.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import ( 4 | "bytes" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | "encoding/base64" 8 | "errors" 9 | 10 | "github.com/gaia-pipeline/gaia" 11 | ) 12 | 13 | const ( 14 | secretCheckKey = "GAIA_CHECK_SECRET" 15 | secretCheckValue = "!CHECK_ME!" 16 | ) 17 | 18 | func (v *Vault) legacyDecrypt(data []byte) ([]byte, error) { 19 | if len(data) < 1 { 20 | gaia.Cfg.Logger.Info("the vault is empty") 21 | return []byte{}, nil 22 | } 23 | paddedPassword := v.pad(v.cert) 24 | ci := base64.URLEncoding.EncodeToString(paddedPassword) 25 | block, err := aes.NewCipher([]byte(ci[:aes.BlockSize])) 26 | if err != nil { 27 | return []byte{}, err 28 | } 29 | 30 | decodedMsg, err := base64.URLEncoding.DecodeString(string(data)) 31 | if err != nil { 32 | return []byte{}, err 33 | } 34 | 35 | if (len(decodedMsg) % aes.BlockSize) != 0 { 36 | return []byte{}, errors.New("blocksize must be multiple of decoded message length") 37 | } 38 | 39 | iv := decodedMsg[:aes.BlockSize] 40 | msg := decodedMsg[aes.BlockSize:] 41 | 42 | cfb := cipher.NewCFBDecrypter(block, iv) 43 | cfb.XORKeyStream(msg, msg) 44 | 45 | unpadMsg, err := v.unpad(msg) 46 | if err != nil { 47 | return []byte{}, err 48 | } 49 | 50 | if !bytes.Contains(unpadMsg, []byte(secretCheckValue)) { 51 | return []byte{}, errors.New("possible mistyped password") 52 | } 53 | return unpadMsg, nil 54 | } 55 | 56 | // Pad pads the src with 0x04 until block length. 57 | func (v *Vault) pad(src []byte) []byte { 58 | padding := aes.BlockSize - len(src)%aes.BlockSize 59 | padtext := bytes.Repeat([]byte{byte(padding)}, padding) 60 | return append(src, padtext...) 61 | } 62 | 63 | // Unpad removes the padding from pad. 64 | func (v *Vault) unpad(src []byte) ([]byte, error) { 65 | length := len(src) 66 | unpadding := int(src[length-1]) 67 | 68 | if unpadding > length { 69 | return nil, errors.New("possible mistyped password") 70 | } 71 | 72 | return src[:(length - unpadding)], nil 73 | } 74 | -------------------------------------------------------------------------------- /security/rbac/endpoint_enforcer.go: -------------------------------------------------------------------------------- 1 | package rbac 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/gaia-pipeline/gaia" 8 | ) 9 | 10 | // EndpointEnforcer represents the interface for enforcing RBAC using the echo.Context. 11 | type EndpointEnforcer interface { 12 | Enforce(username, method, path string, params map[string]string) error 13 | } 14 | 15 | // Enforce uses the echo.Context to enforce RBAC. Uses the APILookup to apply policies to specific endpoints. 16 | func (e *enforcerService) Enforce(username, method, path string, params map[string]string) error { 17 | group := e.rbacAPILookup 18 | 19 | endpoint, ok := group[path] 20 | if !ok { 21 | gaia.Cfg.Logger.Warn("path not mapped to api group", "method", method, "path", path) 22 | return nil 23 | } 24 | 25 | perm, ok := endpoint.Methods[method] 26 | if !ok { 27 | gaia.Cfg.Logger.Warn("method not mapped to api group path", "path", path, "method", method) 28 | return nil 29 | } 30 | 31 | splitAction := strings.Split(perm, "/") 32 | namespace := splitAction[0] 33 | action := splitAction[1] 34 | 35 | fullResource := "*" 36 | if endpoint.Param != "" { 37 | param := params[endpoint.Param] 38 | if param == "" { 39 | return fmt.Errorf("error param %s missing", endpoint.Param) 40 | } 41 | fullResource = param 42 | } 43 | 44 | allow, err := e.enforcer.Enforce(username, namespace, action, fullResource) 45 | if err != nil { 46 | return fmt.Errorf("error enforcing rbac: %w", err) 47 | } 48 | if !allow { 49 | return NewErrPermissionDenied(namespace, action, fullResource) 50 | } 51 | 52 | return nil 53 | } 54 | 55 | // ErrPermissionDenied is for when the RBAC enforcement check fails. 56 | type ErrPermissionDenied struct { 57 | namespace string 58 | action string 59 | resource string 60 | } 61 | 62 | // NewErrPermissionDenied creates a new ErrPermissionDenied. 63 | func NewErrPermissionDenied(namespace string, action string, resource string) *ErrPermissionDenied { 64 | return &ErrPermissionDenied{namespace: namespace, action: action, resource: resource} 65 | } 66 | 67 | func (e *ErrPermissionDenied) Error() string { 68 | msg := fmt.Sprintf("Permission denied. Must have %s/%s", e.namespace, e.action) 69 | if e.resource != "*" { 70 | msg = fmt.Sprintf("%s %s", msg, e.resource) 71 | } 72 | return msg 73 | } 74 | -------------------------------------------------------------------------------- /security/rbac/endpoint_enforcer_test.go: -------------------------------------------------------------------------------- 1 | package rbac 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/casbin/casbin/v2" 8 | "github.com/hashicorp/go-hclog" 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/gaia-pipeline/gaia" 12 | ) 13 | 14 | type mockEnforcer struct { 15 | casbin.IEnforcer 16 | } 17 | 18 | var mappings = APILookup{ 19 | "/api/v1/pipeline/:pipelineid": { 20 | Methods: map[string]string{ 21 | "GET": "pipelines/get", 22 | }, 23 | Param: "pipelineid", 24 | }, 25 | } 26 | 27 | func (m *mockEnforcer) Enforce(rvals ...interface{}) (bool, error) { 28 | role := rvals[0].(string) 29 | if role == "admin" { 30 | return true, nil 31 | } 32 | if role == "failed" { 33 | return false, nil 34 | } 35 | return false, errors.New("error test") 36 | } 37 | 38 | func Test_EnforcerService_Enforce_ValidEnforcement(t *testing.T) { 39 | gaia.Cfg = &gaia.Config{ 40 | Logger: hclog.NewNullLogger(), 41 | } 42 | defer func() { 43 | gaia.Cfg = nil 44 | }() 45 | 46 | svc := enforcerService{ 47 | enforcer: &mockEnforcer{}, 48 | rbacAPILookup: mappings, 49 | } 50 | 51 | err := svc.Enforce("admin", "GET", "/api/v1/pipelines/:pipelineid", map[string]string{"pipelineid": "test"}) 52 | assert.NoError(t, err) 53 | } 54 | 55 | func Test_EnforcerService_Enforce_FailedEnforcement(t *testing.T) { 56 | gaia.Cfg = &gaia.Config{ 57 | Logger: hclog.NewNullLogger(), 58 | } 59 | defer func() { 60 | gaia.Cfg = nil 61 | }() 62 | 63 | svc := enforcerService{ 64 | enforcer: &mockEnforcer{}, 65 | rbacAPILookup: mappings, 66 | } 67 | 68 | err := svc.Enforce("failed", "GET", "/api/v1/pipeline/:pipelineid", map[string]string{"pipelineid": "test"}) 69 | assert.EqualError(t, err, "Permission denied. Must have pipelines/get test") 70 | } 71 | 72 | func Test_EnforcerService_Enforce_ErrorEnforcement(t *testing.T) { 73 | gaia.Cfg = &gaia.Config{ 74 | Logger: hclog.NewNullLogger(), 75 | } 76 | defer func() { 77 | gaia.Cfg = nil 78 | }() 79 | 80 | svc := enforcerService{ 81 | enforcer: &mockEnforcer{}, 82 | rbacAPILookup: mappings, 83 | } 84 | 85 | err := svc.Enforce("error", "GET", "/api/v1/pipeline/:pipelineid", map[string]string{"pipelineid": "test"}) 86 | assert.EqualError(t, err, "error enforcing rbac: error test") 87 | } 88 | 89 | func Test_EnforcerService_Enforce_EndpointParamMissing(t *testing.T) { 90 | gaia.Cfg = &gaia.Config{ 91 | Logger: hclog.NewNullLogger(), 92 | } 93 | defer func() { 94 | gaia.Cfg = nil 95 | }() 96 | 97 | svc := enforcerService{ 98 | enforcer: &mockEnforcer{}, 99 | rbacAPILookup: mappings, 100 | } 101 | 102 | err := svc.Enforce("readonly", "GET", "/api/v1/pipeline/:pipelineid", map[string]string{}) 103 | assert.EqualError(t, err, "error param pipelineid missing") 104 | } 105 | -------------------------------------------------------------------------------- /security/rbac/noop.go: -------------------------------------------------------------------------------- 1 | package rbac 2 | 3 | type noOpService struct{} 4 | 5 | // NewNoOpService is used to instantiated a noOpService for when rbac enabled=false. 6 | func NewNoOpService() Service { 7 | return &noOpService{} 8 | } 9 | 10 | // Enforce no-op enforcement. Allows everything. 11 | func (n noOpService) Enforce(username, method, path string, params map[string]string) error { 12 | // Allow all 13 | return nil 14 | } 15 | 16 | // AddRole that errors since rbac is not enabled. 17 | func (n noOpService) AddRole(role string, roleRules []RoleRule) error { 18 | return nil 19 | } 20 | 21 | // DeleteRole that errors since rbac is not enabled. 22 | func (n noOpService) DeleteRole(role string) error { 23 | return nil 24 | } 25 | 26 | // GetAllRoles that returns nothing since rbac is not enabled. 27 | func (n noOpService) GetAllRoles() []string { 28 | return []string{} 29 | } 30 | 31 | // GetUserAttachedRoles that errors since rbac is not enabled. 32 | func (n noOpService) GetUserAttachedRoles(username string) ([]string, error) { 33 | return nil, nil 34 | } 35 | 36 | // GetRoleAttachedUsers that errors since rbac is not enabled. 37 | func (n noOpService) GetRoleAttachedUsers(role string) ([]string, error) { 38 | return nil, nil 39 | } 40 | 41 | // AttachRole that errors since rbac is not enabled. 42 | func (n noOpService) AttachRole(username string, role string) error { 43 | return nil 44 | } 45 | 46 | // DetachRole that errors since rbac is not enabled. 47 | func (n noOpService) DetachRole(username string, role string) error { 48 | return nil 49 | } 50 | 51 | func (n noOpService) DeleteUser(username string) error { 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /security/rbac/noop_test.go: -------------------------------------------------------------------------------- 1 | package rbac 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestNoOpService_Enforce_AlwaysReturnsNoError(t *testing.T) { 10 | svc := NewNoOpService() 11 | err := svc.Enforce("", "", "", map[string]string{}) 12 | assert.NoError(t, err) 13 | } 14 | 15 | func TestNoOpService_AddRole_ReturnsErrNotEnabled(t *testing.T) { 16 | svc := NewNoOpService() 17 | err := svc.AddRole("", []RoleRule{}) 18 | assert.NoError(t, err) 19 | } 20 | 21 | func TestNoOpService_DeleteRole_ReturnsErrNotEnabled(t *testing.T) { 22 | svc := NewNoOpService() 23 | err := svc.DeleteRole("") 24 | assert.NoError(t, err) 25 | } 26 | 27 | func TestNoOpService_GetAllRoles_ReturnsEmptySlice(t *testing.T) { 28 | svc := NewNoOpService() 29 | roles := svc.GetAllRoles() 30 | assert.Equal(t, roles, []string{}) 31 | } 32 | 33 | func TestNoOpService_GetUserAttachedRoles_ReturnsErrNotEnabled(t *testing.T) { 34 | svc := NewNoOpService() 35 | _, err := svc.GetUserAttachedRoles("") 36 | assert.NoError(t, err) 37 | } 38 | 39 | func TestNoOpService_GetRoleAttachedUsers_ReturnsErrNotEnabled(t *testing.T) { 40 | svc := NewNoOpService() 41 | _, err := svc.GetRoleAttachedUsers("") 42 | assert.NoError(t, err) 43 | } 44 | 45 | func TestNoOpService_AttachRole_ReturnsErrNotEnabled(t *testing.T) { 46 | svc := NewNoOpService() 47 | err := svc.AttachRole("", "") 48 | assert.NoError(t, err) 49 | } 50 | 51 | func TestNoOpService_DetachRole_ReturnsErrNotEnabled(t *testing.T) { 52 | svc := NewNoOpService() 53 | err := svc.DetachRole("", "") 54 | assert.NoError(t, err) 55 | } 56 | -------------------------------------------------------------------------------- /security/secret_generator.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import ( 4 | "crypto/rand" 5 | 6 | "github.com/gofrs/uuid" 7 | ) 8 | 9 | // GenerateRandomUUIDV5 will return a 32bit random seeded UUID based on 10 | // a randomly generated UUID v4. 11 | func GenerateRandomUUIDV5() string { 12 | nsUUID, _ := uuid.NewV4() 13 | token := make([]byte, 32) 14 | _, _ = rand.Read(token) 15 | namespace := string(token) 16 | t := uuid.NewV5(nsUUID, namespace) 17 | return t.String() 18 | } 19 | -------------------------------------------------------------------------------- /security/secret_generator_test.go: -------------------------------------------------------------------------------- 1 | package security 2 | 3 | import "testing" 4 | 5 | func TestRandomGeneratingUUIDV5(t *testing.T) { 6 | uuid1 := GenerateRandomUUIDV5() 7 | uuid2 := GenerateRandomUUIDV5() 8 | if uuid1 == uuid2 { 9 | t.Fatalf("the two random generated uuids should not have equalled: uuid1: %s, uuid2: %s\n", uuid1, uuid2) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /security/testdata/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDVCwxvazrjIbap 3 | k4fP0gFThXS1xaWNucwp47w0xMCH0mOi5B1JGfHlRAJmfQOn4jqzEm+2nTzwQmR/ 4 | EN0wV5MPWg1cLTiSpDrFN9nmkJZiZiN5NROJpUPw4sozIkSBli055TSaNErf2KHF 5 | 1lylR4ZyMV313zS+hb5wPfzbSPrmey6f8eY6b1FoS/5caTGweoxk764qRm6UU25Z 6 | cfGaTfpI3KsgN+Vot9qjSfdI7dDs9jRCf7BYCPhm0DfH9e3JBC/HyRQrbxghpjqQ 7 | 8Van3a8OIWeV/AV1BI2gTrzeCWIAhatNAfjRaEVIpTU+1KJc+bGlxUKldqviGICC 8 | YKFnxLSTAgMBAAECggEAHS9gSrsz2/24Wk69ojiwudJkhKpI3buAPpTWKZxyi6jE 9 | wYHiiSsmujOw6H1jzNHvHKz/5NJxkLBnuAiFZKP6n3XEssX3JA+fhXj7Pty73UsE 10 | vQwKWybqwcsvzAV7wQzjsTS3GhDj2PqCXunY00OTJX2h05b6UMddqV60jw3WYVBq 11 | qQ00MuZWV/rwF4Gmt28PCSnMxooLvNB6pJlH0q64prwiJoe/oEDd8yFDAIaOEDw9 12 | 5E6aPnPfCkj4pu3fraHdcGYCyLGsidS+EZ/hPxD1XL/Z0K/SU0o06jw/WV056Op2 13 | ejcptcMXRXKdWy9+blUk1nniwBAE/dEbyssKulADAQKBgQD1KCxeq02Rr8i0cMQw 14 | XaVYgCHzqDb6NmrE7AHAq/ouO3oGVH4a1r/ApVcp2/h8DZYWFuw7E2EoSCS9QrBk 15 | 2UB6h2zF471LE99JH9wPLhWD7LOsuAiunNzpgkysNbmBW7bJQvWiCoBja3gyJyMm 16 | F587vEsKkFRQxh1/ElWfTK0MwQKBgQDed0MgkCSfNKB8NO7oUHGsyZPKH8b2tX+E 17 | Yql+F19HXOrb7FlLgte0I7lB05CaT3wARZEarVsucK0bDt2rSB1E11SEJtVLYrZ2 18 | /dSVk9aHAvDhAOtTj+KEVwDSgfj8spkt+0nCtIUUbsW1/vnO6OwxCF26G8jZNBnB 19 | SJzX3cQSUwKBgCZcfezmY0Hrvr01dA2Zabkae7WT2d53S2e7Al8yyfgYCHUbHYx3 20 | lBPCC4yaRhyrR5P3TEnGM4rJFy6iU9XEBQnnTQb+Ju2rk2Hu4VFixa0aCdd6CKnC 21 | E/NaF0NPONLcFhMSLjuH5yUneOxoIWDhi2IeiaOCiB8HkTAEH2/I4L9BAoGBAL0I 22 | ijm5QeUmSthAAmHVOUKhZrtxlRc90kUjsPI72fJBui91/cp0O+YOFPUiWNVGhQ+W 23 | DV6lv70OcYl0cFeCx5wffOluNgAAuRsTRPh0zu2aSiRnK4+ty8S4STKWzoOrHw47 24 | YMnZqttZ5RZousxej5R6j2n9AgXOh7P9h4jGID2RAoGAPCvBqiJco4udQsosuyt/ 25 | Z0dOjF6sSaiAnx7W2+0aKMRZOhtChlCq0SFvqTvKB7X9UvEVFSyfGEBOawiNXWDu 26 | PHeDB1DWcqToDTKqS5V4dqS+3ZaMHjvX39YhnjLCS0c3nofVWW+Pay11cC9i8tV1 27 | OILObT2tga2txO4JxUrs+NU= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /security/testdata/gaia_vault: -------------------------------------------------------------------------------- 1 | 0xn40kNax9KkbKhUZZGYmIUZJaFDQ1ajEkyB1OgnRf6MSUhe2RK8xqlVj9mBcTMVin3_UPBlXMCrrDt5fbw0kw== -------------------------------------------------------------------------------- /services/service_provider.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/gaia-pipeline/gaia" 7 | "github.com/gaia-pipeline/gaia/security" 8 | "github.com/gaia-pipeline/gaia/store" 9 | "github.com/gaia-pipeline/gaia/store/memdb" 10 | ) 11 | 12 | // storeService is an instance of store. 13 | // Use this to talk to the store. 14 | var storeService store.GaiaStore 15 | 16 | // vaultService is an instance of the internal Vault. 17 | var vaultService security.GaiaVault 18 | 19 | // memDBService is an instance of the internal memdb. 20 | var memDBService memdb.GaiaMemDB 21 | 22 | // StorageService initializes and keeps track of a storage service. 23 | // If the internal storage service is a singleton. This function retruns an error 24 | // but most of the times we don't care about it, because it's only ever 25 | // initialized once in the main.go. If it wouldn't work, main would 26 | // os.Exit(1) and the rest of the application would just stop. 27 | func StorageService() (store.GaiaStore, error) { 28 | if storeService != nil && !reflect.ValueOf(storeService).IsNil() { 29 | return storeService, nil 30 | } 31 | storeService = store.NewBoltStore() 32 | err := storeService.Init(gaia.Cfg.DataPath) 33 | if err != nil { 34 | gaia.Cfg.Logger.Error("cannot initialize store", "error", err.Error()) 35 | return storeService, err 36 | } 37 | return storeService, nil 38 | } 39 | 40 | // MockStorageService sets the internal store singleton to the give 41 | // mock implementation. A mock needs to be created in the test. The 42 | // provider will make sure that everything that would use the store 43 | // will use the mock instead. 44 | func MockStorageService(store store.GaiaStore) { 45 | storeService = store 46 | } 47 | 48 | // DefaultVaultService provides a vault with a FileStorer backend. 49 | func DefaultVaultService() (security.GaiaVault, error) { 50 | return VaultService(&security.FileVaultStorer{}) 51 | } 52 | 53 | // VaultService creates a vault manager service. 54 | func VaultService(vaultStore security.VaultStorer) (security.GaiaVault, error) { 55 | if vaultService != nil && !reflect.ValueOf(vaultService).IsNil() { 56 | return vaultService, nil 57 | } 58 | 59 | // TODO: For now use this to keep the refactor of certificate out of the refactor of Vault Service. 60 | ca, err := security.InitCA() 61 | if err != nil { 62 | gaia.Cfg.Logger.Error("cannot initialize certificate:", "error", err.Error()) 63 | return nil, err 64 | } 65 | v, err := security.NewVault(ca, vaultStore) 66 | if err != nil { 67 | gaia.Cfg.Logger.Error("cannot initialize vault manager:", "error", err.Error()) 68 | return nil, err 69 | } 70 | vaultService = v 71 | return vaultService, nil 72 | } 73 | 74 | // MockVaultService provides a way to create and set a mock 75 | // for the internal vault service manager. 76 | func MockVaultService(service security.GaiaVault) { 77 | vaultService = service 78 | } 79 | 80 | // DefaultMemDBService provides a default memDBService with an underlying storer. 81 | func DefaultMemDBService() (memdb.GaiaMemDB, error) { 82 | s, err := StorageService() 83 | if err != nil { 84 | return nil, err 85 | } 86 | return MemDBService(s) 87 | } 88 | 89 | // MemDBService creates a memdb service instance. 90 | func MemDBService(store store.GaiaStore) (memdb.GaiaMemDB, error) { 91 | if memDBService != nil && !reflect.ValueOf(memDBService).IsNil() { 92 | return memDBService, nil 93 | } 94 | 95 | db, err := memdb.InitMemDB(store) 96 | if err != nil { 97 | gaia.Cfg.Logger.Error("cannot initialize memdb service", "error", err.Error()) 98 | return nil, err 99 | } 100 | memDBService = db 101 | return memDBService, nil 102 | } 103 | 104 | // MockMemDBService provides a way to create and set a mock 105 | // for the internal memdb service manager. 106 | func MockMemDBService(db memdb.GaiaMemDB) { 107 | memDBService = db 108 | } 109 | -------------------------------------------------------------------------------- /static/rbac-model.conf: -------------------------------------------------------------------------------- 1 | [request_definition] 2 | r = sub, nam, act, res 3 | 4 | [policy_definition] 5 | p = sub, nam, act, res, eft 6 | 7 | [role_definition] 8 | g = _, _ 9 | 10 | [policy_effect] 11 | e = some(where (p.eft == allow)) && !some(where (p.eft == deny)) 12 | 13 | [matchers] 14 | m = g(r.sub, p.sub) && keyMatch(r.nam, p.nam) && keyMatch(r.act, p.act) && keyMatch(r.res, p.res) 15 | 16 | -------------------------------------------------------------------------------- /static/rbac-policy.csv: -------------------------------------------------------------------------------- 1 | # Built-in policy which defines two roles - role:admin and role:readonly. 2 | # role:admin is automatically added to the admin user. 3 | # Policy format: 4 | # p, , , , , 5 | 6 | p, role:admin, *, *, *, allow 7 | 8 | p, role:readonly, pipelines, list-created, *, allow 9 | p, role:readonly, pipelines, list, *, allow 10 | p, role:readonly, pipelines, list-latest, *, allow 11 | p, role:readonly, pipelines, get, *, allow 12 | p, role:readonly, pipelines:runs, get-latest-run, *, allow 13 | p, role:readonly, pipelines:runs, get-run, *, allow 14 | p, role:readonly, pipelines:runs, get-latest, *, allow 15 | p, role:readonly, pipelines:runs, get-latest-run, *, allow 16 | p, role:readonly, secrets, list, *, allow 17 | p, role:readonly, users, list, *, allow 18 | p, role:readonly, workers, status-list, *, allow 19 | p, role:readonly, workers, list, *, allow 20 | p, role:readonly, workers, get-secret, *, allow 21 | p, role:readonly, workers, get-status, *, allow 22 | p, role:readonly, settings, get, *, allow 23 | p, role:readonly, rbac:roles, list, *, allow 24 | p, role:readonly, rbac:roles, get-attached, *, allow 25 | p, role:readonly, users, get-roles, *, allow 26 | 27 | g, admin, role:admin 28 | -------------------------------------------------------------------------------- /store/memdb/memdb_schema.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "github.com/hashicorp/go-memdb" 5 | ) 6 | 7 | var memDBSchema = &memdb.DBSchema{ 8 | Tables: map[string]*memdb.TableSchema{ 9 | workerTableName: { 10 | Name: workerTableName, 11 | Indexes: map[string]*memdb.IndexSchema{ 12 | "id": { 13 | Name: "id", 14 | Unique: true, 15 | Indexer: &memdb.StringFieldIndex{Field: "UniqueID"}, 16 | }, 17 | }, 18 | }, 19 | pipelineRunTable: { 20 | Name: pipelineRunTable, 21 | Indexes: map[string]*memdb.IndexSchema{ 22 | "id": { 23 | Name: "id", 24 | Unique: true, 25 | Indexer: &memdb.StringFieldIndex{Field: "UniqueID"}, 26 | }, 27 | }, 28 | }, 29 | dockerWorkerTableName: { 30 | Name: dockerWorkerTableName, 31 | Indexes: map[string]*memdb.IndexSchema{ 32 | "id": { 33 | Name: "id", 34 | Unique: true, 35 | Indexer: &memdb.StringFieldIndex{Field: "WorkerID"}, 36 | }, 37 | }, 38 | }, 39 | }, 40 | } 41 | -------------------------------------------------------------------------------- /store/permission.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | bolt "go.etcd.io/bbolt" 7 | 8 | "github.com/gaia-pipeline/gaia" 9 | ) 10 | 11 | // UserPermissionsGet gets the permission data for a given username. 12 | func (s *BoltStore) UserPermissionsGet(username string) (*gaia.UserPermission, error) { 13 | var perms *gaia.UserPermission 14 | 15 | err := s.db.View(func(tx *bolt.Tx) error { 16 | b := tx.Bucket(userPermsBucket) 17 | 18 | g := b.Get([]byte(username)) 19 | if g == nil { 20 | return nil 21 | } 22 | 23 | return json.Unmarshal(g, &perms) 24 | }) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | return perms, nil 30 | } 31 | 32 | // UserPermissionsPut adds or updates user permissions. 33 | func (s *BoltStore) UserPermissionsPut(perms *gaia.UserPermission) error { 34 | return s.db.Update(func(tx *bolt.Tx) error { 35 | b := tx.Bucket(userPermsBucket) 36 | 37 | m, err := json.Marshal(perms) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | return b.Put([]byte(perms.Username), m) 43 | }) 44 | } 45 | 46 | // UserPermissionsDelete deletes permission data for a given username. 47 | func (s *BoltStore) UserPermissionsDelete(username string) error { 48 | return s.db.Update(func(tx *bolt.Tx) error { 49 | b := tx.Bucket(userPermsBucket) 50 | return b.Delete([]byte(username)) 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /store/settings.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | bolt "go.etcd.io/bbolt" 7 | 8 | "github.com/gaia-pipeline/gaia" 9 | ) 10 | 11 | const ( 12 | configSettings = "gaia_config_settings" 13 | ) 14 | 15 | // SettingsPut puts settings into the store. 16 | func (s *BoltStore) SettingsPut(c *gaia.StoreConfig) error { 17 | return s.db.Update(func(tx *bolt.Tx) error { 18 | // Get settings bucket 19 | b := tx.Bucket(settingsBucket) 20 | 21 | // Marshal pipeline data into bytes. 22 | buf, err := json.Marshal(c) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | // Persist bytes to settings bucket. 28 | return b.Put([]byte(configSettings), buf) 29 | }) 30 | } 31 | 32 | // SettingsGet gets a pipeline by given id. 33 | func (s *BoltStore) SettingsGet() (*gaia.StoreConfig, error) { 34 | var config = &gaia.StoreConfig{} 35 | 36 | return config, s.db.View(func(tx *bolt.Tx) error { 37 | // Get bucket 38 | b := tx.Bucket(settingsBucket) 39 | 40 | // Get pipeline 41 | v := b.Get([]byte(configSettings)) 42 | 43 | // Check if we found the pipeline 44 | if v == nil { 45 | return nil 46 | } 47 | 48 | // Unmarshal pipeline object 49 | err := json.Unmarshal(v, config) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | return nil 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /store/settings_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/gaia-pipeline/gaia" 11 | ) 12 | 13 | func TestBoltStore_Settings(t *testing.T) { 14 | tmp, err := ioutil.TempDir("", "TestBoltStore_SettingsGet") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | defer os.RemoveAll(tmp) 19 | 20 | store := NewBoltStore() 21 | gaia.Cfg.Bolt.Mode = 0600 22 | err = store.Init(tmp) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | defer store.Close() 27 | 28 | empty := &gaia.StoreConfig{ 29 | ID: 0, 30 | Poll: false, 31 | RBACEnabled: false, 32 | } 33 | 34 | config, err := store.SettingsGet() 35 | assert.NoError(t, err) 36 | assert.EqualValues(t, empty, config) 37 | 38 | cfg := &gaia.StoreConfig{ 39 | ID: 1, 40 | Poll: true, 41 | RBACEnabled: true, 42 | } 43 | 44 | err = store.SettingsPut(cfg) 45 | assert.NoError(t, err) 46 | 47 | config, err = store.SettingsGet() 48 | assert.NoError(t, err) 49 | assert.EqualValues(t, config, cfg) 50 | } 51 | -------------------------------------------------------------------------------- /store/sha_pair.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | bolt "go.etcd.io/bbolt" 7 | 8 | "github.com/gaia-pipeline/gaia" 9 | ) 10 | 11 | // UpsertSHAPair creates or updates a record for a SHA pair of the original SHA and the 12 | // rebuilt Worker SHA for a pipeline. 13 | func (s *BoltStore) UpsertSHAPair(pair gaia.SHAPair) error { 14 | return s.db.Update(func(tx *bolt.Tx) error { 15 | // Get bucket 16 | b := tx.Bucket(shaPairBucket) 17 | 18 | // Marshal SHAPair struct 19 | m, err := json.Marshal(pair) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | // Put SHAPair 25 | return b.Put(itob(pair.PipelineID), m) 26 | }) 27 | } 28 | 29 | // GetSHAPair returns a pair of shas for this pipeline run. 30 | func (s *BoltStore) GetSHAPair(pipelineID int) (ok bool, pair gaia.SHAPair, err error) { 31 | return ok, pair, s.db.View(func(tx *bolt.Tx) error { 32 | // Get bucket 33 | b := tx.Bucket(shaPairBucket) 34 | 35 | // Get SHAPair 36 | v := b.Get(itob(pipelineID)) 37 | 38 | // Check if we found the SHAPair 39 | if v == nil { 40 | ok = false 41 | return nil 42 | } 43 | 44 | // Unmarshal SHAPair struct 45 | err := json.Unmarshal(v, &pair) 46 | if err != nil { 47 | return err 48 | } 49 | ok = true 50 | return nil 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /store/sha_pair_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/gaia-pipeline/gaia" 10 | ) 11 | 12 | func TestGetSHAPair(t *testing.T) { 13 | // Create tmp folder 14 | tmp, err := ioutil.TempDir("", "TestGetSHAPAir") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | defer os.RemoveAll(tmp) 19 | 20 | store := NewBoltStore() 21 | gaia.Cfg.Bolt.Mode = 0600 22 | err = store.Init(tmp) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | defer store.Close() 27 | 28 | pair := gaia.SHAPair{} 29 | pair.PipelineID = 1 30 | pair.Original = []byte("original") 31 | pair.Worker = []byte("worker") 32 | err = store.UpsertSHAPair(pair) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | ok, p, err := store.GetSHAPair(1) 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | if !ok { 42 | t.Fatalf("sha pair not found") 43 | } 44 | 45 | if p.PipelineID != pair.PipelineID { 46 | t.Fatalf("pipeline id match error. want %d got %d", pair.PipelineID, p.PipelineID) 47 | } 48 | if !bytes.Equal(p.Worker, pair.Worker) { 49 | t.Fatalf("worker sha match error. want %s got %s", pair.Worker, p.Worker) 50 | } 51 | if !bytes.Equal(p.Original, pair.Original) { 52 | t.Fatalf("original sha match error. want %s got %s", pair.Original, p.Original) 53 | } 54 | } 55 | 56 | func TestUpsertSHAPair(t *testing.T) { 57 | // Create tmp folder 58 | tmp, err := ioutil.TempDir("", "TestUpsertSHAPair") 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | defer os.RemoveAll(tmp) 63 | 64 | store := NewBoltStore() 65 | gaia.Cfg.Bolt.Mode = 0600 66 | err = store.Init(tmp) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | defer store.Close() 71 | 72 | pair := gaia.SHAPair{} 73 | pair.PipelineID = 1 74 | pair.Original = []byte("original") 75 | pair.Worker = []byte("worker") 76 | err = store.UpsertSHAPair(pair) 77 | if err != nil { 78 | t.Fatal(err) 79 | } 80 | // Test is upsert overwrites existing records. 81 | pair.Original = []byte("original2") 82 | err = store.UpsertSHAPair(pair) 83 | if err != nil { 84 | t.Fatal(err) 85 | } 86 | 87 | ok, p, err := store.GetSHAPair(1) 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | if !ok { 92 | t.Fatalf("sha pair not found") 93 | } 94 | 95 | if p.PipelineID != pair.PipelineID { 96 | t.Fatalf("pipeline id match error. want %d got %d", pair.PipelineID, p.PipelineID) 97 | } 98 | if !bytes.Equal(p.Worker, pair.Worker) { 99 | t.Fatalf("worker sha match error. want %s got %s", pair.Worker, p.Worker) 100 | } 101 | if !bytes.Equal(p.Original, pair.Original) { 102 | t.Fatalf("original sha match error. want %s got %s", pair.Original, p.Original) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /store/upgrade.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "github.com/gaia-pipeline/gaia" 5 | "github.com/gaia-pipeline/gaia/helper/rolehelper" 6 | ) 7 | 8 | // CreatePermissionsIfNotExisting iterates any existing users and creates default permissions if they don't exist. 9 | // This is most probably when they have upgraded to the Gaia version where permissions was added. 10 | func (s *BoltStore) CreatePermissionsIfNotExisting() error { 11 | users, _ := s.UserGetAll() 12 | for _, user := range users { 13 | perms, err := s.UserPermissionsGet(user.Username) 14 | if err != nil { 15 | return err 16 | } 17 | if perms == nil { 18 | perms := &gaia.UserPermission{ 19 | Username: user.Username, 20 | Roles: rolehelper.FlattenUserCategoryRoles(rolehelper.DefaultUserRoles), 21 | Groups: []string{}, 22 | } 23 | err := s.UserPermissionsPut(perms) 24 | if err != nil { 25 | return err 26 | } 27 | } 28 | } 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /store/user.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | bolt "go.etcd.io/bbolt" 8 | "golang.org/x/crypto/bcrypt" 9 | 10 | "github.com/gaia-pipeline/gaia" 11 | ) 12 | 13 | // UserPut takes the given user and saves it 14 | // to the bolt database. User will be overwritten 15 | // if it already exists. 16 | // It also clears the password field afterwards. 17 | func (s *BoltStore) UserPut(u *gaia.User, encryptPassword bool) error { 18 | // Encrypt password before we save it 19 | if encryptPassword { 20 | hash, err := bcrypt.GenerateFromPassword([]byte(u.Password), bcrypt.MinCost) 21 | if err != nil { 22 | return err 23 | } 24 | u.Password = string(hash) 25 | } 26 | 27 | return s.db.Update(func(tx *bolt.Tx) error { 28 | // Get bucket 29 | b := tx.Bucket(userBucket) 30 | 31 | // Marshal user object 32 | m, err := json.Marshal(u) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | // Clear password from origin object 38 | u.Password = "" 39 | 40 | // Put user 41 | return b.Put([]byte(u.Username), m) 42 | }) 43 | } 44 | 45 | // UserAuth looks up a user by given username. 46 | // Then it compares passwords and returns user obj if 47 | // given password is valid. Returns nil if password was 48 | // wrong or user not found. 49 | func (s *BoltStore) UserAuth(u *gaia.User, updateLastLogin bool) (*gaia.User, error) { 50 | // Look up user 51 | user, err := s.UserGet(u.Username) 52 | 53 | // Error occurred and/or user not found 54 | if err != nil || user == nil { 55 | return nil, err 56 | } 57 | 58 | // Check if password is valid 59 | if err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(u.Password)); err != nil { 60 | return nil, err 61 | } 62 | 63 | // Update last login field 64 | if updateLastLogin { 65 | user.LastLogin = time.Now() 66 | err = s.UserPut(user, false) 67 | if err != nil { 68 | return nil, err 69 | } 70 | } 71 | 72 | // We will use the user object later. 73 | // But we don't need the password anymore. 74 | user.Password = "" 75 | 76 | // Return user 77 | return user, nil 78 | } 79 | 80 | // UserGet looks up a user by given username. 81 | // Returns nil if user was not found. 82 | func (s *BoltStore) UserGet(username string) (*gaia.User, error) { 83 | user := &gaia.User{} 84 | err := s.db.View(func(tx *bolt.Tx) error { 85 | // Get bucket 86 | b := tx.Bucket(userBucket) 87 | 88 | // Lookup user 89 | userRaw := b.Get([]byte(username)) 90 | 91 | // User found? 92 | if userRaw == nil { 93 | // Nope. That is not an error so just leave 94 | user = nil 95 | return nil 96 | } 97 | 98 | // Unmarshal 99 | return json.Unmarshal(userRaw, user) 100 | }) 101 | 102 | return user, err 103 | } 104 | 105 | // UserGetAll returns all stored users. 106 | func (s *BoltStore) UserGetAll() ([]gaia.User, error) { 107 | var users []gaia.User 108 | 109 | return users, s.db.View(func(tx *bolt.Tx) error { 110 | // Get bucket 111 | b := tx.Bucket(userBucket) 112 | 113 | // Iterate all users and add them to slice 114 | return b.ForEach(func(k, v []byte) error { 115 | // create single user object 116 | u := &gaia.User{} 117 | 118 | // Unmarshal 119 | err := json.Unmarshal(v, u) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | // Remove password for security reasons 125 | u.Password = "" 126 | 127 | users = append(users, *u) 128 | return nil 129 | }) 130 | }) 131 | } 132 | 133 | // UserDelete deletes the given user. 134 | func (s *BoltStore) UserDelete(u string) error { 135 | return s.db.Update(func(tx *bolt.Tx) error { 136 | // Get bucket 137 | b := tx.Bucket(userBucket) 138 | 139 | // Delete user 140 | return b.Delete([]byte(u)) 141 | }) 142 | } 143 | -------------------------------------------------------------------------------- /store/worker.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | bolt "go.etcd.io/bbolt" 7 | 8 | "github.com/gaia-pipeline/gaia" 9 | ) 10 | 11 | // WorkerPut stores the given worker in the bolt database. 12 | // Worker object will be overwritten in case it already exist. 13 | func (s *BoltStore) WorkerPut(w *gaia.Worker) error { 14 | return s.db.Update(func(tx *bolt.Tx) error { 15 | // Get bucket 16 | b := tx.Bucket(workerBucket) 17 | 18 | // Marshal worker object 19 | m, err := json.Marshal(*w) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | // Put worker 25 | return b.Put([]byte(w.UniqueID), m) 26 | }) 27 | } 28 | 29 | // WorkerGetAll returns all existing worker objects from the store. 30 | // It returns an error when the action failed. 31 | func (s *BoltStore) WorkerGetAll() ([]*gaia.Worker, error) { 32 | var worker []*gaia.Worker 33 | 34 | return worker, s.db.View(func(tx *bolt.Tx) error { 35 | // Get bucket 36 | b := tx.Bucket(workerBucket) 37 | 38 | // Iterate all worker. 39 | return b.ForEach(func(k, v []byte) error { 40 | // Unmarshal 41 | w := &gaia.Worker{} 42 | err := json.Unmarshal(v, w) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | // Append to our list 48 | worker = append(worker, w) 49 | 50 | return nil 51 | }) 52 | }) 53 | } 54 | 55 | // WorkerDeleteAll deletes all worker objects in the bucket. 56 | func (s *BoltStore) WorkerDeleteAll() error { 57 | return s.db.Update(func(tx *bolt.Tx) error { 58 | // Delete bucket 59 | if err := tx.DeleteBucket(workerBucket); err != nil { 60 | return err 61 | } 62 | 63 | _, err := tx.CreateBucket(workerBucket) 64 | return err 65 | }) 66 | } 67 | 68 | // WorkerDelete deletes a worker by the given identifier. 69 | func (s *BoltStore) WorkerDelete(id string) error { 70 | return s.db.Update(func(tx *bolt.Tx) error { 71 | // Get bucket 72 | b := tx.Bucket(workerBucket) 73 | 74 | // Delete entry 75 | return b.Delete([]byte(id)) 76 | }) 77 | } 78 | 79 | // WorkerGet gets a worker by the given identifier. 80 | func (s *BoltStore) WorkerGet(id string) (*gaia.Worker, error) { 81 | var worker *gaia.Worker 82 | 83 | return worker, s.db.View(func(tx *bolt.Tx) error { 84 | // Get bucket 85 | b := tx.Bucket(workerBucket) 86 | 87 | // Get worker 88 | v := b.Get([]byte(id)) 89 | 90 | // Check if we found the worker 91 | if v == nil { 92 | return nil 93 | } 94 | 95 | // Unmarshal pipeline object 96 | worker = &gaia.Worker{} 97 | err := json.Unmarshal(v, worker) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | return nil 103 | }) 104 | } 105 | -------------------------------------------------------------------------------- /workers/agent/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/url" 9 | 10 | "github.com/gaia-pipeline/gaia" 11 | ) 12 | 13 | // RegisterResponse represents a response from API registration 14 | // call. 15 | type RegisterResponse struct { 16 | UniqueID string `json:"uniqueid"` 17 | Cert string `json:"cert"` 18 | Key string `json:"key"` 19 | CACert string `json:"cacert"` 20 | } 21 | 22 | // RegisterWorker registers a new worker at a Gaia instance. 23 | // It uses the given secret for authentication and returns certs 24 | // which can be used for a future mTLS connection. 25 | func RegisterWorker(host, secret, name string, tags []string) (*RegisterResponse, error) { 26 | fullURL := fmt.Sprintf("%s/api/%s/worker/register", host, gaia.APIVersion) 27 | resp, err := http.PostForm(fullURL, 28 | url.Values{ 29 | "secret": {secret}, 30 | "tags": tags, 31 | "name": {name}, 32 | }) 33 | if err != nil { 34 | return nil, err 35 | } 36 | defer resp.Body.Close() 37 | 38 | // Read the content 39 | body, err := ioutil.ReadAll(resp.Body) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | // Check the return code first 45 | if resp.StatusCode != http.StatusOK { 46 | return nil, fmt.Errorf("unable to register worker. Return code was '%d' and message was: %s", resp.StatusCode, string(body)) 47 | } 48 | 49 | // Unmarshal the json response 50 | regResp := RegisterResponse{} 51 | if err = json.Unmarshal(body, ®Resp); err != nil { 52 | return nil, fmt.Errorf("cannot unmarshal registration response: %s", string(body)) 53 | } 54 | 55 | return ®Resp, nil 56 | } 57 | -------------------------------------------------------------------------------- /workers/agent/api/api_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | 10 | "github.com/gaia-pipeline/gaia" 11 | "github.com/gaia-pipeline/gaia/helper/stringhelper" 12 | ) 13 | 14 | func TestRegisterWorker(t *testing.T) { 15 | // Define returned data 16 | uniqueID := "my-unique-id" 17 | cert := "test-cert" 18 | key := "test-key" 19 | caCert := "test-cacert" 20 | 21 | // Define test data 22 | name := "my-worker" 23 | secret := "12345-test-secret" 24 | tags := []string{"tag1", "tag2", "tag3"} 25 | 26 | // Start a local HTTP server 27 | server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { 28 | // Check request parameters 29 | if req.URL.String() != fmt.Sprintf("/api/%s/worker/register", gaia.APIVersion) { 30 | t.Fatalf("wrong request parameters provided: %s", req.URL.String()) 31 | } 32 | 33 | // Check form values 34 | if err := req.ParseForm(); err != nil { 35 | t.Fatal(err) 36 | } 37 | if req.Form.Get("name") != name { 38 | t.Fatalf("expected %s but got %s", name, req.Form.Get("name")) 39 | } 40 | if req.Form.Get("secret") != secret { 41 | t.Fatalf("expected %s but got %s", secret, req.Form.Get("secret")) 42 | } 43 | reqTags := req.Form["tags"] 44 | for _, tag := range tags { 45 | if !stringhelper.IsContainedInSlice(reqTags, tag, false) { 46 | t.Fatalf("expected tag %s to be in slice but it is not: %s", tag, reqTags) 47 | } 48 | } 49 | 50 | // Response 51 | resp := RegisterResponse{ 52 | UniqueID: uniqueID, 53 | Cert: cert, 54 | Key: key, 55 | CACert: caCert, 56 | } 57 | 58 | // Marshal 59 | mResp, err := json.Marshal(resp) 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | 64 | // Return response 65 | if _, err := rw.Write(mResp); err != nil { 66 | t.Fatal(err) 67 | } 68 | })) 69 | defer server.Close() 70 | 71 | // Run call 72 | resp, err := RegisterWorker(server.URL, secret, name, tags) 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | 77 | // Validate returned data 78 | if resp.UniqueID != uniqueID { 79 | t.Fatalf("expected %s but got %s", uniqueID, resp.UniqueID) 80 | } 81 | if resp.Cert != cert { 82 | t.Fatalf("expected %s but got %s", cert, resp.Cert) 83 | } 84 | if resp.Key != key { 85 | t.Fatalf("expected %s but got %s", key, resp.Key) 86 | } 87 | if resp.CACert != caCert { 88 | t.Fatalf("expected %s but got %s", caCert, resp.CACert) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /workers/agent/fixtures/caCert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDIjCCAgqgAwIBAgIRAPU9jaT0VpwaHOmp8oqz0fwwDQYJKoZIhvcNAQELBQAw 3 | GDEWMBQGA1UEChMNZ2FpYS1waXBlbGluZTAeFw0xOTA2MDcwODE1MzBaFw0zOTA2 4 | MDIwODE1MzBaMBgxFjAUBgNVBAoTDWdhaWEtcGlwZWxpbmUwggEiMA0GCSqGSIb3 5 | DQEBAQUAA4IBDwAwggEKAoIBAQCstfrKAoATId6ATS2y8AsPaqAvwh8kRFWbyv5x 6 | iYt/MPELgmS1J80BY5f8A9S9EU0CNaHDvnOj4iaWNzyQA+GOyL99RMWxhoikrc0y 7 | z3nNSPnOvFQKkgVCMMmoQO/nEWqQJPCLg8NWKy/Z5u1pXPSqEx7mI83qV3pirO1x 8 | d1p/yRaNytnv+1i6wMok0AR/uXNw62VV29ynQ/EWFchaYezVq6CCavZWtFMheRSv 9 | TFT64tnH7K5rRfC7tnLaS+Z/I1MfSO+ecd1yqK37SWlLWYch7y4QBjZM+9wdY0Wy 10 | P/Dt7+qM+qYrMp6zK0X/T1wa2RX/n6j/8b1cWbc/UH8sUEIvAgMBAAGjZzBlMA4G 11 | A1UdDwEB/wQEAwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDwYD 12 | VR0TAQH/BAUwAwEB/zAjBgNVHREEHDAaghBnYWlhLXBpcGVsaW5lLmlvggZ1bnVz 13 | ZWQwDQYJKoZIhvcNAQELBQADggEBAG+vjGSnAQ8DxjqEVXYTn7wXNUpRAyh7d6Ek 14 | oewB8/DzfBYCDBz5PD8NzFCSZiKr6QaxbgtLbeypa83wGoyD5NC+uSsrrk88i7Wy 15 | f8hd7lTY68ApQ+zpmjzQKVxgUD9+hvexELJRzTVgug4M63LzTURRx1uv+O6+2LhO 16 | PQn44D/agVdXzim5A8YeYEqpexDx0Rfp0JPnEhawXV1wlQzTsVTurauJbxpW7iFC 17 | i7kaTZkSLESjtiDyJTlUjrhAB7OKGod/in+E0A8Jar59+fW0fnmOgsmdyua3dA1T 18 | IEDYH9yIztvL+q+Fz2PVs+OexG1sCKA5718V/pUsqK65ivaDnyY= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /workers/agent/fixtures/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDIDCCAgigAwIBAgIQMUa0DWQMb93+Y/66o5yNNzANBgkqhkiG9w0BAQsFADAY 3 | MRYwFAYDVQQKEw1nYWlhLXBpcGVsaW5lMB4XDTE5MDYwNzA2MjMyNVoXDTI5MDYw 4 | NDA4MjMyNVowGDEWMBQGA1UEChMNZ2FpYS1waXBlbGluZTCCASIwDQYJKoZIhvcN 5 | AQEBBQADggEPADCCAQoCggEBAKKZC2IT6/nPJn3jsYC6UU6YC/e8rLGFizqb2F37 6 | Wj5LxTUyrQn8X2FvupCg6RmiDi7FkLFc3Wyvanplo50Tnar6sPxT3RhAmNnRaO35 7 | qPqojnqPY5Vtpp2yOj9/FJO2DzPcpQA9SVJW6n+Oi+jwDFk7UETAveeQHvUI1Mss 8 | RtUWDe+J2VUgjkY2Fcwx3NGrJQ1cNuvfgnOj9vbHwXwgwRSVoAMmkWQw8y7et89W 9 | exd81I0SSd7eIboq7PAokc2HsgbNHU2WQ4PAWL166hJI6IcojXtAxqHwu9UZyLzX 10 | R3RXhJQsl/SNgGOJ/NH0XSO7cwge6ab1SbnfCMmqsRZYoZ0CAwEAAaNmMGQwDgYD 11 | VR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAOBgNV 12 | HQ4EBwQFAQIDBAYwIwYDVR0RBBwwGoIQZ2FpYS1waXBlbGluZS5pb4IGdW51c2Vk 13 | MA0GCSqGSIb3DQEBCwUAA4IBAQAWaFqJPWeqoV4iHtFWST5LvLxa+fTL5Mw4/1dh 14 | t6qOX/NT8tH6f4ehR/RBqms3yd1gOU6rZwq6zqSXMPUOTncUiqpBaHfZ6uI5oaYA 15 | ZdhXuITuQ618jAUgj99Yp9psizXqjj+tNxdXm6IQ82NgMuOVXUtOTwXftVC2/1gC 16 | RIB+uMQgkEKPQOMR1Xk1ulcUBU/KLxfJu1HQiRHtD9oe3MkqgsB2qKK60jt7/VKA 17 | RrHOV6RDYQhoBwWiBOScCIzLJDfkKfFAkRQcY9Jdcw06OAxLcuiCCVixVe+cfK0y 18 | okNaUzb5atRzEGUndnapV4HNjtFG4qmLi6s87dUgOZrM+lm6 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /workers/agent/fixtures/key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCimQtiE+v5zyZ9 3 | 47GAulFOmAv3vKyxhYs6m9hd+1o+S8U1Mq0J/F9hb7qQoOkZog4uxZCxXN1sr2p6 4 | ZaOdE52q+rD8U90YQJjZ0Wjt+aj6qI56j2OVbaadsjo/fxSTtg8z3KUAPUlSVup/ 5 | jovo8AxZO1BEwL3nkB71CNTLLEbVFg3vidlVII5GNhXMMdzRqyUNXDbr34Jzo/b2 6 | x8F8IMEUlaADJpFkMPMu3rfPVnsXfNSNEkne3iG6KuzwKJHNh7IGzR1NlkODwFi9 7 | euoSSOiHKI17QMah8LvVGci810d0V4SULJf0jYBjifzR9F0ju3MIHumm9Um53wjJ 8 | qrEWWKGdAgMBAAECggEAMQwLc/1gbE2BZe4eKB+L0TJqQcOnMDGBax+Bp+/fK/V9 9 | Omvb6Kw5NVIXq/Lt/a51qsQrvmSt4lATxXINZZ96Kw5N/v9pLRynPuU9SrPJtjrM 10 | J8pDFtsprF4L1gWGdnwvn8xJ9YWHLZBMUkf0ikKG5w/OSb7+SfCedfnA/ryPbU0r 11 | iooExEnNRT/kZwNjgQM98W3normI/olKANZCenhvsrFrevPC8cPoA7lpMzVD++cL 12 | jv38wubWjWjj3778WR9hzaYmw2AAXi/SIVL90tOStpj8PcsPK9egTQW86YBEiSD6 13 | 2C8dRDxO2WPI5Vu26mqpF1YMW7P+AHssxyqSmvBEfQKBgQDBgEr5yPyt6fK90GSp 14 | pyp2p/KIu14SCIOsukCf0VNfSHa7T8PtuUv4MVuY6L2BSxN+ehQiD4F7Zfll0anm 15 | O3LFQL2oN/xeFTw6lcxa13L5eGbAHxfWcvuXpWfcMX2RgVfC2odNsHCERxA2qCoB 16 | 3WxgtLK1lq0lwSb/JcP0gXim+wKBgQDXHYAsh1QX5oKyjdl4J/fEGpeZR91LRzab 17 | qWQ3JB1rzkF+C3Bx5UKVL4FFGA8SxqlXCC+PsWA7Gm7DPFhqfBV7kXJmUKd70AYd 18 | SRcWJAoP0V32jjFyUJ262+x7mFl210EgYmerk9lmYRmlTu9S8kWMDUXnCvR6KGJL 19 | kzRsjz5WRwKBgQCTWlI+IysgeT6MA40UkKNKlpygOSbqzqcPOwqJact2jOU1wQlw 20 | F6Jdj6C0MdBWDEj9EJQSWAJ/aOlh+ybJl83PnnyPBItfTgZ+iqKDLvx0M5bziPoL 21 | KaakFaagzONVkcy2KtnMdKdKZB9Zr/fFUcv9XL2WgPa8AHnk5OpzYMDzTwKBgEy0 22 | h0TZiKHxHz+eFyKiVdYGiXItzvoNzaoZ79M9vIP5ix7v40uprWFXDChgGNfgIPgY 23 | wiTh4eeRWFejx/9IebyTM5DKR759ggClVGcfuLrFNFU7hOQ3XNcJnry/qX8X6HAs 24 | xrzGvqmkDCoHCI2yOBxlizyEioKYrdw3BGWFenv3AoGAJkUleh4Dn/ujTmNlxGYZ 25 | T8/KYjLc1A/CQYFf27TzE9UGtCH3b+42smTTS+vgE1TH4XoToJL1mplOArOzEVX4 26 | R/SvFm5jXepdbxStm7zYbmLePt5cyFSgIjQzUAlCVyJOCIiSwOnznfdx51XyFTGl 27 | VwkzyLzWy8Zd+wXlnhLUvxQ= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /workers/agent/tags.go: -------------------------------------------------------------------------------- 1 | package agent 2 | 3 | import ( 4 | "fmt" 5 | "github.com/gaia-pipeline/gaia" 6 | "github.com/gaia-pipeline/gaia/helper/stringhelper" 7 | "os/exec" 8 | "strings" 9 | ) 10 | 11 | var ( 12 | supportedBinaries = map[gaia.PipelineType]string{ 13 | gaia.PTypePython: "python", 14 | gaia.PTypeJava: "mvn", 15 | gaia.PTypeCpp: "make", 16 | gaia.PTypeGolang: "go", 17 | gaia.PTypeRuby: "gem", 18 | } 19 | ) 20 | 21 | // findLocalBinaries finds all supported local binaries for local execution 22 | // or build of pipelines in the local "path" variable. If the tags list contains 23 | // a negative language value, the language check is skipped. 24 | func findLocalBinaries(tags []string) []string { 25 | var foundSuppBinary []string 26 | 27 | // Iterate all supported binary names 28 | for key, binName := range supportedBinaries { 29 | // Check if negative tags value has been set 30 | if stringhelper.IsContainedInSlice(tags, fmt.Sprintf("-%s", key.String()), true) { 31 | continue 32 | } 33 | 34 | // Check if the binary name is available 35 | if _, err := exec.LookPath(binName); err == nil { 36 | // It is available. Add the tag to the list. 37 | foundSuppBinary = append(foundSuppBinary, key.String()) 38 | } 39 | } 40 | 41 | // Add given tags 42 | for _, tag := range tags { 43 | if !strings.HasPrefix(tag, "-") { 44 | foundSuppBinary = append(foundSuppBinary, tag) 45 | } 46 | } 47 | 48 | return foundSuppBinary 49 | } 50 | -------------------------------------------------------------------------------- /workers/agent/tags_test.go: -------------------------------------------------------------------------------- 1 | package agent 2 | 3 | import ( 4 | "github.com/gaia-pipeline/gaia/helper/stringhelper" 5 | "testing" 6 | ) 7 | 8 | func TestFindLocalBinaries(t *testing.T) { 9 | testTags := []string{"tag1", "tag2", "tag3"} 10 | 11 | // Simple lookup. Go binary should always exist in dev/test environments. 12 | tags := findLocalBinaries(testTags) 13 | 14 | // Check output 15 | if len(tags) < 4 { 16 | t.Fatalf("expected at least 4 tags but got %d", len(tags)) 17 | } 18 | if len(stringhelper.DiffSlices(append(testTags, "golang"), tags, false)) != 0 { 19 | t.Fatalf("expected different output: %#v", tags) 20 | } 21 | 22 | // Negative language tag 23 | testTags = append(testTags, "-golang") 24 | tags = findLocalBinaries(testTags) 25 | 26 | // Check output 27 | if len(tags) < 3 { 28 | t.Fatalf("expected at least 3 tags but got %d", len(tags)) 29 | } 30 | if stringhelper.IsContainedInSlice(tags, "golang", false) { 31 | t.Fatalf("golang should not be included: %#v", tags) 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /workers/pipeline/build_cpp.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/gaia-pipeline/gaia/helper/filehelper" 11 | "github.com/gaia-pipeline/gaia/helper/pipelinehelper" 12 | 13 | "github.com/gaia-pipeline/gaia" 14 | "github.com/gaia-pipeline/gaia/services" 15 | "github.com/gofrs/uuid" 16 | ) 17 | 18 | const ( 19 | cppBinaryName = "make" 20 | cppFinalBinaryName = "pipeline.out" 21 | ) 22 | 23 | // BuildPipelineCpp is the real implementation of BuildPipeline for C++ 24 | type BuildPipelineCpp struct { 25 | Type gaia.PipelineType 26 | } 27 | 28 | // PrepareEnvironment prepares the environment before we start the build process. 29 | func (b *BuildPipelineCpp) PrepareEnvironment(p *gaia.CreatePipeline) error { 30 | // create uniqueName for destination folder 31 | v4, err := uuid.NewV4() 32 | if err != nil { 33 | return err 34 | } 35 | uniqueName := uuid.Must(v4, nil) 36 | 37 | // Create local temp folder for clone 38 | cloneFolder := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpCppFolder, gaia.SrcFolder, uniqueName.String()) 39 | err = os.MkdirAll(cloneFolder, 0700) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | // Set new generated path in pipeline obj for later usage 45 | if p.Pipeline.Repo == nil { 46 | p.Pipeline.Repo = &gaia.GitRepo{} 47 | } 48 | p.Pipeline.Repo.LocalDest = cloneFolder 49 | p.Pipeline.UUID = uniqueName.String() 50 | return nil 51 | } 52 | 53 | // ExecuteBuild executes the c++ build process 54 | func (b *BuildPipelineCpp) ExecuteBuild(p *gaia.CreatePipeline) error { 55 | // Look for c++ binary executable 56 | path, err := exec.LookPath(cppBinaryName) 57 | if err != nil { 58 | gaia.Cfg.Logger.Debug("cannot find c++ binary executable", "error", err.Error()) 59 | return err 60 | } 61 | 62 | // Set command args for build 63 | args := []string{} 64 | 65 | // Set local destination 66 | localDest := "" 67 | if p.Pipeline.Repo != nil { 68 | localDest = p.Pipeline.Repo.LocalDest 69 | } 70 | 71 | // Execute and wait until finish or timeout 72 | output, err := executeCmd(path, args, os.Environ(), localDest) 73 | p.Output = string(output) 74 | if err != nil { 75 | gaia.Cfg.Logger.Debug("cannot build pipeline", "error", err.Error(), "output", string(output)) 76 | return err 77 | } 78 | 79 | // Build has been finished. Set execution path to the build result archive. 80 | // This will be used during pipeline verification phase which will happen after this step. 81 | p.Pipeline.ExecPath = filepath.Join(localDest, cppFinalBinaryName) 82 | 83 | return nil 84 | } 85 | 86 | // CopyBinary copies the final compiled binary to the 87 | // destination folder. 88 | func (b *BuildPipelineCpp) CopyBinary(p *gaia.CreatePipeline) error { 89 | // Define src and destination 90 | src := filepath.Join(p.Pipeline.Repo.LocalDest, cppFinalBinaryName) 91 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)) 92 | 93 | // Copy binary 94 | if err := filehelper.CopyFileContents(src, dest); err != nil { 95 | return err 96 | } 97 | 98 | // Set +x (execution right) for pipeline 99 | return os.Chmod(dest, gaia.ExecutablePermission) 100 | } 101 | 102 | // SavePipeline saves the current pipeline configuration. 103 | func (b *BuildPipelineCpp) SavePipeline(p *gaia.Pipeline) error { 104 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Name, p.Type)) 105 | p.ExecPath = dest 106 | p.Type = gaia.PTypeCpp 107 | p.Name = strings.TrimSuffix(filepath.Base(dest), typeDelimiter+gaia.PTypeCpp.String()) 108 | p.Created = time.Now() 109 | // Our pipeline is finished constructing. Save it. 110 | storeService, _ := services.StorageService() 111 | return storeService.PipelinePut(p) 112 | } 113 | -------------------------------------------------------------------------------- /workers/pipeline/build_golang.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/gaia-pipeline/gaia/helper/filehelper" 11 | "github.com/gaia-pipeline/gaia/helper/pipelinehelper" 12 | 13 | "github.com/gaia-pipeline/gaia" 14 | "github.com/gaia-pipeline/gaia/services" 15 | "github.com/gofrs/uuid" 16 | ) 17 | 18 | const ( 19 | golangBinaryName = "go" 20 | ) 21 | 22 | // BuildPipelineGolang is the real implementation of BuildPipeline for golang 23 | type BuildPipelineGolang struct { 24 | Type gaia.PipelineType 25 | } 26 | 27 | // PrepareEnvironment prepares the environment before we start the build process. 28 | func (b *BuildPipelineGolang) PrepareEnvironment(p *gaia.CreatePipeline) error { 29 | // create uniqueName for destination folder 30 | v4, err := uuid.NewV4() 31 | if err != nil { 32 | return err 33 | } 34 | uniqueName := uuid.Must(v4, nil) 35 | 36 | // Create local temp folder for clone 37 | goPath := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpGoFolder) 38 | cloneFolder := filepath.Join(goPath, gaia.SrcFolder, uniqueName.String()) 39 | err = os.MkdirAll(cloneFolder, 0700) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | // Set new generated path in pipeline obj for later usage 45 | if p.Pipeline.Repo == nil { 46 | p.Pipeline.Repo = &gaia.GitRepo{} 47 | } 48 | p.Pipeline.Repo.LocalDest = cloneFolder 49 | p.Pipeline.UUID = uniqueName.String() 50 | return nil 51 | } 52 | 53 | // ExecuteBuild executes the golang build process 54 | func (b *BuildPipelineGolang) ExecuteBuild(p *gaia.CreatePipeline) error { 55 | // Look for golang executable 56 | path, err := exec.LookPath(golangBinaryName) 57 | if err != nil { 58 | gaia.Cfg.Logger.Debug("cannot find go executable", "error", err.Error()) 59 | return err 60 | } 61 | goPath := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpGoFolder) 62 | 63 | // Set command args for get dependencies 64 | args := []string{ 65 | "get", 66 | "-d", 67 | "./...", 68 | } 69 | 70 | env := append(os.Environ(), "GOPATH="+goPath) 71 | 72 | // Set local destination 73 | localDest := "" 74 | if p.Pipeline.Repo != nil { 75 | localDest = p.Pipeline.Repo.LocalDest 76 | } 77 | 78 | // Execute and wait until finish or timeout 79 | output, err := executeCmd(path, args, env, localDest) 80 | if err != nil { 81 | gaia.Cfg.Logger.Debug("cannot get dependencies", "error", err.Error(), "output", string(output)) 82 | p.Output = string(output) 83 | return err 84 | } 85 | 86 | // Set command args for build 87 | args = []string{ 88 | "build", 89 | "-o", 90 | pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type), 91 | } 92 | 93 | // Execute and wait until finish or timeout 94 | output, err = executeCmd(path, args, env, localDest) 95 | p.Output = string(output) 96 | if err != nil { 97 | gaia.Cfg.Logger.Debug("cannot build pipeline", "error", err.Error(), "output", string(output)) 98 | return err 99 | } 100 | 101 | // Build has been finished. Set execution path to the build result archive. 102 | // This will be used during pipeline verification phase which will happen after this step. 103 | p.Pipeline.ExecPath = filepath.Join(localDest, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)) 104 | 105 | return nil 106 | } 107 | 108 | // CopyBinary copies the final compiled archive to the 109 | // destination folder. 110 | func (b *BuildPipelineGolang) CopyBinary(p *gaia.CreatePipeline) error { 111 | // Define src and destination 112 | src := filepath.Join(p.Pipeline.Repo.LocalDest, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)) 113 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)) 114 | 115 | // Copy binary 116 | if err := filehelper.CopyFileContents(src, dest); err != nil { 117 | return err 118 | } 119 | 120 | // Set +x (execution right) for pipeline 121 | return os.Chmod(dest, gaia.ExecutablePermission) 122 | } 123 | 124 | // SavePipeline saves the current pipeline configuration. 125 | func (b *BuildPipelineGolang) SavePipeline(p *gaia.Pipeline) error { 126 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Name, p.Type)) 127 | p.ExecPath = dest 128 | p.Type = gaia.PTypeGolang 129 | p.Name = strings.TrimSuffix(filepath.Base(dest), typeDelimiter+gaia.PTypeGolang.String()) 130 | p.Created = time.Now() 131 | // Our pipeline is finished constructing. Save it. 132 | storeService, _ := services.StorageService() 133 | return storeService.PipelinePut(p) 134 | } 135 | -------------------------------------------------------------------------------- /workers/pipeline/build_java.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/gaia-pipeline/gaia/helper/filehelper" 11 | "github.com/gaia-pipeline/gaia/helper/pipelinehelper" 12 | 13 | "github.com/gaia-pipeline/gaia" 14 | "github.com/gaia-pipeline/gaia/services" 15 | "github.com/gofrs/uuid" 16 | ) 17 | 18 | var ( 19 | mavenBinaryName = "mvn" 20 | ) 21 | 22 | const ( 23 | javaFinalJarName = "plugin-jar-with-dependencies.jar" 24 | mavenTargetFolder = "target" 25 | ) 26 | 27 | // BuildPipelineJava is the real implementation of BuildPipeline for java 28 | type BuildPipelineJava struct { 29 | Type gaia.PipelineType 30 | } 31 | 32 | // PrepareEnvironment prepares the environment before we start the build process. 33 | func (b *BuildPipelineJava) PrepareEnvironment(p *gaia.CreatePipeline) error { 34 | // create uniqueName for destination folder 35 | v4, err := uuid.NewV4() 36 | if err != nil { 37 | return err 38 | } 39 | uniqueName := uuid.Must(v4, nil) 40 | 41 | // Create local temp folder for clone 42 | rootPath := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpJavaFolder) 43 | cloneFolder := filepath.Join(rootPath, gaia.SrcFolder, uniqueName.String()) 44 | err = os.MkdirAll(cloneFolder, 0700) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | // Set new generated path in pipeline obj for later usage 50 | if p.Pipeline.Repo == nil { 51 | p.Pipeline.Repo = &gaia.GitRepo{} 52 | } 53 | p.Pipeline.Repo.LocalDest = cloneFolder 54 | p.Pipeline.UUID = uniqueName.String() 55 | return nil 56 | } 57 | 58 | // ExecuteBuild executes the java build process 59 | func (b *BuildPipelineJava) ExecuteBuild(p *gaia.CreatePipeline) error { 60 | // Look for maven executable 61 | path, err := exec.LookPath(mavenBinaryName) 62 | if err != nil { 63 | gaia.Cfg.Logger.Debug("cannot find maven executeable", "error", err.Error()) 64 | return err 65 | } 66 | env := os.Environ() 67 | 68 | // Set command args for build 69 | args := []string{ 70 | "clean", 71 | "compile", 72 | "assembly:single", 73 | } 74 | 75 | // Set local destination 76 | localDest := "" 77 | if p.Pipeline.Repo != nil { 78 | localDest = p.Pipeline.Repo.LocalDest 79 | } 80 | 81 | // Execute and wait until finish or timeout 82 | output, err := executeCmd(path, args, env, localDest) 83 | p.Output = string(output) 84 | if err != nil { 85 | gaia.Cfg.Logger.Debug("cannot build pipeline", "error", err.Error(), "output", string(output)) 86 | return err 87 | } 88 | 89 | // Build has been finished. Set execution path to the build result archive. 90 | // This will be used during pipeline verification phase which will happen after this step. 91 | p.Pipeline.ExecPath = filepath.Join(localDest, mavenTargetFolder, javaFinalJarName) 92 | 93 | return nil 94 | } 95 | 96 | // CopyBinary copies the final compiled archive to the 97 | // destination folder. 98 | func (b *BuildPipelineJava) CopyBinary(p *gaia.CreatePipeline) error { 99 | // Define src and destination 100 | src := filepath.Join(p.Pipeline.Repo.LocalDest, mavenTargetFolder, javaFinalJarName) 101 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)) 102 | 103 | // Copy binary 104 | if err := filehelper.CopyFileContents(src, dest); err != nil { 105 | return err 106 | } 107 | 108 | // Set +x (execution right) for pipeline 109 | return os.Chmod(dest, gaia.ExecutablePermission) 110 | } 111 | 112 | // SavePipeline saves the current pipeline configuration. 113 | func (b *BuildPipelineJava) SavePipeline(p *gaia.Pipeline) error { 114 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Name, p.Type)) 115 | p.ExecPath = dest 116 | p.Type = gaia.PTypeJava 117 | p.Name = strings.TrimSuffix(filepath.Base(dest), typeDelimiter+gaia.PTypeJava.String()) 118 | p.Created = time.Now() 119 | // Our pipeline is finished constructing. Save it. 120 | storeService, _ := services.StorageService() 121 | return storeService.PipelinePut(p) 122 | } 123 | -------------------------------------------------------------------------------- /workers/pipeline/build_nodejs.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/gaia-pipeline/gaia/helper/filehelper" 11 | "github.com/gaia-pipeline/gaia/helper/pipelinehelper" 12 | 13 | "github.com/gaia-pipeline/gaia" 14 | "github.com/gaia-pipeline/gaia/services" 15 | "github.com/gofrs/uuid" 16 | ) 17 | 18 | const nodeJSInternalCloneFolder = "jsclone" 19 | 20 | // BuildPipelineNodeJS is the real implementation of BuildPipeline for NodeJS 21 | type BuildPipelineNodeJS struct { 22 | Type gaia.PipelineType 23 | } 24 | 25 | // PrepareEnvironment prepares the environment before we start the build process. 26 | func (b *BuildPipelineNodeJS) PrepareEnvironment(p *gaia.CreatePipeline) error { 27 | // create uniqueName for destination folder 28 | v4, err := uuid.NewV4() 29 | if err != nil { 30 | gaia.Cfg.Logger.Debug("unable to generate uuid", "error", err.Error()) 31 | return err 32 | } 33 | uniqueName := uuid.Must(v4, nil) 34 | 35 | // Create local temp folder for clone 36 | cloneFolder := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpNodeJSFolder, gaia.SrcFolder, uniqueName.String(), nodeJSInternalCloneFolder) 37 | err = os.MkdirAll(cloneFolder, 0700) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | // Set new generated path in pipeline obj for later usage 43 | if p.Pipeline.Repo == nil { 44 | p.Pipeline.Repo = &gaia.GitRepo{} 45 | } 46 | p.Pipeline.Repo.LocalDest = cloneFolder 47 | p.Pipeline.UUID = uniqueName.String() 48 | return nil 49 | } 50 | 51 | // ExecuteBuild executes the NodeJS build process 52 | func (b *BuildPipelineNodeJS) ExecuteBuild(p *gaia.CreatePipeline) error { 53 | // Look for Tar binary executable 54 | path, err := exec.LookPath(tarName) 55 | if err != nil { 56 | gaia.Cfg.Logger.Debug("cannot find tar binary executable", "error", err.Error()) 57 | return err 58 | } 59 | 60 | // Set local destination 61 | localDest := "" 62 | if p.Pipeline.Repo != nil { 63 | localDest = p.Pipeline.Repo.LocalDest 64 | } 65 | 66 | // Set command args for archive process 67 | pipelineFileName := pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type) 68 | args := []string{ 69 | "--exclude=.git", 70 | "-czvf", 71 | pipelineFileName, 72 | "-C", 73 | localDest, 74 | ".", 75 | } 76 | 77 | // Execute and wait until finish or timeout 78 | uniqueFolder := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpNodeJSFolder, gaia.SrcFolder, p.Pipeline.UUID) 79 | output, err := executeCmd(path, args, os.Environ(), uniqueFolder) 80 | p.Output = string(output[:]) 81 | if err != nil { 82 | gaia.Cfg.Logger.Debug("cannot build pipeline", "error", err.Error(), "output", string(output[:])) 83 | return err 84 | } 85 | 86 | // Build has been finished. Set execution path to the build result archive. 87 | // This will be used during pipeline verification phase which will happen after this step. 88 | p.Pipeline.ExecPath = filepath.Join(uniqueFolder, pipelineFileName) 89 | 90 | // Set the the local destination variable to the unique folder because this is now the place 91 | // where our binary is located. 92 | p.Pipeline.Repo.LocalDest = uniqueFolder 93 | 94 | return nil 95 | } 96 | 97 | // CopyBinary copies the final compiled binary to the 98 | // destination folder. 99 | func (b *BuildPipelineNodeJS) CopyBinary(p *gaia.CreatePipeline) error { 100 | // Define src and destination 101 | src := filepath.Join(p.Pipeline.Repo.LocalDest, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)) 102 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)) 103 | 104 | // Copy binary 105 | if err := filehelper.CopyFileContents(src, dest); err != nil { 106 | return err 107 | } 108 | 109 | // Set +x (execution right) for pipeline 110 | return os.Chmod(dest, gaia.ExecutablePermission) 111 | } 112 | 113 | // SavePipeline saves the current pipeline configuration. 114 | func (b *BuildPipelineNodeJS) SavePipeline(p *gaia.Pipeline) error { 115 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Name, p.Type)) 116 | p.ExecPath = dest 117 | p.Type = gaia.PTypeNodeJS 118 | p.Name = strings.TrimSuffix(filepath.Base(dest), typeDelimiter+gaia.PTypeNodeJS.String()) 119 | p.Created = time.Now() 120 | // Our pipeline is finished constructing. Save it. 121 | storeService, _ := services.StorageService() 122 | return storeService.PipelinePut(p) 123 | } 124 | -------------------------------------------------------------------------------- /workers/pipeline/build_python.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "errors" 5 | "io/ioutil" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | "strings" 10 | "time" 11 | 12 | "github.com/gaia-pipeline/gaia/helper/filehelper" 13 | "github.com/gaia-pipeline/gaia/helper/pipelinehelper" 14 | 15 | "github.com/gaia-pipeline/gaia" 16 | "github.com/gaia-pipeline/gaia/services" 17 | "github.com/gofrs/uuid" 18 | ) 19 | 20 | var ( 21 | pythonBinaryName = "python" 22 | ) 23 | 24 | // BuildPipelinePython is the real implementation of BuildPipeline for python 25 | type BuildPipelinePython struct { 26 | Type gaia.PipelineType 27 | } 28 | 29 | // PrepareEnvironment prepares the environment before we start the build process. 30 | func (b *BuildPipelinePython) PrepareEnvironment(p *gaia.CreatePipeline) error { 31 | // create uniqueName for destination folder 32 | v4, err := uuid.NewV4() 33 | if err != nil { 34 | gaia.Cfg.Logger.Debug("unable to generate uuid", "error", err.Error()) 35 | return err 36 | } 37 | uniqueName := uuid.Must(v4, nil) 38 | 39 | // Create local temp folder for clone 40 | rootPath := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpPythonFolder) 41 | cloneFolder := filepath.Join(rootPath, gaia.SrcFolder, uniqueName.String()) 42 | err = os.MkdirAll(cloneFolder, 0700) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | // Set new generated path in pipeline obj for later usage 48 | if p.Pipeline.Repo == nil { 49 | p.Pipeline.Repo = &gaia.GitRepo{} 50 | } 51 | p.Pipeline.Repo.LocalDest = cloneFolder 52 | p.Pipeline.UUID = uniqueName.String() 53 | return nil 54 | } 55 | 56 | // ExecuteBuild executes the python build process 57 | func (b *BuildPipelinePython) ExecuteBuild(p *gaia.CreatePipeline) error { 58 | // Look for python executeable 59 | path, err := exec.LookPath(pythonBinaryName) 60 | if err != nil { 61 | gaia.Cfg.Logger.Debug("cannot find python executeable", "error", err.Error()) 62 | return err 63 | } 64 | 65 | // Set command args for build distribution package 66 | args := []string{ 67 | "setup.py", 68 | "sdist", 69 | } 70 | 71 | // Set local destination 72 | localDest := "" 73 | if p.Pipeline.Repo != nil { 74 | localDest = p.Pipeline.Repo.LocalDest 75 | } 76 | 77 | // Execute and wait until finish or timeout 78 | output, err := executeCmd(path, args, os.Environ(), localDest) 79 | if err != nil { 80 | gaia.Cfg.Logger.Debug("cannot generate python distribution package", "error", err.Error(), "output", string(output)) 81 | p.Output = string(output) 82 | return err 83 | } 84 | 85 | // Build has been finished. Set execution path to the build result archive. 86 | // This will be used during pipeline verification phase which will happen after this step. 87 | p.Pipeline.ExecPath, err = findPythonArchivePath(p) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | return nil 93 | } 94 | 95 | // findPythonArchivePath filters the archives in the generated dist folder 96 | // and looks for the final archive. It will return an error if less or more 97 | // than one file(s) are found otherwise the full path to the file. 98 | func findPythonArchivePath(p *gaia.CreatePipeline) (src string, err error) { 99 | // find all files in dist folder 100 | distFolder := filepath.Join(p.Pipeline.Repo.LocalDest, "dist") 101 | files, err := ioutil.ReadDir(distFolder) 102 | if err != nil { 103 | return 104 | } 105 | 106 | // filter for archives 107 | archive := []os.FileInfo{} 108 | for _, file := range files { 109 | if strings.HasSuffix(file.Name(), ".tar.gz") { 110 | archive = append(archive, file) 111 | } 112 | } 113 | 114 | // if we found more or less than one archive we have a problem 115 | if len(archive) != 1 { 116 | gaia.Cfg.Logger.Debug("cannot find python package", "foundPackages", len(archive), "archives", files) 117 | err = errors.New("cannot find python package") 118 | return 119 | } 120 | 121 | // Return full path 122 | src = filepath.Join(distFolder, archive[0].Name()) 123 | return 124 | } 125 | 126 | // CopyBinary copies the final compiled archive to the 127 | // destination folder. 128 | func (b *BuildPipelinePython) CopyBinary(p *gaia.CreatePipeline) error { 129 | // Define src and destination 130 | src, err := findPythonArchivePath(p) 131 | if err != nil { 132 | return err 133 | } 134 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Pipeline.Name, p.Pipeline.Type)) 135 | 136 | // Copy binary 137 | if err := filehelper.CopyFileContents(src, dest); err != nil { 138 | return err 139 | } 140 | 141 | // Set +x (execution right) for pipeline 142 | return os.Chmod(dest, gaia.ExecutablePermission) 143 | } 144 | 145 | // SavePipeline saves the current pipeline configuration. 146 | func (b *BuildPipelinePython) SavePipeline(p *gaia.Pipeline) error { 147 | dest := filepath.Join(gaia.Cfg.PipelinePath, pipelinehelper.AppendTypeToName(p.Name, p.Type)) 148 | p.ExecPath = dest 149 | p.Type = gaia.PTypePython 150 | p.Name = strings.TrimSuffix(filepath.Base(dest), typeDelimiter+gaia.PTypePython.String()) 151 | p.Created = time.Now() 152 | // Our pipeline is finished constructing. Save it. 153 | storeService, _ := services.StorageService() 154 | return storeService.PipelinePut(p) 155 | } 156 | -------------------------------------------------------------------------------- /workers/pipeline/service.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "github.com/gaia-pipeline/gaia" 5 | "github.com/gaia-pipeline/gaia/workers/scheduler/service" 6 | ) 7 | 8 | // Dependencies defines dependencies which this service needs to operate. 9 | type Dependencies struct { 10 | Scheduler service.GaiaScheduler 11 | } 12 | 13 | // GaiaPipelineService defines a pipeline service provider providing pipeline related functions. 14 | type GaiaPipelineService struct { 15 | deps Dependencies 16 | } 17 | 18 | // Servicer defines a scheduler service. 19 | type Servicer interface { 20 | CreatePipeline(p *gaia.CreatePipeline) 21 | InitTicker() 22 | CheckActivePipelines() 23 | UpdateRepository(p *gaia.Pipeline) error 24 | UpdateAllCurrentPipelines() 25 | StartPoller() error 26 | StopPoller() error 27 | } 28 | 29 | // NewGaiaPipelineService creates a pipeline service with its required dependencies already wired up 30 | func NewGaiaPipelineService(deps Dependencies) *GaiaPipelineService { 31 | return &GaiaPipelineService{deps: deps} 32 | } 33 | -------------------------------------------------------------------------------- /workers/pipeline/test_helper_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strconv" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | var buildKillContext = false 14 | 15 | func fakeExecCommandContext(ctx context.Context, name string, args ...string) *exec.Cmd { 16 | if buildKillContext { 17 | c, cancel := context.WithTimeout(context.Background(), 0) 18 | defer cancel() 19 | ctx = c 20 | } 21 | cs := []string{"-test.run=TestExecCommandContextHelper", "--", name} 22 | cs = append(cs, args...) 23 | cmd := exec.CommandContext(ctx, os.Args[0], cs...) 24 | arg := strings.Join(cs, ",") 25 | envArgs := os.Getenv("CMD_ARGS") 26 | if len(envArgs) != 0 { 27 | envArgs += ":" + arg 28 | } else { 29 | envArgs = arg 30 | } 31 | _ = os.Setenv("CMD_ARGS", envArgs) 32 | return cmd 33 | } 34 | 35 | func TestExecCommandContextHelper(t *testing.T) { 36 | if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { 37 | return 38 | } 39 | _, _ = fmt.Fprintf(os.Stdout, os.Getenv("STDOUT")) 40 | i, _ := strconv.Atoi(os.Getenv("EXIT_STATUS")) 41 | os.Exit(i) 42 | } 43 | -------------------------------------------------------------------------------- /workers/pipeline/testacc/test.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEoQIBAAKCAQBzdZKIs/r7NL/nZoBe7iibPfZCr+Cp2mJOvb6pq2kvj/zkZXjh 3 | GDs9wW5vD1WHnyyaxBuS1+Juo4gn8Tlf3BladWrU1H7zi7kN246uSb2zf6mYOnWy 4 | G6+I3kLSmy27EgRs15V0bL570siXhAlO0OkogQL4y1cfZ3eH8upuBvsO5YCcMPfW 5 | 4l4S3kSrwobOYzr2zZzYoQmrbsyFFtdEgVJTaeljeuYkxdYyYH5S+Mqhe3mIwMjh 6 | m1ANnZUt1fSAgJVzr87vXfZsVTMEKOIJarGhXr1GQ3XXRiYYLxfWBuwOkqIY8WkI 7 | 6syIKXAwc1pFgs0UC3nqIFswUbJ/lutYG1K/AgMBAAECggEAYJfAG7XXB0o+Mi1C 8 | wCimuBnCaFATuIYHMLD1TaUlvrDLdZie5FINXcDxzuAZQfkcq+3c06Dgwob1ZdUd 9 | luDgJRmOYrfT7ZS7IKjKWW0/02e+Tqi5xmP7Gpo3dVJ1k8ejBBvn4RsI0Taqsne4 10 | AcQtC8HC5rnXDWLWUVocXihg6ThfyXLNOYLo1dK4h9km20IIVuBecjvYm6ZbKlo8 11 | G69iqob93/r2KErM/8diL0FyFQDNqDhbha5fDJgV8BtSp+j7ymFw8J3gFfzRAikK 12 | rtvIA6gBvbAB4AEG0uYfHpkoFT45lLqihBIFBbfJZ36/e+rRIXGPK7EWIZ35dzTD 13 | Q/5TgQKBgQCzuwxChFuUNI2CwfRGMz+5/U/Oc2ge3af5gRb70FGysjzGAw1Z2lWw 14 | ZACvGXHpg8JCV+653IP4ytKklFM7ONTVCSms5XjK6pc3mT8R4Xgow0QOOzJj9WMd 15 | DCi1cn6JEt+2GC6rhmBZcI5CCrp9qNtnaSUMmeFGHfhV8d8cKNFG7wKBgQCkdG79 16 | xHP9K2ob+n6j3sMwxedESd59leHpLnHdx4rBG5f3Zeo5EkXp9E37m0zwNsrtYESy 17 | X96F9QvPfDOA4Vfzf+JccwzHmztAK8IaTXUIgYr/jNLN3Lf0NEmJMgUT5TOXJDU2 18 | bZ7bpR1qjQDAslhosSTqfy3yhqqXHivHhhcxMQKBgFZPTOPki9XwJsTUP2o20jOO 19 | 4fRSl327FB9NTqw+rf0SevzcGl077Ep4u5tarMlm7LLPZ2T58KZZQC6ozA90i0CC 20 | 1fChghBv945LmW4MyJdKrjPnWZuHC8G3RRVdlkZdOfFIS6EzNrG8y5QLcuNFY5eV 21 | dqVGgFgbrFYZgPsU3ushAoGAVmyd+5ynO+/51nRA40tpFpOaYufTFfqTe2CeFGgO 22 | AkfHCAu2GIIC3d02sjg+KasR38eMspTxM0LBDyv9QQirmNqnEeCgYbGxZJraaco0 23 | 6+BwNLZD/k21Go/z6TaxNnBoOVCc6lqXdmSCXgF12M6g6XvWo6lscxzUP5Bqf3N3 24 | crECgYAkNEdmXVFIRL0PREGys2BuNNM8X0QBbnQf+EG8Iu7h41z9kQv8FMJXFq9M 25 | O96TVirC6IOiwY78y2X7wfiOH6PSQp+ABEp393aUETm5q+vRZ20k1QW7rKZgucUH 26 | 4y75Kfobcp7bqpkoQkNoEM8X261pWW2q5JBjxFytfz7irVgZrw== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /workers/pipeline/update_pipeline.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | 10 | "github.com/gaia-pipeline/gaia/helper/filehelper" 11 | 12 | "github.com/gaia-pipeline/gaia" 13 | ) 14 | 15 | var ( 16 | // virtualEnvName is the binary name of virtual environment app. 17 | virtualEnvName = "virtualenv" 18 | 19 | // pythonPipInstallCmd is the command used to install the python distribution 20 | // package. 21 | pythonPipInstallCmd = ". bin/activate; python -m pip install '%s.tar.gz'" 22 | 23 | // Ruby gem binary name. 24 | rubyGemName = "gem" 25 | 26 | // Tar binary name. 27 | tarName = "tar" 28 | 29 | // NPM binary name. 30 | npmName = "npm" 31 | ) 32 | 33 | // updatePipeline executes update steps dependent on the pipeline type. 34 | // Some pipeline types may don't require this. 35 | func updatePipeline(p *gaia.Pipeline) error { 36 | switch p.Type { 37 | case gaia.PTypePython: 38 | // Remove virtual environment if exists 39 | virtualEnvPath := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpPythonFolder, p.Name) 40 | _ = os.RemoveAll(virtualEnvPath) 41 | 42 | // Create virtual environment 43 | path, err := exec.LookPath(virtualEnvName) 44 | if err != nil { 45 | return errors.New("cannot find virtualenv executable") 46 | } 47 | cmd := exec.Command(path, virtualEnvPath) 48 | if err := cmd.Run(); err != nil { 49 | return err 50 | } 51 | 52 | // copy distribution file to environment and remove pipeline type at the end. 53 | // we have to do this otherwise pip will fail. 54 | err = filehelper.CopyFileContents(p.ExecPath, filepath.Join(virtualEnvPath, p.Name+".tar.gz")) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | // install plugin in this environment 60 | cmd = exec.Command("/bin/sh", "-c", fmt.Sprintf(pythonPipInstallCmd, filepath.Join(virtualEnvPath, p.Name))) 61 | cmd.Dir = virtualEnvPath 62 | if out, err := cmd.CombinedOutput(); err != nil { 63 | return fmt.Errorf("cannot install python plugin: %s", string(out[:])) 64 | } 65 | case gaia.PTypeRuby: 66 | // Find gem binary in path variable. 67 | path, err := exec.LookPath(rubyGemName) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | // Gem expects that the file suffix is ".gem". 73 | // Copy gem file to temp folder before we install it. 74 | tmpFolder := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpRubyFolder) 75 | err = os.MkdirAll(tmpFolder, 0700) 76 | if err != nil { 77 | return err 78 | } 79 | pipelineCopyPath := filepath.Join(tmpFolder, filepath.Base(p.ExecPath)+".gem") 80 | err = filehelper.CopyFileContents(p.ExecPath, pipelineCopyPath) 81 | if err != nil { 82 | return err 83 | } 84 | defer os.Remove(pipelineCopyPath) 85 | 86 | // Install gem forcefully. 87 | cmd := exec.Command(path, "install", "-f", pipelineCopyPath) 88 | if out, err := cmd.CombinedOutput(); err != nil { 89 | return fmt.Errorf("cannot install ruby gem: %s", string(out[:])) 90 | } 91 | case gaia.PTypeNodeJS: 92 | // Find tar binary in path 93 | path, err := exec.LookPath(tarName) 94 | if err != nil { 95 | return err 96 | } 97 | 98 | // Delete old folders if exist 99 | tmpFolder := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpNodeJSFolder, p.Name) 100 | _ = os.RemoveAll(tmpFolder) 101 | 102 | // Recreate the temp folder 103 | if err := os.MkdirAll(tmpFolder, 0700); err != nil { 104 | return err 105 | } 106 | 107 | // Unpack it 108 | cmd := exec.Command(path, "-xzvf", p.ExecPath, "-C", tmpFolder) 109 | if out, err := cmd.CombinedOutput(); err != nil { 110 | return fmt.Errorf("cannot unpack nodejs archive: %s", string(out[:])) 111 | } 112 | 113 | // Find npm binary in path 114 | path, err = exec.LookPath(npmName) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | // Install dependencies 120 | cmd = &exec.Cmd{ 121 | Path: path, 122 | Dir: tmpFolder, 123 | Args: []string{path, "install"}, 124 | } 125 | if out, err := cmd.CombinedOutput(); err != nil { 126 | return fmt.Errorf("cannot install dependencies: %s", string(out[:])) 127 | } 128 | } 129 | 130 | // Update checksum 131 | checksum, err := filehelper.GetSHA256Sum(p.ExecPath) 132 | if err != nil { 133 | return err 134 | } 135 | p.SHA256Sum = checksum 136 | 137 | return nil 138 | } 139 | -------------------------------------------------------------------------------- /workers/pipeline/update_pipeline_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | "time" 10 | 11 | "github.com/gaia-pipeline/gaia" 12 | hclog "github.com/hashicorp/go-hclog" 13 | ) 14 | 15 | func TestUpdatePipelinePython(t *testing.T) { 16 | tmp, _ := ioutil.TempDir("", "TestUpdatePipelinePython") 17 | gaia.Cfg = new(gaia.Config) 18 | gaia.Cfg.HomePath = tmp 19 | buf := new(bytes.Buffer) 20 | gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ 21 | Level: hclog.Trace, 22 | Output: buf, 23 | Name: "Gaia", 24 | }) 25 | 26 | p1 := gaia.Pipeline{ 27 | Name: "PipelinA", 28 | Type: gaia.PTypePython, 29 | Created: time.Now(), 30 | } 31 | 32 | // Create fake virtualenv folder with temp file 33 | virtualEnvPath := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpPythonFolder, p1.Name) 34 | err := os.MkdirAll(virtualEnvPath, 0700) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | src := filepath.Join(tmp, "PipelineA_python") 39 | p1.ExecPath = src 40 | defer os.RemoveAll(tmp) 41 | _ = ioutil.WriteFile(src, []byte("testcontent"), 0666) 42 | 43 | // fake execution commands 44 | virtualEnvName = "mkdir" 45 | pythonPipInstallCmd = "echo %s" 46 | 47 | // run 48 | err = updatePipeline(&p1) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | 53 | // check if file has been copied to correct place 54 | if _, err = os.Stat(filepath.Join(virtualEnvPath, p1.Name+".tar.gz")); err != nil { 55 | t.Fatalf("distribution file does not exist: %s", err.Error()) 56 | } 57 | } 58 | 59 | func TestUpdatePipelineRuby(t *testing.T) { 60 | tmp, _ := ioutil.TempDir("", "TestUpdatePipelineRuby") 61 | gaia.Cfg = new(gaia.Config) 62 | gaia.Cfg.HomePath = tmp 63 | buf := new(bytes.Buffer) 64 | gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ 65 | Level: hclog.Trace, 66 | Output: buf, 67 | Name: "Gaia", 68 | }) 69 | 70 | p1 := gaia.Pipeline{ 71 | Name: "PipelinA", 72 | Type: gaia.PTypeRuby, 73 | Created: time.Now(), 74 | } 75 | 76 | // Create fake test gem file. 77 | src := filepath.Join(tmp, "PipelineA_ruby") 78 | p1.ExecPath = src 79 | defer os.RemoveAll(tmp) 80 | _ = ioutil.WriteFile(src, []byte("testcontent"), 0666) 81 | 82 | // fake execution commands 83 | rubyGemName = "echo" 84 | 85 | // run 86 | err := updatePipeline(&p1) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | } 91 | 92 | func TestUpdatePipelineNodeJS(t *testing.T) { 93 | tmp, err := ioutil.TempDir("", "TestUpdatePipelineNodeJS") 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | defer os.RemoveAll(tmp) 98 | gaia.Cfg = new(gaia.Config) 99 | gaia.Cfg.HomePath = tmp 100 | buf := new(bytes.Buffer) 101 | gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ 102 | Level: hclog.Trace, 103 | Output: buf, 104 | Name: "Gaia", 105 | }) 106 | 107 | p1 := gaia.Pipeline{ 108 | Name: "PipelinA", 109 | Type: gaia.PTypeNodeJS, 110 | Created: time.Now(), 111 | } 112 | 113 | // Create fake test nodejs archive file. 114 | src := filepath.Join(tmp, "PipelineA_nodejs") 115 | p1.ExecPath = src 116 | if err := ioutil.WriteFile(src, []byte("testcontent"), 0666); err != nil { 117 | t.Fatal(err) 118 | } 119 | 120 | // fake execution commands 121 | tarName = "echo" 122 | npmName = "echo" 123 | 124 | // run 125 | err = updatePipeline(&p1) 126 | if err != nil { 127 | t.Fatal(err) 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /workers/proto/README.md: -------------------------------------------------------------------------------- 1 | # proto 2 | This folder contains gRPC proto files and their generated language defintions for the gaia worker interface. 3 | 4 | You can use protoc to compile these on your own: 5 | `protoc -I ./ ./worker.proto --go_out=plugins=grpc:./` 6 | 7 | -------------------------------------------------------------------------------- /workers/proto/worker.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package protobuf; 4 | 5 | import "google/protobuf/empty.proto"; 6 | 7 | // WorkerInstance represents the identity of 8 | // a worker instance. 9 | message WorkerInstance { 10 | string unique_id = 1; 11 | int32 worker_slots = 2; 12 | repeated string tags = 3; 13 | } 14 | 15 | // PipelineRun represents one pipeline run. 16 | message PipelineRun { 17 | string unique_id = 1; 18 | int64 id = 2; 19 | string status = 3; 20 | int64 start_date = 4; 21 | int64 finish_date = 5; 22 | int64 schedule_date = 6; 23 | int64 pipeline_id = 7; 24 | string pipeline_name = 8; 25 | string pipeline_type = 9; 26 | bytes sha_sum = 10; 27 | repeated Job jobs = 11; 28 | bool docker = 12; 29 | } 30 | 31 | // PrivateKey represents a key. 32 | message PrivateKey { 33 | string key = 1; 34 | string username = 2; 35 | string password = 3; 36 | } 37 | 38 | // GitRepo is a git repo. 39 | message GitRepo { 40 | PrivateKey private_key = 1; 41 | string username = 2; 42 | string password = 3; 43 | string url = 4; 44 | string selected_branch = 5; 45 | repeated string branches = 6; 46 | string localdest = 7; 47 | } 48 | 49 | message PipelineID { 50 | int64 id = 1; 51 | } 52 | 53 | // Job represents one job from a pipeline run. 54 | message Job { 55 | uint32 unique_id = 1; 56 | string title = 2; 57 | string description = 3; 58 | repeated Job depends_on = 4; 59 | string status = 5; 60 | repeated Argument args = 6; 61 | } 62 | 63 | // Argument represents one argument from a job. 64 | message Argument { 65 | string description = 1; 66 | string type = 2; 67 | string key = 3; 68 | string value = 4; 69 | } 70 | 71 | // LogChunk represents one chunk of a log file. 72 | message LogChunk { 73 | int64 run_id = 1; 74 | int64 pipeline_id = 2; 75 | bytes chunk = 3; 76 | } 77 | 78 | // FileChunk represents one chunk of a file. 79 | message FileChunk { 80 | bytes chunk = 1; 81 | } 82 | 83 | service Worker { 84 | // GetWork pulls work from the primary instance. 85 | rpc GetWork (WorkerInstance) returns (stream PipelineRun); 86 | 87 | // UpdateWork updates work information at the primary instance. 88 | rpc UpdateWork (PipelineRun) returns (google.protobuf.Empty); 89 | 90 | // StreamBinary streams a pipeline binary back to a worker instance. 91 | rpc StreamBinary (PipelineRun) returns (stream FileChunk); 92 | 93 | // StreamLogs streams pipeline run logs to the primary instance. 94 | rpc StreamLogs (stream LogChunk) returns (google.protobuf.Empty); 95 | 96 | // Deregister deregister a registered worker from the primary instance. 97 | rpc Deregister (WorkerInstance) returns (google.protobuf.Empty); 98 | 99 | // GetGitRepo returns git repo information to the worker based on a pipeline name. 100 | rpc GetGitRepo (PipelineID) returns (GitRepo); 101 | } 102 | -------------------------------------------------------------------------------- /workers/scheduler/gaiascheduler/create_cmd.go: -------------------------------------------------------------------------------- 1 | package gaiascheduler 2 | 3 | import ( 4 | "os/exec" 5 | "path/filepath" 6 | 7 | "github.com/gaia-pipeline/gaia" 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | // createPipelineCmd creates the execute command for the plugin system 12 | // dependent on the plugin type. 13 | func createPipelineCmd(p *gaia.Pipeline) *exec.Cmd { 14 | if p == nil { 15 | return nil 16 | } 17 | c := &exec.Cmd{} 18 | 19 | // Dependent on the pipeline type 20 | switch p.Type { 21 | case gaia.PTypeGolang: 22 | c.Path = p.ExecPath 23 | case gaia.PTypeJava: 24 | // Look for java executable 25 | path, err := exec.LookPath(javaExecName) 26 | if err != nil { 27 | gaia.Cfg.Logger.Error("cannot find java executable", "error", err.Error()) 28 | return nil 29 | } 30 | 31 | // Build start command 32 | c.Path = path 33 | c.Args = []string{ 34 | path, 35 | "-jar", 36 | p.ExecPath, 37 | } 38 | case gaia.PTypePython: 39 | // Build start command 40 | c.Path = "/bin/sh" 41 | c.Args = []string{ 42 | "/bin/sh", 43 | "-c", 44 | ". bin/activate; exec " + pythonExecName + " -c \"import pipeline; pipeline.main()\"", 45 | } 46 | c.Dir = filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpPythonFolder, p.Name) 47 | case gaia.PTypeCpp: 48 | c.Path = p.ExecPath 49 | case gaia.PTypeRuby: 50 | // Look for ruby executable 51 | path, err := exec.LookPath(rubyExecName) 52 | if err != nil { 53 | gaia.Cfg.Logger.Error("cannot find ruby executable", "error", err.Error()) 54 | return nil 55 | } 56 | 57 | // Get the gem name from the gem file. 58 | gemName, err := findRubyGemName(p.ExecPath) 59 | if err != nil { 60 | gaia.Cfg.Logger.Error("cannot find the gem name from the gem file", "error", err.Error()) 61 | return nil 62 | } 63 | 64 | // Build start command 65 | c.Path = path 66 | c.Args = []string{ 67 | path, 68 | "-r", 69 | gemName, 70 | "-e", 71 | "Main.main", 72 | } 73 | case gaia.PTypeNodeJS: 74 | // Look for node executable 75 | path, err := exec.LookPath(nodeJSExecName) 76 | if err != nil { 77 | gaia.Cfg.Logger.Error("cannot find NodeJS executable", "error", err) 78 | return nil 79 | } 80 | 81 | // Define the folder where the nodejs plugin is located unpacked 82 | unpackedFolder := filepath.Join(gaia.Cfg.HomePath, gaia.TmpFolder, gaia.TmpNodeJSFolder, p.Name) 83 | 84 | // Build start command 85 | c.Path = path 86 | c.Args = []string{ 87 | path, 88 | "index.js", 89 | } 90 | c.Dir = unpackedFolder 91 | default: 92 | c = nil 93 | } 94 | 95 | return c 96 | } 97 | 98 | var findRubyGemCommands = []string{"specification", "--yaml"} 99 | 100 | // findRubyGemName finds the gem name of a ruby gem file. 101 | func findRubyGemName(execPath string) (name string, err error) { 102 | // Find the gem binary path. 103 | path, err := exec.LookPath(rubyGemName) 104 | if err != nil { 105 | return 106 | } 107 | 108 | // Get the gem specification in YAML format. 109 | gemCommands := append(findRubyGemCommands, execPath) 110 | cmd := exec.Command(path, gemCommands...) 111 | output, err := cmd.CombinedOutput() 112 | if err != nil { 113 | gaia.Cfg.Logger.Debug("output", "output", string(output[:])) 114 | return 115 | } 116 | 117 | // Struct helper to filter for what we need. 118 | type gemSpecOutput struct { 119 | Name string 120 | } 121 | 122 | // Transform and filter the gem specification. 123 | gemSpec := gemSpecOutput{} 124 | err = yaml.Unmarshal(output, &gemSpec) 125 | if err != nil { 126 | return 127 | } 128 | name = gemSpec.Name 129 | return 130 | } 131 | -------------------------------------------------------------------------------- /workers/scheduler/gaiascheduler/create_cmd_test.go: -------------------------------------------------------------------------------- 1 | package gaiascheduler 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestFindRubyGemName(t *testing.T) { 8 | // Adjust gubyGemName (might be not available in test container) 9 | rubyGemName = "echo" 10 | 11 | // Overwrite gem commands 12 | findRubyGemCommands = []string{"name: testruby"} 13 | 14 | // Run command and compare output 15 | gemName, err := findRubyGemName("") 16 | if err != nil { 17 | t.Errorf("error thrown during findRubyGemName: %s", err.Error()) 18 | } 19 | if gemName != "testruby" { 20 | t.Errorf("Gem name should be 'testruby' but was %s", gemName) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /workers/scheduler/gaiascheduler/workload.go: -------------------------------------------------------------------------------- 1 | package gaiascheduler 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/gaia-pipeline/gaia" 7 | ) 8 | 9 | // workload is a wrapper around a single job object. 10 | type workload struct { 11 | finishedSig chan bool 12 | done bool 13 | started bool 14 | job *gaia.Job 15 | } 16 | 17 | // managedWorkloads holds workloads. 18 | // managedWorkloads can be safely shared between goroutines. 19 | type managedWorkloads struct { 20 | sync.RWMutex 21 | 22 | workloads []workload 23 | } 24 | 25 | // newManagedWorkloads creates a new instance of managedWorkloads. 26 | func newManagedWorkloads() *managedWorkloads { 27 | mw := &managedWorkloads{ 28 | workloads: make([]workload, 0), 29 | } 30 | 31 | return mw 32 | } 33 | 34 | // Append appends a new workload to managedWorkloads. 35 | func (mw *managedWorkloads) Append(wl workload) { 36 | mw.Lock() 37 | defer mw.Unlock() 38 | 39 | mw.workloads = append(mw.workloads, wl) 40 | } 41 | 42 | // GetByID looks up the workload by the given id. 43 | func (mw *managedWorkloads) GetByID(id uint32) *workload { 44 | var foundWorkload *workload 45 | for wl := range mw.Iter() { 46 | if wl.job.ID == id { 47 | copyWL := wl 48 | foundWorkload = ©WL 49 | } 50 | } 51 | 52 | return foundWorkload 53 | } 54 | 55 | // Replace takes the given workload and replaces it in the managedWorkloads 56 | // slice. Return true when success otherwise false. 57 | func (mw *managedWorkloads) Replace(wl workload) bool { 58 | mw.Lock() 59 | defer mw.Unlock() 60 | 61 | // Search for the id 62 | i := -1 63 | for id, currWL := range mw.workloads { 64 | if currWL.job.ID == wl.job.ID { 65 | i = id 66 | break 67 | } 68 | } 69 | 70 | // We got it? 71 | if i == -1 { 72 | return false 73 | } 74 | 75 | // Yes 76 | mw.workloads[i] = wl 77 | return true 78 | } 79 | 80 | // Iter iterates over the workloads in the concurrent slice. 81 | func (mw *managedWorkloads) Iter() <-chan workload { 82 | c := make(chan workload) 83 | 84 | go func() { 85 | mw.RLock() 86 | defer mw.RUnlock() 87 | for _, mw := range mw.workloads { 88 | c <- mw 89 | } 90 | close(c) 91 | }() 92 | 93 | return c 94 | } 95 | -------------------------------------------------------------------------------- /workers/scheduler/gaiascheduler/workload_test.go: -------------------------------------------------------------------------------- 1 | package gaiascheduler 2 | 3 | import ( 4 | "strconv" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/gaia-pipeline/gaia" 9 | ) 10 | 11 | func TestNewWorkload(t *testing.T) { 12 | mw := newManagedWorkloads() 13 | var wg sync.WaitGroup 14 | 15 | for i := 0; i < 10; i++ { 16 | wg.Add(1) 17 | go func(j int) { 18 | defer wg.Done() 19 | finished := make(chan bool) 20 | title := strconv.Itoa(j) 21 | wl := workload{ 22 | done: true, 23 | finishedSig: finished, 24 | job: &gaia.Job{ 25 | Description: "Test job", 26 | ID: uint32(j), 27 | Title: "Test " + title, 28 | }, 29 | started: true, 30 | } 31 | mw.Append(wl) 32 | }(i) 33 | } 34 | wg.Wait() 35 | if len(mw.workloads) != 10 { 36 | t.Fatal("workload len want: 10, was:", len(mw.workloads)) 37 | } 38 | 39 | for i := 0; i < 10; i++ { 40 | wg.Add(1) 41 | go func(j int) { 42 | defer wg.Done() 43 | wl := mw.GetByID(uint32(j)) 44 | if wl == nil { 45 | t.Error("failed to find a job that was created previously. failed id: ", j) 46 | } 47 | }(i) 48 | } 49 | if t.Failed() { 50 | t.Fatal("there were errors in the above function. ") 51 | } 52 | wg.Wait() 53 | } 54 | 55 | func TestReplaceWorkloadFlow(t *testing.T) { 56 | mw := newManagedWorkloads() 57 | finished := make(chan bool) 58 | wl := workload{ 59 | done: true, 60 | finishedSig: finished, 61 | job: &gaia.Job{ 62 | Description: "Test job", 63 | ID: 1, 64 | Title: "Test", 65 | }, 66 | started: true, 67 | } 68 | mw.Append(wl) 69 | t.Run("replace works", func(t *testing.T) { 70 | replaceWl := workload{ 71 | done: true, 72 | finishedSig: finished, 73 | job: &gaia.Job{ 74 | Description: "Test job replaced", 75 | ID: 1, 76 | Title: "Test replaced", 77 | }, 78 | started: true, 79 | } 80 | v := mw.Replace(replaceWl) 81 | if !v { 82 | t.Fatalf("return should be true. was false.") 83 | } 84 | l := mw.GetByID(1) 85 | if l.job.Title != "Test replaced" { 86 | t.Fatalf("got title: %s. wanted: 'Test replaced'", l.job.Title) 87 | } 88 | }) 89 | 90 | t.Run("returns false if workload was not found", func(t *testing.T) { 91 | replaceWl := workload{ 92 | done: true, 93 | finishedSig: finished, 94 | job: &gaia.Job{ 95 | Description: "Test job replaced", 96 | ID: 2, 97 | Title: "Test replaced", 98 | }, 99 | started: true, 100 | } 101 | v := mw.Replace(replaceWl) 102 | if v { 103 | t.Fatalf("return should be false. was true.") 104 | } 105 | l := mw.GetByID(2) 106 | if l != nil { 107 | t.Fatal("should have not found id 2 which was replaced:", l) 108 | } 109 | }) 110 | } 111 | -------------------------------------------------------------------------------- /workers/scheduler/service/scheduler.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import "github.com/gaia-pipeline/gaia" 4 | 5 | // GaiaScheduler is a job scheduler for gaia pipeline runs. 6 | type GaiaScheduler interface { 7 | Init() 8 | SchedulePipeline(p *gaia.Pipeline, startReason string, args []*gaia.Argument) (*gaia.PipelineRun, error) 9 | SetPipelineJobs(p *gaia.Pipeline) error 10 | StopPipelineRun(p *gaia.Pipeline, runID int) error 11 | GetFreeWorkers() int32 12 | CountScheduledRuns() int 13 | } 14 | -------------------------------------------------------------------------------- /workers/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/gaia-pipeline/gaia/helper/filehelper" 11 | "github.com/gaia-pipeline/gaia/security" 12 | 13 | "github.com/gaia-pipeline/gaia" 14 | pb "github.com/gaia-pipeline/gaia/workers/proto" 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/credentials" 17 | ) 18 | 19 | const ( 20 | hoursBeforeValid = 2 21 | hoursAfterValid = 87600 // 10 years 22 | ) 23 | 24 | // Dependencies defines dependencies of this service. 25 | type Dependencies struct { 26 | Certificate security.CAAPI 27 | } 28 | 29 | // WorkerServer represents an instance of the worker server implementation 30 | type WorkerServer struct { 31 | Dependencies 32 | } 33 | 34 | // InitWorkerServer creates a new worker server instance. 35 | func InitWorkerServer(deps Dependencies) *WorkerServer { 36 | return &WorkerServer{ 37 | Dependencies: deps, 38 | } 39 | } 40 | 41 | // Start starts the gRPC worker server. 42 | // It returns an error when something badly happens. 43 | func (w *WorkerServer) Start() error { 44 | lis, err := net.Listen("tcp", ":"+gaia.Cfg.WorkerServerPort) 45 | if err != nil { 46 | gaia.Cfg.Logger.Error("cannot start worker gRPC server", "error", err) 47 | return err 48 | } 49 | 50 | // Print info message 51 | gaia.Cfg.Logger.Info("worker gRPC server about to start on port: " + gaia.Cfg.WorkerServerPort) 52 | 53 | // Check if certificates exist for the gRPC server 54 | certPath := filepath.Join(gaia.Cfg.DataPath, "worker_cert.pem") 55 | keyPath := filepath.Join(gaia.Cfg.DataPath, "worker_key.pem") 56 | _, certErr := os.Stat(certPath) 57 | _, keyErr := os.Stat(keyPath) 58 | if os.IsNotExist(certErr) || os.IsNotExist(keyErr) { 59 | // Parse hostname for the certificate 60 | s := strings.Split(gaia.Cfg.WorkerGRPCHostURL, ":") 61 | if len(s) != 2 { 62 | gaia.Cfg.Logger.Error("failed to parse configured gRPC worker host url", "url", gaia.Cfg.WorkerGRPCHostURL) 63 | return fmt.Errorf("failed to parse configured gRPC worker host url: %s", gaia.Cfg.WorkerGRPCHostURL) 64 | } 65 | 66 | // Generate certs 67 | certTmpPath, keyTmpPath, err := w.Certificate.CreateSignedCertWithValidOpts(s[0], hoursBeforeValid, hoursAfterValid) 68 | if err != nil { 69 | gaia.Cfg.Logger.Error("failed to generate cert pair for gRPC server", "error", err.Error()) 70 | return err 71 | } 72 | 73 | // Move certs to correct place 74 | if err = filehelper.CopyFileContents(certTmpPath, certPath); err != nil { 75 | gaia.Cfg.Logger.Error("failed to copy gRPC server cert to data folder", "error", err.Error()) 76 | return err 77 | } 78 | if err = filehelper.CopyFileContents(keyTmpPath, keyPath); err != nil { 79 | gaia.Cfg.Logger.Error("failed to copy gRPC server key to data folder", "error", err.Error()) 80 | return err 81 | } 82 | if err = os.Remove(certTmpPath); err != nil { 83 | gaia.Cfg.Logger.Error("failed to remove temporary server cert file", "error", err) 84 | return err 85 | } 86 | if err = os.Remove(keyTmpPath); err != nil { 87 | gaia.Cfg.Logger.Error("failed to remove temporary key cert file", "error", err) 88 | return err 89 | } 90 | } 91 | 92 | // Generate tls config 93 | tlsConfig, err := w.Certificate.GenerateTLSConfig(certPath, keyPath) 94 | if err != nil { 95 | gaia.Cfg.Logger.Error("failed to generate tls config for gRPC server", "error", err.Error()) 96 | return err 97 | } 98 | 99 | s := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig))) 100 | pb.RegisterWorkerServer(s, &WorkServer{}) 101 | if err := s.Serve(lis); err != nil { 102 | gaia.Cfg.Logger.Error("cannot start worker gRPC server", "error", err) 103 | return err 104 | } 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /workers/server/server_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/gaia-pipeline/gaia/security" 10 | 11 | "github.com/gaia-pipeline/gaia" 12 | hclog "github.com/hashicorp/go-hclog" 13 | ) 14 | 15 | func TestStart(t *testing.T) { 16 | // Create tmp folder 17 | tmpFolder, err := ioutil.TempDir("", "TestStart") 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | defer os.RemoveAll(tmpFolder) 22 | 23 | gaia.Cfg = &gaia.Config{ 24 | Mode: gaia.ModeServer, 25 | WorkerGRPCHostURL: "myhost:12345", 26 | HomePath: tmpFolder, 27 | DataPath: tmpFolder, 28 | CAPath: tmpFolder, 29 | } 30 | gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ 31 | Level: hclog.Trace, 32 | Name: "Gaia", 33 | }) 34 | 35 | ca, _ := security.InitCA() 36 | // Init worker server 37 | server := InitWorkerServer(Dependencies{Certificate: ca}) 38 | 39 | // Start server 40 | errChan := make(chan error) 41 | go func() { 42 | if err := server.Start(); err != nil { 43 | errChan <- err 44 | } 45 | }() 46 | time.Sleep(3 * time.Second) 47 | select { 48 | case err := <-errChan: 49 | t.Fatal(err) 50 | default: 51 | } 52 | } 53 | --------------------------------------------------------------------------------