├── docs
├── CNAME
├── img
│ ├── favicon.ico
│ ├── piper-demo-1080.mov
│ └── piper-demo-1080.mp4
├── index.md
├── configuration
│ ├── health_check.md
│ └── environment_variables.md
├── usage
│ ├── workflows_config.md
│ ├── global_variables.md
│ └── workflows_folder.md
├── getting_started
│ └── installation.md
└── CONTRIBUTING.md
├── .github
├── CODEOWNERS
├── workflows
│ ├── lint-commit.yaml
│ ├── release-please.yaml
│ ├── docs.yaml
│ ├── lint-pr.yaml
│ ├── release.yaml
│ ├── ci.yaml
│ ├── snyk.yaml
│ └── e2e.yaml
├── dependabot.yml
└── pull_request_template.md
├── examples
├── .workflows
│ ├── parameters.yaml
│ ├── exit.yaml
│ ├── templates.yaml
│ ├── main.yaml
│ └── triggers.yaml
├── template.values.dev.yaml
├── config.yaml
└── workflow.yaml
├── .dockerignore
├── helm-chart
├── Chart.yaml
├── templates
│ ├── config.yaml
│ ├── serviceaccount.yaml
│ ├── service.yaml
│ ├── rookout-token.yaml
│ ├── git-token-secret.yaml
│ ├── argo-token-secret.yaml
│ ├── webhook-secret.yaml
│ ├── hpa.yaml
│ ├── security.yaml
│ ├── ingress.yaml
│ ├── _helpers.tpl
│ └── deployment.yaml
├── .helmignore
├── README.md
└── values.yaml
├── scripts
├── init-piper.sh
├── init-argo-workflows.sh
├── init-nginx.sh
└── init-kind.sh
├── pkg
├── clients
│ └── types.go
├── webhook_creator
│ ├── types.go
│ ├── mocks.go
│ └── main.go
├── server
│ ├── routes
│ │ ├── readyz.go
│ │ ├── healthz.go
│ │ └── webhook.go
│ ├── main.go
│ ├── types.go
│ ├── shutdown.go
│ └── server.go
├── common
│ └── types.go
├── event_handler
│ ├── types.go
│ ├── main.go
│ ├── workflow_event_handler.go
│ ├── github_event_notifier.go
│ └── github_event_notifier_test.go
├── webhook_handler
│ ├── types.go
│ └── webhook_handler.go
├── conf
│ ├── rookout.go
│ ├── conf.go
│ ├── workflow_server.go
│ ├── workflows_config.go
│ └── git_provider.go
├── git_provider
│ ├── main.go
│ ├── types.go
│ ├── bitbucket_utils.go
│ ├── github_utils_test.go
│ ├── test_utils.go
│ ├── github_utils.go
│ ├── bitbucket_test.go
│ └── bitbucket.go
├── workflow_handler
│ ├── types.go
│ ├── workflows_utils.go
│ ├── workflows_test.go
│ ├── workflows_utils_test.go
│ └── workflows.go
└── utils
│ ├── os.go
│ ├── os_test.go
│ └── common.go
├── workflows.values.yaml
├── Dockerfile
├── mkdocs.yml
├── makefile
├── .gitignore
├── cmd
└── piper
│ └── piper.go
├── README.md
├── go.mod
└── LICENSE
/docs/CNAME:
--------------------------------------------------------------------------------
1 | piper.rookout.com
--------------------------------------------------------------------------------
/.github/CODEOWNERS:
--------------------------------------------------------------------------------
1 | * @gosharo
2 | **/.github @gosharo
--------------------------------------------------------------------------------
/examples/.workflows/parameters.yaml:
--------------------------------------------------------------------------------
1 | - name: global
2 | value: multi-branch-pipeline
--------------------------------------------------------------------------------
/docs/img/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Rookout/piper/HEAD/docs/img/favicon.ico
--------------------------------------------------------------------------------
/docs/img/piper-demo-1080.mov:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Rookout/piper/HEAD/docs/img/piper-demo-1080.mov
--------------------------------------------------------------------------------
/docs/img/piper-demo-1080.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Rookout/piper/HEAD/docs/img/piper-demo-1080.mp4
--------------------------------------------------------------------------------
/.dockerignore:
--------------------------------------------------------------------------------
1 | # Keep ignoring .git
2 | .git
3 | # Allow specific files with !
4 | !.git/HEAD
5 | !.git/config
6 | !.git/refs
--------------------------------------------------------------------------------
/helm-chart/Chart.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v2
2 | name: piper
3 | description: A Helm chart for Piper
4 | type: application
5 | version: 1.0.1
6 | appVersion: 1.0.1
--------------------------------------------------------------------------------
/examples/.workflows/exit.yaml:
--------------------------------------------------------------------------------
1 | - name: github-status
2 | template: exit-handler
3 | arguments:
4 | parameters:
5 | - name: param1
6 | value: "{{ workflow.labels.repo }}"
--------------------------------------------------------------------------------
/scripts/init-piper.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | set -o errexit
3 |
4 | if [ -z "$(helm list | grep piper)" ]; then
5 | # 8. Install Piper
6 | helm upgrade --install piper ./helm-chart -f values.dev.yaml
7 | else
8 | echo "Piper release exists, skipping installation"
9 | fi
10 |
--------------------------------------------------------------------------------
/pkg/clients/types.go:
--------------------------------------------------------------------------------
1 | package clients
2 |
3 | import (
4 | "github.com/rookout/piper/pkg/git_provider"
5 | "github.com/rookout/piper/pkg/workflow_handler"
6 | )
7 |
8 | type Clients struct {
9 | GitProvider git_provider.Client
10 | Workflows workflow_handler.WorkflowsClient
11 | }
12 |
--------------------------------------------------------------------------------
/pkg/webhook_creator/types.go:
--------------------------------------------------------------------------------
1 | package webhook_creator
2 |
3 | import "golang.org/x/net/context"
4 |
5 | type WebhookCreator interface {
6 | Stop(ctx *context.Context)
7 | Start()
8 | SetWebhookHealth(status bool, hookID *int64) error
9 | RunDiagnosis(ctx *context.Context) error
10 | }
11 |
--------------------------------------------------------------------------------
/examples/.workflows/templates.yaml:
--------------------------------------------------------------------------------
1 | - name: local-step
2 | inputs:
3 | parameters:
4 | - name: message
5 | script:
6 | image: alpine
7 | command: [ sh ]
8 | source: |
9 | echo "wellcome to {{ workflow.parameters.global }}
10 | echo "{{ inputs.parameters.message }}"
--------------------------------------------------------------------------------
/pkg/server/routes/readyz.go:
--------------------------------------------------------------------------------
1 | package routes
2 |
3 | import (
4 | "net/http"
5 |
6 | "github.com/gin-gonic/gin"
7 | )
8 |
9 | func AddReadyRoutes(rg *gin.RouterGroup) {
10 | health := rg.Group("/readyz")
11 |
12 | health.GET("", func(c *gin.Context) {
13 | c.JSON(http.StatusOK, "ready")
14 | })
15 | }
16 |
--------------------------------------------------------------------------------
/examples/.workflows/main.yaml:
--------------------------------------------------------------------------------
1 | - name: local-step1
2 | template: local-step
3 | arguments:
4 | parameters:
5 | - name: message
6 | value: step-1
7 | - name: local-step2
8 | template: local-step
9 | arguments:
10 | parameters:
11 | - name: message
12 | value: step-2
13 | dependencies:
14 | - local-step1
15 |
--------------------------------------------------------------------------------
/helm-chart/templates/config.yaml:
--------------------------------------------------------------------------------
1 | {{- if .Values.piper.workflowsConfig }}
2 | apiVersion: v1
3 | kind: ConfigMap
4 | metadata:
5 | name: piper-workflows-config
6 | labels:
7 | {{- include "piper.labels" . | nindent 4 }}
8 | data:
9 | {{- with .Values.piper.workflowsConfig }}
10 | {{- toYaml . | nindent 2 }}
11 | {{- end }}
12 | {{- end }}
13 |
--------------------------------------------------------------------------------
/examples/.workflows/triggers.yaml:
--------------------------------------------------------------------------------
1 | - events:
2 | - push
3 | - pull_request.synchronize
4 | branches: ["main"]
5 | onStart: ["main.yaml"]
6 | onExit: ["exit.yaml"]
7 | templates: ["templates.yaml"]
8 | config: "default"
9 |
10 | - events:
11 | - pull_request
12 | branches: ["*"]
13 | onStart: ["main.yaml"]
14 | onExit: ["exit.yaml"]
15 | templates: ["templates.yaml"]
--------------------------------------------------------------------------------
/pkg/common/types.go:
--------------------------------------------------------------------------------
1 | package common
2 |
3 | import (
4 | "github.com/rookout/piper/pkg/git_provider"
5 | )
6 |
7 | type WorkflowsBatch struct {
8 | OnStart []*git_provider.CommitFile
9 | OnExit []*git_provider.CommitFile
10 | Templates []*git_provider.CommitFile
11 | Parameters *git_provider.CommitFile
12 | Config *string
13 | Payload *git_provider.WebhookPayload
14 | }
15 |
--------------------------------------------------------------------------------
/helm-chart/templates/serviceaccount.yaml:
--------------------------------------------------------------------------------
1 | {{- if .Values.serviceAccount.create -}}
2 | apiVersion: v1
3 | kind: ServiceAccount
4 | metadata:
5 | name: {{ include "piper.serviceAccountName" . }}
6 | labels:
7 | {{- include "piper.labels" . | nindent 4 }}
8 | {{- with .Values.serviceAccount.annotations }}
9 | annotations:
10 | {{- toYaml . | nindent 4 }}
11 | {{- end }}
12 | {{- end }}
13 |
--------------------------------------------------------------------------------
/scripts/init-argo-workflows.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | set -o errexit
3 |
4 | if [ -z "$(helm list -n workflows | grep argo-workflow)" ]; then
5 | # 7. Install argo workflows
6 | helm repo add argo https://argoproj.github.io/argo-helm
7 | helm upgrade --install argo-workflow argo/argo-workflows -n workflows --create-namespace -f workflows.values.yaml
8 | else
9 | echo "Workflows release exists, skipping installation"
10 | fi
11 |
--------------------------------------------------------------------------------
/pkg/event_handler/types.go:
--------------------------------------------------------------------------------
1 | package event_handler
2 |
3 | import (
4 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
5 | "golang.org/x/net/context"
6 | "k8s.io/apimachinery/pkg/watch"
7 | )
8 |
9 | type EventHandler interface {
10 | Handle(ctx context.Context, event *watch.Event) error
11 | }
12 |
13 | type EventNotifier interface {
14 | Notify(ctx *context.Context, workflow *v1alpha1.Workflow) error
15 | }
16 |
--------------------------------------------------------------------------------
/helm-chart/templates/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: {{ include "piper.fullname" . }}
5 | labels:
6 | {{- include "piper.labels" . | nindent 4 }}
7 | spec:
8 | type: {{ .Values.service.type }}
9 | ports:
10 | - port: {{ .Values.service.port }}
11 | targetPort: 8080
12 | protocol: TCP
13 | name: http
14 | selector:
15 | {{- include "piper.selectorLabels" . | nindent 4 }}
--------------------------------------------------------------------------------
/.github/workflows/lint-commit.yaml:
--------------------------------------------------------------------------------
1 | name: Lint Commit
2 | on: pull_request
3 | jobs:
4 | conventional:
5 | name: Conventional Commit Linter
6 | runs-on: ubuntu-latest
7 | steps:
8 | - uses: actions/checkout@v2
9 | - uses: actions/setup-node@v2
10 | - uses: taskmedia/action-conventional-commits@v1.1.8
11 | with:
12 | token: ${{ github.token }}
13 | types: "fix|feat|revert|ci|docs|chore"
14 |
--------------------------------------------------------------------------------
/workflows.values.yaml:
--------------------------------------------------------------------------------
1 | controller:
2 | workflowNamespaces:
3 | - workflows
4 | server:
5 | baseHref: /argo/
6 | serviceAccount:
7 | create: true
8 | extraArgs:
9 | - server
10 | - --auth-mode=server
11 | ingress:
12 | enabled: true
13 | annotations:
14 | nginx.ingress.kubernetes.io/rewrite-target: /$1
15 | nginx.ingress.kubernetes.io/backend-protocol: HTTP
16 | paths:
17 | - /argo/(.*)
18 | - /argo
--------------------------------------------------------------------------------
/.github/workflows/release-please.yaml:
--------------------------------------------------------------------------------
1 | name: Release Please
2 |
3 | on:
4 | push:
5 | branches:
6 | - main
7 |
8 | permissions:
9 | contents: write
10 | pull-requests: write
11 |
12 | jobs:
13 | release-please:
14 | runs-on: ubuntu-latest
15 | steps:
16 | - uses: google-github-actions/release-please-action@v3
17 | with:
18 | release-type: helm
19 | package-name: piper
20 | token: ${{ secrets.GIT_TOKEN }}
--------------------------------------------------------------------------------
/helm-chart/templates/rookout-token.yaml:
--------------------------------------------------------------------------------
1 | {{- if and .Values.rookout.token (not .Values.rookout.existingSecret) }}
2 | apiVersion: v1
3 | kind: Secret
4 | metadata:
5 | name: {{ template "rookout.secretName" . }}
6 | namespace: {{ .Release.Namespace }}
7 | labels:
8 | app.kubernetes.io/name: {{ .Chart.Name }}
9 | app.kubernetes.io/instance: {{ .Release.Name }}
10 | type: Opaque
11 | data:
12 | token: {{ .Values.rookout.token | b64enc | quote }}
13 | {{- end }}
14 |
--------------------------------------------------------------------------------
/helm-chart/.helmignore:
--------------------------------------------------------------------------------
1 | # Patterns to ignore when building packages.
2 | # This supports shell glob matching, relative path matching, and
3 | # negation (prefixed with !). Only one pattern per line.
4 | .DS_Store
5 | # Common VCS dirs
6 | .git/
7 | .gitignore
8 | .bzr/
9 | .bzrignore
10 | .hg/
11 | .hgignore
12 | .svn/
13 | # Common backup files
14 | *.swp
15 | *.bak
16 | *.tmp
17 | *.orig
18 | *~
19 | # Various IDEs
20 | .project
21 | .idea/
22 | *.tmproj
23 | .vscode/
24 | _lint.yaml
--------------------------------------------------------------------------------
/helm-chart/templates/git-token-secret.yaml:
--------------------------------------------------------------------------------
1 | {{- if and .Values.piper.gitProvider.token (not .Values.piper.gitProvider.existingSecret) }}
2 | apiVersion: v1
3 | kind: Secret
4 | metadata:
5 | name: {{ template "piper.gitProvider.tokenSecretName" . }}
6 | namespace: {{ .Release.Namespace }}
7 | labels:
8 | app.kubernetes.io/name: {{ .Chart.Name }}
9 | app.kubernetes.io/instance: {{ .Release.Name }}
10 | type: Opaque
11 | data:
12 | token: {{ .Values.piper.gitProvider.token | b64enc | quote }}
13 | {{- end }}
14 |
--------------------------------------------------------------------------------
/pkg/server/main.go:
--------------------------------------------------------------------------------
1 | package server
2 |
3 | import (
4 | "github.com/rookout/piper/pkg/clients"
5 | "github.com/rookout/piper/pkg/conf"
6 | "golang.org/x/net/context"
7 | "log"
8 | )
9 |
10 | func Start(ctx context.Context, stop context.CancelFunc, cfg *conf.GlobalConfig, clients *clients.Clients) {
11 |
12 | srv := NewServer(cfg, clients)
13 | gracefulShutdownHandler := NewGracefulShutdown(ctx, stop)
14 | srv.Start()
15 |
16 | gracefulShutdownHandler.Shutdown(srv)
17 |
18 | log.Println("Server exiting")
19 | }
20 |
--------------------------------------------------------------------------------
/helm-chart/templates/argo-token-secret.yaml:
--------------------------------------------------------------------------------
1 | {{- if and .Values.piper.argoWorkflows.server.token (not .Values.piper.argoWorkflows.server.existingSecret) }}
2 | apiVersion: v1
3 | kind: Secret
4 | metadata:
5 | name: {{ template "piper.argoWorkflows.tokenSecretName" . }}
6 | namespace: {{ .Release.Namespace }}
7 | labels:
8 | app.kubernetes.io/name: {{ .Chart.Name }}
9 | app.kubernetes.io/instance: {{ .Release.Name }}
10 | type: Opaque
11 | data:
12 | token: {{ .Values.piper.argoWorkflows.server.token | b64enc | quote }}
13 | {{- end }}
14 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | # To get started with Dependabot version updates, you'll need to specify which
2 | # package ecosystems to update and where the package manifests are located.
3 | # Please see the documentation for all configuration options:
4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
5 |
6 | version: 2
7 | updates:
8 | - package-ecosystem: "" # See documentation for possible values
9 | directory: "/" # Location of package manifests
10 | schedule:
11 | interval: "weekly"
12 |
--------------------------------------------------------------------------------
/pkg/server/types.go:
--------------------------------------------------------------------------------
1 | package server
2 |
3 | import (
4 | "github.com/gin-gonic/gin"
5 | "github.com/rookout/piper/pkg/clients"
6 | "github.com/rookout/piper/pkg/conf"
7 | "github.com/rookout/piper/pkg/webhook_creator"
8 | "net/http"
9 | )
10 |
11 | type Server struct {
12 | router *gin.Engine
13 | config *conf.GlobalConfig
14 | clients *clients.Clients
15 | webhookCreator *webhook_creator.WebhookCreatorImpl
16 | httpServer *http.Server
17 | }
18 |
19 | type Interface interface {
20 | startServer() *http.Server
21 | registerMiddlewares()
22 | getRoutes()
23 | Start() *http.Server
24 | }
25 |
--------------------------------------------------------------------------------
/pkg/webhook_handler/types.go:
--------------------------------------------------------------------------------
1 | package webhook_handler
2 |
3 | import (
4 | "context"
5 | "github.com/rookout/piper/pkg/common"
6 | )
7 |
8 | type Trigger struct {
9 | Events *[]string `yaml:"events"`
10 | Branches *[]string `yaml:"branches"`
11 | OnStart *[]string `yaml:"onStart"`
12 | Templates *[]string `yaml:"templates"`
13 | OnExit *[]string `yaml:"onExit"`
14 | Config string `yaml:"config" default:"default"`
15 | }
16 |
17 | type WebhookHandler interface {
18 | RegisterTriggers(ctx *context.Context) error
19 | PrepareBatchForMatchingTriggers(ctx *context.Context) ([]*common.WorkflowsBatch, error)
20 | }
21 |
--------------------------------------------------------------------------------
/helm-chart/templates/webhook-secret.yaml:
--------------------------------------------------------------------------------
1 | {{- if not .Values.piper.gitProvider.webhook.existingSecret }}
2 | apiVersion: v1
3 | kind: Secret
4 | metadata:
5 | name: {{ template "piper.gitProvider.webhook.secretName" . }}
6 | namespace: {{ .Release.Namespace }}
7 | labels:
8 | app.kubernetes.io/name: {{ .Chart.Name }}
9 | app.kubernetes.io/instance: {{ .Release.Name }}
10 | type: Opaque
11 | data:
12 | {{- if and .Values.piper.gitProvider.webhook.secret }}
13 | secret: {{ .Values.piper.gitProvider.webhook.secret | b64enc | quote }}
14 | {{- else }}
15 | secret: {{ randAlphaNum 30 | b64enc | quote }}
16 | {{- end }}
17 | {{- end }}
18 |
--------------------------------------------------------------------------------
/pkg/conf/rookout.go:
--------------------------------------------------------------------------------
1 | package conf
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/kelseyhightower/envconfig"
7 | )
8 |
9 | type RookoutConfig struct {
10 | Token string `envconfig:"ROOKOUT_TOKEN" default:""`
11 | Labels string `envconfig:"ROOKOUT_LABELS" default:"service:piper"`
12 | RemoteOrigin string `envconfig:"ROOKOUT_REMOTE_ORIGIN" default:"https://github.com/Rookout/piper.git"`
13 | }
14 |
15 | func (cfg *RookoutConfig) RookoutConfLoad() error {
16 | err := envconfig.Process("", cfg)
17 | if err != nil {
18 | return fmt.Errorf("failed to load the Rookout configuration, error: %v", err)
19 | }
20 |
21 | return nil
22 | }
23 |
--------------------------------------------------------------------------------
/pkg/conf/conf.go:
--------------------------------------------------------------------------------
1 | package conf
2 |
3 | import (
4 | "fmt"
5 | "github.com/kelseyhightower/envconfig"
6 | )
7 |
8 | type GlobalConfig struct {
9 | GitProviderConfig
10 | WorkflowServerConfig
11 | RookoutConfig
12 | WorkflowsConfig
13 | }
14 |
15 | func (cfg *GlobalConfig) Load() error {
16 | err := envconfig.Process("", cfg)
17 | if err != nil {
18 | return fmt.Errorf("failed to load the configuration, error: %v", err)
19 | }
20 |
21 | return nil
22 | }
23 |
24 | func LoadConfig() (*GlobalConfig, error) {
25 | cfg := new(GlobalConfig)
26 |
27 | err := cfg.Load()
28 | if err != nil {
29 | return nil, err
30 | }
31 |
32 | return cfg, nil
33 | }
34 |
--------------------------------------------------------------------------------
/pkg/git_provider/main.go:
--------------------------------------------------------------------------------
1 | package git_provider
2 |
3 | import (
4 | "fmt"
5 | "github.com/rookout/piper/pkg/conf"
6 | )
7 |
8 | func NewGitProviderClient(cfg *conf.GlobalConfig) (Client, error) {
9 |
10 | switch cfg.GitProviderConfig.Provider {
11 | case "github":
12 | gitClient, err := NewGithubClient(cfg)
13 | if err != nil {
14 | return nil, err
15 | }
16 | return gitClient, nil
17 | case "bitbucket":
18 | gitClient, err := NewBitbucketServerClient(cfg)
19 | if err != nil {
20 | return nil, err
21 | }
22 | return gitClient, nil
23 | }
24 |
25 | return nil, fmt.Errorf("didn't find matching git provider %s", cfg.GitProviderConfig.Provider)
26 | }
27 |
--------------------------------------------------------------------------------
/examples/template.values.dev.yaml:
--------------------------------------------------------------------------------
1 | piper:
2 | gitProvider:
3 | name: github
4 | token: "GIT_TOKEN"
5 | organization:
6 | name: "ORG_NAME"
7 | webhook:
8 | url: https://NGROK_ADDRESS/piper/webhook
9 | orgLevel: false
10 | repoList: ["REPO_NAME"]
11 | argoWorkflows:
12 | server:
13 | namespace: "workflows"
14 | address: "ARGO_ADDRESS"
15 | token: "ARGO_TOKEN"
16 | image:
17 | name: piper
18 | repository: localhost:5001
19 | pullPolicy: Always
20 | tag: latest
21 | ingress:
22 | enabled: true
23 | annotations:
24 | nginx.ingress.kubernetes.io/rewrite-target: /$2
25 | hosts:
26 | - paths:
27 | - path: /piper(/|$)(.*)
28 | pathType: ImplementationSpecific
--------------------------------------------------------------------------------
/scripts/init-nginx.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | set -o errexit
3 |
4 | if [ -z "$(kubectl get pods --all-namespaces | grep ingress-nginx-controller)" ]; then
5 | # 6. Deploy of nginx ingress controller to the cluster
6 | kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml && \
7 | kubectl wait --namespace ingress-nginx \
8 | --for=condition=complete job/ingress-nginx-admission-create \
9 | --timeout=180s && \
10 | kubectl wait --namespace ingress-nginx \
11 | --for=condition=ready pod \
12 | --selector=app.kubernetes.io/component=controller \
13 | --timeout=360s
14 | else
15 | echo "Nginx already exists, skipping installation"
16 | fi
17 |
--------------------------------------------------------------------------------
/pkg/conf/workflow_server.go:
--------------------------------------------------------------------------------
1 | package conf
2 |
3 | import (
4 | "fmt"
5 | "github.com/kelseyhightower/envconfig"
6 | )
7 |
8 | type WorkflowServerConfig struct {
9 | ArgoToken string `envconfig:"ARGO_WORKFLOWS_TOKEN" required:"false"`
10 | ArgoAddress string `envconfig:"ARGO_WORKFLOWS_ADDRESS" required:"false"`
11 | CreateCRD bool `envconfig:"ARGO_WORKFLOWS_CREATE_CRD" default:"true"`
12 | Namespace string `envconfig:"ARGO_WORKFLOWS_NAMESPACE" default:"default"`
13 | KubeConfig string `envconfig:"KUBE_CONFIG" default:""`
14 | }
15 |
16 | func (cfg *WorkflowServerConfig) ArgoConfLoad() error {
17 | err := envconfig.Process("", cfg)
18 | if err != nil {
19 | return fmt.Errorf("failed to load the Argo configuration, error: %v", err)
20 | }
21 |
22 | return nil
23 | }
24 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM golang:1.20-alpine3.16 as builder
2 |
3 | WORKDIR /piper
4 |
5 | RUN apk update && apk add --no-cache \
6 | git \
7 | make \
8 | wget \
9 | curl \
10 | gcc \
11 | bash \
12 | ca-certificates \
13 | musl-dev \
14 | zlib-static \
15 | build-base
16 |
17 | COPY go.mod .
18 | COPY go.sum .
19 | RUN --mount=type=cache,target=/go/pkg/mod go mod download
20 |
21 | COPY . .
22 |
23 | RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod tidy
24 |
25 | RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go build -gcflags='all=-N -l' -tags=alpine -buildvcs=false -trimpath ./cmd/piper
26 |
27 |
28 | FROM alpine:3.16 as piper-release
29 |
30 | ENV GIN_MODE=release
31 |
32 | USER 1001
33 |
34 | COPY .git /.git
35 |
36 | COPY --chown=1001 --from=builder /piper/piper /bin
37 |
38 | ENTRYPOINT [ "piper" ]
--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
1 | # Introduction
2 |
3 |
4 | 
5 |
6 |
7 | Welcome to Piper!
8 |
9 | Piper is an open source project that aimed at providing multibranch pipeline functionality to Argo Workflows, allows users to create distinct Workflows based on Git branches. Supports GitHub and Bitbucket.
10 |
11 | ## General explanation
12 |
13 |
14 | 
15 |
16 |
17 | To achieve multibranch pipeline functionality Piper will do the hard works for us.
18 | At initialization, it will load all configuration and create a webhook in repository or organization scope.
19 | Then each branch that have `.workflows` folder will create a Workflow CRD out of the files in this folder.
20 |
21 | 
--------------------------------------------------------------------------------
/pkg/server/routes/healthz.go:
--------------------------------------------------------------------------------
1 | package routes
2 |
3 | import (
4 | "github.com/rookout/piper/pkg/conf"
5 | "github.com/rookout/piper/pkg/webhook_creator"
6 | "golang.org/x/net/context"
7 | "log"
8 | "net/http"
9 | "time"
10 |
11 | "github.com/gin-gonic/gin"
12 | )
13 |
14 | func AddHealthRoutes(rg *gin.RouterGroup, wc *webhook_creator.WebhookCreatorImpl, cfg *conf.GlobalConfig) {
15 | health := rg.Group("/healthz")
16 |
17 | health.GET("", func(c *gin.Context) {
18 | if cfg.GitProviderConfig.FullHealthCheck {
19 | ctx := c.Copy().Request.Context()
20 | ctx2, cancel := context.WithTimeout(ctx, 5*time.Second)
21 | defer cancel()
22 | err := wc.RunDiagnosis(&ctx2)
23 | if err != nil {
24 | log.Printf("error from healthz endpoint:%s\n", err)
25 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
26 | return
27 | }
28 | }
29 | c.JSON(http.StatusOK, "healthy")
30 | })
31 | }
32 |
--------------------------------------------------------------------------------
/docs/configuration/health_check.md:
--------------------------------------------------------------------------------
1 | ## Health Check
2 |
3 | Health check executed every 1 minute as configured in the helm chart under `livenessProbe`, and triggered by `/healthz` endpoint:
4 | ```yaml
5 | livenessProbe:
6 | httpGet:
7 | path: /healthz
8 | port: 8080
9 | scheme: HTTP
10 | initialDelaySeconds: 10
11 | timeoutSeconds: 10
12 | periodSeconds: 60
13 | successThreshold: 1
14 | failureThreshold: 4
15 | ```
16 |
17 | The mechanism for checking the health of Piper is:
18 |
19 | 1. Piper set health status of all webhooks to not-healthy.
20 |
21 | 2. Piper requests ping from all the webhooks configured.
22 |
23 | 3. Git Provider send ping to `/webhook` endpoint, this will set the health status to `healthy` with timeout of 5 seconds.
24 |
25 | 4. Piper check the status of all webhooks configured.
26 |
27 | Therefore, the criteria for health checking are:
28 | 1. The registered webhook exists.
29 | 2. The webhook send a ping in 5 seconds.
30 |
31 |
32 |
--------------------------------------------------------------------------------
/helm-chart/templates/hpa.yaml:
--------------------------------------------------------------------------------
1 | {{- if .Values.autoscaling.enabled }}
2 | apiVersion: autoscaling/v2beta1
3 | kind: HorizontalPodAutoscaler
4 | metadata:
5 | name: {{ include "piper.fullname" . }}
6 | labels:
7 | {{- include "piper.labels" . | nindent 4 }}
8 | spec:
9 | scaleTargetRef:
10 | apiVersion: apps/v1
11 | kind: Deployment
12 | name: {{ include "piper.fullname" . }}
13 | minReplicas: {{ .Values.autoscaling.minReplicas }}
14 | maxReplicas: {{ .Values.autoscaling.maxReplicas }}
15 | metrics:
16 | {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
17 | - type: Resource
18 | resource:
19 | name: cpu
20 | targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
21 | {{- end }}
22 | {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
23 | - type: Resource
24 | resource:
25 | name: memory
26 | targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
27 | {{- end }}
28 | {{- end }}
29 |
--------------------------------------------------------------------------------
/examples/config.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ConfigMap
3 | metadata:
4 | name: piper-workflows-config
5 | data:
6 | default: |
7 | spec:
8 | volumes:
9 | - name: shared-volume
10 | emptyDir: { }
11 | serviceAccountName: argo-wf
12 | activeDeadlineSeconds: 7200 # (seconds) == 2 hours
13 | ttlStrategy:
14 | secondsAfterCompletion: 28800 # (seconds) == 8 hours
15 | podGC:
16 | strategy: OnPodSuccess
17 | archiveLogs: true
18 | artifactRepositoryRef:
19 | configMap: artifact-repositories
20 | nodeSelector:
21 | node_pool: workflows
22 | tolerations:
23 | - effect: NoSchedule
24 | key: node_pool
25 | operator: Equal
26 | value: workflows
27 | onExit: # optinal, will be overwritten if specifc in .wokrflows/exit.yaml.
28 | - name: github-status
29 | template: exit-handler
30 | arguments:
31 | parameters:
32 | - name: param1
33 | value: "{{ workflow.labels.repo }}"
--------------------------------------------------------------------------------
/docs/usage/workflows_config.md:
--------------------------------------------------------------------------------
1 | ## Workflow Configuration
2 |
3 | Piper can inject configuration for Workflows that Piper creates.
4 |
5 | `default` config used as a convention for all Workflows that piper will create, even if not explicitly mentioned in triggers.yaml file.
6 |
7 | ### ConfigMap
8 | Piper will mount a configMap when helm used.
9 | `piper.workflowsConfig` variable in helm chart, will create a configMap that hold set of configuration for Piper.
10 | Here is an [examples](https://github.com/Rookout/piper/tree/main/examples/config.yaml) of such configuration.
11 |
12 | ### Spec
13 | This will be injected to Workflow spec field. can hold all configuration of the Workflow.
14 | > :warning: Please notice that the fields `entrypoint` and `onExit` should not exist in spec. both of them are managed fields.
15 |
16 | ### onExit
17 | This is the exit handler for each of the Workflows create by piper.
18 | It configures a DAG that will be executed when the workflow ends.
19 | You can provide the templates to it us in the following [Examples](https://github.com/Rookout/piper/tree/main/examples/config.yaml).
--------------------------------------------------------------------------------
/helm-chart/templates/security.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRole
3 | metadata:
4 | name: {{ include "piper.fullname" . }}
5 | rules:
6 | - apiGroups:
7 | - ""
8 | resources:
9 | - events
10 | verbs:
11 | - list
12 | - watch
13 | - create
14 | - patch
15 | - apiGroups:
16 | - argoproj.io
17 | resources:
18 | - workflows
19 | - workflows/finalizers
20 | - workflowtasksets
21 | - workflowtasksets/finalizers
22 | - workflowartifactgctasks
23 | verbs:
24 | - get
25 | - list
26 | - watch
27 | - update
28 | - patch
29 | - delete
30 | - create
31 | ---
32 | apiVersion: rbac.authorization.k8s.io/v1
33 | kind: ClusterRoleBinding
34 | metadata:
35 | name: {{ include "piper.fullname" . }}
36 | subjects:
37 | - kind: ServiceAccount
38 | name: {{ include "piper.fullname" . }}
39 | namespace: {{ .Release.Namespace | quote }}
40 | roleRef:
41 | apiGroup: rbac.authorization.k8s.io
42 | kind: ClusterRole
43 | name: {{ include "piper.fullname" . }}
44 |
--------------------------------------------------------------------------------
/.github/pull_request_template.md:
--------------------------------------------------------------------------------
1 | ## Pull Request
2 |
3 | ### Description
4 |
5 | Please provide a brief description of the changes made in this pull request.
6 |
7 | ### Related Issue(s)
8 |
9 | If this pull request addresses or relates to any open issues, please mention them here using the syntax `Fixes #` or `Resolves #`.
10 |
11 | ### Checklist
12 |
13 | Before submitting this pull request, please ensure that you have completed the following tasks:
14 |
15 | - [ ] Reviewed the [Contributing Guidelines](../docs/CONTRIBUTING.md) for the Piper project.
16 | - [ ] Ensured that your changes follow the [coding guidelines and style](../docs/CONTRIBUTING.md#coding-guidelines) of the project.
17 | - [ ] Run the tests locally and made sure they pass.
18 | - [ ] Updated the relevant documentation, if applicable.
19 |
20 | ### Testing Instructions
21 |
22 | Please provide clear instructions on how to test and verify the changes made in this pull request.
23 |
24 | ### Additional Information
25 |
26 | Add any additional information or context that would be helpful in understanding and reviewing this pull request.
27 |
--------------------------------------------------------------------------------
/pkg/workflow_handler/types.go:
--------------------------------------------------------------------------------
1 | package workflow_handler
2 |
3 | import (
4 | "context"
5 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
6 | "github.com/rookout/piper/pkg/common"
7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
8 | "k8s.io/apimachinery/pkg/watch"
9 | )
10 |
11 | type WorkflowsClient interface {
12 | ConstructTemplates(workflowsBatch *common.WorkflowsBatch, configName string) ([]v1alpha1.Template, error)
13 | ConstructSpec(templates []v1alpha1.Template, params []v1alpha1.Parameter, configName string) (*v1alpha1.WorkflowSpec, error)
14 | CreateWorkflow(spec *v1alpha1.WorkflowSpec, workflowsBatch *common.WorkflowsBatch) (*v1alpha1.Workflow, error)
15 | SelectConfig(workflowsBatch *common.WorkflowsBatch) (string, error)
16 | Lint(wf *v1alpha1.Workflow) error
17 | Submit(ctx *context.Context, wf *v1alpha1.Workflow) error
18 | HandleWorkflowBatch(ctx *context.Context, workflowsBatch *common.WorkflowsBatch) error
19 | Watch(ctx *context.Context, labelSelector *metav1.LabelSelector) (watch.Interface, error)
20 | UpdatePiperWorkflowLabel(ctx *context.Context, workflowName string, label string, value string) error
21 | }
22 |
--------------------------------------------------------------------------------
/pkg/conf/workflows_config.go:
--------------------------------------------------------------------------------
1 | package conf
2 |
3 | import (
4 | "encoding/json"
5 | "log"
6 |
7 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
8 | "github.com/rookout/piper/pkg/utils"
9 | )
10 |
11 | type WorkflowsConfig struct {
12 | Configs map[string]*ConfigInstance
13 | }
14 |
15 | type ConfigInstance struct {
16 | Spec v1alpha1.WorkflowSpec `yaml:"spec"`
17 | OnExit []v1alpha1.DAGTask `yaml:"onExit"`
18 | }
19 |
20 | func (wfc *WorkflowsConfig) WorkflowsSpecLoad(configPath string) error {
21 | var jsonBytes []byte
22 | wfc.Configs = make(map[string]*ConfigInstance)
23 |
24 | configs, err := utils.GetFilesData(configPath)
25 | if len(configs) == 0 {
26 | log.Printf("No config files to load at %s", configPath)
27 | return nil
28 | }
29 | if err != nil {
30 | return err
31 | }
32 |
33 | for key, config := range configs {
34 | tmp := new(ConfigInstance)
35 | jsonBytes, err = utils.ConvertYAMLToJSON(config)
36 | if err != nil {
37 | return err
38 | }
39 | err = json.Unmarshal(jsonBytes, &tmp)
40 | if err != nil {
41 | return err
42 | }
43 | wfc.Configs[key] = tmp
44 | }
45 |
46 | return nil
47 | }
48 |
--------------------------------------------------------------------------------
/.github/workflows/docs.yaml:
--------------------------------------------------------------------------------
1 | name: docs
2 | on:
3 | push:
4 | branches:
5 | - main
6 | paths:
7 | - 'docs/**'
8 | - '.github/workflows/docs.yaml'
9 | - 'mkdocs.yml'
10 |
11 | permissions:
12 | contents: write
13 | jobs:
14 | publish:
15 | runs-on: ubuntu-latest
16 | steps:
17 | - name: Checkout main
18 | uses: actions/checkout@v2
19 | - name: Install mkdocs
20 | run: |
21 | pip install mkdocs-material
22 | pip install mkdocs-video
23 | - name: Generate docs artifacts
24 | run: mkdocs build -s -d /tmp/docs
25 | - uses: actions/checkout@v2
26 | with:
27 | ref: gh-pages
28 | path: gh-pages
29 | - name: Publish docs artifacts to gh-pages
30 | run: |
31 | cd gh-pages
32 | shopt -s extglob
33 | rm -rf !(index.yaml|LICENSE|*.tgz)
34 | cp -R /tmp/docs/** .
35 | git config --local user.email "action@github.com"
36 | git config --local user.name "GitHub Action"
37 | git add -A
38 | git commit -m "Publish docs from $GITHUB_SHA"
39 | git push https://github.com/$GITHUB_REPOSITORY.git gh-pages
--------------------------------------------------------------------------------
/pkg/server/shutdown.go:
--------------------------------------------------------------------------------
1 | package server
2 |
3 | import (
4 | "golang.org/x/net/context"
5 | "log"
6 | "time"
7 | )
8 |
9 | type GracefulShutdown struct {
10 | ctx context.Context
11 | stop context.CancelFunc
12 | }
13 |
14 | func NewGracefulShutdown(ctx context.Context, stop context.CancelFunc) *GracefulShutdown {
15 | return &GracefulShutdown{
16 | ctx: ctx,
17 | stop: stop,
18 | }
19 | }
20 |
21 | func (s *GracefulShutdown) StopServices(ctx *context.Context, server *Server) {
22 | server.webhookCreator.Stop(ctx)
23 | }
24 |
25 | func (s *GracefulShutdown) Shutdown(server *Server) {
26 | // Listen for the interrupt signal.
27 | <-s.ctx.Done()
28 |
29 | // Restore default behavior on the interrupt signal and notify user of shutdown.
30 | s.stop()
31 |
32 | log.Println("shutting down gracefully...")
33 | // The context is used to inform the server it has 10 seconds to finish
34 | // the request it is currently handling
35 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
36 | defer cancel()
37 |
38 | s.StopServices(&ctx, server)
39 |
40 | err := server.httpServer.Shutdown(ctx)
41 | if err != nil {
42 | log.Fatal("Server forced to shutdown: ", err)
43 | }
44 |
45 | }
46 |
--------------------------------------------------------------------------------
/pkg/event_handler/main.go:
--------------------------------------------------------------------------------
1 | package event_handler
2 |
3 | import (
4 | "context"
5 | "github.com/rookout/piper/pkg/clients"
6 | "github.com/rookout/piper/pkg/conf"
7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
8 | "log"
9 | )
10 |
11 | func Start(ctx context.Context, stop context.CancelFunc, cfg *conf.GlobalConfig, clients *clients.Clients) {
12 | labelSelector := &metav1.LabelSelector{
13 | MatchExpressions: []metav1.LabelSelectorRequirement{
14 | {Key: "piper.rookout.com/notified",
15 | Operator: metav1.LabelSelectorOpExists},
16 | },
17 | }
18 | watcher, err := clients.Workflows.Watch(&ctx, labelSelector)
19 | if err != nil {
20 | log.Printf("[event handler] Failed to watch workflow error:%s", err)
21 | return
22 | }
23 |
24 | notifier := NewGithubEventNotifier(cfg, clients)
25 | handler := &workflowEventHandler{
26 | Clients: clients,
27 | Notifier: notifier,
28 | }
29 | go func() {
30 | for event := range watcher.ResultChan() {
31 | err2 := handler.Handle(ctx, &event)
32 | if err2 != nil {
33 | log.Printf("[event handler] failed to Handle workflow event %s", err2) // ERROR
34 | }
35 | }
36 | log.Print("[event handler] stopped work, closing watcher")
37 | watcher.Stop()
38 | stop()
39 | }()
40 | }
41 |
--------------------------------------------------------------------------------
/docs/usage/global_variables.md:
--------------------------------------------------------------------------------
1 | ## Global variables
2 |
3 | Piper will automatically add Workflow scope parameters that can be referenced from any template.
4 | The parameters taken from webhook metadata, and will be populated respectively to GitProvider and event that triggered the workflow.
5 |
6 | 1. `{{ workflow.parameters.event }}` the event that triggered the workflow.
7 |
8 | 2. `{{ workflow.parameters.action }}` the action that triggered the workflow.
9 |
10 | 3. `{{ workflow.parameters.dest_branch }}` the destination branch for pull request.
11 |
12 | 4. `{{ workflow.parameters.commit }}` the commit that triggered the workflow.
13 |
14 | 5. `{{ workflow.parameters.repo }}` repository name that triggered the workflow.
15 |
16 | 6. `{{ workflow.parameters.user }}` the username that triggered the workflow.
17 |
18 | 7. `{{ workflow.parameters.user_email }}` the user's email that triggered the workflow.
19 |
20 | 8. `{{ workflow.parameters.pull_request_url }}` the url of the pull request that triggered the workflow.
21 |
22 | 9. `{{workflow.parameters.pull_request_title }}` the tile of the pull request that triggered the workflow.
23 |
24 | 10. `{{workflow.parameters.pull_request_labels }}` comma seperated labels of the pull request that triggered the workflow.
25 |
--------------------------------------------------------------------------------
/pkg/conf/git_provider.go:
--------------------------------------------------------------------------------
1 | package conf
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/kelseyhightower/envconfig"
7 | )
8 |
9 | type GitProviderConfig struct {
10 | Provider string `envconfig:"GIT_PROVIDER" required:"true"`
11 | Token string `envconfig:"GIT_TOKEN" required:"true"`
12 | OrgName string `envconfig:"GIT_ORG_NAME" required:"true"`
13 | OrgLevelWebhook bool `envconfig:"GIT_ORG_LEVEL_WEBHOOK" default:"false" required:"false"`
14 | RepoList string `envconfig:"GIT_WEBHOOK_REPO_LIST" required:"false"`
15 | WebhookURL string `envconfig:"GIT_WEBHOOK_URL" required:"false"`
16 | WebhookSecret string `envconfig:"GIT_WEBHOOK_SECRET" required:"false"`
17 | WebhookAutoCleanup bool `envconfig:"GIT_WEBHOOK_AUTO_CLEANUP" default:"false" required:"false"`
18 | EnforceOrgBelonging bool `envconfig:"GIT_ENFORCE_ORG_BELONGING" default:"false" required:"false"`
19 | OrgID int64
20 | FullHealthCheck bool `envconfig:"GIT_FULL_HEALTH_CHECK" default:"false" required:"false"`
21 | }
22 |
23 | func (cfg *GitProviderConfig) GitConfLoad() error {
24 | err := envconfig.Process("", cfg)
25 | if err != nil {
26 | return fmt.Errorf("failed to load the Git provider configuration, error: %v", err)
27 | }
28 |
29 | return nil
30 | }
31 |
--------------------------------------------------------------------------------
/.github/workflows/lint-pr.yaml:
--------------------------------------------------------------------------------
1 | name: Check PR title
2 |
3 | on:
4 | pull_request_target:
5 | types:
6 | - opened
7 | - reopened
8 | - edited
9 | - synchronize
10 |
11 | permissions:
12 | contents: read
13 |
14 | jobs:
15 | lint:
16 | permissions:
17 | pull-requests: read # for amannn/action-semantic-pull-request to analyze PRs
18 | statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR
19 | name: Validate PR title
20 | runs-on: ubuntu-latest
21 | steps:
22 | - uses: amannn/action-semantic-pull-request@v5
23 | with:
24 | # Configure which types are allowed (newline delimited).
25 | # Default: https://github.com/commitizen/conventional-commit-types
26 | types: |
27 | feat
28 | fix
29 | docs
30 | chore
31 | revert
32 | ci
33 | # Configure which scopes are allowed (newline delimited).
34 | scopes: |
35 | deps
36 | core
37 | RK-\d+
38 | # Configure that a scope must always be provided.
39 | requireScope: false
40 | ignoreLabels: |
41 | autorelease: pending
42 | env:
43 | GITHUB_TOKEN: ${{ github.token }}
44 |
--------------------------------------------------------------------------------
/mkdocs.yml:
--------------------------------------------------------------------------------
1 | site_name: Piper - Multibranch Pipeline for ArgoWorkflows
2 | site_description: 'Piper project for multibranch pipeline in Argo Workflows'
3 | site_author: 'George Dozoretz'
4 | docs_dir: docs/
5 | repo_url: https://github.com/rookout/piper
6 | repo_name: rookout/piper
7 | theme:
8 | name: material
9 | icon:
10 | repo: fontawesome/brands/github
11 | palette:
12 | - scheme: default
13 | toggle:
14 | icon: material/weather-night
15 | name: Switch to dark mode
16 | - scheme: slate
17 | toggle:
18 | icon: material/weather-sunny
19 | name: Switch to light mode
20 | features:
21 | - content.code.annotate
22 | plugins:
23 | - mkdocs-video:
24 | is_video: True
25 | video_loop: True
26 | video_muted: True
27 | video_autoplay: True
28 | markdown_extensions:
29 | - pymdownx.highlight:
30 | anchor_linenums: true
31 | - pymdownx.inlinehilite
32 | - pymdownx.snippets
33 | - pymdownx.superfences
34 |
35 | nav:
36 | - Introduction: index.md
37 | - Getting Started: getting_started/installation.md
38 | - Configuration:
39 | - configuration/environment_variables.md
40 | - configuration/health_check.md
41 | - Use piper:
42 | - usage/workflows_folder.md
43 | - usage/global_variables.md
44 | - usage/workflows_config.md
45 | - Developers: CONTRIBUTING.md
--------------------------------------------------------------------------------
/pkg/utils/os.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "os"
5 | "path/filepath"
6 | )
7 |
8 | func GetFilesData(directory string) (map[string][]byte, error) {
9 | fileData := make(map[string][]byte)
10 | var data []byte
11 |
12 | path, dirList, err := GetFilesInLinkDirectory(directory)
13 | if err != nil {
14 | return nil, err
15 | }
16 | for _, dir := range dirList {
17 | data, err = GetFileData(*path + "/" + dir)
18 | if err != nil {
19 | return nil, err
20 | }
21 | fileData[dir] = data
22 | }
23 |
24 | return fileData, nil
25 | }
26 |
27 | func GetFileData(filePath string) ([]byte, error) {
28 | data, err := os.ReadFile(filePath)
29 | if err != nil {
30 | return nil, err
31 | }
32 | return data, nil
33 | }
34 |
35 | func GetFilesInLinkDirectory(linkPath string) (*string, []string, error) {
36 | realPath, err := filepath.EvalSymlinks(linkPath)
37 | if err != nil {
38 | return nil, nil, err
39 | }
40 |
41 | var fileNames []string
42 | err = filepath.WalkDir(realPath, func(path string, d os.DirEntry, err error) error {
43 | if err != nil {
44 | return err
45 | }
46 | if !d.IsDir() {
47 | info, err := d.Info()
48 | if err != nil {
49 | return err
50 | }
51 | if info.Mode()&os.ModeSymlink != 0 {
52 | return nil // Skip symbolic links
53 | }
54 | relPath, err := filepath.Rel(realPath, path)
55 | if err != nil {
56 | return err
57 | }
58 | fileNames = append(fileNames, relPath)
59 | }
60 | return nil
61 | })
62 |
63 | if err != nil {
64 | return nil, nil, err
65 | }
66 |
67 | return &realPath, fileNames, nil
68 | }
69 |
--------------------------------------------------------------------------------
/pkg/server/server.go:
--------------------------------------------------------------------------------
1 | package server
2 |
3 | import (
4 | "github.com/gin-gonic/gin"
5 | "github.com/rookout/piper/pkg/clients"
6 | "github.com/rookout/piper/pkg/conf"
7 | "github.com/rookout/piper/pkg/server/routes"
8 | "github.com/rookout/piper/pkg/webhook_creator"
9 | "log"
10 | "net/http"
11 | )
12 |
13 | func NewServer(config *conf.GlobalConfig, clients *clients.Clients) *Server {
14 | srv := &Server{
15 | router: gin.New(),
16 | config: config,
17 | clients: clients,
18 | webhookCreator: webhook_creator.NewWebhookCreator(config, clients),
19 | }
20 |
21 | return srv
22 | }
23 |
24 | func (s *Server) startServer() *http.Server {
25 | srv := &http.Server{
26 | Addr: ":8080",
27 | Handler: s.router,
28 | }
29 |
30 | go func() {
31 | if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
32 | log.Fatalf("listen: %s\n", err)
33 | }
34 | }()
35 |
36 | return srv
37 | }
38 |
39 | func (s *Server) registerMiddlewares() {
40 | s.router.Use(
41 | gin.LoggerWithConfig(gin.LoggerConfig{
42 | SkipPaths: []string{"/healthz", "/readyz"},
43 | }),
44 | gin.Recovery(),
45 | )
46 |
47 | }
48 |
49 | func (s *Server) getRoutes() {
50 | v1 := s.router.Group("/")
51 | routes.AddReadyRoutes(v1)
52 | routes.AddHealthRoutes(v1, s.webhookCreator, s.config)
53 | routes.AddWebhookRoutes(s.config, s.clients, v1, s.webhookCreator)
54 | }
55 |
56 | func (s *Server) startServices() {
57 | s.webhookCreator.Start()
58 | }
59 |
60 | func (s *Server) Start() {
61 |
62 | s.registerMiddlewares()
63 |
64 | s.getRoutes()
65 |
66 | s.httpServer = s.startServer()
67 |
68 | s.startServices()
69 |
70 | }
71 |
--------------------------------------------------------------------------------
/makefile:
--------------------------------------------------------------------------------
1 | SHELL := /bin/sh
2 | CLUSTER_DEPLOYED := $(shell kind get clusters -q | grep piper)
3 |
4 | .PHONY: ngrok
5 | ngrok:
6 | ngrok http 80
7 |
8 | .PHONY: local-build
9 | local-build:
10 | DOCKER_BUILDKIT=1 docker build -t localhost:5001/piper:latest .
11 |
12 | .PHONY: local-push
13 | local-push:
14 | docker push localhost:5001/piper:latest
15 |
16 | .PHONY: init-kind
17 | init-kind:
18 | ifndef CLUSTER_DEPLOYED
19 | sh ./scripts/init-kind.sh
20 | else
21 | $(info Kind piper cluster exists, skipping cluster installation)
22 | endif
23 | kubectl config set-context kind-piper
24 |
25 | .PHONY: init-nginx
26 | init-nginx: init-kind
27 | sh ./scripts/init-nginx.sh
28 |
29 | .PHONY: init-argo-workflows
30 | init-argo-workflows: init-kind
31 | sh ./scripts/init-argo-workflows.sh
32 |
33 | .PHONY: init-piper
34 | init-piper: init-kind local-build
35 | sh ./scripts/init-piper.sh
36 |
37 | .PHONY: deploy
38 | deploy: init-kind init-nginx init-argo-workflows local-build local-push init-piper
39 |
40 | .PHONY: restart
41 | restart: local-build
42 | docker push localhost:5001/piper:latest
43 | kubectl rollout restart deployment piper
44 |
45 | .PHONY: clean
46 | clean:
47 | kind delete cluster --name piper
48 | docker stop kind-registry && docker rm kind-registry
49 |
50 | .PHONY: helm
51 | helm:
52 | helm lint ./helm-chart
53 | helm template ./helm-chart --debug > _lint.yaml
54 | helm-docs
55 |
56 | .PHONY: test
57 | test:
58 | go test -short ./pkg/...
59 |
60 | $(GOPATH)/bin/golangci-lint:
61 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b `go env GOPATH`/bin v1.52.2
62 |
63 | .PHONY: lint
64 | lint: $(GOPATH)/bin/golangci-lint
65 | $(GOPATH)/bin/golangci-lint run --fix --verbose
--------------------------------------------------------------------------------
/pkg/git_provider/types.go:
--------------------------------------------------------------------------------
1 | package git_provider
2 |
3 | import (
4 | "context"
5 | "net/http"
6 | )
7 |
8 | type HookWithStatus struct {
9 | HookID int64
10 | Uuid string
11 | HealthStatus bool
12 | RepoName *string
13 | }
14 |
15 | type CommitFile struct {
16 | Path *string `json:"path"`
17 | Content *string `json:"content"`
18 | }
19 |
20 | type WebhookPayload struct {
21 | Event string `json:"event"`
22 | Action string `json:"action"`
23 | Repo string `json:"repoName"`
24 | Branch string `json:"branch"`
25 | Commit string `json:"commit"`
26 | User string `json:"user"`
27 | UserEmail string `json:"user_email"`
28 | PullRequestURL string `json:"pull_request_url"`
29 | PullRequestTitle string `json:"pull_request_title"`
30 | DestBranch string `json:"dest_branch"`
31 | Labels []string `json:"labels"`
32 | HookID int64 `json:"hookID"`
33 | OwnerID int64 `json:"ownerID"`
34 | }
35 |
36 | type Client interface {
37 | ListFiles(ctx *context.Context, repo string, branch string, path string) ([]string, error)
38 | GetFile(ctx *context.Context, repo string, branch string, path string) (*CommitFile, error)
39 | GetFiles(ctx *context.Context, repo string, branch string, paths []string) ([]*CommitFile, error)
40 | SetWebhook(ctx *context.Context, repo *string) (*HookWithStatus, error)
41 | UnsetWebhook(ctx *context.Context, hook *HookWithStatus) error
42 | HandlePayload(ctx *context.Context, request *http.Request, secret []byte) (*WebhookPayload, error)
43 | SetStatus(ctx *context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error
44 | PingHook(ctx *context.Context, hook *HookWithStatus) error
45 | }
46 |
--------------------------------------------------------------------------------
/.github/workflows/release.yaml:
--------------------------------------------------------------------------------
1 | name: Release Workflow
2 |
3 | on:
4 | release:
5 | types:
6 | - published
7 | - edited
8 |
9 | jobs:
10 | piper-image:
11 | name: piper-image
12 | runs-on: ubuntu-latest
13 | steps:
14 | - uses: actions/checkout@v3
15 | - uses: docker/setup-qemu-action@v2
16 | - uses: docker/setup-buildx-action@v2
17 | - name: Login to Docker Hub
18 | uses: docker/login-action@v2
19 | with:
20 | username: ${{ secrets.DOCKERHUB_USERNAME }}
21 | password: ${{ secrets.DOCKERHUB_TOKEN }}
22 | - name: Docker meta
23 | id: meta
24 | uses: docker/metadata-action@v4
25 | with:
26 | images: rookout/piper
27 | - name: Build and export
28 | uses: docker/build-push-action@v4
29 | with:
30 | context: .
31 | platforms: linux/amd64,linux/arm64
32 | push: true
33 | tags: rookout/piper:${{ github.ref_name }},latest
34 | labels: ${{ steps.meta.outputs.labels }}
35 | cache-from: type=gha
36 | cache-to: type=gha,mode=max
37 | helm:
38 | name: helm
39 | runs-on: ubuntu-latest
40 | steps:
41 | - uses: actions/checkout@v3
42 | - name: Install Helm
43 | uses: azure/setup-helm@v3
44 | - name: Install Helm Docs
45 | uses: envoy/install-helm-docs@v1.0.0
46 | with:
47 | version: 1.11.0
48 | - name: Helm lint and template
49 | run: |
50 | make helm
51 | - name: Publish Helm chart
52 | uses: stefanprodan/helm-gh-pages@master
53 | with:
54 | chart_version: ${{ github.ref_name }}
55 | app_version: ${{ github.ref_name }}
56 | token: ${{ secrets.GIT_TOKEN }}
57 | charts_dir: .
--------------------------------------------------------------------------------
/pkg/event_handler/workflow_event_handler.go:
--------------------------------------------------------------------------------
1 | package event_handler
2 |
3 | import (
4 | "fmt"
5 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
6 | "github.com/rookout/piper/pkg/clients"
7 | "golang.org/x/net/context"
8 | "k8s.io/apimachinery/pkg/watch"
9 | "log"
10 | )
11 |
12 | type workflowEventHandler struct {
13 | Clients *clients.Clients
14 | Notifier EventNotifier
15 | }
16 |
17 | func (weh *workflowEventHandler) Handle(ctx context.Context, event *watch.Event) error {
18 | workflow, ok := event.Object.(*v1alpha1.Workflow)
19 | if !ok {
20 | return fmt.Errorf(
21 | "event object is not a Workflow object, got: %v\n",
22 | event.DeepCopy().Object,
23 | )
24 | }
25 |
26 | currentPiperNotifyLabelStatus, ok := workflow.GetLabels()["piper.rookout.com/notified"]
27 | if !ok {
28 | return fmt.Errorf(
29 | "workflow %s missing piper.rookout.com/notified label\n",
30 | workflow.GetName(),
31 | )
32 | }
33 |
34 | if currentPiperNotifyLabelStatus == string(workflow.Status.Phase) {
35 | log.Printf(
36 | "workflow %s already informed for %s status. skiping... \n",
37 | workflow.GetName(),
38 | workflow.Status.Phase,
39 | ) //INFO
40 | return nil
41 | }
42 |
43 | err := weh.Notifier.Notify(&ctx, workflow)
44 | if err != nil {
45 | return fmt.Errorf("failed to Notify workflow to git provider, error:%s\n", err)
46 | }
47 |
48 | err = weh.Clients.Workflows.UpdatePiperWorkflowLabel(&ctx, workflow.GetName(), "notified", string(workflow.Status.Phase))
49 | if err != nil {
50 | return fmt.Errorf("error in workflow %s status patch: %s", workflow.GetName(), err)
51 | }
52 | log.Printf(
53 | "[event handler] done with event of type: %s for worklfow: %s phase: %s message: %s\n",
54 | event.Type,
55 | workflow.GetName(),
56 | workflow.Status.Phase,
57 | workflow.Status.Message) //INFO
58 |
59 | return nil
60 | }
61 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | .history/
3 | .vscode
4 | # Byte-compiled / optimized / DLL files
5 | __pycache__/
6 | *.py[cod]
7 | *$py.class
8 |
9 | # C extensions
10 | *.so
11 |
12 | # Distribution / packaging
13 | .Python
14 | build/
15 | develop-eggs/
16 | dist/
17 | downloads/
18 | eggs/
19 | .eggs/
20 | lib/
21 | lib64/
22 | parts/
23 | sdist/
24 | var/
25 | wheels/
26 | *.egg-info/
27 | .installed.cfg
28 | *.egg
29 | MANIFEST
30 |
31 | # PyInstaller
32 | # Usually these files are written by a python script from a template
33 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
34 | *.manifest
35 | *.spec
36 |
37 | # Installer logs
38 | pip-log.txt
39 | pip-delete-this-directory.txt
40 |
41 | # Unit test / coverage reports
42 | htmlcov/
43 | .tox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | .hypothesis/
51 | .pytest_cache/
52 |
53 | # Translations
54 | *.mo
55 | *.pot
56 |
57 | # Django stuff:
58 | *.log
59 | local_settings.py
60 | db.sqlite3
61 |
62 | # Flask stuff:
63 | instance/
64 | .webassets-cache
65 |
66 | # Scrapy stuff:
67 | .scrapy
68 |
69 | # Sphinx documentation
70 | docs/_build/
71 |
72 | # PyBuilder
73 | target/
74 |
75 | # Jupyter Notebook
76 | .ipynb_checkpoints
77 |
78 | # pyenv
79 | .python-version
80 |
81 | # celery beat schedule file
82 | celerybeat-schedule
83 |
84 | # SageMath parsed files
85 | *.sage.py
86 |
87 | # Environments
88 | dev.env
89 | *.env
90 | .venv
91 | env/
92 | venv/
93 | ENV/
94 | env.bak/
95 | venv.bak/
96 |
97 | # Spyder project settings
98 | .spyderproject
99 | .spyproject
100 |
101 | # Rope project settings
102 | .ropeproject
103 |
104 | # mkdocs documentation
105 | /site
106 |
107 | # mypy
108 | .mypy_cache/
109 | *.iml
110 |
111 | _lint.yaml
112 | values.dev.yaml
--------------------------------------------------------------------------------
/cmd/piper/piper.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | rookout "github.com/Rookout/GoSDK"
5 | "github.com/rookout/piper/pkg/clients"
6 | "github.com/rookout/piper/pkg/conf"
7 | "github.com/rookout/piper/pkg/event_handler"
8 | "github.com/rookout/piper/pkg/git_provider"
9 | "github.com/rookout/piper/pkg/server"
10 | "github.com/rookout/piper/pkg/utils"
11 | workflowHandler "github.com/rookout/piper/pkg/workflow_handler"
12 | "golang.org/x/net/context"
13 | "log"
14 | "os/signal"
15 | "syscall"
16 | )
17 |
18 | func main() {
19 | cfg, err := conf.LoadConfig()
20 | if err != nil {
21 | log.Panicf("failed to load the configuration for Piper, error: %v", err)
22 | }
23 |
24 | if cfg.RookoutConfig.Token != "" {
25 | labels := utils.StringToMap(cfg.RookoutConfig.Labels)
26 | err = rookout.Start(rookout.RookOptions{Token: cfg.RookoutConfig.Token, Labels: labels})
27 | if err != nil {
28 | log.Printf("failed to start Rookout, error: %v\n", err)
29 | }
30 | }
31 |
32 | err = cfg.WorkflowsConfig.WorkflowsSpecLoad("/piper-config/..data")
33 | if err != nil {
34 | log.Panicf("Failed to load workflow spec configuration, error: %v", err)
35 | }
36 |
37 | gitProvider, err := git_provider.NewGitProviderClient(cfg)
38 | if err != nil {
39 | log.Panicf("failed to load the Git client for Piper, error: %v", err)
40 | }
41 | workflows, err := workflowHandler.NewWorkflowsClient(cfg)
42 | if err != nil {
43 | log.Panicf("failed to load the Argo Workflows client for Piper, error: %v", err)
44 | }
45 |
46 | globalClients := &clients.Clients{
47 | GitProvider: gitProvider,
48 | Workflows: workflows,
49 | }
50 |
51 | // Create context that listens for the interrupt signal from the OS.
52 | ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
53 | defer stop()
54 | event_handler.Start(ctx, stop, cfg, globalClients)
55 | server.Start(ctx, stop, cfg, globalClients)
56 | }
57 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yaml:
--------------------------------------------------------------------------------
1 | name: CI
2 |
3 | on:
4 | push:
5 | branches:
6 | - "main"
7 | paths:
8 | - '**'
9 | - '!docs/**'
10 | pull_request:
11 | branches:
12 | - "main"
13 |
14 | concurrency:
15 | group: ${{ github.workflow }}-${{ github.ref }}
16 | cancel-in-progress: true
17 |
18 | permissions:
19 | contents: read
20 |
21 | jobs:
22 | tests:
23 | name: Unit Tests
24 | runs-on: ubuntu-latest
25 | timeout-minutes: 10
26 | steps:
27 | - uses: actions/checkout@v3
28 | - uses: actions/setup-go@v4
29 | with:
30 | go-version: "1.20"
31 | cache: true
32 | - run: make test
33 | lint:
34 | name: Go Lint
35 | runs-on: ubuntu-latest
36 | timeout-minutes: 10
37 | steps:
38 | - uses: actions/checkout@v3
39 | - uses: actions/setup-go@v4
40 | with:
41 | go-version: '1.20'
42 | cache: true
43 | - name: golangci-lint
44 | uses: golangci/golangci-lint-action@v3
45 | with:
46 | version: v1.53
47 | only-new-issues: true
48 | skip-pkg-cache: true
49 | args: --timeout=10m
50 | helm:
51 | name: Helm Lint
52 | runs-on: ubuntu-latest
53 | timeout-minutes: 10
54 | steps:
55 | - uses: actions/checkout@v3
56 | with:
57 | fetch-depth: 0
58 | - name: Check Git diff in /helm-chart
59 | run: |
60 | if [ "$(git diff --exit-code --name-only --diff-filter=d origin/main -- helm-chart/)" != "" ]; then
61 | echo "There are Git diffs in the /helm-chart folder."
62 | echo "CHART_UPDATED=true" >> $GITHUB_ENV
63 | else
64 | echo "There are no Git diffs in the /helm-chart folder."
65 | fi
66 | - name: Install Helm Docs
67 | uses: envoy/install-helm-docs@v1.0.0
68 | with:
69 | version: 1.11.0
70 | - name: Helm lint and template
71 | run: |
72 | make helm
73 | if: ${{ env.CHART_UPDATED }}
--------------------------------------------------------------------------------
/examples/workflow.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: argoproj.io/v1alpha1
2 | kind: Workflow
3 | metadata:
4 | generateName: test-
5 | labels:
6 | branch: test-branch
7 | commit: xxxxxxxxxxxxxx
8 | repo: somerepo
9 | user: gosharo
10 | spec:
11 | volumes:
12 | - name: shared-volume
13 | emptyDir: { }
14 | activeDeadlineSeconds: 7200 # (seconds) == 2 hours
15 | ttlStrategy:
16 | secondsAfterCompletion: 28800 # (seconds) == 8 hours
17 | podGC:
18 | strategy: OnPodSuccess
19 | archiveLogs: true
20 | arguments:
21 | parameters:
22 | - name: PLACHOLDER
23 | artifactRepositoryRef:
24 | configMap: artifact-repositories
25 | onExit: exit-handler
26 | entrypoint: entrypoint
27 | nodeSelector:
28 | node_pool: workflows
29 | serviceAccountName: argo-wf
30 | tolerations:
31 | - effect: NoSchedule
32 | key: node_pool
33 | operator: Equal
34 | value: workflows
35 | templates:
36 | - dag:
37 | name: exit-handler
38 | tasks:
39 | - name: github-status
40 | template: exit-handler
41 | arguments:
42 | parameters:
43 | - name: param1
44 | value: '{{ workflow.labels.repo }}'
45 | - name: local-step
46 | inputs:
47 | parameters:
48 | - name: message
49 | script:
50 | image: alpine
51 | command: [sh]
52 | source: |
53 | echo "wellcome to {{ workflow.parameters.global }}
54 | echo "{{ inputs.parameters.message }}"
55 | - name: exit-handler
56 | script:
57 | image: alpine
58 | command: [sh]
59 | source: |
60 | echo "exit"
61 | - dag:
62 | name: entrypoint
63 | tasks:
64 | - name: local-step1
65 | template: local-step
66 | arguments:
67 | parameters:
68 | - name: message
69 | value: step-1
70 | - name: local-step2
71 | template: local-step
72 | arguments:
73 | parameters:
74 | - name: message
75 | value: step-2
76 | dependencies:
77 | - local-step1
78 |
79 |
80 |
--------------------------------------------------------------------------------
/pkg/server/routes/webhook.go:
--------------------------------------------------------------------------------
1 | package routes
2 |
3 | import (
4 | "github.com/rookout/piper/pkg/webhook_creator"
5 | "log"
6 | "net/http"
7 |
8 | "github.com/gin-gonic/gin"
9 | "github.com/rookout/piper/pkg/clients"
10 | "github.com/rookout/piper/pkg/conf"
11 | webhookHandler "github.com/rookout/piper/pkg/webhook_handler"
12 | )
13 |
14 | func AddWebhookRoutes(cfg *conf.GlobalConfig, clients *clients.Clients, rg *gin.RouterGroup, wc *webhook_creator.WebhookCreatorImpl) {
15 | webhook := rg.Group("/webhook")
16 |
17 | webhook.POST("", func(c *gin.Context) {
18 | ctx := c.Request.Context()
19 | webhookPayload, err := clients.GitProvider.HandlePayload(&ctx, c.Request, []byte(cfg.GitProviderConfig.WebhookSecret))
20 | if err != nil {
21 | log.Println(err)
22 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
23 | return
24 | }
25 |
26 | if webhookPayload.Event == "ping" {
27 | if cfg.GitProviderConfig.FullHealthCheck {
28 | err = wc.SetWebhookHealth(webhookPayload.HookID, true)
29 | if err != nil {
30 | c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
31 | return
32 | }
33 | }
34 | c.JSON(http.StatusOK, gin.H{"status": "ok"})
35 | return
36 | }
37 |
38 | wh, err := webhookHandler.NewWebhookHandler(cfg, clients, webhookPayload)
39 | if err != nil {
40 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
41 | log.Printf("failed to create webhook handler, error: %v", err)
42 | return
43 | }
44 |
45 | workflowsBatches, err := webhookHandler.HandleWebhook(&ctx, wh)
46 | if err != nil {
47 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
48 | log.Printf("failed to handle webhook, error: %v", err)
49 | return
50 | }
51 |
52 | for _, wf := range workflowsBatches {
53 | err = clients.Workflows.HandleWorkflowBatch(&ctx, wf)
54 | if err != nil {
55 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
56 | log.Printf("failed to handle workflow, error: %v", err)
57 | return
58 | }
59 | }
60 |
61 | c.JSON(http.StatusOK, gin.H{"status": "ok"})
62 | })
63 | }
64 |
--------------------------------------------------------------------------------
/docs/configuration/environment_variables.md:
--------------------------------------------------------------------------------
1 | ## Environment Variables
2 |
3 | The environment variables used by Piper to configure its functionality.
4 | The helm chart populate them using [values.yaml](https://github.com/Rookout/piper/tree/main/helm-chart/values.yaml) file
5 |
6 | ### Git
7 |
8 | * GIT_PROVIDER
9 | The git provider that Piper will use, possible variables: GitHub (will support bitbucket and gitlab)
10 |
11 | * GIT_TOKEN
12 | The git token that will be used.
13 |
14 | * GIT_ORG_NAME
15 | The organization name.
16 |
17 | * GIT_ORG_LEVEL_WEBHOOK
18 | Boolean variable, whether to config webhook in organization level. default `false`
19 |
20 | * GIT_WEBHOOK_REPO_LIST
21 | List of repositories to configure webhooks to.
22 |
23 | * GIT_WEBHOOK_URL
24 | URL of piper ingress, to configure webhooks.
25 |
26 | * GIT_WEBHOOK_AUTO_CLEANUP
27 | Will cleanup all webhook that were created with piper.
28 | Notice that there will be a race conditions between pod that being terminated and the new one.
29 |
30 | * GIT_ENFORCE_ORG_BELONGING
31 | Boolean variable, whether to enforce organizational belonging of git event creator. default `false`
32 |
33 | * GIT_FULL_HEALTH_CHECK
34 | Enables full health check of webhook. Full health check contains expecting and validating ping event from a webhook.
35 | Doesn't work for bitbucket, because the API call don't
36 |
37 |
38 | ### Argo Workflows Server
39 | * ARGO_WORKFLOWS_TOKEN
40 | The token of Argo Workflows server.
41 |
42 | * ARGO_WORKFLOWS_ADDRESS
43 | The address of Argo Workflows Server.
44 |
45 | * ARGO_WORKFLOWS_CREATE_CRD
46 | Whether to directly send Workflows instruction or create a CRD in the Cluster.
47 |
48 | * ARGO_WORKFLOWS_NAMESPACE
49 | The namespace of Workflows creation for Argo Workflows.
50 |
51 | * KUBE_CONFIG
52 | Used to configure Argo Workflows client with local kube configurations.
53 |
54 | ### Rookout
55 | * ROOKOUT_TOKEN
56 | The token used to configure Rookout agent. If not provided, will not start the agent.
57 | * ROOKOUT_LABELS
58 | The labels to label instances at Rookout, default to "service:piper"
59 | * ROOKOUT_REMOTE_ORIGIN
60 | The repo URL for source code fetching, default:"https://github.com/Rookout/piper.git".
--------------------------------------------------------------------------------
/.github/workflows/snyk.yaml:
--------------------------------------------------------------------------------
1 | name: Snyk Scan
2 |
3 | on:
4 | pull_request:
5 | branches: ["main"]
6 |
7 | permissions:
8 | contents: read
9 | jobs:
10 | snyk-container:
11 | permissions:
12 | contents: read # for actions/checkout to fetch code
13 | security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
14 | runs-on: ubuntu-latest
15 | steps:
16 | - uses: actions/checkout@v3
17 | - uses: actions/setup-go@v4
18 | with:
19 | go-version: "1.20"
20 | cache: true
21 | - uses: docker/setup-qemu-action@v2
22 | - uses: docker/setup-buildx-action@v2
23 | - name: Build Docker Image without push
24 | uses: docker/build-push-action@v4
25 | with:
26 | context: .
27 | push: false
28 | tags: rookout/piper:latest
29 | cache-from: type=gha
30 | cache-to: type=gha,mode=max
31 | load: true
32 | - name: Run Snyk to check Docker image for vulnerabilities
33 | uses: snyk/actions/docker@master
34 | env:
35 | SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
36 | with:
37 | image: rookout/piper:latest
38 | args: --file=Dockerfile --severity-threshold=high --sarif-file-output=snyk.sarif
39 | - name: Upload result to GitHub Code Scanning
40 | uses: github/codeql-action/upload-sarif@v2
41 | with:
42 | sarif_file: snyk.sarif
43 | snyk-golang:
44 | permissions:
45 | contents: read # for actions/checkout to fetch code
46 | security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
47 | runs-on: ubuntu-latest
48 | steps:
49 | - uses: actions/checkout@v3
50 | - uses: actions/setup-go@v4
51 | with:
52 | go-version: "1.20"
53 | cache: true
54 | - name: Run Snyk to check for vulnerabilities
55 | uses: snyk/actions/golang@master
56 | env:
57 | SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
58 | with:
59 | args: --sarif-file-output=snyk.sarif --severity-threshold=high
60 | - name: Upload result to GitHub Code Scanning
61 | uses: github/codeql-action/upload-sarif@v2
62 | with:
63 | sarif_file: snyk.sarif
--------------------------------------------------------------------------------
/helm-chart/templates/ingress.yaml:
--------------------------------------------------------------------------------
1 | {{- if .Values.ingress.enabled -}}
2 | {{- $fullName := include "piper.fullname" . -}}
3 | {{- $svcPort := .Values.service.port -}}
4 | {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
5 | {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
6 | {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
7 | {{- end }}
8 | {{- end }}
9 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
10 | apiVersion: networking.k8s.io/v1
11 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
12 | apiVersion: networking.k8s.io/v1beta1
13 | {{- else -}}
14 | apiVersion: extensions/v1beta1
15 | {{- end }}
16 | kind: Ingress
17 | metadata:
18 | name: {{ $fullName }}
19 | labels:
20 | {{- include "piper.labels" . | nindent 4 }}
21 | {{- with .Values.ingress.annotations }}
22 | annotations:
23 | {{- toYaml . | nindent 4 }}
24 | {{- end }}
25 | spec:
26 | {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
27 | ingressClassName: {{ .Values.ingress.className }}
28 | {{- end }}
29 | {{- if .Values.ingress.tls }}
30 | tls:
31 | {{- range .Values.ingress.tls }}
32 | - hosts:
33 | {{- range .hosts }}
34 | - {{ . | quote }}
35 | {{- end }}
36 | secretName: {{ .secretName }}
37 | {{- end }}
38 | {{- end }}
39 | rules:
40 | {{- range .Values.ingress.hosts }}
41 | - host: {{ .host | quote }}
42 | http:
43 | paths:
44 | {{- range .paths }}
45 | - path: {{ .path }}
46 | {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
47 | pathType: {{ .pathType }}
48 | {{- end }}
49 | backend:
50 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
51 | service:
52 | name: {{ $fullName }}
53 | port:
54 | number: {{ $svcPort }}
55 | {{- else }}
56 | serviceName: {{ $fullName }}
57 | servicePort: {{ $svcPort }}
58 | {{- end }}
59 | {{- end }}
60 | {{- end }}
61 | {{- end }}
62 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Piper - MOVED
2 |
3 | ⚠️ **Important Notice:** Piper has moved to a new and improved repository. Please visit the following link for the latest updates and contributions:
4 |
5 | [New repository](https://github.com/quickube/piper)
6 |
7 | ## Support and Contributions
8 |
9 | As we focus our efforts on the new repository, we won't be actively addressing issues or accepting pull requests here. Feel free to reach out to us on the new repository for any questions, issues, or contributions.
10 |
11 | Thank you for your support and understanding.
12 |
13 | Happy coding!
14 |
15 | [New Piper Repository](https://github.com/quickube/piper)
16 |
17 | ## Table of Contents
18 |
19 | - [Getting Started](#getting-started)
20 | - [Reporting Issues](#reporting-issues)
21 | - [How to Contribute](docs/CONTRIBUTING.md#how-to-contribute)
22 | - [License](#license)
23 |
24 | ## Getting Started
25 |
26 | Piper configures a webhook in git provider and listens to the webhooks sends. It will create a Workflow CRD out of branches that contains `.workflows` folder.
27 | This folder should contain declarations of the templates and main DAG that will be running.
28 | Finally, it will submit the Workflow as a K8s resource in the cluster.
29 | To access more detailed explanations, please navigate to the [Documentation site](https://piper.rookout.com).
30 |
31 | https://github.com/Rookout/piper/assets/106976988/09b3a5d8-3428-4bdc-9146-3034d81164bf
32 |
33 | ## Reporting Issues
34 |
35 | If you encounter any issues or bugs while using Piper, please help us improve by reporting them. Follow these steps to report an issue:
36 |
37 | 1. Go to the [Piper Issues](https://github.com/Rookout/Piper/issues) page on GitHub.
38 | 2. Click on the "New Issue" button.
39 | 3. Provide a descriptive title and detailed description of the issue, including any relevant error messages or steps to reproduce the problem.
40 | 4. Add appropriate labels to categorize the issue (e.g., bug, enhancement, question).
41 | 5. Submit the issue, and our team will review and address it as soon as possible.
42 |
43 | ## How to Contribute
44 |
45 | If you're interested in contributing to this project, please feel free to submit a pull request. We welcome all contributions and feedback.
46 | Please check out our [Contribution guidelines for this project](docs/CONTRIBUTING.md)
47 |
48 | ## License
49 |
50 | This project is licensed under the Apache License. Please see the [LICENSE](LICENSE) file for details.
51 |
--------------------------------------------------------------------------------
/pkg/git_provider/bitbucket_utils.go:
--------------------------------------------------------------------------------
1 | package git_provider
2 |
3 | import (
4 | "fmt"
5 | bitbucket "github.com/ktrysmt/go-bitbucket"
6 | "github.com/rookout/piper/pkg/conf"
7 | "github.com/rookout/piper/pkg/utils"
8 | "log"
9 | "net/http"
10 | "strings"
11 | )
12 |
13 | func ValidateBitbucketPermissions(client *bitbucket.Client, cfg *conf.GlobalConfig) error {
14 |
15 | repoAdminScopes := []string{"webhook", "repository:admin", "pullrequest:write"}
16 | repoGranularScopes := []string{"webhook", "repository", "pullrequest"}
17 |
18 | scopes, err := GetBitbucketTokenScopes(client, cfg)
19 |
20 | if err != nil {
21 | return fmt.Errorf("failed to get scopes: %v", err)
22 | }
23 | if len(scopes) == 0 {
24 | return fmt.Errorf("permissions error: no scopes found for the github client")
25 | }
26 |
27 | if utils.ListContains(repoAdminScopes, scopes) {
28 | return nil
29 | }
30 | if utils.ListContains(repoGranularScopes, scopes) {
31 | return nil
32 | }
33 |
34 | return fmt.Errorf("permissions error: %v is not a valid scopes", scopes)
35 | }
36 |
37 | func GetBitbucketTokenScopes(client *bitbucket.Client, cfg *conf.GlobalConfig) ([]string, error) {
38 |
39 | req, err := http.NewRequest("GET", fmt.Sprintf("%s/repositories/%s", client.GetApiBaseURL(), cfg.GitProviderConfig.OrgName), nil)
40 | if err != nil {
41 | log.Println("Error creating request:", err)
42 | return nil, err
43 | }
44 | req.Header.Set("Accept", "application/json")
45 | req.Header.Set("Authorization", "Bearer "+cfg.GitProviderConfig.Token)
46 |
47 | resp, err := client.HttpClient.Do(req)
48 | if err != nil {
49 | log.Println("Error making request:", err)
50 | return nil, err
51 | }
52 | defer resp.Body.Close()
53 |
54 | if resp.StatusCode != 200 {
55 | return nil, fmt.Errorf("token validation failed: %v", resp.Status)
56 | }
57 |
58 | // Check the "X-OAuth-Scopes" header to get the token scopes
59 | acceptedScopes := resp.Header.Get("X-Accepted-OAuth-Scopes")
60 | scopes := resp.Header.Get("X-OAuth-Scopes")
61 | log.Println("Bitbucket Token Scopes are:", scopes, acceptedScopes)
62 |
63 | scopes = strings.ReplaceAll(scopes, " ", "")
64 | return append(strings.Split(scopes, ","), acceptedScopes), nil
65 |
66 | }
67 |
68 | func addHookToHashTable(hookUuid string, hookHashTable map[string]int64) {
69 | hookHashTable[hookUuid] = utils.StringToInt64(hookUuid)
70 | }
71 |
72 | func getHookByUUID(hookUuid string, hookHashTable map[string]int64) (int64, error) {
73 | res, ok := hookHashTable[hookUuid]
74 | if !ok {
75 | return 0, fmt.Errorf("hookUuid %s not found", hookUuid)
76 | }
77 | return res, nil
78 | }
79 |
--------------------------------------------------------------------------------
/pkg/event_handler/github_event_notifier.go:
--------------------------------------------------------------------------------
1 | package event_handler
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
7 | "github.com/rookout/piper/pkg/clients"
8 | "github.com/rookout/piper/pkg/conf"
9 | "github.com/rookout/piper/pkg/utils"
10 | )
11 |
12 | var workflowTranslationToGithubMap = map[string]string{
13 | "": "pending",
14 | "Pending": "pending",
15 | "Running": "pending",
16 | "Succeeded": "success",
17 | "Failed": "failure",
18 | "Error": "error",
19 | }
20 |
21 | var workflowTranslationToBitbucketMap = map[string]string{
22 | "": "INPROGRESS",
23 | "Pending": "INPROGRESS",
24 | "Running": "INPROGRESS",
25 | "Succeeded": "SUCCESSFUL",
26 | "Failed": "FAILED",
27 | "Error": "STOPPED",
28 | }
29 |
30 | type githubNotifier struct {
31 | cfg *conf.GlobalConfig
32 | clients *clients.Clients
33 | }
34 |
35 | func NewGithubEventNotifier(cfg *conf.GlobalConfig, clients *clients.Clients) EventNotifier {
36 | return &githubNotifier{
37 | cfg: cfg,
38 | clients: clients,
39 | }
40 | }
41 |
42 | func (gn *githubNotifier) Notify(ctx *context.Context, workflow *v1alpha1.Workflow) error {
43 | fmt.Printf("Notifing workflow, %s\n", workflow.GetName())
44 |
45 | repo, ok := workflow.GetLabels()["repo"]
46 | if !ok {
47 | return fmt.Errorf("failed get repo label for workflow: %s", workflow.GetName())
48 | }
49 | commit, ok := workflow.GetLabels()["commit"]
50 | if !ok {
51 | return fmt.Errorf("failed get commit label for workflow: %s", workflow.GetName())
52 | }
53 |
54 | workflowLink := fmt.Sprintf("%s/workflows/%s/%s", gn.cfg.WorkflowServerConfig.ArgoAddress, gn.cfg.Namespace, workflow.GetName())
55 |
56 | status, err := gn.translateWorkflowStatus(string(workflow.Status.Phase), workflow.GetName())
57 | if err != nil {
58 | return err
59 | }
60 |
61 | message := utils.TrimString(workflow.Status.Message, 140) // Max length of message is 140 characters
62 | err = gn.clients.GitProvider.SetStatus(ctx, &repo, &commit, &workflowLink, &status, &message)
63 | if err != nil {
64 | return fmt.Errorf("failed to set status for workflow %s: %s", workflow.GetName(), err)
65 | }
66 |
67 | return nil
68 | }
69 |
70 | func (gn *githubNotifier) translateWorkflowStatus(status string, workflowName string) (string, error) {
71 | switch gn.cfg.GitProviderConfig.Provider {
72 | case "github":
73 | result, ok := workflowTranslationToGithubMap[status]
74 | if !ok {
75 | return "", fmt.Errorf("failed to translate workflow status to github stasuts for %s status: %s", workflowName, status)
76 | }
77 | return result, nil
78 | case "bitbucket":
79 | result, ok := workflowTranslationToBitbucketMap[status]
80 | if !ok {
81 | return "", fmt.Errorf("failed to translate workflow status to bitbucket stasuts for %s status: %s", workflowName, status)
82 | }
83 | return result, nil
84 | }
85 | return "", fmt.Errorf("failed to translate workflow status")
86 | }
87 |
--------------------------------------------------------------------------------
/pkg/git_provider/github_utils_test.go:
--------------------------------------------------------------------------------
1 | package git_provider
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | "net/http"
8 | "testing"
9 |
10 | "github.com/google/go-github/v52/github"
11 | "github.com/rookout/piper/pkg/conf"
12 | "github.com/rookout/piper/pkg/utils"
13 | assertion "github.com/stretchr/testify/assert"
14 | )
15 |
16 | func TestIsOrgWebhookEnabled(t *testing.T) {
17 | //
18 | // Prepare
19 | //
20 | client, mux, _, teardown := setup()
21 | defer teardown()
22 |
23 | config := make(map[string]interface{})
24 | config["url"] = "https://bla.com"
25 | Hooks := github.Hook{
26 | Active: utils.BPtr(true),
27 | Name: utils.SPtr("web"),
28 | Config: config,
29 | }
30 | jsonBytes, _ := json.Marshal(&[]github.Hook{Hooks})
31 |
32 | mux.HandleFunc("/orgs/test/hooks", func(w http.ResponseWriter, r *http.Request) {
33 | testMethod(t, r, "GET")
34 | testFormValues(t, r, values{})
35 | _, _ = fmt.Fprint(w, string(jsonBytes))
36 | })
37 |
38 | c := GithubClientImpl{
39 | client: client,
40 | cfg: &conf.GlobalConfig{
41 | GitProviderConfig: conf.GitProviderConfig{
42 | OrgLevelWebhook: true,
43 | OrgName: "test",
44 | WebhookURL: "https://bla.com",
45 | },
46 | },
47 | }
48 | ctx := context.Background()
49 |
50 | //
51 | // Execute
52 | //
53 | hooks, isEnabled := isOrgWebhookEnabled(ctx, &c)
54 |
55 | //
56 | // Assert
57 | //
58 | assert := assertion.New(t)
59 | assert.True(isEnabled)
60 | assert.NotNil(t, hooks)
61 | }
62 |
63 | func TestIsRepoWebhookEnabled(t *testing.T) {
64 | //
65 | // Prepare
66 | //
67 | client, mux, _, teardown := setup()
68 | defer teardown()
69 |
70 | config := make(map[string]interface{})
71 | config["url"] = "https://bla.com"
72 | Hooks := github.Hook{
73 | Active: utils.BPtr(true),
74 | Name: utils.SPtr("web"),
75 | Config: config,
76 | }
77 | jsonBytes, _ := json.Marshal(&[]github.Hook{Hooks})
78 |
79 | mux.HandleFunc("/repos/test/test-repo2/hooks", func(w http.ResponseWriter, r *http.Request) {
80 | testMethod(t, r, "GET")
81 | testFormValues(t, r, values{})
82 | _, _ = fmt.Fprint(w, string(jsonBytes))
83 | })
84 |
85 | c := GithubClientImpl{
86 | client: client,
87 | cfg: &conf.GlobalConfig{
88 | GitProviderConfig: conf.GitProviderConfig{
89 | OrgLevelWebhook: false,
90 | OrgName: "test",
91 | WebhookURL: "https://bla.com",
92 | RepoList: "test-repo1,test-repo2",
93 | },
94 | },
95 | }
96 | ctx := context.Background()
97 |
98 | //
99 | // Execute
100 | //
101 | hook, isEnabled := isRepoWebhookEnabled(ctx, &c, "test-repo2")
102 |
103 | //
104 | // Assert
105 | //
106 | assert := assertion.New(t)
107 | assert.True(isEnabled)
108 | assert.NotNil(t, hook)
109 |
110 | //
111 | // Execute
112 | //
113 | hook, isEnabled = isRepoWebhookEnabled(ctx, &c, "test-repo3")
114 |
115 | //
116 | // Assert
117 | //
118 | assert = assertion.New(t)
119 | assert.False(isEnabled)
120 | assert.NotNil(t, hook)
121 | }
122 |
--------------------------------------------------------------------------------
/helm-chart/templates/_helpers.tpl:
--------------------------------------------------------------------------------
1 | {{/*
2 | Expand the name of the chart.
3 | */}}
4 | {{- define "piper.name" -}}
5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
6 | {{- end }}
7 |
8 | {{/*
9 | Return secret name to be used based on provided values.
10 | */}}
11 | {{- define "piper.argoWorkflows.tokenSecretName" -}}
12 | {{- $fullName := printf "%s-argo-token" .Release.Name -}}
13 | {{- default $fullName .Values.piper.argoWorkflows.server.existingSecret | quote -}}
14 | {{- end -}}
15 |
16 | {{/*
17 | Return secret name to be used based on provided values.
18 | */}}
19 | {{- define "piper.gitProvider.tokenSecretName" -}}
20 | {{- $fullName := printf "%s-git-token" .Release.Name -}}
21 | {{- default $fullName .Values.piper.gitProvider.existingSecret | quote -}}
22 | {{- end -}}
23 |
24 | {{/*
25 | Return secret name to be used based on provided values.
26 | */}}
27 | {{- define "piper.gitProvider.webhook.secretName" -}}
28 | {{- $fullName := printf "%s-webhook-secret" .Release.Name -}}
29 | {{- default $fullName .Values.piper.gitProvider.webhook.existingSecret | quote -}}
30 | {{- end -}}
31 |
32 | {{/*
33 | Return secret name to be used based on provided values.
34 | */}}
35 | {{- define "rookout.secretName" -}}
36 | {{- $fullName := printf "%s-rookout-token" .Release.Name -}}
37 | {{- default $fullName .Values.rookout.existingSecret | quote -}}
38 | {{- end -}}
39 |
40 |
41 | {{/*
42 | Create a default fully qualified app name.
43 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
44 | If release name contains chart name it will be used as a full name.
45 | */}}
46 | {{- define "piper.fullname" -}}
47 | {{- if .Values.fullnameOverride }}
48 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
49 | {{- else }}
50 | {{- $name := default .Chart.Name .Values.nameOverride }}
51 | {{- if contains $name .Release.Name }}
52 | {{- .Release.Name | trunc 63 | trimSuffix "-" }}
53 | {{- else }}
54 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
55 | {{- end }}
56 | {{- end }}
57 | {{- end }}
58 |
59 | {{/*
60 | Create chart name and version as used by the chart label.
61 | */}}
62 | {{- define "piper.chart" -}}
63 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
64 | {{- end }}
65 |
66 | {{/*
67 | Common labels
68 | */}}
69 | {{- define "piper.labels" -}}
70 | helm.sh/chart: {{ include "piper.chart" . }}
71 | {{ include "piper.selectorLabels" . }}
72 |
73 | {{- if .Chart.AppVersion }}
74 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
75 | {{- end }}
76 | app.kubernetes.io/managed-by: {{ .Release.Service }}
77 | {{- end }}
78 |
79 | {{/*
80 | Selector labels
81 | */}}
82 | {{- define "piper.selectorLabels" -}}
83 | app.kubernetes.io/name: {{ include "piper.name" . }}
84 | app.kubernetes.io/instance: {{ .Release.Name }}
85 | {{- end }}
86 |
87 | {{/*
88 | Create the name of the service account to use
89 | */}}
90 | {{- define "piper.serviceAccountName" -}}
91 | {{- if .Values.serviceAccount.create }}
92 | {{- default (include "piper.fullname" .) .Values.serviceAccount.name }}
93 | {{- else }}
94 | {{- default "default" .Values.serviceAccount.name }}
95 | {{- end }}
96 | {{- end }}
97 |
--------------------------------------------------------------------------------
/pkg/git_provider/test_utils.go:
--------------------------------------------------------------------------------
1 | package git_provider
2 |
3 | import (
4 | "fmt"
5 | "github.com/google/go-cmp/cmp"
6 | "github.com/google/go-github/v52/github"
7 | "github.com/ktrysmt/go-bitbucket"
8 | "net/http"
9 | "net/http/httptest"
10 | "net/url"
11 | "os"
12 | "testing"
13 | )
14 |
15 | const (
16 | // baseURLPath is a non-empty Client.BaseURL path to use during tests,
17 | // to ensure relative URLs are used for all endpoints. See issue #752.
18 | baseURLPath = "/api-v3"
19 | bitbucketBaseURLPath = "/2.0"
20 | )
21 |
22 | func setup() (client *github.Client, mux *http.ServeMux, serverURL string, teardown func()) {
23 | mux = http.NewServeMux()
24 |
25 | apiHandler := http.NewServeMux()
26 | apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, mux))
27 | apiHandler.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
28 | fmt.Fprintln(os.Stderr, "FAIL: Client.BaseURL path prefix is not preserved in the request URL:")
29 | fmt.Fprintln(os.Stderr)
30 | fmt.Fprintln(os.Stderr, "\t"+req.URL.String())
31 | fmt.Fprintln(os.Stderr)
32 | fmt.Fprintln(os.Stderr, "\tDid you accidentally use an absolute endpoint URL rather than relative?")
33 | fmt.Fprintln(os.Stderr, "\tSee https://github.com/google/go-github/issues/752 for information.")
34 | http.Error(w, "Client.BaseURL path prefix is not preserved in the request URL.", http.StatusInternalServerError)
35 | })
36 |
37 | // server is a test HTTP server used to provide mock API responses.
38 | server := httptest.NewServer(apiHandler)
39 |
40 | // client is the GitHub client being tested and is
41 | // configured to use test server.
42 | client = github.NewClient(nil)
43 | url, _ := url.Parse(server.URL + baseURLPath + "/")
44 | client.BaseURL = url
45 | client.UploadURL = url
46 |
47 | return client, mux, server.URL, server.Close
48 | }
49 |
50 | func testMethod(t *testing.T, r *http.Request, want string) {
51 | t.Helper()
52 | if got := r.Method; got != want {
53 | t.Errorf("Request method: %v, want %v", got, want)
54 | }
55 | }
56 |
57 | type values map[string]string
58 |
59 | func testFormValues(t *testing.T, r *http.Request, values values) {
60 | t.Helper()
61 | want := url.Values{}
62 | for k, v := range values {
63 | want.Set(k, v)
64 | }
65 |
66 | err := r.ParseForm()
67 | if err != nil {
68 | t.Errorf("Go error parsing form: %v", err)
69 | }
70 | if got := r.Form; !cmp.Equal(got, want) {
71 | t.Errorf("Request parameters: %v, want %v", got, want)
72 | }
73 | }
74 |
75 | func setupBitbucket() (client *bitbucket.Client, mux *http.ServeMux, serverURL string, teardown func()) {
76 | mux = http.NewServeMux()
77 |
78 | apiHandler := http.NewServeMux()
79 | apiHandler.Handle(bitbucketBaseURLPath+"/", http.StripPrefix(bitbucketBaseURLPath, mux))
80 | apiHandler.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
81 | http.Error(w, "Client.BaseURL path prefix is not preserved in the request URL.", http.StatusInternalServerError)
82 | })
83 |
84 | server := httptest.NewServer(apiHandler)
85 | url, _ := url.Parse(server.URL + bitbucketBaseURLPath)
86 | client = bitbucket.NewBasicAuth("username", "password")
87 | client.SetApiBaseURL(*url)
88 |
89 | return client, mux, server.URL, server.Close
90 | }
91 |
--------------------------------------------------------------------------------
/scripts/init-kind.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | set -o errexit
3 |
4 | # 1. Create registry container unless it already exists
5 | reg_name='kind-registry'
6 | reg_port='5001'
7 | if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
8 | docker run \
9 | -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
10 | registry:2
11 | fi
12 |
13 | # 2. Create kind cluster with containerd registry config dir enabled
14 |
15 | if [ $(kind get clusters | grep "piper") = ""]; then
16 | cat <" ]; do
80 | sleep 0.1;
81 | done;
82 |
83 | # waiting for core dns to up - to indicate that scheduling can happen
84 | echo "waiting core dns to up"
85 | until [ "`kubectl rollout status deployment --namespace kube-system | grep coredns`"=="deployment "coredns" successfully rolled out" ]; do
86 | sleep 0.1;
87 | done;
88 |
89 | kubectl wait --namespace kube-system \
90 | --for=condition=ready pod \
91 | --selector=k8s-app=kube-dns \
92 | --timeout=30s
93 |
--------------------------------------------------------------------------------
/pkg/webhook_creator/mocks.go:
--------------------------------------------------------------------------------
1 | package webhook_creator
2 |
3 | import (
4 | "errors"
5 | "github.com/rookout/piper/pkg/git_provider"
6 | "golang.org/x/net/context"
7 | "net/http"
8 | )
9 |
10 | type MockGitProviderClient struct {
11 | ListFilesFunc func(ctx context.Context, repo string, branch string, path string) ([]string, error)
12 | GetFileFunc func(ctx context.Context, repo string, branch string, path string) (*git_provider.CommitFile, error)
13 | GetFilesFunc func(ctx context.Context, repo string, branch string, paths []string) ([]*git_provider.CommitFile, error)
14 | SetWebhookFunc func(ctx context.Context, repo *string) (*git_provider.HookWithStatus, error)
15 | UnsetWebhookFunc func(ctx context.Context, hook *git_provider.HookWithStatus) error
16 | HandlePayloadFunc func(request *http.Request, secret []byte) (*git_provider.WebhookPayload, error)
17 | SetStatusFunc func(ctx context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error
18 | PingHookFunc func(ctx context.Context, hook *git_provider.HookWithStatus) error
19 | }
20 |
21 | func (m *MockGitProviderClient) ListFiles(ctx *context.Context, repo string, branch string, path string) ([]string, error) {
22 | if m.ListFilesFunc != nil {
23 | return m.ListFilesFunc(*ctx, repo, branch, path)
24 | }
25 | return nil, errors.New("unimplemented")
26 | }
27 |
28 | func (m *MockGitProviderClient) GetFile(ctx *context.Context, repo string, branch string, path string) (*git_provider.CommitFile, error) {
29 | if m.GetFileFunc != nil {
30 | return m.GetFileFunc(*ctx, repo, branch, path)
31 | }
32 | return nil, errors.New("unimplemented")
33 | }
34 |
35 | func (m *MockGitProviderClient) GetFiles(ctx *context.Context, repo string, branch string, paths []string) ([]*git_provider.CommitFile, error) {
36 | if m.GetFilesFunc != nil {
37 | return m.GetFilesFunc(*ctx, repo, branch, paths)
38 | }
39 | return nil, errors.New("unimplemented")
40 | }
41 |
42 | func (m *MockGitProviderClient) SetWebhook(ctx *context.Context, repo *string) (*git_provider.HookWithStatus, error) {
43 | if m.SetWebhookFunc != nil {
44 | return m.SetWebhookFunc(*ctx, repo)
45 | }
46 | return nil, errors.New("unimplemented")
47 | }
48 |
49 | func (m *MockGitProviderClient) UnsetWebhook(ctx *context.Context, hook *git_provider.HookWithStatus) error {
50 | if m.UnsetWebhookFunc != nil {
51 | return m.UnsetWebhookFunc(*ctx, hook)
52 | }
53 | return errors.New("unimplemented")
54 | }
55 |
56 | func (m *MockGitProviderClient) HandlePayload(ctx *context.Context, request *http.Request, secret []byte) (*git_provider.WebhookPayload, error) {
57 | if m.HandlePayloadFunc != nil {
58 | return m.HandlePayloadFunc(request, secret)
59 | }
60 | return nil, errors.New("unimplemented")
61 | }
62 |
63 | func (m *MockGitProviderClient) SetStatus(ctx *context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error {
64 | if m.SetStatusFunc != nil {
65 | return m.SetStatusFunc(*ctx, repo, commit, linkURL, status, message)
66 | }
67 | return errors.New("unimplemented")
68 | }
69 |
70 | func (m *MockGitProviderClient) PingHook(ctx *context.Context, hook *git_provider.HookWithStatus) error {
71 | if m.PingHookFunc != nil {
72 | return m.PingHookFunc(*ctx, hook)
73 | }
74 | return errors.New("unimplemented")
75 | }
76 |
--------------------------------------------------------------------------------
/pkg/utils/os_test.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "bytes"
5 | "io"
6 | "os"
7 | "path/filepath"
8 | "testing"
9 | )
10 |
11 | func createFileWithContent(path string, content string) error {
12 | file, err := os.Create(path)
13 | if err != nil {
14 | return err
15 | }
16 | defer file.Close()
17 |
18 | _, err = io.WriteString(file, content)
19 | return err
20 | }
21 |
22 | func TestGetFilesData(t *testing.T) {
23 | // Create a temporary directory for testing
24 | tempDir := os.TempDir()
25 | testDir := filepath.Join(tempDir, "test")
26 | err := os.Mkdir(testDir, 0755)
27 | if err != nil {
28 | t.Fatalf("Error creating temporary directory: %v", err)
29 | }
30 | defer os.RemoveAll(testDir) // Clean up the temporary directory
31 |
32 | // Create some dummy files in the test directory
33 | file1Path := filepath.Join(testDir, "file1.txt")
34 | err = createFileWithContent(file1Path, "File 1 data")
35 | if err != nil {
36 | t.Fatalf("Error creating file1: %v", err)
37 | }
38 |
39 | file2Path := filepath.Join(testDir, "file2.txt")
40 | err = createFileWithContent(file2Path, "File 2 data")
41 | if err != nil {
42 | t.Fatalf("Error creating file2: %v", err)
43 | }
44 |
45 | // Call the function being tested
46 | fileData, err := GetFilesData(testDir)
47 | if err != nil {
48 | t.Fatalf("Error calling GetFilesData: %v", err)
49 | }
50 |
51 | // Verify the results
52 | expectedData := map[string][]byte{
53 | "file1.txt": []byte("File 1 data"),
54 | "file2.txt": []byte("File 2 data"),
55 | }
56 |
57 | for fileName, expected := range expectedData {
58 | actual, ok := fileData[fileName]
59 | if !ok {
60 | t.Errorf("Missing file data for %s", fileName)
61 | }
62 |
63 | if !bytes.Equal(actual, expected) {
64 | t.Errorf("File data mismatch for %s: expected '%s', got '%s'", fileName, expected, actual)
65 | }
66 | }
67 | }
68 |
69 | func stringSlicesEqual(a, b []string) bool {
70 | if len(a) != len(b) {
71 | return false
72 | }
73 | for i := range a {
74 | if a[i] != b[i] {
75 | return false
76 | }
77 | }
78 | return true
79 | }
80 |
81 | func TestGetFilesInLinkDirectory(t *testing.T) {
82 | tempDir := t.TempDir()
83 |
84 | fileNames := []string{"file1.txt", "file2.txt", "file3.txt"}
85 | for _, name := range fileNames {
86 | filePath := filepath.Join(tempDir, name)
87 | _, err := os.Create(filePath)
88 | if err != nil {
89 | t.Fatalf("Failed to create test file: %v", err)
90 | }
91 | }
92 |
93 | linkPath := filepath.Join(tempDir, "symlink")
94 | err := os.Symlink(tempDir, linkPath)
95 | if err != nil {
96 | t.Fatalf("Failed to create symbolic link: %v", err)
97 | }
98 |
99 | realPath, result, err := GetFilesInLinkDirectory(linkPath)
100 | if err != nil {
101 | t.Fatalf("Unexpected error: %v", err)
102 | }
103 |
104 | expectedRealPath, err := filepath.EvalSymlinks(linkPath)
105 | if err != nil {
106 | t.Fatalf("Failed to evaluate symlink: %v", err)
107 | }
108 | if *realPath != expectedRealPath {
109 | t.Errorf("Unexpected real path. Expected: %s, got: %s", expectedRealPath, *realPath)
110 | }
111 |
112 | t.Logf("Result: %v", result) // Add this line for debugging
113 |
114 | if !stringSlicesEqual(result, fileNames) {
115 | t.Errorf("Unexpected file names. Expected: %v, got: %v", fileNames, result)
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/pkg/workflow_handler/workflows_utils.go:
--------------------------------------------------------------------------------
1 | package workflow_handler
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
7 | "github.com/rookout/piper/pkg/conf"
8 | "github.com/rookout/piper/pkg/git_provider"
9 | "github.com/rookout/piper/pkg/utils"
10 | "gopkg.in/yaml.v3"
11 | "log"
12 | "regexp"
13 | "strings"
14 | )
15 |
16 | func CreateDAGTemplate(fileList []*git_provider.CommitFile, name string) (*v1alpha1.Template, error) {
17 | if len(fileList) == 0 {
18 | log.Printf("empty file list for %s", name)
19 | return nil, nil
20 | }
21 | DAGs := make([]v1alpha1.DAGTask, 0)
22 | for _, file := range fileList {
23 | if file.Content == nil || file.Path == nil {
24 | return nil, fmt.Errorf("missing content or path for %s", name)
25 | }
26 | DAGTask := make([]v1alpha1.DAGTask, 0)
27 | jsonBytes, err := utils.ConvertYAMLListToJSONList(*file.Content)
28 | if err != nil {
29 | return nil, err
30 | }
31 | err = json.Unmarshal(jsonBytes, &DAGTask)
32 | if err != nil {
33 | return nil, err
34 | }
35 | err = ValidateDAGTasks(DAGTask)
36 | if err != nil {
37 | return nil, err
38 | }
39 | DAGs = append(DAGs, DAGTask...)
40 | }
41 |
42 | if len(DAGs) == 0 {
43 | return nil, fmt.Errorf("no tasks for %s", name)
44 | }
45 |
46 | template := &v1alpha1.Template{
47 | Name: name,
48 | DAG: &v1alpha1.DAGTemplate{
49 | Tasks: DAGs,
50 | },
51 | }
52 |
53 | return template, nil
54 | }
55 |
56 | func AddFilesToTemplates(templates []v1alpha1.Template, files []*git_provider.CommitFile) ([]v1alpha1.Template, error) {
57 | for _, f := range files {
58 | t := make([]v1alpha1.Template, 0)
59 | jsonBytes, err := utils.ConvertYAMLListToJSONList(*f.Content)
60 | if err != nil {
61 | return nil, err
62 | }
63 |
64 | err = json.Unmarshal(jsonBytes, &t)
65 | if err != nil {
66 | return nil, err
67 | }
68 | templates = append(templates, t...)
69 | }
70 | return templates, nil
71 | }
72 |
73 | func GetParameters(paramsFile *git_provider.CommitFile) ([]v1alpha1.Parameter, error) {
74 | var params []v1alpha1.Parameter
75 | err := yaml.Unmarshal([]byte(*paramsFile.Content), ¶ms)
76 | if err != nil {
77 | return nil, err
78 | }
79 | return params, nil
80 | }
81 |
82 | func IsConfigExists(cfg *conf.WorkflowsConfig, config string) bool {
83 | _, ok := cfg.Configs[config]
84 | return ok
85 | }
86 |
87 | func IsConfigsOnExitExists(cfg *conf.WorkflowsConfig, config string) bool {
88 | return len(cfg.Configs[config].OnExit) != 0
89 | }
90 |
91 | func ValidateDAGTasks(tasks []v1alpha1.DAGTask) error {
92 | for _, task := range tasks {
93 | if task.Name == "" {
94 | return fmt.Errorf("task name cannot be empty: %+v\n", task)
95 | }
96 |
97 | if task.Template == "" && task.TemplateRef == nil {
98 | return fmt.Errorf("task template or templateRef cannot be empty: %+v\n", task)
99 | }
100 |
101 | }
102 | return nil
103 | }
104 |
105 | func ConvertToValidString(input string) string {
106 | // Convert to lowercase
107 | lowercase := strings.ToLower(input)
108 |
109 | // Replace underscores with hyphens
110 | converted := strings.ReplaceAll(lowercase, "_", "-")
111 |
112 | // Remove symbols except . and -
113 | pattern := `[^a-z0-9.\-]`
114 | re := regexp.MustCompile(pattern)
115 | validString := re.ReplaceAllString(converted, "")
116 |
117 | return validString
118 | }
119 |
--------------------------------------------------------------------------------
/docs/usage/workflows_folder.md:
--------------------------------------------------------------------------------
1 | ## .workflows Folder
2 |
3 | Piper will look in each of the target branches for a `.workflows` folder. [example](https://github.com/Rookout/piper/tree/main/examples/.workflows).
4 | We will explain each of the files that should be included in the `.workflows` folder.
5 |
6 | ### triggers.yaml (convention name)
7 |
8 | This file holds a list of triggers that will be executed `onStart` by `events` from specific `branches`.
9 | Piper will execute each of matching triggers, so configure it wisely.
10 | ```yaml
11 | - events:
12 | - push
13 | - pull_request.synchronize
14 | branches: ["main"]
15 | onStart: ["main.yaml"]
16 | onExit: ["exit.yaml"]
17 | templates: ["templates.yaml"]
18 | config: "default"
19 | ```
20 | Can be found [here](https://github.com/Rookout/piper/tree/main/examples/.workflows/triggers.yaml).
21 |
22 | In this example `main.yaml` will be executed as DAG when `push` or `pull_request.synchronize` events will be applied in `main` branch.
23 | `onExit` will be executed `exit.yaml` when finished the workflow as exit handler.
24 |
25 |
26 | `onExit` can overwrite the default `onExit` configuration from by reference existing DAG tasks as in the [example](https://github.com/Rookout/piper/tree/main/examples/.workflows/exit.yaml).
27 |
28 | `config` field used for workflow configuration selection. the default value is `default` configuration.
29 |
30 | #### events
31 | Events field used to terminate when the trigger will be executed. name of the event depends on the git provider.
32 |
33 | For instance, GitHub pull_request event have few action, one of them is synchronize.
34 |
35 | #### branches
36 | For which branch that trigger will be executed.
37 |
38 | #### onStart
39 | This [file](https://github.com/Rookout/piper/tree/main/examples/.workflows/main.yaml) can be named as you wish and will be referenced in `triggers.yaml` file. It will define an entrypoint DAG that the Workflow will execute.
40 |
41 | As a best practice, this file should contain the dependencies logic and parametrization of each of referenced templates. It should not implement new templates, for this, use template.yaml file.
42 |
43 | #### onExit
44 | This field used to pass verbose exitHandler to the triggered workflow.
45 | It will override the default onExit from the provided `config` or the default `config`.
46 |
47 | In the provided `exit.yaml` describes a DAG that will overwrite the default `onExit` configuration.
48 | [Example](https://github.com/Rookout/piper/tree/main/examples/.workflows/exit.yaml)
49 |
50 | #### templates
51 | This field will have additional templates that will be injected to the workflows.
52 | The purpose of this field is to create repository scope templates that can be referenced from the DAGs templates at `onStart` or `onExit`.
53 | [Example](https://github.com/Rookout/piper/tree/main/examples/.workflows/templates.yaml)
54 |
55 | As a best practice, use this field for template implementation and reference them from executed.
56 | [Example](https://github.com/Rookout/piper/tree/main/examples/.workflows/main.yaml).
57 |
58 | ### config
59 | configured by `piper-workflows-config` [configMap](workflows_config.md).
60 | Can be passed explicitly, or will use `deafault` configuration.
61 |
62 | ### parameters.yaml (convention name)
63 | Will hold a list of global parameters of the Workflow.
64 | can be referenced from any template with `{{ workflow.parameters.___ }}.`
65 |
66 | [Example](https://github.com/Rookout/piper/tree/main/examples/.workflows/parameters.yaml)
--------------------------------------------------------------------------------
/pkg/git_provider/github_utils.go:
--------------------------------------------------------------------------------
1 | package git_provider
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "net/http"
7 | "strings"
8 |
9 | "github.com/rookout/piper/pkg/utils"
10 |
11 | "github.com/google/go-github/v52/github"
12 | "github.com/rookout/piper/pkg/conf"
13 | )
14 |
15 | func isOrgWebhookEnabled(ctx context.Context, c *GithubClientImpl) (*github.Hook, bool) {
16 | emptyHook := github.Hook{}
17 | hooks, resp, err := c.client.Organizations.ListHooks(ctx, c.cfg.GitProviderConfig.OrgName, &github.ListOptions{})
18 | if err != nil {
19 | return &emptyHook, false
20 | }
21 | if resp.StatusCode != 200 {
22 | return &emptyHook, false
23 | }
24 | if len(hooks) == 0 {
25 | return &emptyHook, false
26 | }
27 | for _, hook := range hooks {
28 | if hook.GetActive() && hook.GetName() == "web" && hook.Config["url"] == c.cfg.GitProviderConfig.WebhookURL {
29 | return hook, true
30 | }
31 | }
32 | return &emptyHook, false
33 | }
34 |
35 | func isRepoWebhookEnabled(ctx context.Context, c *GithubClientImpl, repo string) (*github.Hook, bool) {
36 | emptyHook := github.Hook{}
37 | hooks, resp, err := c.client.Repositories.ListHooks(ctx, c.cfg.GitProviderConfig.OrgName, repo, &github.ListOptions{})
38 | if err != nil {
39 | return &emptyHook, false
40 | }
41 | if resp.StatusCode != 200 {
42 | return &emptyHook, false
43 | }
44 | if len(hooks) == 0 {
45 | return &emptyHook, false
46 | }
47 |
48 | for _, hook := range hooks {
49 | if hook.GetActive() && hook.GetName() == "web" && hook.Config["url"] == c.cfg.GitProviderConfig.WebhookURL {
50 | return hook, true
51 | }
52 | }
53 |
54 | return &emptyHook, false
55 | }
56 |
57 | func GetScopes(ctx context.Context, client *github.Client) ([]string, error) {
58 | // Make a request to the "Get the authenticated user" endpoint
59 | req, err := http.NewRequest("GET", "https://api.github.com/user", nil)
60 | if err != nil {
61 | fmt.Println("Error creating request:", err)
62 | return nil, err
63 | }
64 | resp, err := client.Do(ctx, req, nil)
65 | if err != nil {
66 | fmt.Println("Error making request:", err)
67 | return nil, err
68 | }
69 | defer resp.Body.Close()
70 |
71 | // Check the "X-OAuth-Scopes" header to get the token scopes
72 | scopes := resp.Header.Get("X-OAuth-Scopes")
73 | fmt.Println("Github Token Scopes are:", scopes)
74 |
75 | scopes = strings.ReplaceAll(scopes, " ", "")
76 | return strings.Split(scopes, ","), nil
77 |
78 | }
79 |
80 | func ValidatePermissions(ctx context.Context, client *github.Client, cfg *conf.GlobalConfig) error {
81 |
82 | orgScopes := []string{"admin:org_hook"}
83 | repoAdminScopes := []string{"admin:repo_hook"}
84 | repoGranularScopes := []string{"write:repo_hook", "read:repo_hook"}
85 |
86 | scopes, err := GetScopes(ctx, client)
87 |
88 | if err != nil {
89 | return fmt.Errorf("failed to get scopes: %v", err)
90 | }
91 | if len(scopes) == 0 {
92 | return fmt.Errorf("permissions error: no scopes found for the github client")
93 | }
94 |
95 | if cfg.GitProviderConfig.OrgLevelWebhook {
96 | if utils.ListContains(orgScopes, scopes) {
97 | return nil
98 | }
99 | return fmt.Errorf("permissions error: %v is not a valid scope for the org level permissions", scopes)
100 | }
101 |
102 | if utils.ListContains(repoAdminScopes, scopes) {
103 | return nil
104 | }
105 | if utils.ListContains(repoGranularScopes, scopes) {
106 | return nil
107 | }
108 |
109 | return fmt.Errorf("permissions error: %v is not a valid scope for the repo level permissions", scopes)
110 | }
111 |
--------------------------------------------------------------------------------
/docs/getting_started/installation.md:
--------------------------------------------------------------------------------
1 | ## Instalation
2 |
3 | Piper should be deployed in the cluster with Argo Workflows.
4 | Piper will create a CRD that Argo Workflows will pick, so install or configure Piper to create those CRDs in the right namespace.
5 |
6 | Please check out [values.yaml](https://github.com/Rookout/piper/tree/main/helm-chart/values.yaml) file of the helm chart configurations.
7 |
8 | To add piper helm repo run:
9 | ```bash
10 | helm repo add piper https://piper.rookout.com
11 | ```
12 |
13 | After configuring Piper [values.yaml](https://github.com/Rookout/piper/tree/main/helm-chart/values.yaml), run the following command for installation:
14 | ```bash
15 | helm upgrade --install piper piper/piper \
16 | -f YOUR_VALUES_FILE.yaml
17 | ```
18 |
19 | ---
20 |
21 | ## Required Configuration
22 |
23 | ### Ingress
24 |
25 | Piper should listen to webhooks from your git provider.
26 | Expose it using ingress or service, then provide the address to `piper.webhook.url` as followed:
27 | `https://PIPER_EXPOESED_URL/webhook`
28 |
29 | Checkout [values.yaml](https://github.com/Rookout/piper/tree/main/helm-chart/values.yaml)
30 |
31 | ### Git
32 |
33 | Piper will use git for fetching `.workflows` folder and receiving events using webhooks.
34 |
35 | To pick which git provider you are using provide `gitProvider.name` configuration in helm chart (Now only supports GitHub and Bitbucket).
36 |
37 | Also configure you organization (Github) or workspace (Bitbucket) name using `gitProvider.organization.name` in helm chart.
38 |
39 | #### Git Token Permissions
40 |
41 | The token should have access for creating webhooks and read repositories content.
42 | For GitHub configure `admin:org` and `write:org` permissions in Classic Token.
43 | For Bitbucket configure `Repositories:read`, `Webhooks:read and write` and `Pull requests:read` permissions (for multiple repos use workspace token).
44 |
45 | #### Token
46 |
47 | The git token should be passed as secret in the helm chart at `gitProvider.token`.
48 | Can be passed as parameter in helm install command using `--set piper.gitProvider.token=YOUR_GIT_TOKEN`
49 |
50 | Alternatively, you can consume already existing secret and fill up `piper.gipProvider.existingSecret`.
51 | The key should be name `token`. Can be created using
52 | ```bash
53 | kubectl create secret generic piper-git-token --from-literal=token=YOUR_GIT_OKEN
54 | ```
55 |
56 | #### Webhook creation
57 |
58 | Piper will create a webhook configuration for you, for the whole organization or for each repo you configure.
59 |
60 | Configure `piper.webhook.url` the address of piper that exposed with ingress with `/webhook` postfix.
61 |
62 | For organization level configure: `gitProvider.webhook.orgLevel` to `true`.
63 |
64 | For granular repo webhook provide list of repos at: `gitProvider.webhook.repoList`.
65 |
66 | Piper implements graceful shutdown, it will delete all the webhooks when terminated.
67 |
68 | #### Status check
69 |
70 | Piper will handle status checks for you.
71 | It will notify the GitProvider for the status of Workflow for specific commit that triggered Piper.
72 | For linking provide valid URL of your Argo Workflows server address at: `argoWorkflows.server.address`
73 |
74 | ---
75 |
76 | ### Argo Workflow Server (On development)
77 |
78 | Piper will use REST API to communicate with Argo Workflows server for linting or for creation of workflows (ARGO_WORKFLOWS_CREATE_CRD). Please follow this [configuration](https://argoproj.github.io/argo-workflows/rest-api/).
79 |
80 | To lint the workflow before submitting it, please configure the internal address of Argo Workflows server (for example, `argo-server.workflows.svc.cluster.local`) in the field: `argoWorkflows.server.address`. Argo will need a [token](https://argoproj.github.io/argo-workflows/access-token/) to authenticate. please provide the secret in `argoWorkflows.server.token`, Better to pass as a references to a secret in the field `argoWorkflows.server.token`.
81 |
82 | #### Skip CRD Creation (On development)
83 |
84 | Piper can communicate directly to Argo Workflow using ARGO_WORKFLOWS_CREATE_CRD environment variable, if you want to skip the creation of CRD change `argoWorkflows.crdCreation` to `false`.
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/rookout/piper
2 |
3 | go 1.20
4 |
5 | require (
6 | github.com/Rookout/GoSDK v0.1.45
7 | github.com/argoproj/argo-workflows/v3 v3.4.8
8 | github.com/emicklei/go-restful/v3 v3.8.0
9 | github.com/gin-gonic/gin v1.9.1
10 | github.com/google/go-cmp v0.5.9
11 | github.com/google/go-github/v52 v52.0.0
12 | github.com/kelseyhightower/envconfig v1.4.0
13 | github.com/ktrysmt/go-bitbucket v0.9.66
14 | github.com/stretchr/testify v1.8.4
15 | github.com/tidwall/gjson v1.16.0
16 | golang.org/x/net v0.17.0
17 | gopkg.in/yaml.v3 v3.0.1
18 | k8s.io/apimachinery v0.24.3
19 | k8s.io/client-go v0.24.3
20 | )
21 |
22 | require (
23 | github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
24 | github.com/bytedance/sonic v1.9.1 // indirect
25 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
26 | github.com/cloudflare/circl v1.3.3 // indirect
27 | github.com/davecgh/go-spew v1.1.1 // indirect
28 | github.com/fallais/logrus-lumberjack-hook v0.0.0-20210917073259-3227e1ab93b0 // indirect
29 | github.com/gabriel-vasile/mimetype v1.4.2 // indirect
30 | github.com/gin-contrib/sse v0.1.0 // indirect
31 | github.com/go-errors/errors v1.4.1 // indirect
32 | github.com/go-logr/logr v1.2.3 // indirect
33 | github.com/go-openapi/jsonpointer v0.19.6 // indirect
34 | github.com/go-openapi/jsonreference v0.20.2 // indirect
35 | github.com/go-openapi/swag v0.22.3 // indirect
36 | github.com/go-playground/locales v0.14.1 // indirect
37 | github.com/go-playground/universal-translator v0.18.1 // indirect
38 | github.com/go-playground/validator/v10 v10.14.0 // indirect
39 | github.com/goccy/go-json v0.10.2 // indirect
40 | github.com/gogo/protobuf v1.3.2 // indirect
41 | github.com/golang/protobuf v1.5.3 // indirect
42 | github.com/google/gnostic v0.5.7-v3refs // indirect
43 | github.com/google/go-querystring v1.1.0 // indirect
44 | github.com/google/gofuzz v1.2.0 // indirect
45 | github.com/google/uuid v1.3.0 // indirect
46 | github.com/gorilla/websocket v1.5.0 // indirect
47 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
48 | github.com/hashicorp/golang-lru v0.5.4 // indirect
49 | github.com/imdario/mergo v0.3.13 // indirect
50 | github.com/josharian/intern v1.0.0 // indirect
51 | github.com/json-iterator/go v1.1.12 // indirect
52 | github.com/klauspost/cpuid/v2 v2.2.4 // indirect
53 | github.com/kr/pretty v0.3.1 // indirect
54 | github.com/leodido/go-urn v1.2.4 // indirect
55 | github.com/mailru/easyjson v0.7.7 // indirect
56 | github.com/mattn/go-isatty v0.0.19 // indirect
57 | github.com/mitchellh/mapstructure v1.5.0 // indirect
58 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
59 | github.com/modern-go/reflect2 v1.0.2 // indirect
60 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
61 | github.com/pelletier/go-toml/v2 v2.0.8 // indirect
62 | github.com/pkg/errors v0.9.1 // indirect
63 | github.com/pmezard/go-difflib v1.0.0 // indirect
64 | github.com/sirupsen/logrus v1.9.2 // indirect
65 | github.com/spf13/pflag v1.0.5 // indirect
66 | github.com/tidwall/match v1.1.1 // indirect
67 | github.com/tidwall/pretty v1.2.0 // indirect
68 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
69 | github.com/ugorji/go/codec v1.2.11 // indirect
70 | github.com/yhirose/go-peg v0.0.0-20210804202551-de25d6753cf1 // indirect
71 | golang.org/x/arch v0.3.0 // indirect
72 | golang.org/x/crypto v0.14.0 // indirect
73 | golang.org/x/oauth2 v0.11.0 // indirect
74 | golang.org/x/sys v0.13.0 // indirect
75 | golang.org/x/term v0.13.0 // indirect
76 | golang.org/x/text v0.13.0 // indirect
77 | golang.org/x/time v0.3.0 // indirect
78 | google.golang.org/appengine v1.6.7 // indirect
79 | google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
80 | google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
81 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
82 | google.golang.org/grpc v1.56.3 // indirect
83 | google.golang.org/protobuf v1.31.0 // indirect
84 | gopkg.in/inf.v0 v0.9.1 // indirect
85 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
86 | gopkg.in/yaml.v2 v2.4.0 // indirect
87 | k8s.io/api v0.24.3 // indirect
88 | k8s.io/klog/v2 v2.60.1 // indirect
89 | k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 // indirect
90 | k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
91 | sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
92 | sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
93 | sigs.k8s.io/yaml v1.3.0 // indirect
94 | )
95 |
--------------------------------------------------------------------------------
/docs/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | ## How To Contribute
2 |
3 | We appreciate contributions from the community to make Piper even better. To contribute, follow the steps below:
4 |
5 | 1. Fork the Piper repository to your GitHub account.
6 | 2. Clone the forked repository to your local machine:
7 | ```bash
8 | git clone https://github.com/your-username/Piper.git
9 | ```
10 | 3. Create a new branch to work on your feature or bug fix:
11 | ```bash
12 | git checkout -b my-feature
13 | ```
14 | 4. Make your changes, following the coding guidelines outlined in this document.
15 | 5. Commit your changes with clear and descriptive commit messages and sign it:
16 | ```bash
17 | git commit -s -m "fix: Add new feature"
18 | ```
19 | * please make sure you commit as described in [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/)
20 | 6. Push your changes to your forked repository:
21 | ```bash
22 | git push origin my-feature
23 | ```
24 | 7. Open a [pull request](#pull-requests) against the main branch of the original Piper repository.
25 |
26 | ## Pull Requests
27 |
28 | We welcome and appreciate contributions from the community. If you have developed a new feature, improvement, or bug fix for Piper, follow these steps to submit a pull request:
29 |
30 | 1. Make sure you have forked the Piper repository and created a new branch for your changes. Checkout [How To Contribute](#How-to-contribute).
31 | 2. commit your changes and push them to your forked repository.
32 | 3. Go to the Piper repository on GitHub.
33 | 4. Click on the "New Pull Request" button.
34 | 5. Select your branch and provide a [descriptive title](#pull-request-nameing) and detailed description of your changes.
35 | 6. If your pull request relates to an open issue, reference the issue in the description using the GitHub issue syntax (e.g., Fixes #123).
36 | 7. Submit the pull request, and our team will review your changes. We appreciate your patience during the review process and may provide feedback or request further modifications.
37 |
38 | ### Pull Request Naming
39 |
40 | The name should follow conventional commit naming.
41 |
42 | ## Coding Guidelines
43 |
44 | To maintain a consistent codebase and ensure readability, we follow a set of coding guidelines in Piper. Please adhere to the following guidelines when making changes:
45 |
46 | * Follow the [Effective Go](https://go.dev/doc/effective_go) guide for Go code.
47 | * Follow the [Folder convention](https://github.com/golang-standards/project-layout) guide for Go code.
48 | * Write clear and concise comments to explain the code's functionality.
49 | * Use meaningful variable and function names.
50 | * Make sure your code is properly formatted and free of syntax errors.
51 | * Run tests locally.
52 | * Check that the feature documented.
53 | * Add new packages only if necessary and already existing one, can't be used.
54 | * Add tests for new features or modification.
55 |
56 | ## Helm Chart Development
57 |
58 | To make sure that the documentation is updated use [helm-docs](https://github.com/norwoodj/helm-docs) comment convention. The pipeline will execute `helm-docs` command and update the version of the chart.
59 |
60 | Also, please make sure to run those commands locally to debug the chart before merging:
61 |
62 | ```bash
63 | make helm
64 | ```
65 |
66 | ## Local deployment
67 |
68 | To make it easy to develop locally, please run the following
69 |
70 | Prerequisites :
71 | 1. install helm
72 | ```bash
73 | brew install helm
74 | ```
75 | 2. install kubectl
76 | ```bash
77 | brew install kubectl
78 | ```
79 | 3. isntall docker
80 |
81 | 4. install ngrok
82 | ```bash
83 | brew install ngrok
84 | ```
85 | 6. install 5
86 | ```bash
87 | brew install kind
88 | ```
89 |
90 | Deployment:
91 | 1. make sure docker are running.
92 | 2. create tunnel with ngrok using `make ngrok`, save the `Forwarding` address.
93 | 3. create `values.dev.yaml` file that contains subset of chart's `value.yaml` file. check [example of values file](https://github.com/Rookout/piper/tree/main/examples/template.values.dev.yaml) rename it to `values.dev.yaml` and put in root directory.
94 | 4. use `make deploy`. it will do the following:
95 | * deploy a local registry as container
96 | * deploy a kind cluster as container with configuration
97 | * deploy nginx reverse proxy in the kind cluster
98 | * deploy Piper with the local helm chart
99 | 5. validate using `curl localhost/piper/healthz`.
100 |
101 | ## Debugging
102 |
103 | For debugging the best practice is to use Rookout. To enable this function pass a Rookout token in the chart `rookout.token` or as existing secret `rookout.existingSecret`
--------------------------------------------------------------------------------
/pkg/utils/common.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "hash/fnv"
7 | "regexp"
8 | "strings"
9 |
10 | "gopkg.in/yaml.v3"
11 | "k8s.io/client-go/rest"
12 | "k8s.io/client-go/tools/clientcmd"
13 | )
14 |
15 | func ListContains(subList, list []string) bool {
16 | if len(subList) > len(list) {
17 | return false
18 | }
19 | for _, element := range subList {
20 | found := false
21 | for _, b := range list {
22 | if element == b {
23 | found = true
24 | break
25 | }
26 | }
27 | if !found {
28 | return false
29 | }
30 | }
31 | return true
32 | }
33 |
34 | func IsElementExists(list []string, element string) bool {
35 | for _, item := range list {
36 | if item == element {
37 | return true
38 | }
39 | }
40 | return false
41 | }
42 |
43 | func IsElementMatch(element string, elements []string) bool {
44 | if IsElementExists(elements, "*") {
45 | return true
46 | }
47 |
48 | return IsElementExists(elements, element)
49 | }
50 |
51 | func GetClientConfig(kubeConfig string) (*rest.Config, error) {
52 | if kubeConfig != "" {
53 | return clientcmd.BuildConfigFromFlags("", kubeConfig)
54 | }
55 | return rest.InClusterConfig()
56 | }
57 |
58 | func AddPrefixToList(list []string, prefix string) []string {
59 | result := make([]string, len(list))
60 |
61 | for i, item := range list {
62 | result[i] = prefix + item
63 | }
64 |
65 | return result
66 | }
67 |
68 | func StringToMap(str string) map[string]string {
69 | pairs := strings.Split(str, ",")
70 | m := make(map[string]string)
71 |
72 | for _, pair := range pairs {
73 | keyValue := strings.Split(pair, ":")
74 | if len(keyValue) == 2 {
75 | key := strings.TrimSpace(keyValue[0])
76 | value := strings.TrimSpace(keyValue[1])
77 | m[key] = value
78 | }
79 | }
80 |
81 | return m
82 | }
83 |
84 | func ConvertYAMLListToJSONList(yamlString string) ([]byte, error) {
85 | // Unmarshal YAML into a map[string]interface{}
86 | yamlData := make([]map[string]interface{}, 0)
87 | err := yaml.Unmarshal([]byte(yamlString), &yamlData)
88 | if err != nil {
89 | return nil, fmt.Errorf("failed to unmarshal YAML: %v", err)
90 | }
91 |
92 | // Marshal the YAML data as JSON
93 | jsonBytes, err := json.Marshal(&yamlData)
94 | if err != nil {
95 | return nil, fmt.Errorf("failed to marshal JSON: %v", err)
96 | }
97 |
98 | return jsonBytes, nil
99 | }
100 |
101 | func ConvertYAMLToJSON(yamlString []byte) ([]byte, error) {
102 | // Unmarshal YAML into a map[string]interface{}
103 | yamlData := make(map[string]interface{})
104 | err := yaml.Unmarshal(yamlString, &yamlData)
105 | if err != nil {
106 | return nil, fmt.Errorf("failed to unmarshal YAML: %v", err)
107 | }
108 |
109 | // Marshal the YAML data as JSON
110 | jsonBytes, err := json.Marshal(&yamlData)
111 | if err != nil {
112 | return nil, fmt.Errorf("failed to marshal JSON: %v", err)
113 | }
114 |
115 | return jsonBytes, nil
116 | }
117 |
118 | func SPtr(str string) *string {
119 | return &str
120 | }
121 |
122 | func BPtr(b bool) *bool {
123 | return &b
124 | }
125 |
126 | func IPtr(i int64) *int64 {
127 | return &i
128 | }
129 |
130 | func ValidateHTTPFormat(input string) bool {
131 | regex := `^(https?://)([\w-]+(\.[\w-]+)*)(:\d+)?(/[\w-./?%&=]*)?$`
132 | match, _ := regexp.MatchString(regex, input)
133 | return match
134 | }
135 |
136 | func TrimString(s string, maxLength int) string {
137 | if maxLength >= len(s) {
138 | return s
139 | }
140 | return s[:maxLength]
141 | }
142 |
143 | func StringToInt64(input string) int64 {
144 | h := fnv.New64a()
145 | h.Write([]byte(input))
146 | hashValue := h.Sum64()
147 |
148 | // Convert the hash value to int64
149 | int64Value := int64(hashValue)
150 | // Make sure the value is positive (int64 can represent only non-negative values)
151 | if int64Value < 0 {
152 | int64Value = int64Value * -1
153 | }
154 |
155 | return int64Value
156 | }
157 |
158 | func RemoveBraces(input string) string {
159 | output := strings.ReplaceAll(input, "{", "")
160 | output = strings.ReplaceAll(output, "}", "")
161 | return output
162 | }
163 |
164 | func ExtractStringsBetweenTags(input string) []string {
165 | re := regexp.MustCompile(`<([^>]+)>`)
166 | matches := re.FindAllStringSubmatch(input, -1)
167 |
168 | var result []string
169 | for _, match := range matches {
170 | result = append(result, match[1])
171 | }
172 |
173 | return result
174 | }
175 |
176 | func SanitizeString(input string) string {
177 | // Replace whitespace with "-"
178 | input = strings.ReplaceAll(input, " ", "-")
179 |
180 | // Define a regular expression pattern to match characters that are not a-z, A-Z, _, or .
181 | // Use a negated character class to allow a-z, A-Z, _, and .
182 | pattern := regexp.MustCompile(`[^a-zA-Z_1-9\.-]+`)
183 |
184 | // Remove characters that don't match the pattern
185 | sanitized := pattern.ReplaceAllString(input, "")
186 |
187 | return sanitized
188 | }
189 |
--------------------------------------------------------------------------------
/pkg/git_provider/bitbucket_test.go:
--------------------------------------------------------------------------------
1 | package git_provider
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "fmt"
7 | "github.com/ktrysmt/go-bitbucket"
8 | "github.com/rookout/piper/pkg/conf"
9 | "github.com/rookout/piper/pkg/utils"
10 | assertion "github.com/stretchr/testify/assert"
11 | "golang.org/x/net/context"
12 | "net/http"
13 | "testing"
14 | )
15 |
16 | func TestBitbucketListFiles(t *testing.T) {
17 | // Prepare
18 | client, mux, _, teardown := setupBitbucket()
19 | defer teardown()
20 |
21 | repoContent := &bitbucket.RepositoryFile{
22 | Type: "file",
23 | Path: ".workflows/exit.yaml",
24 | }
25 |
26 | repoContent2 := &bitbucket.RepositoryFile{
27 | Type: "file",
28 | Path: ".workflows/main.yaml",
29 | }
30 |
31 | data := map[string]interface{}{"values": []bitbucket.RepositoryFile{*repoContent, *repoContent2}}
32 | jsonBytes, _ := json.Marshal(data)
33 |
34 | mux.HandleFunc("/repositories/test/test-repo1/src/branch1/.workflows/", func(w http.ResponseWriter, r *http.Request) {
35 | testMethod(t, r, "GET")
36 | //testFormValues(t, r, values{})
37 |
38 | _, _ = fmt.Fprint(w, string(jsonBytes))
39 | })
40 |
41 | c := BitbucketClientImpl{
42 | client: client,
43 | cfg: &conf.GlobalConfig{
44 | GitProviderConfig: conf.GitProviderConfig{
45 | OrgLevelWebhook: false,
46 | OrgName: "test",
47 | RepoList: "test-repo1",
48 | },
49 | },
50 | }
51 | ctx := context.Background()
52 |
53 | // Execute
54 | actualContent, err := c.ListFiles(&ctx, "test-repo1", "branch1", ".workflows")
55 | expectedContent := []string{"exit.yaml", "main.yaml"}
56 |
57 | // Assert
58 | assert := assertion.New(t)
59 | assert.NotNil(t, err)
60 | assert.Equal(expectedContent, actualContent)
61 |
62 | }
63 |
64 | func TestBitbucketSetStatus(t *testing.T) {
65 | // Prepare
66 | ctx := context.Background()
67 | assert := assertion.New(t)
68 | client, mux, _, teardown := setupBitbucket()
69 | defer teardown()
70 |
71 | mux.HandleFunc("/repositories/test/test-repo1/commit/test-commit/statuses/build", func(w http.ResponseWriter, r *http.Request) {
72 | testMethod(t, r, "POST")
73 | testFormValues(t, r, values{})
74 |
75 | w.WriteHeader(http.StatusCreated)
76 | jsonBytes := []byte(`{"status": "ok"}`)
77 | _, _ = fmt.Fprint(w, string(jsonBytes))
78 | })
79 |
80 | c := BitbucketClientImpl{
81 | client: client,
82 | cfg: &conf.GlobalConfig{
83 | GitProviderConfig: conf.GitProviderConfig{
84 | Provider: "bitbucket",
85 | OrgLevelWebhook: false,
86 | OrgName: "test",
87 | RepoList: "test-repo1",
88 | },
89 | },
90 | }
91 |
92 | // Define test cases
93 | tests := []struct {
94 | name string
95 | repo *string
96 | commit *string
97 | linkURL *string
98 | status *string
99 | message *string
100 | wantedError error
101 | }{
102 | {
103 | name: "Notify success",
104 | repo: utils.SPtr("test-repo1"),
105 | commit: utils.SPtr("test-commit"),
106 | linkURL: utils.SPtr("https://argo"),
107 | status: utils.SPtr("success"),
108 | message: utils.SPtr(""),
109 | wantedError: nil,
110 | },
111 | {
112 | name: "Notify pending",
113 | repo: utils.SPtr("test-repo1"),
114 | commit: utils.SPtr("test-commit"),
115 | linkURL: utils.SPtr("https://argo"),
116 | status: utils.SPtr("pending"),
117 | message: utils.SPtr(""),
118 | wantedError: nil,
119 | },
120 | {
121 | name: "Notify error",
122 | repo: utils.SPtr("test-repo1"),
123 | commit: utils.SPtr("test-commit"),
124 | linkURL: utils.SPtr("https://argo"),
125 | status: utils.SPtr("error"),
126 | message: utils.SPtr("some message"),
127 | wantedError: nil,
128 | },
129 | {
130 | name: "Notify failure",
131 | repo: utils.SPtr("test-repo1"),
132 | commit: utils.SPtr("test-commit"),
133 | linkURL: utils.SPtr("https://argo"),
134 | status: utils.SPtr("failure"),
135 | message: utils.SPtr(""),
136 | wantedError: nil,
137 | },
138 | {
139 | name: "Non managed repo",
140 | repo: utils.SPtr("non-existing-repo"),
141 | commit: utils.SPtr("test-commit"),
142 | linkURL: utils.SPtr("https://argo"),
143 | status: utils.SPtr("error"),
144 | message: utils.SPtr(""),
145 | wantedError: errors.New("some error"),
146 | },
147 | {
148 | name: "Non existing commit",
149 | repo: utils.SPtr("test-repo1"),
150 | commit: utils.SPtr("not-exists"),
151 | linkURL: utils.SPtr("https://argo"),
152 | status: utils.SPtr("error"),
153 | message: utils.SPtr(""),
154 | wantedError: errors.New("some error"),
155 | },
156 | }
157 | // Run test cases
158 | for _, test := range tests {
159 | t.Run(test.name, func(t *testing.T) {
160 |
161 | // Call the function being tested
162 | err := c.SetStatus(&ctx, test.repo, test.commit, test.linkURL, test.status, test.message)
163 |
164 | // Use assert to check the equality of the error
165 | if test.wantedError != nil {
166 | assert.Error(err)
167 | assert.NotNil(err)
168 | } else {
169 | assert.NoError(err)
170 | assert.Nil(err)
171 | }
172 | })
173 | }
174 |
175 | }
176 |
--------------------------------------------------------------------------------
/pkg/workflow_handler/workflows_test.go:
--------------------------------------------------------------------------------
1 | package workflow_handler
2 |
3 | import (
4 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
5 | "github.com/rookout/piper/pkg/common"
6 | "github.com/rookout/piper/pkg/conf"
7 | "github.com/rookout/piper/pkg/git_provider"
8 | assertion "github.com/stretchr/testify/assert"
9 | "testing"
10 | )
11 |
12 | func TestSelectConfig(t *testing.T) {
13 | var wfc *conf.WorkflowsConfig
14 |
15 | assert := assertion.New(t)
16 | // Create a sample WorkflowsBatch object for testing
17 | configName := "default"
18 | workflowsBatch := &common.WorkflowsBatch{
19 | Config: &configName, // Set the desired config name here
20 | Payload: &git_provider.WebhookPayload{},
21 | }
22 |
23 | // Create a mock WorkflowsClientImpl object with necessary dependencies
24 | wfc = &conf.WorkflowsConfig{Configs: map[string]*conf.ConfigInstance{
25 | "default": {Spec: v1alpha1.WorkflowSpec{},
26 | OnExit: []v1alpha1.DAGTask{}},
27 | "config1": {Spec: v1alpha1.WorkflowSpec{},
28 | OnExit: []v1alpha1.DAGTask{}},
29 | }}
30 |
31 | wfcImpl := &WorkflowsClientImpl{
32 | cfg: &conf.GlobalConfig{
33 | WorkflowsConfig: *wfc,
34 | },
35 | }
36 |
37 | // Call the SelectConfig function
38 | returnConfigName, err := wfcImpl.SelectConfig(workflowsBatch)
39 |
40 | // Assert the expected output
41 | assert.Equal("default", returnConfigName)
42 | assert.Nil(err)
43 |
44 | // Test case 2
45 | configName = "config1"
46 | workflowsBatch = &common.WorkflowsBatch{
47 | Config: &configName, // Set the desired config name here
48 | Payload: &git_provider.WebhookPayload{},
49 | }
50 |
51 | // Call the SelectConfig function
52 | returnConfigName, err = wfcImpl.SelectConfig(workflowsBatch)
53 |
54 | // Assert the expected output
55 | assert.Equal("config1", returnConfigName)
56 | assert.Nil(err)
57 |
58 | // Test case 3 - selection of non-existing config when default config exists
59 | configName = "notInConfigs"
60 | workflowsBatch = &common.WorkflowsBatch{
61 | Config: &configName, // Set the desired config name here
62 | Payload: &git_provider.WebhookPayload{},
63 | }
64 |
65 | // Call the SelectConfig function
66 | returnConfigName, err = wfcImpl.SelectConfig(workflowsBatch)
67 |
68 | // Assert the expected output
69 | assert.Equal("default", returnConfigName)
70 | assert.NotNil(err)
71 |
72 | // Test case 4 - selection of non-existing config when default config not exists
73 | configName = "notInConfig"
74 | workflowsBatch = &common.WorkflowsBatch{
75 | Config: &configName, // Set the desired config name here
76 | Payload: &git_provider.WebhookPayload{},
77 | }
78 |
79 | wfc4 := &conf.WorkflowsConfig{Configs: map[string]*conf.ConfigInstance{
80 | "config1": {Spec: v1alpha1.WorkflowSpec{},
81 | OnExit: []v1alpha1.DAGTask{}},
82 | }}
83 |
84 | wfcImpl4 := &WorkflowsClientImpl{
85 | cfg: &conf.GlobalConfig{
86 | WorkflowsConfig: *wfc4,
87 | },
88 | }
89 |
90 | // Call the SelectConfig function
91 | returnConfigName, err = wfcImpl4.SelectConfig(workflowsBatch)
92 |
93 | // Assert the expected output
94 | assert.NotNil(returnConfigName)
95 | assert.NotNil(err)
96 | }
97 |
98 | func TestCreateWorkflow(t *testing.T) {
99 | var wfc *conf.WorkflowsConfig
100 | var wfs *conf.WorkflowServerConfig
101 |
102 | // Create a WorkflowsClientImpl instance
103 | assert := assertion.New(t)
104 | // Create a mock WorkflowsClientImpl object with necessary dependencies
105 | wfc = &conf.WorkflowsConfig{Configs: map[string]*conf.ConfigInstance{
106 | "default": {Spec: v1alpha1.WorkflowSpec{},
107 | OnExit: []v1alpha1.DAGTask{}},
108 | "config1": {Spec: v1alpha1.WorkflowSpec{},
109 | OnExit: []v1alpha1.DAGTask{}},
110 | }}
111 |
112 | wfs = &conf.WorkflowServerConfig{Namespace: "default"}
113 |
114 | wfcImpl := &WorkflowsClientImpl{
115 | cfg: &conf.GlobalConfig{
116 | WorkflowsConfig: *wfc,
117 | WorkflowServerConfig: *wfs,
118 | },
119 | }
120 |
121 | // Create a sample WorkflowSpec
122 | spec := &v1alpha1.WorkflowSpec{
123 | // Assign values to the fields of WorkflowSpec
124 | // ...
125 |
126 | // Example assignments:
127 | Entrypoint: "my-entrypoint",
128 | }
129 |
130 | // Create a sample WorkflowsBatch
131 | workflowsBatch := &common.WorkflowsBatch{
132 | Payload: &git_provider.WebhookPayload{
133 | Repo: "my-repo",
134 | Branch: "my-branch",
135 | User: "my-user",
136 | Commit: "my-commit",
137 | },
138 | }
139 |
140 | // Call the CreateWorkflow method
141 | workflow, err := wfcImpl.CreateWorkflow(spec, workflowsBatch)
142 |
143 | // Assert that no error occurred
144 | assert.NoError(err)
145 |
146 | // Assert that the returned workflow is not nil
147 | assert.NotNil(workflow)
148 |
149 | // Assert that the workflow's GenerateName, Namespace, and Labels are assigned correctly
150 | assert.Equal("my-repo-my-branch-", workflow.ObjectMeta.GenerateName)
151 | assert.Equal(wfcImpl.cfg.Namespace, workflow.ObjectMeta.Namespace)
152 | assert.Equal(map[string]string{
153 | "piper.rookout.com/notified": "false",
154 | "repo": "my-repo",
155 | "branch": "my-branch",
156 | "user": "my-user",
157 | "commit": "my-commit",
158 | }, workflow.ObjectMeta.Labels)
159 |
160 | // Assert that the workflow's Spec is assigned correctly
161 | assert.Equal(*spec, workflow.Spec)
162 | }
163 |
--------------------------------------------------------------------------------
/helm-chart/README.md:
--------------------------------------------------------------------------------
1 | # piper
2 |
3 |   
4 |
5 | A Helm chart for Piper
6 |
7 | ## Values
8 |
9 | | Key | Type | Default | Description |
10 | |-----|------|---------|-------------|
11 | | affinity | object | `{}` | Assign custom [affinity] rules to the deployment |
12 | | autoscaling.enabled | bool | `false` | Wheter to enable auto-scaling of piper. |
13 | | autoscaling.maxReplicas | int | `5` | Maximum reoplicas of Piper. |
14 | | autoscaling.minReplicas | int | `1` | Minimum reoplicas of Piper. |
15 | | autoscaling.targetCPUUtilizationPercentage | int | `85` | CPU utilization percentage threshold. |
16 | | autoscaling.targetMemoryUtilizationPercentage | int | `85` | Memory utilization percentage threshold. |
17 | | env | list | `[]` | Additional environment variables for Piper. A list of name/value maps. |
18 | | extraLabels | object | `{}` | Deployment and pods extra labels |
19 | | fullnameOverride | string | `""` | String to fully override "piper.fullname" template |
20 | | image.name | string | `"piper"` | Piper image name |
21 | | image.pullPolicy | string | `"IfNotPresent"` | Piper image pull policy |
22 | | image.repository | string | `"rookout"` | Piper public dockerhub repo |
23 | | image.tag | string | `""` | Piper image tag |
24 | | imagePullSecrets | list | `[]` | secret to use for image pulling |
25 | | ingress.annotations | object | `{}` | Piper ingress annotations |
26 | | ingress.className | string | `""` | Piper ingress class name |
27 | | ingress.enabled | bool | `false` | Enable Piper ingress support |
28 | | ingress.hosts | list | `[{"host":"piper.example.local","paths":[{"path":"/","pathType":"ImplementationSpecific"}]}]` | Piper ingress hosts # Hostnames must be provided if Ingress is enabled. |
29 | | ingress.tls | list | `[]` | Controller ingress tls |
30 | | lifecycle | object | `{}` | Specify postStart and preStop lifecycle hooks for Piper container |
31 | | nameOverride | string | `""` | String to partially override "piper.fullname" template |
32 | | nodeSelector | object | `{}` | [Node selector] |
33 | | piper.argoWorkflows.crdCreation | bool | `true` | Whether create Workflow CRD or send direct commands to Argo Workflows server. |
34 | | piper.argoWorkflows.server.address | string | `""` | The DNS address of Argo Workflow server that Piper can address. |
35 | | piper.argoWorkflows.server.existingSecret | string | `nil` | |
36 | | piper.argoWorkflows.server.namespace | string | `""` | The namespace in which the Workflow CRD will be created. |
37 | | piper.argoWorkflows.server.token | string | `""` | This will create a secret named -token and with the key 'token' |
38 | | piper.gitProvider.existingSecret | string | `nil` | |
39 | | piper.gitProvider.name | string | `"github"` | Name of your git provider (github/gitlab/bitbucket). for now, only github supported. |
40 | | piper.gitProvider.organization.name | string | `""` | Name of your Git Organization |
41 | | piper.gitProvider.token | string | `nil` | This will create a secret named -git-token and with the key 'token' |
42 | | piper.gitProvider.webhook.existingSecret | string | `nil` | |
43 | | piper.gitProvider.webhook.orgLevel | bool | `false` | Whether config webhook on org level |
44 | | piper.gitProvider.webhook.repoList | list | `[]` | Used of orgLevel=false, to configure webhook for each of the repos provided. |
45 | | piper.gitProvider.webhook.secret | string | `""` | This will create a secret named -webhook-secret and with the key 'secret' |
46 | | piper.gitProvider.webhook.url | string | `""` | The url in which piper listens for webhook, the path should be /webhook |
47 | | piper.workflowsConfig | object | `{}` | |
48 | | podAnnotations | object | `{}` | Annotations to be added to the Piper pods |
49 | | podSecurityContext | object | `{"fsGroup":1001,"runAsGroup":1001,"runAsUser":1001}` | Security Context to set on the pod level |
50 | | replicaCount | int | `1` | Piper number of replicas |
51 | | resources | object | `{"requests":{"cpu":"200m","memory":"512Mi"}}` | Resource limits and requests for the pods. |
52 | | rookout.existingSecret | string | `""` | |
53 | | rookout.token | string | `""` | Rookout token for agent configuration and enablement. |
54 | | securityContext | object | `{"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"runAsUser":1001}` | Security Context to set on the container level |
55 | | service.annotations | object | `{}` | Piper service extra annotations |
56 | | service.labels | object | `{}` | Piper service extra labels |
57 | | service.port | int | `80` | Service port For TLS mode change the port to 443 |
58 | | service.type | string | `"ClusterIP"` | Sets the type of the Service |
59 | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account |
60 | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created |
61 | | serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template |
62 | | tolerations | list | `[]` | [Tolerations] for use with node taints |
63 | | volumeMounts | list | `[]` | Volumes to mount to Piper container. |
64 | | volumes | list | `[]` | Volumes of Piper Pod. |
65 |
66 | ----------------------------------------------
67 | Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)
68 |
--------------------------------------------------------------------------------
/pkg/webhook_creator/main.go:
--------------------------------------------------------------------------------
1 | package webhook_creator
2 |
3 | import (
4 | "fmt"
5 | "github.com/emicklei/go-restful/v3/log"
6 | "github.com/rookout/piper/pkg/clients"
7 | "github.com/rookout/piper/pkg/conf"
8 | "github.com/rookout/piper/pkg/git_provider"
9 | "github.com/rookout/piper/pkg/utils"
10 | "golang.org/x/net/context"
11 | "strconv"
12 | "strings"
13 | "sync"
14 | "time"
15 | )
16 |
17 | type WebhookCreatorImpl struct {
18 | clients *clients.Clients
19 | cfg *conf.GlobalConfig
20 | hooks map[int64]*git_provider.HookWithStatus
21 | mu sync.Mutex
22 | }
23 |
24 | func NewWebhookCreator(cfg *conf.GlobalConfig, clients *clients.Clients) *WebhookCreatorImpl {
25 | wr := &WebhookCreatorImpl{
26 | clients: clients,
27 | cfg: cfg,
28 | hooks: make(map[int64]*git_provider.HookWithStatus, 0),
29 | }
30 |
31 | return wr
32 | }
33 |
34 | func (wc *WebhookCreatorImpl) Start() {
35 |
36 | err := wc.initWebhooks()
37 | if err != nil {
38 | log.Print(err)
39 | panic("failed in initializing webhooks")
40 | }
41 | }
42 |
43 | func (wc *WebhookCreatorImpl) setWebhook(hookID int64, healthStatus bool, repoName string) {
44 | wc.mu.Lock()
45 | defer wc.mu.Unlock()
46 | wc.hooks[hookID] = &git_provider.HookWithStatus{HookID: hookID, HealthStatus: healthStatus, RepoName: &repoName}
47 | }
48 |
49 | func (wc *WebhookCreatorImpl) getWebhook(hookID int64) *git_provider.HookWithStatus {
50 | wc.mu.Lock()
51 | defer wc.mu.Unlock()
52 | hook, ok := wc.hooks[hookID]
53 | if !ok {
54 | return nil
55 | }
56 | return hook
57 | }
58 |
59 | func (wc *WebhookCreatorImpl) deleteWebhook(hookID int64) {
60 | wc.mu.Lock()
61 | defer wc.mu.Unlock()
62 |
63 | delete(wc.hooks, hookID)
64 | }
65 |
66 | func (wc *WebhookCreatorImpl) SetWebhookHealth(hookID int64, status bool) error {
67 |
68 | hook, ok := wc.hooks[hookID]
69 | if !ok {
70 | return fmt.Errorf("unable to find hookID: %d in internal hooks map %v", hookID, wc.hooks)
71 | }
72 | wc.setWebhook(hookID, status, *hook.RepoName)
73 | log.Printf("set health status to %s for hook id: %d", strconv.FormatBool(status), hookID)
74 | return nil
75 | }
76 |
77 | func (wc *WebhookCreatorImpl) setAllHooksHealth(status bool) {
78 | for hookID, hook := range wc.hooks {
79 | wc.setWebhook(hookID, status, *hook.RepoName)
80 | }
81 | log.Printf("set all hooks health status for to %s", strconv.FormatBool(status))
82 | }
83 |
84 | func (wc *WebhookCreatorImpl) initWebhooks() error {
85 |
86 | ctx := context.Background()
87 | if wc.cfg.GitProviderConfig.OrgLevelWebhook && len(wc.cfg.GitProviderConfig.RepoList) != 0 {
88 | return fmt.Errorf("org level webhook wanted but provided repositories list")
89 | }
90 | for _, repo := range strings.Split(wc.cfg.GitProviderConfig.RepoList, ",") {
91 | if wc.cfg.GitProviderConfig.Provider == "bitbucket" {
92 | repo = utils.SanitizeString(repo)
93 | }
94 | hook, err := wc.clients.GitProvider.SetWebhook(&ctx, &repo)
95 | if err != nil {
96 | return err
97 | }
98 | wc.setWebhook(hook.HookID, hook.HealthStatus, *hook.RepoName)
99 | }
100 |
101 | return nil
102 | }
103 |
104 | func (wc *WebhookCreatorImpl) Stop(ctx *context.Context) {
105 | if wc.cfg.GitProviderConfig.WebhookAutoCleanup {
106 | err := wc.deleteWebhooks(ctx)
107 | if err != nil {
108 | log.Printf("Failed to delete webhooks, error: %v", err)
109 | }
110 | }
111 | }
112 |
113 | func (wc *WebhookCreatorImpl) deleteWebhooks(ctx *context.Context) error {
114 |
115 | for hookID, hook := range wc.hooks {
116 | err := wc.clients.GitProvider.UnsetWebhook(ctx, hook)
117 | if err != nil {
118 | return err
119 | }
120 | wc.deleteWebhook(hookID)
121 | }
122 |
123 | return nil
124 | }
125 |
126 | func (wc *WebhookCreatorImpl) checkHooksHealth(timeoutSeconds time.Duration) bool {
127 | startTime := time.Now()
128 |
129 | for {
130 | allHealthy := true
131 | for _, hook := range wc.hooks {
132 | if !hook.HealthStatus {
133 | allHealthy = false
134 | break
135 | }
136 | }
137 |
138 | if allHealthy {
139 | return true
140 | }
141 |
142 | if time.Since(startTime) >= timeoutSeconds {
143 | break
144 | }
145 |
146 | time.Sleep(1 * time.Second) // Adjust the sleep duration as per your requirement
147 | }
148 |
149 | return false
150 | }
151 |
152 | func (wc *WebhookCreatorImpl) recoverHook(ctx *context.Context, hookID int64) error {
153 |
154 | log.Printf("started recover of hook %d", hookID)
155 | hook := wc.getWebhook(hookID)
156 | if hook == nil {
157 | return fmt.Errorf("failed to recover hook, hookID %d not found", hookID)
158 | }
159 | newHook, err := wc.clients.GitProvider.SetWebhook(ctx, hook.RepoName)
160 | if err != nil {
161 | return err
162 | }
163 | wc.deleteWebhook(hookID)
164 | wc.setWebhook(newHook.HookID, newHook.HealthStatus, *newHook.RepoName)
165 | log.Printf("successful recover of hook %d", hookID)
166 | return nil
167 |
168 | }
169 |
170 | func (wc *WebhookCreatorImpl) pingHooks(ctx *context.Context) error {
171 | for _, hook := range wc.hooks {
172 | err := wc.clients.GitProvider.PingHook(ctx, hook)
173 | if err != nil {
174 | return err
175 | }
176 | }
177 | return nil
178 | }
179 |
180 | func (wc *WebhookCreatorImpl) RunDiagnosis(ctx *context.Context) error {
181 | log.Printf("Starting webhook diagnostics")
182 | wc.setAllHooksHealth(false)
183 | err := wc.pingHooks(ctx)
184 | if err != nil {
185 | return err
186 | }
187 | if !wc.checkHooksHealth(5 * time.Second) {
188 | for hookID, hook := range wc.hooks {
189 | if !hook.HealthStatus {
190 | return fmt.Errorf("hook %d is not healthy", hookID)
191 | }
192 | }
193 | }
194 |
195 | log.Print("Successful webhook diagnosis")
196 | return nil
197 | }
198 |
--------------------------------------------------------------------------------
/helm-chart/templates/deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: {{ include "piper.fullname" . }}
5 | labels:
6 | {{- include "piper.labels" . | nindent 4 }}
7 | spec:
8 | revisionHistoryLimit: 3
9 | {{- if not .Values.autoscaling.enabled }}
10 | replicas: {{ .Values.replicaCount }}
11 | {{- end }}
12 | selector:
13 | matchLabels:
14 | {{- include "piper.selectorLabels" . | nindent 6 }}
15 | template:
16 | metadata:
17 | {{- with .Values.podAnnotations }}
18 | annotations:
19 | {{- toYaml . | nindent 8 }}
20 | {{- end }}
21 | labels:
22 | {{- include "piper.selectorLabels" . | nindent 8 }}
23 | app: {{ .Chart.Name | trunc 63 | trimSuffix "-" }}
24 | version: {{ .Values.image.tag | default .Chart.AppVersion | trunc 63 | trimSuffix "-" }}
25 | spec:
26 | volumes:
27 | {{- if .Values.piper.workflowsConfig }}
28 | - name: piper-workflows-config
29 | configMap:
30 | name: piper-workflows-config
31 | {{- end }}
32 | {{- with .Values.volumes }}
33 | {{- toYaml . | nindent 8 }}
34 | {{- end }}
35 | {{- with .Values.imagePullSecrets }}
36 | imagePullSecrets:
37 | {{- toYaml . | nindent 8 }}
38 | {{- end }}
39 | serviceAccountName: {{ include "piper.serviceAccountName" . }}
40 | securityContext:
41 | {{- toYaml .Values.podSecurityContext | nindent 8 }}
42 | containers:
43 | - name: {{ .Chart.Name }}
44 | volumeMounts:
45 | {{- if .Values.piper.workflowsConfig }}
46 | - mountPath: /piper-config
47 | name: piper-workflows-config
48 | readOnly: true
49 | {{- end }}
50 | {{- with .Values.volumeMounts }}
51 | {{- toYaml . | nindent 12 }}
52 | {{- end }}
53 | securityContext:
54 | {{- toYaml .Values.securityContext | nindent 12 }}
55 | {{- with .Values.lifecycle }}
56 | lifecycle:
57 | {{- toYaml . | nindent 10 }}
58 | {{- end }}
59 | image: {{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}
60 | imagePullPolicy: {{ .Values.image.pullPolicy }}
61 | ports:
62 | - containerPort: 8080
63 | protocol: TCP
64 | livenessProbe:
65 | httpGet:
66 | path: /healthz
67 | port: 8080
68 | scheme: HTTP
69 | initialDelaySeconds: 10
70 | timeoutSeconds: 10
71 | periodSeconds: 60
72 | successThreshold: 1
73 | failureThreshold: 4
74 | readinessProbe:
75 | httpGet:
76 | path: /readyz
77 | port: 8080
78 | scheme: HTTP
79 | initialDelaySeconds: 2
80 | timeoutSeconds: 1
81 | periodSeconds: 4
82 | successThreshold: 1
83 | failureThreshold: 2
84 | resources:
85 | {{- toYaml .Values.resources | nindent 12 }}
86 | env:
87 | {{- if or .Values.rookout.token .Values.rookout.existingSecret }}
88 | - name: ROOKOUT_TOKEN
89 | valueFrom:
90 | secretKeyRef:
91 | name: {{ template "rookout.secretName" . }}
92 | key: token
93 | {{- end }}
94 | - name: GIT_PROVIDER
95 | value: {{ .Values.piper.gitProvider.name | quote }}
96 | - name: GIT_TOKEN
97 | valueFrom:
98 | secretKeyRef:
99 | name: {{ template "piper.gitProvider.tokenSecretName" . }}
100 | key: token
101 | - name: GIT_ORG_NAME
102 | value: {{ .Values.piper.gitProvider.organization.name | quote }}
103 | - name: GIT_WEBHOOK_URL
104 | value: {{ .Values.piper.gitProvider.webhook.url | quote }}
105 | - name: GIT_WEBHOOK_SECRET
106 | valueFrom:
107 | secretKeyRef:
108 | name: {{ template "piper.gitProvider.webhook.secretName" . }}
109 | key: secret
110 | - name: GIT_ORG_LEVEL_WEBHOOK
111 | value: {{ .Values.piper.gitProvider.webhook.orgLevel | quote }}
112 | - name: GIT_WEBHOOK_REPO_LIST
113 | value: {{ join "," .Values.piper.gitProvider.webhook.repoList | quote }}
114 | {{- if or .Values.piper.argoWorkflows.server.token .Values.piper.argoWorkflows.server.existingSecret }}
115 | - name: ARGO_WORKFLOWS_TOKEN
116 | valueFrom:
117 | secretKeyRef:
118 | name: {{ template "piper.argoWorkflows.tokenSecretName" . }}
119 | key: token
120 | {{- end }}
121 | - name: ARGO_WORKFLOWS_NAMESPACE
122 | value: {{ .Values.piper.argoWorkflows.server.namespace | default .Release.Namespace | quote }}
123 | - name: ARGO_WORKFLOWS_ADDRESS
124 | value: {{ .Values.piper.argoWorkflows.server.address | quote }}
125 | - name: ARGO_WORKFLOWS_CREATE_CRD
126 | value: {{ .Values.piper.argoWorkflows.crdCreation | quote }}
127 | {{- with .Values.env }}
128 | {{- toYaml . | nindent 10 }}
129 | {{- end }}
130 | {{- with .Values.nodeSelector }}
131 | nodeSelector:
132 | {{- toYaml . | nindent 8 }}
133 | {{- end }}
134 | {{- with .Values.affinity }}
135 | affinity:
136 | {{- toYaml . | nindent 8 }}
137 | {{- end }}
138 | {{- with .Values.tolerations }}
139 | tolerations:
140 | {{- toYaml . | nindent 8 }}
141 | {{- end }}
142 |
--------------------------------------------------------------------------------
/.github/workflows/e2e.yaml:
--------------------------------------------------------------------------------
1 | name: E2E tests
2 |
3 | on:
4 | push:
5 | branches:
6 | - "main"
7 | paths:
8 | - '**'
9 | - '!docs/**'
10 | pull_request:
11 | branches:
12 | - "main"
13 |
14 | concurrency:
15 | group: ${{ github.workflow }} # Bottleneck in ngrok tunnel, only one tunnel can exist for specific token -${{ github.ref }}
16 | # cancel-in-progress: true
17 |
18 | permissions:
19 | contents: read
20 |
21 | jobs:
22 | e2e-env-init:
23 | name: E2E Tests (on development)
24 | runs-on: ubuntu-latest
25 | timeout-minutes: 15
26 | steps:
27 | - uses: actions/checkout@v3
28 | - uses: docker/setup-qemu-action@v2
29 | - uses: docker/setup-buildx-action@v2
30 | with:
31 | driver-opts: network=host
32 | - uses: actions/setup-go@v4
33 | with:
34 | go-version: "1.20"
35 | cache: true
36 | - name: Install Ngrok Tunnel
37 | run: |
38 | curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && \
39 | echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | sudo tee /etc/apt/sources.list.d/ngrok.list && \
40 | sudo apt update && \
41 | sudo apt install ngrok
42 | touch ~/ngrok.log
43 | (timeout 30m ngrok http 80 --authtoken ${{ secrets.NGROK_AUTHTOKEN }} --log ~/ngrok.log) &
44 | echo $?
45 | - name: Install kind
46 | run: |
47 | curl -sSLo kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.19.0/kind-linux-amd64"
48 | chmod +x kind
49 | sudo mv kind /usr/local/bin/kind
50 | kind version
51 | - name: Install Kubectl
52 | run: |
53 | curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/v1.26.1/bin/linux/amd64/kubectl"
54 | chmod +x kubectl
55 | sudo mv kubectl /usr/local/bin/kubectl
56 | kubectl version --client --output=yaml
57 | - name: Kubernetes KinD Cluster
58 | run: |
59 | make init-kind
60 | - name: install nginx
61 | run: |
62 | make init-nginx
63 | - name: install workflows
64 | run: |
65 | make init-argo-workflows
66 | - name: Build Docker Image
67 | uses: docker/build-push-action@v4
68 | with:
69 | context: .
70 | push: true
71 | tags: localhost:5001/piper:latest
72 | cache-from: type=gha
73 | cache-to: type=gha,mode=max
74 | - name: Check tunnel existence
75 | run: |
76 | echo "NGROK_URL=$(cat ~/ngrok.log | grep -o 'url=https://.*' | cut -d '=' -f 2)" >> $GITHUB_ENV
77 | cat ~/ngrok.log | grep -o 'url=https://.*' | cut -d '=' -f 2
78 | - name: init piper
79 | run: |
80 | helm upgrade --install piper ./helm-chart \
81 | -f ./examples/template.values.dev.yaml \
82 | --set piper.gitProvider.token="${{ secrets.GIT_TOKEN }}" \
83 | --set piper.gitProvider.webhook.url="${{ env.NGROK_URL }}/piper/webhook" \
84 | --set piper.gitProvider.webhook.repoList={piper-e2e-test} \
85 | --set piper.gitProvider.organization.name="rookout" \
86 | --set image.repository=localhost:5001 \
87 | --set piper.argoWorkflows.server.address="${{ env.NGROK_URL }}/argo" \
88 | --set-string env\[0\].name=GIT_WEBHOOK_AUTO_CLEANUP,env\[0\].value="true" && \
89 | sleep 20 && kubectl logs deployment/piper
90 | kubectl wait \
91 | --for=condition=ready pod \
92 | --selector=app=piper \
93 | --timeout=60s
94 | - uses: actions/checkout@v3
95 | with:
96 | repository: 'rookout/piper-e2e-test'
97 | path: piper-e2e-test
98 | ref: 'main'
99 | - name: inject some changes to piper-e2e-test repo
100 | run: |
101 | cd ./piper-e2e-test
102 | echo "" >> .workflows/triggers.yaml
103 | git config user.name 'e2e-test'
104 | git config user.email 'sonario@rookout.com'
105 | git commit -am "trigger e2e test"
106 | - name: Create Pull Request
107 | id: cpr
108 | uses: peter-evans/create-pull-request@v5
109 | with:
110 | token: ${{ secrets.GIT_TOKEN }}
111 | path: piper-e2e-test
112 | branch: ${{ github.head_ref }}-test
113 | title: ${{ github.head_ref }}-test
114 | delete-branch: true
115 | - name: Close Pull Request
116 | uses: peter-evans/close-pull@v3
117 | with:
118 | token: ${{ secrets.GIT_TOKEN }}
119 | pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
120 | repository: 'rookout/piper-e2e-test'
121 | comment: Auto-closing pull request
122 | delete-branch: true
123 | - name: Check Result
124 | run: |
125 | kubectl logs deployment/piper
126 | kubectl get workflows.argoproj.io -n workflows
127 | BRANCH_VALID_STRING=$(echo ${{ github.head_ref }}-test | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd 'a-z0-9.\-')
128 |
129 | ## check if created
130 | RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers | grep piper-e2e-test)
131 | [ ! -z "$RESULT" ] && echo "CRD created $RESULT" || { echo "Workflow not exists, existing..."; exit 1; }
132 |
133 | ## check if status phase not Failed, if yes, show message
134 | RESULT=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.phase")
135 | MESSAGE=$(kubectl get workflows.argoproj.io -n workflows --selector=branch=$BRANCH_VALID_STRING --no-headers -o custom-columns="Status:status.message")
136 | [ ! "$RESULT" == "Failed" ] && echo "CRD created $MESSAGE" || { echo "Workflow Failed $MESSAGE, existing..."; exit 1; }
--------------------------------------------------------------------------------
/pkg/event_handler/github_event_notifier_test.go:
--------------------------------------------------------------------------------
1 | package event_handler
2 |
3 | import (
4 | "context"
5 | "errors"
6 | "github.com/rookout/piper/pkg/git_provider"
7 | assertion "github.com/stretchr/testify/assert"
8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9 | "net/http"
10 | "testing"
11 |
12 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
13 | "github.com/rookout/piper/pkg/clients"
14 | "github.com/rookout/piper/pkg/conf"
15 | )
16 |
17 | type mockGitProvider struct{}
18 |
19 | func (m *mockGitProvider) GetFile(ctx *context.Context, repo string, branch string, path string) (*git_provider.CommitFile, error) {
20 | return nil, nil
21 | }
22 |
23 | func (m *mockGitProvider) GetFiles(ctx *context.Context, repo string, branch string, paths []string) ([]*git_provider.CommitFile, error) {
24 | return nil, nil
25 | }
26 |
27 | func (m *mockGitProvider) ListFiles(ctx *context.Context, repo string, branch string, path string) ([]string, error) {
28 | return nil, nil
29 | }
30 |
31 | func (m *mockGitProvider) SetWebhook(ctx *context.Context, repo *string) (*git_provider.HookWithStatus, error) {
32 | return nil, nil
33 | }
34 |
35 | func (m *mockGitProvider) UnsetWebhook(ctx *context.Context, hook *git_provider.HookWithStatus) error {
36 | return nil
37 | }
38 |
39 | func (m *mockGitProvider) HandlePayload(ctx *context.Context, request *http.Request, secret []byte) (*git_provider.WebhookPayload, error) {
40 | return nil, nil
41 | }
42 |
43 | func (m *mockGitProvider) SetStatus(ctx *context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error {
44 | return nil
45 | }
46 |
47 | func (m *mockGitProvider) PingHook(ctx *context.Context, hook *git_provider.HookWithStatus) error {
48 | return nil
49 | }
50 |
51 | func (m *mockGitProvider) GetHooks() []*git_provider.HookWithStatus {
52 | return nil
53 | }
54 |
55 | func TestNotify(t *testing.T) {
56 | assert := assertion.New(t)
57 | ctx := context.Background()
58 |
59 | // Define test cases
60 | tests := []struct {
61 | name string
62 | workflow *v1alpha1.Workflow
63 | wantedError error
64 | }{
65 | {
66 | name: "Succeeded workflow",
67 | workflow: &v1alpha1.Workflow{
68 | ObjectMeta: metav1.ObjectMeta{
69 | Name: "test-workflow",
70 | Labels: map[string]string{
71 | "repo": "test-repo",
72 | "commit": "test-commit",
73 | },
74 | },
75 | Status: v1alpha1.WorkflowStatus{
76 | Phase: v1alpha1.WorkflowSucceeded,
77 | Message: "",
78 | },
79 | },
80 | wantedError: nil,
81 | },
82 | {
83 | name: "Failed workflow",
84 | workflow: &v1alpha1.Workflow{
85 | ObjectMeta: metav1.ObjectMeta{
86 | Name: "test-workflow",
87 | Labels: map[string]string{
88 | "repo": "test-repo",
89 | "commit": "test-commit",
90 | },
91 | },
92 | Status: v1alpha1.WorkflowStatus{
93 | Phase: v1alpha1.WorkflowFailed,
94 | Message: "something",
95 | },
96 | },
97 | wantedError: nil,
98 | },
99 | {
100 | name: "Error workflow",
101 | workflow: &v1alpha1.Workflow{
102 | ObjectMeta: metav1.ObjectMeta{
103 | Name: "test-workflow",
104 | Labels: map[string]string{
105 | "repo": "test-repo",
106 | "commit": "test-commit",
107 | },
108 | },
109 | Status: v1alpha1.WorkflowStatus{
110 | Phase: v1alpha1.WorkflowError,
111 | Message: "something",
112 | },
113 | },
114 | wantedError: nil,
115 | },
116 | {
117 | name: "Pending workflow",
118 | workflow: &v1alpha1.Workflow{
119 | ObjectMeta: metav1.ObjectMeta{
120 | Name: "test-workflow",
121 | Labels: map[string]string{
122 | "repo": "test-repo",
123 | "commit": "test-commit",
124 | },
125 | },
126 | Status: v1alpha1.WorkflowStatus{
127 | Phase: v1alpha1.WorkflowPending,
128 | Message: "something",
129 | },
130 | },
131 | wantedError: nil,
132 | },
133 | {
134 | name: "Running workflow",
135 | workflow: &v1alpha1.Workflow{
136 | ObjectMeta: metav1.ObjectMeta{
137 | Name: "test-workflow",
138 | Labels: map[string]string{
139 | "repo": "test-repo",
140 | "commit": "test-commit",
141 | },
142 | },
143 | Status: v1alpha1.WorkflowStatus{
144 | Phase: v1alpha1.WorkflowRunning,
145 | Message: "something",
146 | },
147 | },
148 | wantedError: nil,
149 | },
150 | {
151 | name: "Missing label repo",
152 | workflow: &v1alpha1.Workflow{
153 | ObjectMeta: metav1.ObjectMeta{
154 | Name: "test-workflow",
155 | Labels: map[string]string{
156 | "commit": "test-commit",
157 | },
158 | },
159 | Status: v1alpha1.WorkflowStatus{
160 | Phase: v1alpha1.WorkflowSucceeded,
161 | Message: "something",
162 | },
163 | },
164 | wantedError: errors.New("some error"),
165 | },
166 | {
167 | name: "Missing label commit",
168 | workflow: &v1alpha1.Workflow{
169 | ObjectMeta: metav1.ObjectMeta{
170 | Name: "test-workflow",
171 | Labels: map[string]string{
172 | "repo": "test-repo",
173 | },
174 | },
175 | Status: v1alpha1.WorkflowStatus{
176 | Phase: v1alpha1.WorkflowSucceeded,
177 | Message: "something",
178 | },
179 | },
180 | wantedError: errors.New("some error"),
181 | },
182 | }
183 |
184 | // Create a mock configuration and clients
185 | cfg := &conf.GlobalConfig{
186 | GitProviderConfig: conf.GitProviderConfig{Provider: "github"},
187 | WorkflowServerConfig: conf.WorkflowServerConfig{
188 | ArgoAddress: "http://workflow-server",
189 | Namespace: "test-namespace",
190 | },
191 | }
192 | globalClients := &clients.Clients{
193 | GitProvider: &mockGitProvider{},
194 | }
195 |
196 | // Create a new githubNotifier instance
197 | gn := NewGithubEventNotifier(cfg, globalClients)
198 |
199 | // Call the Notify method
200 |
201 | // Run test cases
202 | for _, test := range tests {
203 | t.Run(test.name, func(t *testing.T) {
204 | // Call the function being tested
205 | err := gn.Notify(&ctx, test.workflow)
206 |
207 | // Use assert to check the equality of the error
208 | if test.wantedError != nil {
209 | assert.Error(err)
210 | assert.NotNil(err)
211 | } else {
212 | assert.NoError(err)
213 | assert.Nil(err)
214 | }
215 | })
216 | }
217 | }
218 |
--------------------------------------------------------------------------------
/pkg/workflow_handler/workflows_utils_test.go:
--------------------------------------------------------------------------------
1 | package workflow_handler
2 |
3 | import (
4 | "fmt"
5 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
6 | "github.com/rookout/piper/pkg/git_provider"
7 | assertion "github.com/stretchr/testify/assert"
8 | "testing"
9 | )
10 |
11 | func TestAddFilesToTemplates(t *testing.T) {
12 | assert := assertion.New(t)
13 |
14 | template := make([]v1alpha1.Template, 0)
15 | files := make([]*git_provider.CommitFile, 0)
16 |
17 | content := `
18 | - name: local-step
19 | inputs:
20 | parameters:
21 | - name: message
22 | script:
23 | image: alpine
24 | command: [ sh ]
25 | source: |
26 | echo "wellcome to {{ workflow.parameters.global }}
27 | echo "{{ inputs.parameters.message }}"
28 | `
29 | path := "path"
30 | files = append(files, &git_provider.CommitFile{
31 | Content: &content,
32 | Path: &path,
33 | })
34 |
35 | template, err := AddFilesToTemplates(template, files)
36 |
37 | assert.Nil(err)
38 | assert.Equal("alpine", template[0].Script.Container.Image)
39 | assert.Equal([]string{"sh"}, template[0].Script.Command)
40 | expectedScript := "echo \"wellcome to {{ workflow.parameters.global }}\necho \"{{ inputs.parameters.message }}\"\n"
41 | assert.Equal(expectedScript, template[0].Script.Source)
42 | }
43 | func TestValidateDAGTasks(t *testing.T) {
44 | assert := assertion.New(t)
45 | // Define test cases
46 | tests := []struct {
47 | name string
48 | tasks []v1alpha1.DAGTask
49 | want error
50 | }{
51 | {
52 | name: "Valid tasks",
53 | tasks: []v1alpha1.DAGTask{
54 | {Name: "Task1", Template: "Template1"},
55 | {Name: "Task2", TemplateRef: &v1alpha1.TemplateRef{Name: "Template2"}},
56 | },
57 | want: nil,
58 | },
59 | {
60 | name: "Empty task name",
61 | tasks: []v1alpha1.DAGTask{
62 | {Name: "", Template: "Template1"},
63 | },
64 | want: fmt.Errorf("task name cannot be empty"),
65 | },
66 | {
67 | name: "Empty template and templateRef",
68 | tasks: []v1alpha1.DAGTask{
69 | {Name: "Task1"},
70 | },
71 | want: fmt.Errorf("task template or templateRef cannot be empty"),
72 | },
73 | }
74 |
75 | // Run test cases
76 | for _, test := range tests {
77 | t.Run(test.name, func(t *testing.T) {
78 | // Call the function being tested
79 | got := ValidateDAGTasks(test.tasks)
80 |
81 | // Use assert to check the equality of the error
82 | if test.want != nil {
83 | assert.Error(got)
84 | assert.NotNil(got)
85 | } else {
86 | assert.NoError(got)
87 | assert.Nil(got)
88 | }
89 | })
90 | }
91 | }
92 |
93 | func TestCreateDAGTemplate(t *testing.T) {
94 | assert := assertion.New(t)
95 | t.Run("Empty file list", func(t *testing.T) {
96 | fileList := []*git_provider.CommitFile{}
97 | name := "template1"
98 | template, err := CreateDAGTemplate(fileList, name)
99 | assert.Nil(template)
100 | assert.Nil(err)
101 | })
102 |
103 | t.Run("Missing content or path", func(t *testing.T) {
104 | file := &git_provider.CommitFile{
105 | Content: nil,
106 | Path: nil,
107 | }
108 | fileList := []*git_provider.CommitFile{file}
109 | name := "template2"
110 | template, err := CreateDAGTemplate(fileList, name)
111 | assert.Nil(template)
112 | assert.NotNil(err)
113 | })
114 |
115 | t.Run("Valid file list", func(t *testing.T) {
116 | path3 := "some-path"
117 | content3 := `- name: local-step1
118 | template: local-step
119 | arguments:
120 | parameters:
121 | - name: message
122 | value: step-1
123 | - name: local-step2
124 | templateRef:
125 | name: common-toolkit
126 | template: versioning
127 | clusterScope: true
128 | arguments:
129 | parameters:
130 | - name: message
131 | value: step-2
132 | dependencies:
133 | - local-step1`
134 | file := &git_provider.CommitFile{
135 | Content: &content3,
136 | Path: &path3,
137 | }
138 | fileList := []*git_provider.CommitFile{file}
139 | name := "template3"
140 | template, err := CreateDAGTemplate(fileList, name)
141 |
142 | assert.Nil(err)
143 | assert.NotNil(template)
144 |
145 | assert.Equal(name, template.Name)
146 | assert.NotNil(template.DAG)
147 | assert.Equal(2, len(template.DAG.Tasks))
148 |
149 | assert.NotNil(template.DAG.Tasks[0])
150 | assert.Equal("local-step1", template.DAG.Tasks[0].Name)
151 | assert.Equal("local-step", template.DAG.Tasks[0].Template)
152 | assert.Equal(1, len(template.DAG.Tasks[0].Arguments.Parameters))
153 | assert.Equal("message", template.DAG.Tasks[0].Arguments.Parameters[0].Name)
154 | assert.Equal("step-1", template.DAG.Tasks[0].Arguments.Parameters[0].Value.String())
155 |
156 | assert.NotNil(template.DAG.Tasks[1])
157 | assert.Equal("local-step2", template.DAG.Tasks[1].Name)
158 | assert.Equal(1, len(template.DAG.Tasks[1].Dependencies))
159 | assert.Equal("local-step1", template.DAG.Tasks[1].Dependencies[0])
160 | assert.NotNil(template.DAG.Tasks[1].TemplateRef)
161 | assert.Equal("common-toolkit", template.DAG.Tasks[1].TemplateRef.Name)
162 | assert.Equal("versioning", template.DAG.Tasks[1].TemplateRef.Template)
163 | assert.True(template.DAG.Tasks[1].TemplateRef.ClusterScope)
164 | })
165 |
166 | t.Run("Invalid configuration", func(t *testing.T) {
167 | path4 := "some-path"
168 | content4 := `- noName: local-step1
169 | wrongkey2: local-step
170 | - noName: local-step2
171 | wrongkey: something
172 | dependencies:
173 | - local-step1`
174 | file := &git_provider.CommitFile{
175 | Content: &content4,
176 | Path: &path4,
177 | }
178 | fileList := []*git_provider.CommitFile{file}
179 | name := "template4"
180 | template, err := CreateDAGTemplate(fileList, name)
181 |
182 | assert.Nil(template)
183 | assert.NotNil(err)
184 | })
185 |
186 | t.Run("YAML syntax error", func(t *testing.T) {
187 | path5 := "some-path"
188 | content5 := `- noName: local-step1
189 | wrongkey2: local-step
190 | error: should be list`
191 | file := &git_provider.CommitFile{
192 | Content: &content5,
193 | Path: &path5,
194 | }
195 | fileList := []*git_provider.CommitFile{file}
196 | name := "template5"
197 | template, err := CreateDAGTemplate(fileList, name)
198 |
199 | assert.Nil(template)
200 | assert.NotNil(err)
201 | })
202 | }
203 |
204 | func TestConvertToValidString(t *testing.T) {
205 | assert := assertion.New(t)
206 |
207 | tests := []struct {
208 | input string
209 | expected string
210 | }{
211 | {"A@bC!-123.def", "abc-123.def"},
212 | {"Hello World!", "helloworld"},
213 | {"123$%^", "123"},
214 | {"abc_123.xyz", "abc-123.xyz"}, // Underscore (_) should be converted to hyphen (-)
215 | {"..--..", "..--.."}, // Only dots (.) and hyphens (-) should remain
216 | }
217 |
218 | for _, test := range tests {
219 | t.Run(test.input, func(t *testing.T) {
220 | converted := ConvertToValidString(test.input)
221 | assert.Equal(converted, test.expected)
222 | })
223 | }
224 | }
225 |
--------------------------------------------------------------------------------
/pkg/webhook_handler/webhook_handler.go:
--------------------------------------------------------------------------------
1 | package webhook_handler
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "github.com/rookout/piper/pkg/clients"
7 | "github.com/rookout/piper/pkg/common"
8 | "github.com/rookout/piper/pkg/conf"
9 | "github.com/rookout/piper/pkg/git_provider"
10 | "github.com/rookout/piper/pkg/utils"
11 | "gopkg.in/yaml.v3"
12 | "log"
13 | )
14 |
15 | type WebhookHandlerImpl struct {
16 | cfg *conf.GlobalConfig
17 | clients *clients.Clients
18 | Triggers *[]Trigger
19 | Payload *git_provider.WebhookPayload
20 | }
21 |
22 | func NewWebhookHandler(cfg *conf.GlobalConfig, clients *clients.Clients, payload *git_provider.WebhookPayload) (*WebhookHandlerImpl, error) {
23 | var err error
24 |
25 | return &WebhookHandlerImpl{
26 | cfg: cfg,
27 | clients: clients,
28 | Triggers: &[]Trigger{},
29 | Payload: payload,
30 | }, err
31 | }
32 |
33 | func (wh *WebhookHandlerImpl) RegisterTriggers(ctx *context.Context) error {
34 | if !IsFileExists(ctx, wh, "", ".workflows") {
35 | return fmt.Errorf(".workflows folder does not exist in %s/%s", wh.Payload.Repo, wh.Payload.Branch)
36 | }
37 |
38 | if !IsFileExists(ctx, wh, ".workflows", "triggers.yaml") {
39 | return fmt.Errorf(".workflows/triggers.yaml file does not exist in %s/%s", wh.Payload.Repo, wh.Payload.Branch)
40 | }
41 |
42 | triggers, err := wh.clients.GitProvider.GetFile(ctx, wh.Payload.Repo, wh.Payload.Branch, ".workflows/triggers.yaml")
43 | if err != nil {
44 | return fmt.Errorf("failed to get triggers content: %v", err)
45 | }
46 |
47 | log.Printf("triggers content is: \n %s \n", *triggers.Content) // DEBUG
48 |
49 | err = yaml.Unmarshal([]byte(*triggers.Content), wh.Triggers)
50 | if err != nil {
51 | return fmt.Errorf("failed to unmarshal triggers content: %v", err)
52 | }
53 | return nil
54 | }
55 |
56 | func (wh *WebhookHandlerImpl) PrepareBatchForMatchingTriggers(ctx *context.Context) ([]*common.WorkflowsBatch, error) {
57 | triggered := false
58 | var workflowBatches []*common.WorkflowsBatch
59 | for _, trigger := range *wh.Triggers {
60 | if trigger.Branches == nil {
61 | return nil, fmt.Errorf("trigger from repo %s branch %s missing branch field", wh.Payload.Repo, wh.Payload.Branch)
62 | }
63 | if trigger.Events == nil {
64 | return nil, fmt.Errorf("trigger from repo %s branch %s missing event field", wh.Payload.Repo, wh.Payload.Branch)
65 | }
66 |
67 | eventToCheck := wh.Payload.Event
68 | if wh.Payload.Action != "" {
69 | eventToCheck += "." + wh.Payload.Action
70 | }
71 | if utils.IsElementMatch(wh.Payload.Branch, *trigger.Branches) && utils.IsElementMatch(eventToCheck, *trigger.Events) {
72 | log.Printf(
73 | "Triggering event %s for repo %s branch %s are triggered.",
74 | wh.Payload.Event,
75 | wh.Payload.Repo,
76 | wh.Payload.Branch,
77 | )
78 | triggered = true
79 | onStartFiles, err := wh.clients.GitProvider.GetFiles(
80 | ctx,
81 | wh.Payload.Repo,
82 | wh.Payload.Branch,
83 | utils.AddPrefixToList(*trigger.OnStart, ".workflows/"),
84 | )
85 | if len(onStartFiles) == 0 {
86 | return nil, fmt.Errorf("one or more of onStart: %s files found in repo: %s branch %s", *trigger.OnStart, wh.Payload.Repo, wh.Payload.Branch)
87 | }
88 | if err != nil {
89 | return nil, err
90 | }
91 |
92 | onExitFiles := make([]*git_provider.CommitFile, 0)
93 | if trigger.OnExit != nil {
94 | onExitFiles, err = wh.clients.GitProvider.GetFiles(
95 | ctx,
96 | wh.Payload.Repo,
97 | wh.Payload.Branch,
98 | utils.AddPrefixToList(*trigger.OnExit, ".workflows/"),
99 | )
100 | if len(onExitFiles) == 0 {
101 | log.Printf("one or more of onExist: %s files not found in repo: %s branch %s", *trigger.OnExit, wh.Payload.Repo, wh.Payload.Branch)
102 | }
103 | if err != nil {
104 | return nil, err
105 | }
106 | }
107 |
108 | templatesFiles := make([]*git_provider.CommitFile, 0)
109 | if trigger.Templates != nil {
110 | templatesFiles, err = wh.clients.GitProvider.GetFiles(
111 | ctx,
112 | wh.Payload.Repo,
113 | wh.Payload.Branch,
114 | utils.AddPrefixToList(*trigger.Templates, ".workflows/"),
115 | )
116 | if len(templatesFiles) == 0 {
117 | log.Printf("one or more of templates: %s files not found in repo: %s branch %s", *trigger.Templates, wh.Payload.Repo, wh.Payload.Branch)
118 | }
119 | if err != nil {
120 | return nil, err
121 | }
122 | }
123 |
124 | parameters := &git_provider.CommitFile{
125 | Path: nil,
126 | Content: nil,
127 | }
128 | if IsFileExists(ctx, wh, ".workflows", "parameters.yaml") {
129 | parameters, err = wh.clients.GitProvider.GetFile(
130 | ctx,
131 | wh.Payload.Repo,
132 | wh.Payload.Branch,
133 | ".workflows/parameters.yaml",
134 | )
135 | if err != nil {
136 | return nil, err
137 | }
138 | } else {
139 | log.Printf("parameters.yaml not found in repo: %s branch %s", wh.Payload.Repo, wh.Payload.Branch)
140 | }
141 |
142 | workflowBatches = append(workflowBatches, &common.WorkflowsBatch{
143 | OnStart: onStartFiles,
144 | OnExit: onExitFiles,
145 | Templates: templatesFiles,
146 | Parameters: parameters,
147 | Config: &trigger.Config,
148 | Payload: wh.Payload,
149 | })
150 | }
151 | }
152 | if !triggered {
153 | return nil, fmt.Errorf("no matching trigger found for event: %s action: %s in branch :%s", wh.Payload.Event, wh.Payload.Action, wh.Payload.Branch)
154 | }
155 | return workflowBatches, nil
156 | }
157 |
158 | func IsFileExists(ctx *context.Context, wh *WebhookHandlerImpl, path string, file string) bool {
159 | files, err := wh.clients.GitProvider.ListFiles(ctx, wh.Payload.Repo, wh.Payload.Branch, path)
160 | if err != nil {
161 | log.Printf("Error listing files in repo: %s branch: %s. %v", wh.Payload.Repo, wh.Payload.Branch, err)
162 | return false
163 | }
164 | if len(files) == 0 {
165 | log.Printf("Empty list of files in repo: %s branch: %s", wh.Payload.Repo, wh.Payload.Branch)
166 | return false
167 | }
168 |
169 | if utils.IsElementExists(files, file) {
170 | return true
171 | }
172 |
173 | return false
174 | }
175 |
176 | func HandleWebhook(ctx *context.Context, wh *WebhookHandlerImpl) ([]*common.WorkflowsBatch, error) {
177 | err := wh.RegisterTriggers(ctx)
178 | if err != nil {
179 | return nil, fmt.Errorf("failed to register triggers, error: %v", err)
180 | } else {
181 | log.Printf("successfully registered triggers for repo: %s branch: %s", wh.Payload.Repo, wh.Payload.Branch)
182 | }
183 |
184 | workflowsBatches, err := wh.PrepareBatchForMatchingTriggers(ctx)
185 | if err != nil {
186 | return nil, fmt.Errorf("failed to prepare matching triggers, error: %v", err)
187 | }
188 |
189 | if len(workflowsBatches) == 0 {
190 | log.Printf("no workflows to execute")
191 | return nil, fmt.Errorf("no workflows to execute for repo: %s branch: %s",
192 | wh.Payload.Repo,
193 | wh.Payload.Branch,
194 | )
195 | }
196 | return workflowsBatches, nil
197 | }
198 |
--------------------------------------------------------------------------------
/helm-chart/values.yaml:
--------------------------------------------------------------------------------
1 | # Default values for Piper.
2 | # For more information head to https:/github.com/rookout/piper
3 |
4 | # Map of Piper configurations.
5 | piper:
6 | gitProvider:
7 | # -- Name of your git provider (github/bitbucket).
8 | name: github
9 | # -- The token for authentication with the Git provider.
10 | # -- This will create a secret named -git-token and with the key 'token'
11 | token:
12 | # -- The token for authentication with the Git provider.
13 | # -- Reference to existing token with 'token' key.
14 | # -- can be created with `kubectl create secret generic piper-git-token --from-literal=token=YOUR_TOKEN`
15 | existingSecret: #piper-git-token
16 | # Map of organization configurations.
17 | organization:
18 | # -- Name of your Git Organization (GitHub) or Workspace (Bitbucket)
19 | name: ""
20 | # Map of webhook configurations.
21 | webhook:
22 | # -- The secret that will be used for webhook authentication
23 | # -- If not provided, will be generated
24 | # -- This will create a secret named -webhook-secret and with the key 'secret'
25 | secret: ""
26 | # -- The secret for webhook encryption
27 | # -- Reference to existing token with 'secret' key.
28 | # -- can be created with `kubectl create secret generic piper-webhook-secret --from-literal=secret=YOUR_TOKEN`
29 | existingSecret: #piper-webhook-secret
30 | # -- The url in which piper listens for webhook, the path should be /webhook
31 | url: "" #https://piper.example.local/webhook
32 | # -- Whether config webhook on org level (GitHub) or at workspace level (Bitbucket - not supported yet)
33 | orgLevel: false
34 | # -- (Github) Used of orgLevel=false, to configure webhook for each of the repos provided.
35 | repoList: []
36 |
37 | # Map of Argo Workflows configurations.
38 | argoWorkflows:
39 | # Map of Argo Workflows server configurations.
40 | server:
41 | # -- The namespace in which the Workflow CRD will be created.
42 | namespace: ""
43 | # -- The DNS address of Argo Workflow server that Piper can address.
44 | address: ""
45 | # -- The token for authentication with Argo Workflows server.
46 | # -- This will create a secret named -token and with the key 'token'
47 | token: ""
48 | # -- The token for authentication with Argo Workflows server.
49 | # -- Reference to existing token with 'token' key.
50 | # -- can be created with `kubectl create secret generic piper-argo-token --from-literal=token=YOUR_TOKEN`
51 | existingSecret: #piper-argo-token
52 | # -- Whether create Workflow CRD or send direct commands to Argo Workflows server.
53 | crdCreation: true
54 |
55 | workflowsConfig: {}
56 | # default: |
57 | # spec:
58 | # volumes:
59 | # - name: shared-volume
60 | # emptyDir: {}
61 | # serviceAccountName: argo-wf
62 | # activeDeadlineSeconds: 7200 # (seconds) == 2 hours
63 | # ttlStrategy:
64 | # secondsAfterCompletion: 28800 # (seconds) == 8 hours
65 | # podGC:
66 | # strategy: OnPodSuccess
67 | # archiveLogs: true
68 | # artifactRepositoryRef:
69 | # configMap: artifact-repositories
70 | # nodeSelector:
71 | # node_pool: workflows
72 | # tolerations:
73 | # - effect: NoSchedule
74 | # key: node_pool
75 | # operator: Equal
76 | # value: workflows
77 | # onExit: # optinal, will be overwritten if specifc in .wokrflows/exit.yaml.
78 | # - name: github-status
79 | # template: exit-handler
80 | # arguments:
81 | # parameters:
82 | # - name: param1
83 | # value: "{{ workflow.labels.repo }}"
84 |
85 | rookout:
86 | # -- Rookout token for agent configuration and enablement.
87 | token: ""
88 | # -- The token for Rookout.
89 | # -- Reference to existing token with 'token' key.
90 | # -- can be created with `kubectl create secret generic piper-rookout-token --from-literal=token=YOUR_TOKEN`
91 | existingSecret: ""
92 |
93 | # -- Piper number of replicas
94 | replicaCount: 1
95 |
96 | image:
97 | # -- Piper image name
98 | name: piper
99 | # -- Piper public dockerhub repo
100 | repository: rookout
101 | # -- Piper image pull policy
102 | pullPolicy: IfNotPresent
103 | # -- Piper image tag
104 | tag: ""
105 |
106 | # -- secret to use for image pulling
107 | imagePullSecrets: []
108 |
109 | # -- String to partially override "piper.fullname" template
110 | nameOverride: ""
111 |
112 | # -- String to fully override "piper.fullname" template
113 | fullnameOverride: ""
114 |
115 | serviceAccount:
116 | # -- Specifies whether a service account should be created
117 | create: true
118 | # -- Annotations to add to the service account
119 | annotations: {}
120 | # -- The name of the service account to use.
121 | # If not set and create is true, a name is generated using the fullname template
122 | name: ""
123 |
124 | # -- Annotations to be added to the Piper pods
125 | podAnnotations: {}
126 |
127 | # -- Security Context to set on the pod level
128 | podSecurityContext:
129 | fsGroup: 1001
130 | runAsUser: 1001
131 | runAsGroup: 1001
132 |
133 | # -- Security Context to set on the container level
134 | securityContext:
135 | runAsUser: 1001
136 | capabilities:
137 | drop:
138 | - ALL
139 | readOnlyRootFilesystem: true
140 | runAsNonRoot: true
141 |
142 |
143 | service:
144 | # -- Sets the type of the Service
145 | type: ClusterIP
146 | # -- Service port
147 | # For TLS mode change the port to 443
148 | port: 80
149 | #nodePort:
150 | # -- Piper service extra labels
151 | labels: {}
152 | # -- Piper service extra annotations
153 | annotations: {}
154 |
155 | ingress:
156 | # -- Enable Piper ingress support
157 | enabled: false
158 | # -- Piper ingress class name
159 | className: ""
160 | # -- Piper ingress annotations
161 | annotations: {}
162 | # kubernetes.io/ingress.class: nginx
163 | # kubernetes.io/tls-acme: "true"
164 |
165 | # -- Piper ingress hosts
166 | ## Hostnames must be provided if Ingress is enabled.
167 | hosts:
168 | - host: piper.example.local
169 | paths:
170 | - path: /
171 | pathType: ImplementationSpecific
172 | # -- Controller ingress tls
173 | tls: []
174 | # - secretName: chart-example-tls
175 | # hosts:
176 | # - chart-example.local
177 |
178 | # -- Additional environment variables for Piper. A list of name/value maps.
179 | env: []
180 |
181 | # -- Resource limits and requests for the pods.
182 | resources:
183 | requests:
184 | cpu: 200m
185 | memory: 512Mi
186 |
187 | # -- [Node selector]
188 | nodeSelector: {}
189 |
190 | # -- [Tolerations] for use with node taints
191 | tolerations: []
192 |
193 | # -- Assign custom [affinity] rules to the deployment
194 | affinity: {}
195 |
196 | # -- Deployment and pods extra labels
197 | extraLabels: {}
198 |
199 | autoscaling:
200 | # -- Wheter to enable auto-scaling of piper.
201 | enabled: false
202 | # -- Minimum reoplicas of Piper.
203 | minReplicas: 1
204 | # -- Maximum reoplicas of Piper.
205 | maxReplicas: 5
206 | # -- CPU utilization percentage threshold.
207 | targetCPUUtilizationPercentage: 85
208 | # -- Memory utilization percentage threshold.
209 | targetMemoryUtilizationPercentage: 85
210 |
211 | # -- Volumes of Piper Pod.
212 | volumes: []
213 |
214 | # -- Volumes to mount to Piper container.
215 | volumeMounts: []
216 |
217 | # -- Specify postStart and preStop lifecycle hooks for Piper container
218 | lifecycle: {}
--------------------------------------------------------------------------------
/pkg/workflow_handler/workflows.go:
--------------------------------------------------------------------------------
1 | package workflow_handler
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
8 | wfClientSet "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned"
9 | "k8s.io/apimachinery/pkg/apis/meta/v1"
10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11 | "k8s.io/apimachinery/pkg/types"
12 | "k8s.io/apimachinery/pkg/watch"
13 | "log"
14 | "strings"
15 |
16 | "github.com/rookout/piper/pkg/common"
17 | "github.com/rookout/piper/pkg/conf"
18 | "github.com/rookout/piper/pkg/utils"
19 | )
20 |
21 | const (
22 | ENTRYPOINT = "entryPoint"
23 | ONEXIT = "exitHandler"
24 | )
25 |
26 | type WorkflowsClientImpl struct {
27 | clientSet *wfClientSet.Clientset
28 | cfg *conf.GlobalConfig
29 | }
30 |
31 | func NewWorkflowsClient(cfg *conf.GlobalConfig) (WorkflowsClient, error) {
32 | restClientConfig, err := utils.GetClientConfig(cfg.WorkflowServerConfig.KubeConfig)
33 | if err != nil {
34 | return nil, err
35 | }
36 |
37 | clientSet := wfClientSet.NewForConfigOrDie(restClientConfig) //.ArgoprojV1alpha1().Workflows(namespace)
38 | return &WorkflowsClientImpl{
39 | clientSet: clientSet,
40 | cfg: cfg,
41 | }, nil
42 | }
43 |
44 | func (wfc *WorkflowsClientImpl) ConstructTemplates(workflowsBatch *common.WorkflowsBatch, configName string) ([]v1alpha1.Template, error) {
45 | finalTemplate := make([]v1alpha1.Template, 0)
46 | onStart, err := CreateDAGTemplate(workflowsBatch.OnStart, ENTRYPOINT)
47 | if err != nil {
48 | return nil, err
49 | }
50 | finalTemplate = append(finalTemplate, *onStart)
51 |
52 | onExit, err := CreateDAGTemplate(workflowsBatch.OnExit, ONEXIT)
53 | if err != nil {
54 | return nil, err
55 | }
56 | if onExit == nil || len(onExit.DAG.Tasks) == 0 {
57 | if IsConfigExists(&wfc.cfg.WorkflowsConfig, configName) && IsConfigsOnExitExists(&wfc.cfg.WorkflowsConfig, configName) {
58 | template := &v1alpha1.Template{
59 | Name: ONEXIT,
60 | DAG: &v1alpha1.DAGTemplate{
61 | Tasks: wfc.cfg.WorkflowsConfig.Configs[configName].OnExit,
62 | },
63 | }
64 |
65 | finalTemplate = append(finalTemplate, *template)
66 | }
67 | } else {
68 | finalTemplate = append(finalTemplate, *onExit)
69 | }
70 |
71 | finalTemplate, err = AddFilesToTemplates(finalTemplate, workflowsBatch.Templates)
72 | if err != nil {
73 | return nil, err
74 | }
75 |
76 | return finalTemplate, nil
77 | }
78 |
79 | func (wfc *WorkflowsClientImpl) ConstructSpec(templates []v1alpha1.Template, params []v1alpha1.Parameter, configName string) (*v1alpha1.WorkflowSpec, error) {
80 | finalSpec := &v1alpha1.WorkflowSpec{}
81 | if IsConfigExists(&wfc.cfg.WorkflowsConfig, configName) {
82 | *finalSpec = wfc.cfg.WorkflowsConfig.Configs[configName].Spec
83 | if len(wfc.cfg.WorkflowsConfig.Configs[configName].OnExit) != 0 {
84 | finalSpec.OnExit = ONEXIT
85 | }
86 | }
87 |
88 | finalSpec.Entrypoint = ENTRYPOINT
89 | finalSpec.Templates = templates
90 | finalSpec.Arguments.Parameters = params
91 |
92 | return finalSpec, nil
93 | }
94 |
95 | func (wfc *WorkflowsClientImpl) CreateWorkflow(spec *v1alpha1.WorkflowSpec, workflowsBatch *common.WorkflowsBatch) (*v1alpha1.Workflow, error) {
96 | workflow := &v1alpha1.Workflow{
97 | ObjectMeta: metav1.ObjectMeta{
98 | GenerateName: ConvertToValidString(workflowsBatch.Payload.Repo + "-" + workflowsBatch.Payload.Branch + "-"),
99 | Namespace: wfc.cfg.Namespace,
100 | Labels: map[string]string{
101 | "piper.rookout.com/notified": "false",
102 | "repo": ConvertToValidString(workflowsBatch.Payload.Repo),
103 | "branch": ConvertToValidString(workflowsBatch.Payload.Branch),
104 | "user": ConvertToValidString(workflowsBatch.Payload.User),
105 | "commit": ConvertToValidString(workflowsBatch.Payload.Commit),
106 | },
107 | },
108 | Spec: *spec,
109 | }
110 |
111 | return workflow, nil
112 | }
113 |
114 | func (wfc *WorkflowsClientImpl) SelectConfig(workflowsBatch *common.WorkflowsBatch) (string, error) {
115 | var configName string
116 | if IsConfigExists(&wfc.cfg.WorkflowsConfig, "default") {
117 | configName = "default"
118 | }
119 |
120 | if *workflowsBatch.Config != "" {
121 | if IsConfigExists(&wfc.cfg.WorkflowsConfig, *workflowsBatch.Config) {
122 | configName = *workflowsBatch.Config
123 | } else {
124 | return configName, fmt.Errorf(
125 | "error in selecting config, staying with default config for repo %s branch %s",
126 | workflowsBatch.Payload.Repo,
127 | workflowsBatch.Payload.Branch,
128 | )
129 | }
130 | }
131 |
132 | log.Printf(
133 | "%s config selected for workflow in repo: %s branch %s",
134 | configName,
135 | workflowsBatch.Payload.Repo,
136 | workflowsBatch.Payload.Branch,
137 | ) // Info
138 |
139 | return configName, nil
140 | }
141 |
142 | func (wfc *WorkflowsClientImpl) Lint(wf *v1alpha1.Workflow) error {
143 | //TODO implement me
144 | panic("implement me")
145 | }
146 |
147 | func (wfc *WorkflowsClientImpl) Submit(ctx *context.Context, wf *v1alpha1.Workflow) error {
148 | workflowsClient := wfc.clientSet.ArgoprojV1alpha1().Workflows(wfc.cfg.Namespace)
149 | _, err := workflowsClient.Create(*ctx, wf, metav1.CreateOptions{})
150 | if err != nil {
151 | return err
152 | }
153 |
154 | return nil
155 | }
156 |
157 | func (wfc *WorkflowsClientImpl) HandleWorkflowBatch(ctx *context.Context, workflowsBatch *common.WorkflowsBatch) error {
158 | var params []v1alpha1.Parameter
159 |
160 | configName, err := wfc.SelectConfig(workflowsBatch)
161 | if err != nil {
162 | return err
163 | }
164 |
165 | templates, err := wfc.ConstructTemplates(workflowsBatch, configName)
166 | if err != nil {
167 | return err
168 | }
169 |
170 | if workflowsBatch.Parameters != nil {
171 | params, err = GetParameters(workflowsBatch.Parameters)
172 | if err != nil {
173 | return err
174 | }
175 | }
176 |
177 | globalParams := []v1alpha1.Parameter{
178 | {Name: "event", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Event)},
179 | {Name: "action", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Action)},
180 | {Name: "repo", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Repo)},
181 | {Name: "branch", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Branch)},
182 | {Name: "commit", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Commit)},
183 | {Name: "user", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.User)},
184 | {Name: "user_email", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.UserEmail)},
185 | {Name: "pull_request_url", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.PullRequestURL)},
186 | {Name: "pull_request_title", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.PullRequestTitle)},
187 | {Name: "dest_branch", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.DestBranch)},
188 | {Name: "pull_request_labels", Value: v1alpha1.AnyStringPtr(strings.Join(workflowsBatch.Payload.Labels, ","))},
189 | }
190 |
191 | params = append(params, globalParams...)
192 |
193 | spec, err := wfc.ConstructSpec(templates, params, configName)
194 | if err != nil {
195 | return err
196 | }
197 |
198 | workflow, err := wfc.CreateWorkflow(spec, workflowsBatch)
199 | if err != nil {
200 | return err
201 | }
202 |
203 | err = wfc.Submit(ctx, workflow)
204 | if err != nil {
205 | return fmt.Errorf("failed to submit workflow, error: %v", err)
206 | }
207 |
208 | log.Printf("submit workflow for branch %s repo %s commit %s", workflowsBatch.Payload.Branch, workflowsBatch.Payload.Repo, workflowsBatch.Payload.Commit)
209 | return nil
210 | }
211 |
212 | func (wfc *WorkflowsClientImpl) Watch(ctx *context.Context, labelSelector *metav1.LabelSelector) (watch.Interface, error) {
213 | workflowsClient := wfc.clientSet.ArgoprojV1alpha1().Workflows(wfc.cfg.Namespace)
214 | opts := v1.ListOptions{
215 | Watch: true,
216 | LabelSelector: metav1.FormatLabelSelector(labelSelector),
217 | }
218 | watcher, err := workflowsClient.Watch(*ctx, opts)
219 | if err != nil {
220 | return nil, err
221 | }
222 |
223 | return watcher, nil
224 | }
225 |
226 | func (wfc *WorkflowsClientImpl) UpdatePiperWorkflowLabel(ctx *context.Context, workflowName string, label string, value string) error {
227 | workflowsClient := wfc.clientSet.ArgoprojV1alpha1().Workflows(wfc.cfg.Namespace)
228 |
229 | patch, err := json.Marshal(map[string]interface{}{"metadata": metav1.ObjectMeta{
230 | Labels: map[string]string{
231 | fmt.Sprintf("piper.rookout.com/%s", label): value,
232 | },
233 | }})
234 | if err != nil {
235 | return err
236 | }
237 | _, err = workflowsClient.Patch(*ctx, workflowName, types.MergePatchType, patch, v1.PatchOptions{})
238 | if err != nil {
239 | return err
240 | }
241 |
242 | fmt.Printf("workflow %s labels piper.rookout.com/%s updated to %s\n", workflowName, label, value)
243 | return nil
244 | }
245 |
--------------------------------------------------------------------------------
/pkg/git_provider/bitbucket.go:
--------------------------------------------------------------------------------
1 | package git_provider
2 |
3 | import (
4 | "bytes"
5 | "encoding/json"
6 | "fmt"
7 | "github.com/ktrysmt/go-bitbucket"
8 | "github.com/rookout/piper/pkg/conf"
9 | "github.com/rookout/piper/pkg/utils"
10 | "github.com/tidwall/gjson"
11 | "golang.org/x/net/context"
12 | "io"
13 | "log"
14 | "net/http"
15 | "strings"
16 | )
17 |
18 | type BitbucketClientImpl struct {
19 | client *bitbucket.Client
20 | cfg *conf.GlobalConfig
21 | HooksHashTable map[string]int64
22 | }
23 |
24 | func NewBitbucketServerClient(cfg *conf.GlobalConfig) (Client, error) {
25 | client := bitbucket.NewOAuthbearerToken(cfg.GitProviderConfig.Token)
26 |
27 | err := ValidateBitbucketPermissions(client, cfg)
28 | if err != nil {
29 | return nil, err
30 | }
31 |
32 | return &BitbucketClientImpl{
33 | client: client,
34 | cfg: cfg,
35 | HooksHashTable: make(map[string]int64),
36 | }, err
37 | }
38 |
39 | func (b BitbucketClientImpl) ListFiles(ctx *context.Context, repo string, branch string, path string) ([]string, error) {
40 | var filesList []string
41 | fileOptions := bitbucket.RepositoryFilesOptions{
42 | Owner: b.cfg.GitProviderConfig.OrgName,
43 | RepoSlug: repo,
44 | Ref: branch,
45 | Path: path,
46 | MaxDepth: 0,
47 | }
48 | files, err := b.client.Repositories.Repository.ListFiles(&fileOptions)
49 | if err != nil {
50 | return nil, err
51 | }
52 |
53 | for _, f := range files {
54 | fileWithoutPath := strings.ReplaceAll(f.Path, path+"/", "")
55 | filesList = append(filesList, fileWithoutPath)
56 | }
57 |
58 | return filesList, nil
59 | }
60 |
61 | func (b BitbucketClientImpl) GetFile(ctx *context.Context, repo string, branch string, path string) (*CommitFile, error) {
62 | fileOptions := bitbucket.RepositoryFilesOptions{
63 | Owner: b.cfg.GitProviderConfig.OrgName,
64 | RepoSlug: repo,
65 | Ref: branch,
66 | Path: path,
67 | MaxDepth: 0,
68 | }
69 | fileContent, err := b.client.Repositories.Repository.GetFileContent(&fileOptions)
70 | if err != nil {
71 | return nil, err
72 | }
73 |
74 | stringContent := string(fileContent[:])
75 | return &CommitFile{
76 | Path: &path,
77 | Content: &stringContent,
78 | }, nil
79 | }
80 |
81 | func (b BitbucketClientImpl) GetFiles(ctx *context.Context, repo string, branch string, paths []string) ([]*CommitFile, error) {
82 | var commitFiles []*CommitFile
83 | for _, path := range paths {
84 | file, err := b.GetFile(ctx, repo, branch, path)
85 | if err != nil {
86 | return nil, err
87 | }
88 | if file == nil {
89 | log.Printf("file %s not found in repo %s branch %s", path, repo, branch)
90 | continue
91 | }
92 | commitFiles = append(commitFiles, file)
93 | }
94 | return commitFiles, nil
95 | }
96 |
97 | func (b BitbucketClientImpl) SetWebhook(ctx *context.Context, repo *string) (*HookWithStatus, error) {
98 | webhookOptions := &bitbucket.WebhooksOptions{
99 | Owner: b.cfg.GitProviderConfig.OrgName,
100 | RepoSlug: *repo,
101 | Uuid: "",
102 | Description: "Piper",
103 | Url: b.cfg.GitProviderConfig.WebhookURL,
104 | Active: true,
105 | Events: []string{"repo:push", "pullrequest:created", "pullrequest:updated", "pullrequest:fulfilled"},
106 | }
107 |
108 | hook, exists := b.isRepoWebhookExists(*repo)
109 | if exists {
110 | log.Printf("webhook already exists for repository %s, skipping creation... \n", *repo)
111 | addHookToHashTable(utils.RemoveBraces(hook.Uuid), b.HooksHashTable)
112 | hookID, err := getHookByUUID(utils.RemoveBraces(hook.Uuid), b.HooksHashTable)
113 | if err != nil {
114 | return nil, err
115 | }
116 | return &HookWithStatus{
117 | HookID: hookID,
118 | HealthStatus: true,
119 | RepoName: repo,
120 | }, nil
121 | }
122 |
123 | hook, err := b.client.Repositories.Webhooks.Create(webhookOptions)
124 | if err != nil {
125 | return nil, err
126 | }
127 | log.Printf("created webhook for repository %s \n", *repo)
128 |
129 | addHookToHashTable(utils.RemoveBraces(hook.Uuid), b.HooksHashTable)
130 | hookID, err := getHookByUUID(utils.RemoveBraces(hook.Uuid), b.HooksHashTable)
131 | if err != nil {
132 | return nil, err
133 | }
134 |
135 | return &HookWithStatus{
136 | HookID: hookID,
137 | HealthStatus: true,
138 | RepoName: repo,
139 | }, nil
140 | }
141 |
142 | func (b BitbucketClientImpl) UnsetWebhook(ctx *context.Context, hook *HookWithStatus) error {
143 | //TODO implement me
144 | panic("implement me")
145 | }
146 |
147 | func (b BitbucketClientImpl) HandlePayload(ctx *context.Context, request *http.Request, secret []byte) (*WebhookPayload, error) {
148 | var webhookPayload *WebhookPayload
149 | var buf bytes.Buffer
150 |
151 | // Used for authentication, hookID generated by bitbucket.
152 | hookID, err := getHookByUUID(request.Header.Get("X-Hook-UUID"), b.HooksHashTable)
153 | if err != nil {
154 | return nil, fmt.Errorf("failed to get hook by UUID, %s", err)
155 | }
156 |
157 | _, err = io.Copy(&buf, request.Body)
158 | if err != nil {
159 | return nil, fmt.Errorf("error reading response: %s", err)
160 | }
161 |
162 | var body map[string]interface{}
163 | err = json.Unmarshal(buf.Bytes(), &body)
164 | if err != nil {
165 | return nil, fmt.Errorf("failed to unmarshal response: %s", err)
166 | }
167 |
168 | // https://support.atlassian.com/bitbucket-cloud/docs/event-payloads
169 | switch request.Header.Get("X-Event-Key") {
170 | case "repo:push":
171 | if gjson.GetBytes(buf.Bytes(), "push.changes.0.new.type").Value().(string) == "tag" {
172 | webhookPayload = &WebhookPayload{
173 | Event: "tag",
174 | Repo: utils.SanitizeString(gjson.GetBytes(buf.Bytes(), "repository.name").Value().(string)),
175 | Branch: gjson.GetBytes(buf.Bytes(), "push.changes.0.new.name").Value().(string),
176 | Commit: gjson.GetBytes(buf.Bytes(), "push.changes.0.new.name").Value().(string),
177 | User: gjson.GetBytes(buf.Bytes(), "actor.display_name").Value().(string),
178 | HookID: hookID,
179 | }
180 | } else {
181 | webhookPayload = &WebhookPayload{
182 | Event: "push",
183 | Repo: utils.SanitizeString(gjson.GetBytes(buf.Bytes(), "repository.name").Value().(string)),
184 | Branch: gjson.GetBytes(buf.Bytes(), "push.changes.0.new.name").Value().(string),
185 | Commit: gjson.GetBytes(buf.Bytes(), "push.changes.0.commits.0.hash").Value().(string),
186 | UserEmail: utils.ExtractStringsBetweenTags(gjson.GetBytes(buf.Bytes(), "push.changes.0.commits.0.author.raw").Value().(string))[0],
187 | User: gjson.GetBytes(buf.Bytes(), "actor.display_name").Value().(string),
188 | HookID: hookID,
189 | }
190 | }
191 |
192 | case "pullrequest:created", "pullrequest:updated":
193 | webhookPayload = &WebhookPayload{
194 | Event: "pull_request",
195 | Repo: utils.SanitizeString(gjson.GetBytes(buf.Bytes(), "repository.name").Value().(string)),
196 | Branch: gjson.GetBytes(buf.Bytes(), "pullrequest.source.branch.name").Value().(string),
197 | Commit: gjson.GetBytes(buf.Bytes(), "pullrequest.source.commit.hash").Value().(string),
198 | User: gjson.GetBytes(buf.Bytes(), "pullrequest.author.display_name").Value().(string),
199 | PullRequestURL: gjson.GetBytes(buf.Bytes(), "pullrequest.links.html.href").Value().(string),
200 | PullRequestTitle: gjson.GetBytes(buf.Bytes(), "pullrequest.title").Value().(string),
201 | DestBranch: gjson.GetBytes(buf.Bytes(), "pullrequest.destination.branch.name").Value().(string),
202 | HookID: hookID,
203 | }
204 | }
205 | return webhookPayload, nil
206 | }
207 |
208 | func (b BitbucketClientImpl) SetStatus(ctx *context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error {
209 | commitOptions := bitbucket.CommitsOptions{
210 | Owner: b.cfg.GitProviderConfig.OrgName,
211 | RepoSlug: *repo,
212 | Revision: *commit,
213 | }
214 | commitStatusOptions := bitbucket.CommitStatusOptions{
215 | Key: "build",
216 | Url: *linkURL,
217 | State: *status,
218 | Description: *message,
219 | }
220 | _, err := b.client.Repositories.Commits.CreateCommitStatus(&commitOptions, &commitStatusOptions)
221 | if err != nil {
222 | return err
223 | }
224 | log.Printf("set status of commit %s in repo %s to %s", *commit, *repo, *status)
225 | return nil
226 | }
227 |
228 | func (b BitbucketClientImpl) PingHook(ctx *context.Context, hook *HookWithStatus) error {
229 | //TODO implement me
230 | panic("implement me")
231 | }
232 |
233 | func (b BitbucketClientImpl) isRepoWebhookExists(repo string) (*bitbucket.Webhook, bool) {
234 | emptyHook := bitbucket.Webhook{}
235 |
236 | webhookOptions := bitbucket.WebhooksOptions{
237 | Owner: b.cfg.GitProviderConfig.OrgName,
238 | RepoSlug: repo,
239 | }
240 | hooks, err := b.client.Repositories.Webhooks.List(&webhookOptions)
241 |
242 | if err != nil {
243 | log.Printf("failed to list existing hooks for repository %s. error:%s", repo, err)
244 | return &emptyHook, false
245 | }
246 |
247 | if len(hooks) == 0 {
248 | return &emptyHook, false
249 | }
250 |
251 | for _, hook := range hooks {
252 | if hook.Description == "Piper" && hook.Url == b.cfg.GitProviderConfig.WebhookURL {
253 | return &hook, true
254 | }
255 | }
256 |
257 | return &emptyHook, false
258 | }
259 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright 2023 George Dozoretz
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------