├── .dockerignore ├── .github ├── CODEOWNERS ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── ci.yaml │ ├── docs.yaml │ ├── e2e.yaml │ ├── lint-commit.yaml │ ├── lint-pr.yaml │ ├── release-please.yaml │ └── release.yaml ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── README.md ├── cmd └── piper │ └── piper.go ├── docs ├── CNAME ├── CONTRIBUTING.md ├── configuration │ ├── environment_variables.md │ └── health_check.md ├── getting_started │ └── installation.md ├── img │ ├── favicon.ico │ ├── flow.svg │ ├── piper-demo-1080.mov │ └── piper-demo-1080.mp4 ├── index.md └── usage │ ├── global_variables.md │ ├── workflows_config.md │ └── workflows_folder.md ├── examples ├── .workflows │ ├── exit.yaml │ ├── main.yaml │ ├── parameters.yaml │ ├── templates.yaml │ └── triggers.yaml ├── config.yaml ├── template.values.dev.yaml └── workflow.yaml ├── gitlab.values.yaml ├── go.mod ├── go.sum ├── helm-chart ├── .helmignore ├── Chart.yaml ├── README.md ├── templates │ ├── _helpers.tpl │ ├── argo-token-secret.yaml │ ├── config.yaml │ ├── deployment.yaml │ ├── git-token-secret.yaml │ ├── hpa.yaml │ ├── ingress.yaml │ ├── rookout-token.yaml │ ├── security.yaml │ ├── service.yaml │ ├── serviceaccount.yaml │ └── webhook-secret.yaml └── values.yaml ├── makefile ├── mkdocs.yml ├── pkg ├── clients │ └── types.go ├── common │ └── types.go ├── conf │ ├── conf.go │ ├── git_provider.go │ ├── rookout.go │ ├── workflow_server.go │ └── workflows_config.go ├── event_handler │ ├── event_notifier.go │ ├── event_notifier_test.go │ ├── main.go │ ├── types.go │ └── workflow_event_handler.go ├── git_provider │ ├── bitbucket.go │ ├── bitbucket_test.go │ ├── bitbucket_utils.go │ ├── github.go │ ├── github_test.go │ ├── github_utils.go │ ├── github_utils_test.go │ ├── gitlab.go │ ├── gitlab_test.go │ ├── gitlab_utils.go │ ├── gitlab_utils_test.go │ ├── main.go │ ├── test_utils.go │ └── types.go ├── server │ ├── main.go │ ├── routes │ │ ├── healthz.go │ │ ├── readyz.go │ │ └── webhook.go │ ├── server.go │ ├── shutdown.go │ └── types.go ├── utils │ ├── common.go │ ├── common_test.go │ ├── os.go │ └── os_test.go ├── webhook_creator │ ├── main.go │ ├── mocks.go │ ├── tests_test.go │ └── types.go ├── webhook_handler │ ├── types.go │ ├── webhook_handler.go │ └── webhook_handler_test.go └── workflow_handler │ ├── types.go │ ├── workflows.go │ ├── workflows_test.go │ ├── workflows_utils.go │ └── workflows_utils_test.go ├── scripts ├── gitlab-setup.yaml ├── init-argo-workflows.sh ├── init-gitlab.sh ├── init-kind.sh ├── init-nginx.sh └── init-piper.sh └── workflows.values.yaml /.dockerignore: -------------------------------------------------------------------------------- 1 | # Keep ignoring .git 2 | .git 3 | # Allow specific files with ! 4 | !.git/HEAD 5 | !.git/config 6 | !.git/refs -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @goshado 2 | **/.github @goshado -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Pull Request 2 | 3 | ### Description 4 | 5 | Please provide a brief description of the changes made in this pull request. 6 | 7 | ### Related Issue(s) 8 | 9 | If this pull request addresses or relates to any open issues, please mention them here using the syntax `Fixes #` or `Resolves #`. 10 | 11 | ### Checklist 12 | 13 | Before submitting this pull request, please ensure that you have completed the following tasks: 14 | 15 | - [ ] Reviewed the [Contributing Guidelines](../docs/CONTRIBUTING.md) for the Piper project. 16 | - [ ] Ensured that your changes follow the [coding guidelines and style](../docs/CONTRIBUTING.md#coding-guidelines) of the project. 17 | - [ ] Run the tests locally and made sure they pass. 18 | - [ ] Updated the relevant documentation, if applicable. 19 | 20 | ### Testing Instructions 21 | 22 | Please provide clear instructions on how to test and verify the changes made in this pull request. 23 | 24 | ### Additional Information 25 | 26 | Add any additional information or context that would be helpful in understanding and reviewing this pull request. 27 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - "main" 7 | paths: 8 | - '**' 9 | - '!docs/**' 10 | pull_request: 11 | branches: 12 | - "main" 13 | 14 | concurrency: 15 | group: ${{ github.workflow }}-${{ github.ref }} 16 | cancel-in-progress: true 17 | 18 | permissions: 19 | contents: read 20 | 21 | jobs: 22 | tests: 23 | name: Unit Tests 24 | runs-on: ubuntu-latest 25 | timeout-minutes: 10 26 | steps: 27 | - uses: actions/checkout@v3 28 | - uses: actions/setup-go@v4 29 | with: 30 | go-version: "1.20" 31 | cache: true 32 | - run: make test 33 | lint: 34 | name: Go Lint 35 | runs-on: ubuntu-latest 36 | timeout-minutes: 10 37 | steps: 38 | - uses: actions/checkout@v3 39 | - uses: actions/setup-go@v4 40 | with: 41 | go-version: '1.20' 42 | cache: true 43 | - name: golangci-lint 44 | uses: golangci/golangci-lint-action@v3 45 | with: 46 | version: v1.53 47 | only-new-issues: true 48 | skip-pkg-cache: true 49 | args: --timeout=10m 50 | helm: 51 | name: Helm Lint 52 | runs-on: ubuntu-latest 53 | timeout-minutes: 10 54 | steps: 55 | - uses: actions/checkout@v3 56 | with: 57 | fetch-depth: 0 58 | - name: Check Git diff in /helm-chart 59 | run: | 60 | if [ "$(git diff --exit-code --name-only --diff-filter=d origin/main -- helm-chart/)" != "" ]; then 61 | echo "There are Git diffs in the /helm-chart folder." 62 | echo "CHART_UPDATED=true" >> $GITHUB_ENV 63 | else 64 | echo "There are no Git diffs in the /helm-chart folder." 65 | fi 66 | - name: Install Helm Docs 67 | uses: envoy/install-helm-docs@v1.0.0 68 | with: 69 | version: 1.11.0 70 | - name: Helm lint and template 71 | run: | 72 | make helm 73 | if: ${{ env.CHART_UPDATED }} -------------------------------------------------------------------------------- /.github/workflows/docs.yaml: -------------------------------------------------------------------------------- 1 | name: docs 2 | on: 3 | push: 4 | branches: 5 | - main 6 | paths: 7 | - 'docs/**' 8 | - '.github/workflows/docs.yaml' 9 | - 'mkdocs.yml' 10 | 11 | permissions: 12 | contents: write 13 | jobs: 14 | publish: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout main 18 | uses: actions/checkout@v2 19 | - name: Install mkdocs 20 | run: | 21 | pip install mkdocs-material 22 | pip install mkdocs-video 23 | - name: Generate docs artifacts 24 | run: mkdocs build -s -d /tmp/docs 25 | - uses: actions/checkout@v2 26 | with: 27 | ref: gh-pages 28 | path: gh-pages 29 | - name: Publish docs artifacts to gh-pages 30 | run: | 31 | cd gh-pages 32 | shopt -s extglob 33 | rm -rf !(index.yaml|LICENSE|*.tgz) 34 | cp -R /tmp/docs/** . 35 | git config --local user.email "action@github.com" 36 | git config --local user.name "GitHub Action" 37 | git add -A 38 | git commit -m "Publish docs from $GITHUB_SHA" 39 | git push https://github.com/$GITHUB_REPOSITORY.git gh-pages -------------------------------------------------------------------------------- /.github/workflows/lint-commit.yaml: -------------------------------------------------------------------------------- 1 | name: Lint Commit 2 | on: pull_request 3 | jobs: 4 | conventional: 5 | name: Conventional Commit Linter 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | - uses: actions/setup-node@v2 10 | - uses: taskmedia/action-conventional-commits@v1.1.8 11 | with: 12 | token: ${{ github.token }} 13 | types: "fix|feat|revert|ci|docs|chore" 14 | -------------------------------------------------------------------------------- /.github/workflows/lint-pr.yaml: -------------------------------------------------------------------------------- 1 | name: Check PR title 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - reopened 8 | - edited 9 | - synchronize 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | lint: 16 | permissions: 17 | pull-requests: read # for amannn/action-semantic-pull-request to analyze PRs 18 | statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR 19 | name: Validate PR title 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: amannn/action-semantic-pull-request@v5 23 | with: 24 | # Configure which types are allowed (newline delimited). 25 | # Default: https://github.com/commitizen/conventional-commit-types 26 | types: | 27 | feat 28 | fix 29 | docs 30 | chore 31 | revert 32 | ci 33 | # Configure which scopes are allowed (newline delimited). 34 | scopes: | 35 | deps 36 | core 37 | RK-\d+ 38 | # Configure that a scope must always be provided. 39 | requireScope: false 40 | ignoreLabels: | 41 | autorelease: pending 42 | env: 43 | GITHUB_TOKEN: ${{ github.token }} 44 | -------------------------------------------------------------------------------- /.github/workflows/release-please.yaml: -------------------------------------------------------------------------------- 1 | name: Release Please 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | pull-requests: write 11 | 12 | jobs: 13 | release-please: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: google-github-actions/release-please-action@v3 17 | with: 18 | release-type: helm 19 | package-name: piper 20 | token: ${{ secrets.GIT_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release Workflow 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | - edited 8 | 9 | jobs: 10 | piper-image: 11 | name: piper-image 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: docker/setup-qemu-action@v2 16 | - uses: docker/setup-buildx-action@v2 17 | - name: Login to Docker Hub 18 | uses: docker/login-action@v2 19 | with: 20 | username: ${{ secrets.DOCKERHUB_USERNAME }} 21 | password: ${{ secrets.DOCKERHUB_TOKEN }} 22 | - name: Docker meta 23 | id: meta 24 | uses: docker/metadata-action@v4 25 | with: 26 | images: quickube/piper 27 | - name: Build and export 28 | uses: docker/build-push-action@v4 29 | with: 30 | context: . 31 | platforms: linux/amd64,linux/arm64 32 | push: true 33 | tags: quickube/piper:${{ github.ref_name }},quickube/piper:latest 34 | labels: ${{ steps.meta.outputs.labels }} 35 | cache-from: type=gha 36 | cache-to: type=gha,mode=max 37 | helm: 38 | name: helm 39 | runs-on: ubuntu-latest 40 | steps: 41 | - uses: actions/checkout@v3 42 | - name: Install Helm 43 | uses: azure/setup-helm@v3 44 | - name: Install Helm Docs 45 | uses: envoy/install-helm-docs@v1.0.0 46 | with: 47 | version: 1.11.0 48 | - name: Helm lint and template 49 | run: | 50 | make helm 51 | - name: Publish Helm chart 52 | uses: stefanprodan/helm-gh-pages@master 53 | with: 54 | chart_version: ${{ github.ref_name }} 55 | app_version: ${{ github.ref_name }} 56 | token: ${{ secrets.GIT_TOKEN }} 57 | charts_dir: . -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea/ 3 | .history/ 4 | .vscode 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | dev.env 90 | *.env 91 | .venv 92 | env/ 93 | venv/ 94 | ENV/ 95 | env.bak/ 96 | venv.bak/ 97 | 98 | # Spyder project settings 99 | .spyderproject 100 | .spyproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | 105 | # mkdocs documentation 106 | /site 107 | 108 | #mirrord config 109 | .mirrord/ 110 | 111 | # mypy 112 | .mypy_cache/ 113 | *.iml 114 | 115 | _lint.yaml 116 | values.dev.yaml 117 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20-alpine3.16 AS builder 2 | 3 | WORKDIR /piper 4 | 5 | RUN apk update && apk add --no-cache \ 6 | git \ 7 | make \ 8 | wget \ 9 | curl \ 10 | gcc \ 11 | bash \ 12 | ca-certificates \ 13 | musl-dev \ 14 | zlib-static \ 15 | build-base 16 | 17 | COPY go.mod . 18 | COPY go.sum . 19 | RUN --mount=type=cache,target=/go/pkg/mod go mod download 20 | 21 | COPY . . 22 | 23 | RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod tidy 24 | 25 | RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go build -gcflags='all=-N -l' -tags=alpine -buildvcs=false -trimpath ./cmd/piper 26 | 27 | 28 | FROM alpine:3.16 AS piper-release 29 | 30 | ENV GIN_MODE=release 31 | 32 | USER 1001 33 | 34 | COPY .git /.git 35 | 36 | COPY --chown=1001 --from=builder /piper/piper /bin 37 | 38 | ENTRYPOINT [ "piper" ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Piper 2 | ![alt text](https://www.rookout.com/wp-content/uploads/2022/10/ArgoPipeline_1.0_Hero.png.webp?raw=true) 3 | 4 | Welcome to Piper! Piper is an open-source project aimed at providing multibranch pipeline functionality to Argo Workflows, allowing users to create distinct Workflows based on Git branches. 5 | 6 | ## Table of Contents 7 | 8 | - [Piper](#piper) 9 | - [Table of Contents](#table-of-contents) 10 | - [Getting Started](#getting-started) 11 | - [Reporting Issues](#reporting-issues) 12 | - [How to Contribute](#how-to-contribute) 13 | - [License](#license) 14 | 15 | ## Getting Started 16 | 17 | Piper configures a webhook in the git provider and listens to the webhooks sent. It will create a Workflow CRD out of branches that contain a `.workflows` folder. 18 | This folder should contain declarations of the templates and the main DAG that will be running. 19 | Finally, it will submit the Workflow as a K8s resource in the cluster. 20 | To access more detailed explanations, please navigate to the [Documentation site](https://piper.quickube.com). 21 | 22 | https://github.com/quickube/piper/assets/106976988/09b3a5d8-3428-4bdc-9146-3034d81164bf 23 | 24 | ## Reporting Issues 25 | 26 | If you encounter any issues or bugs while using Piper, please help us improve by reporting them. Follow these steps to report an issue: 27 | 28 | 1. Go to the [Piper Issues](https://github.com/quickube/Piper/issues) page on GitHub. 29 | 2. Click on the "New Issue" button. 30 | 3. Provide a descriptive title and detailed description of the issue, including any relevant error messages or steps to reproduce the problem. 31 | 4. Add appropriate labels to categorize the issue (e.g., bug, enhancement, question). 32 | 5. Submit the issue, and our team will review and address it as soon as possible. 33 | 34 | ## How to Contribute 35 | 36 | If you're interested in contributing to this project, please feel free to submit a pull request. We welcome all contributions and feedback. 37 | Please check out our [contribution guidelines for this project](docs/CONTRIBUTING.md). 38 | 39 | ## License 40 | 41 | This project is licensed under the Apache License. Please see the [LICENSE](LICENSE) file for details. -------------------------------------------------------------------------------- /cmd/piper/piper.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | rookout "github.com/Rookout/GoSDK" 5 | "github.com/quickube/piper/pkg/clients" 6 | "github.com/quickube/piper/pkg/conf" 7 | "github.com/quickube/piper/pkg/event_handler" 8 | "github.com/quickube/piper/pkg/git_provider" 9 | "github.com/quickube/piper/pkg/server" 10 | "github.com/quickube/piper/pkg/utils" 11 | workflowHandler "github.com/quickube/piper/pkg/workflow_handler" 12 | "golang.org/x/net/context" 13 | "log" 14 | "os/signal" 15 | "syscall" 16 | ) 17 | 18 | func main() { 19 | cfg, err := conf.LoadConfig() 20 | if err != nil { 21 | log.Panicf("failed to load the configuration for Piper, error: %v", err) 22 | } 23 | 24 | if cfg.RookoutConfig.Token != "" { 25 | labels := utils.StringToMap(cfg.RookoutConfig.Labels) 26 | err = rookout.Start(rookout.RookOptions{Token: cfg.RookoutConfig.Token, Labels: labels}) 27 | if err != nil { 28 | log.Printf("failed to start Rookout, error: %v\n", err) 29 | } 30 | } 31 | 32 | err = cfg.WorkflowsConfig.WorkflowsSpecLoad("/piper-config/..data") 33 | if err != nil { 34 | log.Panicf("Failed to load workflow spec configuration, error: %v", err) 35 | } 36 | 37 | gitProvider, err := git_provider.NewGitProviderClient(cfg) 38 | if err != nil { 39 | log.Panicf("failed to load the Git client for Piper, error: %v", err) 40 | } 41 | workflows, err := workflowHandler.NewWorkflowsClient(cfg) 42 | if err != nil { 43 | log.Panicf("failed to load the Argo Workflows client for Piper, error: %v", err) 44 | } 45 | 46 | globalClients := &clients.Clients{ 47 | GitProvider: gitProvider, 48 | Workflows: workflows, 49 | } 50 | 51 | // Create context that listens for the interrupt signal from the OS. 52 | ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) 53 | defer stop() 54 | event_handler.Start(ctx, stop, cfg, globalClients) 55 | server.Start(ctx, stop, cfg, globalClients) 56 | } 57 | -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | piper.quickube.com -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## How To Contribute 2 | 3 | We appreciate contributions from the community to make Piper even better. To contribute, follow the steps below: 4 | 5 | 1. Fork the Piper repository to your GitHub account. 6 | 2. Clone the forked repository to your local machine: 7 | ```bash 8 | git clone https://github.com/your-username/Piper.git 9 | ``` 10 | 3. Create a new branch to work on your feature or bug fix: 11 | ```bash 12 | git checkout -b my-feature 13 | ``` 14 | 4. Make your changes, following the coding guidelines outlined in this document. 15 | 5. Commit your changes with clear and descriptive commit messages and sign it: 16 | ```bash 17 | git commit -s -m "fix: Add new feature" 18 | ``` 19 | * Please make sure you commit as described in [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) 20 | 6. Push your changes to your forked repository: 21 | ```bash 22 | git push origin my-feature 23 | ``` 24 | 7. Open a [pull request](#pull-requests) against the main branch of the original Piper repository. 25 | 26 | ## Pull Requests 27 | 28 | We welcome and appreciate contributions from the community. If you have developed a new feature, improvement, or bug fix for Piper, follow these steps to submit a pull request: 29 | 30 | 1. Make sure you have forked the Piper repository and created a new branch for your changes. Checkout [How To Contribute](#How-to-contribute). 31 | 2. Commit your changes and push them to your forked repository. 32 | 3. Go to the Piper repository on GitHub. 33 | 4. Click on the "New Pull Request" button. 34 | 5. Select your branch and provide a [descriptive title](#pull-request-naming) and a detailed description of your changes. 35 | 6. If your pull request relates to an open issue, reference the issue in the description using the GitHub issue syntax (e.g., Fixes #123). 36 | 7. Submit the pull request, and our team will review your changes. We appreciate your patience during the review process and may provide feedback or request further modifications. 37 | 38 | ### Pull Request Naming 39 | 40 | The name should follow Conventional Commits naming. 41 | 42 | ## Coding Guidelines 43 | 44 | To maintain a consistent codebase and ensure readability, we follow a set of coding guidelines in Piper. Please adhere to the following guidelines when making changes: 45 | 46 | * Follow the [Effective Go](https://go.dev/doc/effective_go) guide for Go code. 47 | * Follow the [Folder convention](https://github.com/golang-standards/project-layout) guide for Go code. 48 | * Write clear and concise comments to explain the code's functionality. 49 | * Use meaningful variable and function names. 50 | * Make sure your code is properly formatted and free of syntax errors. 51 | * Run tests locally. 52 | * Check that the feature is documented. 53 | * Add new packages only if necessary and if an already existing one can't be used. 54 | * Add tests for new features or modification. 55 | 56 | ## Helm Chart Development 57 | 58 | To make sure that the documentation is updated use [helm-docs](https://github.com/norwoodj/helm-docs) comment convention. The pipeline will execute `helm-docs` command and update the version of the chart. 59 | 60 | Also, please make sure to run those commands locally to debug the chart before merging: 61 | 62 | ```bash 63 | make helm 64 | ``` 65 | 66 | ## Local deployment 67 | 68 | To make it easy to develop locally, please run the following 69 | 70 | Prerequisites : 71 | 1. install helm 72 | ```bash 73 | brew install helm 74 | ``` 75 | 2. install kubectl 76 | ```bash 77 | brew install kubectl 78 | ``` 79 | 3. install docker 80 | 81 | 4. install ngrok 82 | ```bash 83 | brew install ngrok 84 | ``` 85 | 5. install kind 86 | ```bash 87 | brew install kind 88 | ``` 89 | 90 | Deployment: 91 | 1. Make sure Docker is running. 92 | 2. Create a tunnel with ngrok using `make ngrok`, and save the `Forwarding` address. 93 | 3. Create a `values.dev.yaml` file that contains a subset of the chart's `values.yaml` file. Check the [example of values file](https://github.com/quickube/piper/tree/main/examples/template.values.dev.yaml), rename it to `values.dev.yaml`, and put it in the root directory. 94 | 4. Use `make deploy`. It will do the following: 95 | * Deploy a local registry as a container 96 | * Deploy a kind cluster as a container with configuration 97 | * Deploy nginx reverse proxy in the kind cluster 98 | * Deploy Piper with the local helm chart 99 | 5. Validate using `curl localhost/piper/healthz`. 100 | 101 | ## Debugging 102 | 103 | For debugging, the best practice is to use Rookout. To enable this function, pass a Rookout token in the chart `rookout.token` or as an existing secret `rookout.existingSecret`. -------------------------------------------------------------------------------- /docs/configuration/environment_variables.md: -------------------------------------------------------------------------------- 1 | ## Environment Variables 2 | 3 | Piper uses the following environment variables to configure its functionality. 4 | The helm chart populates them using the [values.yaml](https://github.com/quickube/piper/tree/main/helm-chart/values.yaml) file. 5 | 6 | ### Git 7 | 8 | - GIT_PROVIDER 9 | The git provider that Piper will use, possible variables: GitHub | GitLab | Bitbucket 10 | 11 | * GIT_TOKEN 12 | The git token that will be used to connect to the git provider. 13 | 14 | - GIT_URL 15 | The git URL that will be used, only relevant when running GitLab self-hosted. 16 | 17 | - GIT_ORG_NAME 18 | The organization name. 19 | 20 | * GIT_ORG_LEVEL_WEBHOOK 21 | Boolean variable, whether to configure the webhook at the organization level. Defaults to `false`. 22 | 23 | * GIT_WEBHOOK_REPO_LIST 24 | List of repositories to configure webhooks for. 25 | 26 | * GIT_WEBHOOK_URL 27 | URL of Piper ingress to configure webhooks. 28 | 29 | * GIT_WEBHOOK_AUTO_CLEANUP 30 | Boolean variable that, if true, will cause Piper to automatically clean up all webhooks it creates when they are no longer necessary. 31 | Note that there is a race condition between a pod being terminated and a new one being scheduled. 32 | 33 | * GIT_ENFORCE_ORG_BELONGING 34 | Boolean variable that, if true, will cause Piper to enforce the organizational belonging of the git event creator. Defaults to `false`. 35 | 36 | * GIT_FULL_HEALTH_CHECK 37 | Boolean variable that, if true, enables full health checks on webhooks. A full health check involves expecting and validating a ping event from a webhook. 38 | This doesn't work for Bitbucket because the API call doesn't exist on that platform. 39 | 40 | ### Argo Workflows Server 41 | 42 | * ARGO_WORKFLOWS_TOKEN 43 | This token is used to authenticate with the Argo Workflows server. 44 | 45 | * ARGO_WORKFLOWS_ADDRESS 46 | The address of the Argo Workflows server. 47 | 48 | * ARGO_WORKFLOWS_CREATE_CRD 49 | Boolean variable that determines whether to directly send Workflows instructions or create a CRD in the Cluster. 50 | 51 | * ARGO_WORKFLOWS_NAMESPACE 52 | The namespace of Workflows creation for Argo Workflows. 53 | 54 | * KUBE_CONFIG 55 | Used to configure the Argo Workflows client with local kube configurations. 56 | 57 | ### Rookout 58 | 59 | * ROOKOUT_TOKEN 60 | The token used to configure the Rookout agent. If not provided, the agent will not start. 61 | 62 | * ROOKOUT_LABELS 63 | The labels to label instances in Rookout, defaults to "service:piper". 64 | 65 | * ROOKOUT_REMOTE_ORIGIN 66 | The repo URL for source code fetching, defaults to "https://github.com/quickube/piper.git". -------------------------------------------------------------------------------- /docs/configuration/health_check.md: -------------------------------------------------------------------------------- 1 | ## Health Check 2 | 3 | Currently not supported for GitLab / Bitbucket 4 | 5 | The following example shows a health check being executed every 1 minute as configured in the helm chart under `livenessProbe`, and triggered by the `/healthz` endpoint: 6 | 7 | ```yaml 8 | livenessProbe: 9 | httpGet: 10 | path: /healthz 11 | port: 8080 12 | scheme: HTTP 13 | initialDelaySeconds: 10 14 | timeoutSeconds: 10 15 | periodSeconds: 60 16 | successThreshold: 1 17 | failureThreshold: 4 18 | ``` 19 | 20 | The mechanism for checking the health of Piper is: 21 | 22 | 1. Piper sets the health status of all webhooks to not healthy 23 | 24 | 2. Piper requests a ping from all the configured webhooks. 25 | 26 | 3. The Git provider sends a ping to the `/webhook` endpoint, which will set the health status to `healthy` with a timeout of 5 seconds. 27 | 28 | 4. Piper checks the status of all configured webhooks. 29 | 30 | Therefore, the criteria for health checking are: 31 | 32 | 1. The registered webhook exists. 33 | 2. The webhook sends a ping within 5 seconds. 34 | -------------------------------------------------------------------------------- /docs/getting_started/installation.md: -------------------------------------------------------------------------------- 1 | ## Installation 2 | 3 | You must deploy Piper to a cluster with a pre-existing Argo Workflows deployment. 4 | Piper will create a CRD that Argo Workflows will pick up, so install or configure Piper to create those CRDs in the right namespace. 5 | 6 | Please check out [values.yaml](https://github.com/quickube/piper/tree/main/helm-chart/values.yaml) file of the helm chart configurations. 7 | 8 | To add piper helm repo run: 9 | 10 | ```bash 11 | helm repo add piper https://piper.quickube.com 12 | ``` 13 | 14 | After configuring Piper [values.yaml](https://github.com/quickube/piper/tree/main/helm-chart/values.yaml), run the following command for installation: 15 | 16 | ```bash 17 | helm upgrade --install piper piper/piper \ 18 | -f YOUR_VALUES_FILE.yaml 19 | ``` 20 | 21 | --- 22 | 23 | ## Required Configuration 24 | 25 | ### Ingress 26 | 27 | Piper works best when it is able to listen to webhooks from your git provider. 28 | Expose Piper using an ingress or service, then provide the address to `piper.webhook.url` as follows: 29 | `https://PIPER_EXPOSED_URL/webhook` 30 | 31 | Refer to [values.yaml](https://github.com/quickube/piper/tree/main/helm-chart/values.yaml) for more information. 32 | 33 | ### Git 34 | 35 | Piper will use git to fetch the `.workflows` folder and receive events using webhooks. 36 | 37 | To pick which git provider you are using provide `gitProvider.name` configuration in helm chart (Currently we only support GitHub and Bitbucket). 38 | 39 | Also configure your organization (GitHub), workspace (Bitbucket) or group (GitLab) name using `gitProvider.organization.name` in helm chart. 40 | 41 | #### Git Token Permissions 42 | 43 | The token should have access to create webhooks and read repository content.
44 | For GitHub, configure `admin:org` and `write:org` permissions in Classic Token.
45 | For Bitbucket, configure `Repositories:read`, `Webhooks:read and write` and `Pull requests:read` permissions (for multiple repos use workspace token).
46 | For Gitlab, configure `read_api`, `write_repository` and `api` (for multiple repos use group token with owner role).
47 | 48 | #### Token 49 | 50 | The git token should be passed as secret in the helm chart at `gitProvider.token`. 51 | The token can be passed as parameter via helm install command using `--set piper.gitProvider.token=YOUR_GIT_TOKEN` 52 | 53 | Alternatively, you can use an already existing secret by configuring `piper.gipProvider.existingSecret`. 54 | The key should be named token `token`. You can create a Secret using this command: 55 | 56 | ```bash 57 | kubectl create secret generic piper-git-token --from-literal=token=YOUR_GIT_OKEN 58 | ``` 59 | 60 | #### Webhook creation 61 | 62 | Piper will create a webhook configuration for you, either for the whole organization or for each repo you configure. 63 | 64 | Configure `piper.webhook.url` with the address of Piper that you exposed using an Ingress or Service with `/webhook` postfix. 65 | 66 | For organization level configuration: `gitProvider.webhook.orgLevel` to `true`. 67 | 68 | For granular repo webhook provide list of repos at: `gitProvider.webhook.repoList`. 69 | 70 | Piper implements a graceful shutdown; it will delete all the webhooks when terminated. 71 | 72 | #### Status check 73 | 74 | Piper will handle status checks for you. 75 | It will notify the GitProvider of the status of the Workflow for the specific commit that triggered Piper. 76 | For linking provide valid URL of your Argo Workflows server address at: `argoWorkflows.server.address` 77 | 78 | --- 79 | 80 | ### Argo Workflow Server (On development) 81 | 82 | Piper will use the REST API to communicate with the Argo Workflows server for linting or creating workflows. 83 | 84 | To lint the workflow before submitting it, please configure the internal address of Argo Workflows server (for example, `argo-server.workflows.svc.cluster.local`) in the field: `argoWorkflows.server.address`. Argo will need a [token](https://argoproj.github.io/argo-workflows/access-token/) to authenticate. Please provide the secret in `argoWorkflows.server.token`. It is better to pass it as a reference to a secret in the field `argoWorkflows.server.token`. 85 | 86 | #### Skip CRD Creation (On development) 87 | 88 | Piper can communicate directly to Argo Workflow using ARGO_WORKFLOWS_CREATE_CRD environment variable, if you want to skip the creation of CRD change `argoWorkflows.crdCreation` to `false`. 89 | -------------------------------------------------------------------------------- /docs/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quickube/piper/cd6e26def7f914261c0badaf4005989016282fd1/docs/img/favicon.ico -------------------------------------------------------------------------------- /docs/img/piper-demo-1080.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quickube/piper/cd6e26def7f914261c0badaf4005989016282fd1/docs/img/piper-demo-1080.mov -------------------------------------------------------------------------------- /docs/img/piper-demo-1080.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quickube/piper/cd6e26def7f914261c0badaf4005989016282fd1/docs/img/piper-demo-1080.mp4 -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 |

4 | ArgoPipeline 5 |

6 | 7 | Welcome to Piper! 8 | 9 | Piper is an open-source project aimed at providing multibranch pipeline functionality to Argo Workflows. This allows users to create distinct Workflows based on Git branches. We support GitHub and Bitbucket. 10 | 11 | ## General Explanation 12 | 13 |

14 | flow 15 |

16 | 17 | Piper handles the hard work of configuring multibranch pipelines for us! At initialization, it will load all configuration and create a webhook in the repository or organization scope. Then, for each branch that has a `.workflows` folder, Piper will create a Workflow CRD out of the files in this folder. Finally, when Piper detects changes in the repository via the webhook, it triggers the workflows that match the branch and event. 18 | 19 | ![type:video](./img/piper-demo-1080.mp4) -------------------------------------------------------------------------------- /docs/usage/global_variables.md: -------------------------------------------------------------------------------- 1 | ## Global Variables 2 | 3 | Piper will automatically add Workflow scope parameters that can be referenced from any template. 4 | The parameters are taken from webhook metadata and will be populated according to the GitProvider and the event that triggered the workflow. 5 | 6 | 1. `{{ workflow.parameters.event }}` The event that triggered the workflow. 7 | 8 | 2. `{{ workflow.parameters.action }}` The action that triggered the workflow. 9 | 10 | 3. `{{ workflow.parameters.dest_branch }}` The destination branch for the pull request. 11 | 12 | 4. `{{ workflow.parameters.commit }}` The commit that triggered the workflow. 13 | 14 | 5. `{{ workflow.parameters.repo }}` The repository name that triggered the workflow. 15 | 16 | 6. `{{ workflow.parameters.user }}` The username that triggered the workflow. 17 | 18 | 7. `{{ workflow.parameters.user_email }}` The user's email that triggered the workflow. 19 | 20 | 8. `{{ workflow.parameters.pull_request_url }}` The URL of the pull request that triggered the workflow. 21 | 22 | 9. `{{ workflow.parameters.pull_request_title }}` The title of the pull request that triggered the workflow. 23 | 24 | 10. `{{ workflow.parameters.pull_request_labels }}` Comma-separated labels of the pull request that triggered the workflow. -------------------------------------------------------------------------------- /docs/usage/workflows_config.md: -------------------------------------------------------------------------------- 1 | ## Workflow Configuration 2 | 3 | Piper can inject configuration for Workflows that Piper creates. 4 | 5 | `default` config is used as a convention for all Workflows that Piper will create, even if not explicitly mentioned in the `triggers.yaml` file. 6 | 7 | ### ConfigMap 8 | 9 | Piper will mount a ConfigMap when Helm is used. 10 | The `piper.workflowsConfig` variable in the Helm chart will create a ConfigMap that holds a set of configurations for Piper. 11 | Here is an [example](https://github.com/quickube/piper/tree/main/examples/config.yaml) of such a configuration. 12 | 13 | ### Spec 14 | 15 | This will be injected into the Workflow spec field and can hold all configurations of the Workflow. 16 | > :warning: Please note that the fields `entrypoint` and `onExit` should not exist in the spec; both of them are managed fields. 17 | 18 | ### onExit 19 | 20 | This is the exit handler for each of the Workflows created by Piper. 21 | It configures a DAG that will be executed when the workflow ends. 22 | You can provide the templates to it as shown in the following [Examples](https://github.com/quickube/piper/tree/main/examples/config.yaml). -------------------------------------------------------------------------------- /docs/usage/workflows_folder.md: -------------------------------------------------------------------------------- 1 | ## .workflows Folder 2 | 3 | Piper will look in each of the target branches for a `.workflows` folder. [Example](https://github.com/quickube/piper/tree/main/examples/.workflows). 4 | We will explain each of the files that should be included in the `.workflows` folder: 5 | 6 | ### triggers.yaml (convention name) 7 | 8 | This file holds a list of triggers that will be executed `onStart` by `events` from specific `branches`. 9 | Piper will execute each of the matching triggers, so configure it wisely. 10 | 11 | ```yaml 12 | - events: 13 | - push 14 | - pull_request.synchronize 15 | branches: ["main"] 16 | onStart: ["main.yaml"] 17 | onExit: ["exit.yaml"] 18 | templates: ["templates.yaml"] 19 | config: "default" 20 | ``` 21 | 22 | This example can be found [here](https://github.com/quickube/piper/tree/main/examples/.workflows/triggers.yaml). 23 | 24 | In this example, `main.yaml` will be executed as a DAG when `push` or `pull_request.synchronize` events are applied in the `main` branch. 25 | `onExit` will execute `exit.yaml` when the workflow finishes as an exit handler. 26 | 27 | `onExit` can overwrite the default `onExit` configuration by referencing existing DAG tasks as in the [example](https://github.com/quickube/piper/tree/main/examples/.workflows/exit.yaml). 28 | 29 | The `config` field is used for workflow configuration selection. The default value is the `default` configuration. 30 | 31 | #### events 32 | 33 | The `events` field is used to determine when the trigger will be executed. The name of the event depends on the git provider. 34 | 35 | For instance, the GitHub `pull_request` event has a few actions, one of which is `synchronize`. 36 | 37 | #### branches 38 | 39 | The branch for which the trigger will be executed. 40 | 41 | #### onStart 42 | 43 | This [file](https://github.com/quickube/piper/tree/main/examples/.workflows/main.yaml) can be named as you wish and will be referenced in the `triggers.yaml` file. It will define an entrypoint DAG that the Workflow will execute. 44 | 45 | As a best practice, this file should contain the dependency logic and parameterization of each referenced template. It should not implement new templates; for this, use the `template.yaml` file. 46 | 47 | #### onExit 48 | 49 | This field is used to pass a verbose exit handler to the triggered workflow. 50 | It will override the default `onExit` from the provided `config` or the default `config`. 51 | 52 | The provided `exit.yaml` describes a DAG that will overwrite the default `onExit` configuration. 53 | [Example](https://github.com/quickube/piper/tree/main/examples/.workflows/exit.yaml) 54 | 55 | #### templates 56 | 57 | This field will have additional templates that will be injected into the workflows. 58 | The purpose of this field is to create repository-scope templates that can be referenced from the DAG templates at `onStart` or `onExit`. 59 | [Example](https://github.com/quickube/piper/tree/main/examples/.workflows/templates.yaml) 60 | 61 | As a best practice, use this field for template implementation and reference them from the executed DAGs. 62 | [Example](https://github.com/quickube/piper/tree/main/examples/.workflows/main.yaml). 63 | 64 | ### config 65 | 66 | Configured by the `piper-workflows-config` [ConfigMap](workflows_config.md). 67 | It can be passed explicitly, or it will use the `default` configuration. 68 | 69 | ### parameters.yaml (convention name) 70 | 71 | It will hold a list of global parameters for the Workflow. 72 | These can be referenced from any template with `{{ workflow.parameters.___ }}`. 73 | 74 | [Example](https://github.com/quickube/piper/tree/main/examples/.workflows/parameters.yaml) -------------------------------------------------------------------------------- /examples/.workflows/exit.yaml: -------------------------------------------------------------------------------- 1 | - name: github-status 2 | template: exit-handler 3 | arguments: 4 | parameters: 5 | - name: param1 6 | value: "{{ workflow.labels.repo }}" -------------------------------------------------------------------------------- /examples/.workflows/main.yaml: -------------------------------------------------------------------------------- 1 | - name: local-step1 2 | template: local-step 3 | arguments: 4 | parameters: 5 | - name: message 6 | value: step-1 7 | - name: local-step2 8 | template: local-step 9 | arguments: 10 | parameters: 11 | - name: message 12 | value: step-2 13 | dependencies: 14 | - local-step1 15 | -------------------------------------------------------------------------------- /examples/.workflows/parameters.yaml: -------------------------------------------------------------------------------- 1 | - name: global 2 | value: multi-branch-pipeline -------------------------------------------------------------------------------- /examples/.workflows/templates.yaml: -------------------------------------------------------------------------------- 1 | - name: local-step 2 | inputs: 3 | parameters: 4 | - name: message 5 | script: 6 | image: alpine 7 | command: [ sh ] 8 | source: | 9 | echo "welcome to {{ workflow.parameters.global }} 10 | echo "{{ inputs.parameters.message }}" 11 | -------------------------------------------------------------------------------- /examples/.workflows/triggers.yaml: -------------------------------------------------------------------------------- 1 | - events: 2 | - push 3 | - pull_request.synchronize 4 | branches: ["main"] 5 | onStart: ["main.yaml"] 6 | onExit: ["exit.yaml"] 7 | templates: ["templates.yaml"] 8 | config: "default" 9 | 10 | - events: 11 | - pull_request 12 | branches: ["*"] 13 | onStart: ["main.yaml"] 14 | onExit: ["exit.yaml"] 15 | templates: ["templates.yaml"] -------------------------------------------------------------------------------- /examples/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: piper-workflows-config 5 | data: 6 | default: | 7 | spec: 8 | volumes: 9 | - name: shared-volume 10 | emptyDir: { } 11 | serviceAccountName: argo-wf 12 | activeDeadlineSeconds: 7200 # (seconds) == 2 hours 13 | ttlStrategy: 14 | secondsAfterCompletion: 28800 # (seconds) == 8 hours 15 | podGC: 16 | strategy: OnPodSuccess 17 | archiveLogs: true 18 | artifactRepositoryRef: 19 | configMap: artifact-repositories 20 | nodeSelector: 21 | node_pool: workflows 22 | tolerations: 23 | - effect: NoSchedule 24 | key: node_pool 25 | operator: Equal 26 | value: workflows 27 | onExit: # optional, will be overwritten if specifc in .workflows/exit.yaml. 28 | - name: github-status 29 | template: exit-handler 30 | arguments: 31 | parameters: 32 | - name: param1 33 | value: "{{ workflow.labels.repo }}" 34 | -------------------------------------------------------------------------------- /examples/template.values.dev.yaml: -------------------------------------------------------------------------------- 1 | piper: 2 | gitProvider: 3 | name: "" # github/bitbucket/gitlab | env: GIT_PROVIDER 4 | token: "GIT_TOKEN" 5 | organization: 6 | name: "ORG_NAME" 7 | webhook: 8 | url: https://NGROK_ADDRESS/piper/webhook 9 | orgLevel: false 10 | repoList: ["REPO_NAME"] 11 | argoWorkflows: 12 | server: 13 | namespace: "workflows" 14 | address: "ARGO_ADDRESS" 15 | token: "ARGO_TOKEN" 16 | image: 17 | name: piper 18 | repository: localhost:5001 19 | pullPolicy: Always 20 | tag: latest 21 | ingress: 22 | enabled: true 23 | annotations: 24 | nginx.ingress.kubernetes.io/rewrite-target: /$2 25 | hosts: 26 | - paths: 27 | - path: /piper(/|$)(.*) 28 | pathType: ImplementationSpecific -------------------------------------------------------------------------------- /examples/workflow.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Workflow 3 | metadata: 4 | generateName: test- 5 | labels: 6 | branch: test-branch 7 | commit: xxxxxxxxxxxxxx 8 | repo: somerepo 9 | user: gosharo 10 | spec: 11 | volumes: 12 | - name: shared-volume 13 | emptyDir: { } 14 | activeDeadlineSeconds: 7200 # (seconds) == 2 hours 15 | ttlStrategy: 16 | secondsAfterCompletion: 28800 # (seconds) == 8 hours 17 | podGC: 18 | strategy: OnPodSuccess 19 | archiveLogs: true 20 | arguments: 21 | parameters: 22 | - name: PLACEHOLDER 23 | artifactRepositoryRef: 24 | configMap: artifact-repositories 25 | onExit: exit-handler 26 | entrypoint: entrypoint 27 | nodeSelector: 28 | node_pool: workflows 29 | serviceAccountName: argo-wf 30 | tolerations: 31 | - effect: NoSchedule 32 | key: node_pool 33 | operator: Equal 34 | value: workflows 35 | templates: 36 | - dag: 37 | name: exit-handler 38 | tasks: 39 | - name: github-status 40 | template: exit-handler 41 | arguments: 42 | parameters: 43 | - name: param1 44 | value: '{{ workflow.labels.repo }}' 45 | - name: local-step 46 | inputs: 47 | parameters: 48 | - name: message 49 | script: 50 | image: alpine 51 | command: [sh] 52 | source: | 53 | echo "welcome to {{ workflow.parameters.global }} 54 | echo "{{ inputs.parameters.message }}" 55 | - name: exit-handler 56 | script: 57 | image: alpine 58 | command: [sh] 59 | source: | 60 | echo "exit" 61 | - dag: 62 | name: entrypoint 63 | tasks: 64 | - name: local-step1 65 | template: local-step 66 | arguments: 67 | parameters: 68 | - name: message 69 | value: step-1 70 | - name: local-step2 71 | template: local-step 72 | arguments: 73 | parameters: 74 | - name: message 75 | value: step-2 76 | dependencies: 77 | - local-step1 78 | -------------------------------------------------------------------------------- /gitlab.values.yaml: -------------------------------------------------------------------------------- 1 | gitlab: 2 | toolbox: 3 | enabled: true 4 | extraVolumes: |- 5 | - name: piper-config 6 | configMap: 7 | name: piper-setup 8 | extraVolumeMounts: |- 9 | - mountPath: /tmp/scripts/piper-setup.rb 10 | name: piper-config 11 | subPath: piper-setup.rb 12 | readOnly: true 13 | gitlab-shell: 14 | enabled: true 15 | gitlab-pages: 16 | enabled: false 17 | gitlab-exporter: 18 | enabled: false 19 | kas: 20 | minReplicas: 1 21 | webservice: 22 | enabled: true 23 | minReplicas: 1 24 | ingress: 25 | requireBasePath: false 26 | global: 27 | gitlab: 28 | license: 29 | key: license_key 30 | secret: gitlab-license 31 | hosts: 32 | domain: localhost 33 | https: false 34 | gitlab: 35 | name: localhost 36 | https: false 37 | ingress: 38 | enabled: true 39 | configureCertmanager: false 40 | tls: 41 | enabled: false 42 | redis: 43 | install: false 44 | traefik: 45 | enabled: false 46 | gitlab-runner: 47 | install: false 48 | registry: 49 | enabled: false 50 | prometheus: 51 | install: false 52 | certmanager: 53 | installCRDs: false 54 | install: false 55 | nginx-ingress: 56 | controller: 57 | ingressClassResource: 58 | name: gitlab-nginx 59 | controllerValue: "k8s.io/ingress-nginx-gitlab" 60 | replicaCount: 1 61 | minAavailable: 1 62 | service: 63 | type: NodePort 64 | nodePorts: 65 | http: 32080 66 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/quickube/piper 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/Rookout/GoSDK v0.1.45 7 | github.com/argoproj/argo-workflows/v3 v3.4.8 8 | github.com/emicklei/go-restful/v3 v3.8.0 9 | github.com/gin-gonic/gin v1.9.1 10 | github.com/google/go-cmp v0.5.9 11 | github.com/google/go-github/v52 v52.0.0 12 | github.com/kelseyhightower/envconfig v1.4.0 13 | github.com/ktrysmt/go-bitbucket v0.9.66 14 | github.com/stretchr/testify v1.8.4 15 | github.com/tidwall/gjson v1.16.0 16 | github.com/xanzy/go-gitlab v0.113.0 17 | golang.org/x/net v0.17.0 18 | gopkg.in/yaml.v3 v3.0.1 19 | k8s.io/apimachinery v0.24.3 20 | k8s.io/client-go v0.24.3 21 | ) 22 | 23 | require ( 24 | github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect 25 | github.com/bytedance/sonic v1.9.1 // indirect 26 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect 27 | github.com/cloudflare/circl v1.3.3 // indirect 28 | github.com/davecgh/go-spew v1.1.1 // indirect 29 | github.com/fallais/logrus-lumberjack-hook v0.0.0-20210917073259-3227e1ab93b0 // indirect 30 | github.com/gabriel-vasile/mimetype v1.4.2 // indirect 31 | github.com/gin-contrib/sse v0.1.0 // indirect 32 | github.com/go-errors/errors v1.4.1 // indirect 33 | github.com/go-logr/logr v1.2.3 // indirect 34 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 35 | github.com/go-openapi/jsonreference v0.20.2 // indirect 36 | github.com/go-openapi/swag v0.22.3 // indirect 37 | github.com/go-playground/locales v0.14.1 // indirect 38 | github.com/go-playground/universal-translator v0.18.1 // indirect 39 | github.com/go-playground/validator/v10 v10.14.0 // indirect 40 | github.com/goccy/go-json v0.10.2 // indirect 41 | github.com/gogo/protobuf v1.3.2 // indirect 42 | github.com/golang/protobuf v1.5.3 // indirect 43 | github.com/google/gnostic v0.5.7-v3refs // indirect 44 | github.com/google/go-querystring v1.1.0 // indirect 45 | github.com/google/gofuzz v1.2.0 // indirect 46 | github.com/google/uuid v1.3.0 // indirect 47 | github.com/gorilla/websocket v1.5.0 // indirect 48 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect 49 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 50 | github.com/hashicorp/go-retryablehttp v0.7.7 // indirect 51 | github.com/hashicorp/golang-lru v0.5.4 // indirect 52 | github.com/imdario/mergo v0.3.13 // indirect 53 | github.com/josharian/intern v1.0.0 // indirect 54 | github.com/json-iterator/go v1.1.12 // indirect 55 | github.com/klauspost/cpuid/v2 v2.2.4 // indirect 56 | github.com/kr/pretty v0.3.1 // indirect 57 | github.com/leodido/go-urn v1.2.4 // indirect 58 | github.com/mailru/easyjson v0.7.7 // indirect 59 | github.com/mattn/go-isatty v0.0.20 // indirect 60 | github.com/mitchellh/mapstructure v1.5.0 // indirect 61 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 62 | github.com/modern-go/reflect2 v1.0.2 // indirect 63 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 64 | github.com/pelletier/go-toml/v2 v2.0.8 // indirect 65 | github.com/pkg/errors v0.9.1 // indirect 66 | github.com/pmezard/go-difflib v1.0.0 // indirect 67 | github.com/sirupsen/logrus v1.9.2 // indirect 68 | github.com/spf13/pflag v1.0.5 // indirect 69 | github.com/tidwall/match v1.1.1 // indirect 70 | github.com/tidwall/pretty v1.2.0 // indirect 71 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 72 | github.com/ugorji/go/codec v1.2.11 // indirect 73 | github.com/yhirose/go-peg v0.0.0-20210804202551-de25d6753cf1 // indirect 74 | golang.org/x/arch v0.3.0 // indirect 75 | golang.org/x/crypto v0.17.0 // indirect 76 | golang.org/x/oauth2 v0.11.0 // indirect 77 | golang.org/x/sys v0.20.0 // indirect 78 | golang.org/x/term v0.15.0 // indirect 79 | golang.org/x/text v0.14.0 // indirect 80 | golang.org/x/time v0.3.0 // indirect 81 | google.golang.org/appengine v1.6.7 // indirect 82 | google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect 83 | google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect 84 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect 85 | google.golang.org/grpc v1.56.3 // indirect 86 | google.golang.org/protobuf v1.31.0 // indirect 87 | gopkg.in/inf.v0 v0.9.1 // indirect 88 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 89 | gopkg.in/yaml.v2 v2.4.0 // indirect 90 | k8s.io/api v0.24.3 // indirect 91 | k8s.io/klog/v2 v2.60.1 // indirect 92 | k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 // indirect 93 | k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect 94 | sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect 95 | sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect 96 | sigs.k8s.io/yaml v1.3.0 // indirect 97 | ) 98 | -------------------------------------------------------------------------------- /helm-chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | _lint.yaml -------------------------------------------------------------------------------- /helm-chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: piper 3 | description: A Helm chart for Piper 4 | type: application 5 | version: 1.0.1 6 | appVersion: 1.0.1 -------------------------------------------------------------------------------- /helm-chart/README.md: -------------------------------------------------------------------------------- 1 | # piper 2 | 3 | ![Version: 1.0.1](https://img.shields.io/badge/Version-1.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.1](https://img.shields.io/badge/AppVersion-1.0.1-informational?style=flat-square) 4 | 5 | A Helm chart for Piper 6 | 7 | ## Values 8 | 9 | | Key | Type | Default | Description | 10 | |-----|------|---------|-------------| 11 | | affinity | object | `{}` | Assign custom [affinity] rules to the deployment | 12 | | autoscaling.enabled | bool | `false` | Wheter to enable auto-scaling of piper. | 13 | | autoscaling.maxReplicas | int | `5` | Maximum reoplicas of Piper. | 14 | | autoscaling.minReplicas | int | `1` | Minimum reoplicas of Piper. | 15 | | autoscaling.targetCPUUtilizationPercentage | int | `85` | CPU utilization percentage threshold. | 16 | | autoscaling.targetMemoryUtilizationPercentage | int | `85` | Memory utilization percentage threshold. | 17 | | env | list | `[]` | Additional environment variables for Piper. A list of name/value maps. | 18 | | extraLabels | object | `{}` | Deployment and pods extra labels | 19 | | fullnameOverride | string | `""` | String to fully override "piper.fullname" template | 20 | | image.name | string | `"piper"` | Piper image name | 21 | | image.pullPolicy | string | `"IfNotPresent"` | Piper image pull policy | 22 | | image.repository | string | `"rookout"` | Piper public dockerhub repo | 23 | | image.tag | string | `""` | Piper image tag | 24 | | imagePullSecrets | list | `[]` | secret to use for image pulling | 25 | | ingress.annotations | object | `{}` | Piper ingress annotations | 26 | | ingress.className | string | `""` | Piper ingress class name | 27 | | ingress.enabled | bool | `false` | Enable Piper ingress support | 28 | | ingress.hosts | list | `[{"host":"piper.example.local","paths":[{"path":"/","pathType":"ImplementationSpecific"}]}]` | Piper ingress hosts # Hostnames must be provided if Ingress is enabled. | 29 | | ingress.tls | list | `[]` | Controller ingress tls | 30 | | lifecycle | object | `{}` | Specify postStart and preStop lifecycle hooks for Piper container | 31 | | nameOverride | string | `""` | String to partially override "piper.fullname" template | 32 | | nodeSelector | object | `{}` | [Node selector] | 33 | | piper.argoWorkflows.crdCreation | bool | `true` | Whether create Workflow CRD or send direct commands to Argo Workflows server. | 34 | | piper.argoWorkflows.server.address | string | `""` | The DNS address of Argo Workflow server that Piper can address. | 35 | | piper.argoWorkflows.server.existingSecret | string | `nil` | | 36 | | piper.argoWorkflows.server.namespace | string | `""` | The namespace in which the Workflow CRD will be created. | 37 | | piper.argoWorkflows.server.token | string | `""` | This will create a secret named -token and with the key 'token' | 38 | | piper.gitProvider.existingSecret | string | `nil` | | 39 | | piper.gitProvider.name | string | `"github"` | Name of your git provider (github/gitlab/bitbucket). for now, only github supported. | 40 | | piper.gitProvider.organization.name | string | `""` | Name of your Git Organization | 41 | | piper.gitProvider.token | string | `nil` | This will create a secret named -git-token and with the key 'token' | 42 | | piper.gitProvider.webhook.existingSecret | string | `nil` | | 43 | | piper.gitProvider.webhook.orgLevel | bool | `false` | Whether config webhook on org level | 44 | | piper.gitProvider.webhook.repoList | list | `[]` | Used of orgLevel=false, to configure webhook for each of the repos provided. | 45 | | piper.gitProvider.webhook.secret | string | `""` | This will create a secret named -webhook-secret and with the key 'secret' | 46 | | piper.gitProvider.webhook.url | string | `""` | The url in which piper listens for webhook, the path should be /webhook | 47 | | piper.workflowsConfig | object | `{}` | | 48 | | podAnnotations | object | `{}` | Annotations to be added to the Piper pods | 49 | | podSecurityContext | object | `{"fsGroup":1001,"runAsGroup":1001,"runAsUser":1001}` | Security Context to set on the pod level | 50 | | replicaCount | int | `1` | Piper number of replicas | 51 | | resources | object | `{"requests":{"cpu":"200m","memory":"512Mi"}}` | Resource limits and requests for the pods. | 52 | | rookout.existingSecret | string | `""` | | 53 | | rookout.token | string | `""` | Rookout token for agent configuration and enablement. | 54 | | securityContext | object | `{"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"runAsUser":1001}` | Security Context to set on the container level | 55 | | service.annotations | object | `{}` | Piper service extra annotations | 56 | | service.labels | object | `{}` | Piper service extra labels | 57 | | service.port | int | `80` | Service port For TLS mode change the port to 443 | 58 | | service.type | string | `"ClusterIP"` | Sets the type of the Service | 59 | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account | 60 | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created | 61 | | serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | 62 | | tolerations | list | `[]` | [Tolerations] for use with node taints | 63 | | volumeMounts | list | `[]` | Volumes to mount to Piper container. | 64 | | volumes | list | `[]` | Volumes of Piper Pod. | 65 | 66 | ---------------------------------------------- 67 | Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) 68 | -------------------------------------------------------------------------------- /helm-chart/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "piper.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Return secret name to be used based on provided values. 10 | */}} 11 | {{- define "piper.argoWorkflows.tokenSecretName" -}} 12 | {{- $fullName := printf "%s-argo-token" .Release.Name -}} 13 | {{- default $fullName .Values.piper.argoWorkflows.server.existingSecret | quote -}} 14 | {{- end -}} 15 | 16 | {{/* 17 | Return secret name to be used based on provided values. 18 | */}} 19 | {{- define "piper.gitProvider.tokenSecretName" -}} 20 | {{- $fullName := printf "%s-git-token" .Release.Name -}} 21 | {{- default $fullName .Values.piper.gitProvider.existingSecret | quote -}} 22 | {{- end -}} 23 | 24 | {{/* 25 | Return secret name to be used based on provided values. 26 | */}} 27 | {{- define "piper.gitProvider.webhook.secretName" -}} 28 | {{- $fullName := printf "%s-webhook-secret" .Release.Name -}} 29 | {{- default $fullName .Values.piper.gitProvider.webhook.existingSecret | quote -}} 30 | {{- end -}} 31 | 32 | {{/* 33 | Return secret name to be used based on provided values. 34 | */}} 35 | {{- define "rookout.secretName" -}} 36 | {{- $fullName := printf "%s-rookout-token" .Release.Name -}} 37 | {{- default $fullName .Values.rookout.existingSecret | quote -}} 38 | {{- end -}} 39 | 40 | 41 | {{/* 42 | Create a default fully qualified app name. 43 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 44 | If release name contains chart name it will be used as a full name. 45 | */}} 46 | {{- define "piper.fullname" -}} 47 | {{- if .Values.fullnameOverride }} 48 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 49 | {{- else }} 50 | {{- $name := default .Chart.Name .Values.nameOverride }} 51 | {{- if contains $name .Release.Name }} 52 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 53 | {{- else }} 54 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 55 | {{- end }} 56 | {{- end }} 57 | {{- end }} 58 | 59 | {{/* 60 | Create chart name and version as used by the chart label. 61 | */}} 62 | {{- define "piper.chart" -}} 63 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 64 | {{- end }} 65 | 66 | {{/* 67 | Common labels 68 | */}} 69 | {{- define "piper.labels" -}} 70 | helm.sh/chart: {{ include "piper.chart" . }} 71 | {{ include "piper.selectorLabels" . }} 72 | 73 | {{- if .Chart.AppVersion }} 74 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 75 | {{- end }} 76 | app.kubernetes.io/managed-by: {{ .Release.Service }} 77 | {{- end }} 78 | 79 | {{/* 80 | Selector labels 81 | */}} 82 | {{- define "piper.selectorLabels" -}} 83 | app.kubernetes.io/name: {{ include "piper.name" . }} 84 | app.kubernetes.io/instance: {{ .Release.Name }} 85 | {{- end }} 86 | 87 | {{/* 88 | Create the name of the service account to use 89 | */}} 90 | {{- define "piper.serviceAccountName" -}} 91 | {{- if .Values.serviceAccount.create }} 92 | {{- default (include "piper.fullname" .) .Values.serviceAccount.name }} 93 | {{- else }} 94 | {{- default "default" .Values.serviceAccount.name }} 95 | {{- end }} 96 | {{- end }} 97 | -------------------------------------------------------------------------------- /helm-chart/templates/argo-token-secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.piper.argoWorkflows.server.token (not .Values.piper.argoWorkflows.server.existingSecret) }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "piper.argoWorkflows.tokenSecretName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | app.kubernetes.io/name: {{ .Chart.Name }} 9 | app.kubernetes.io/instance: {{ .Release.Name }} 10 | type: Opaque 11 | data: 12 | token: {{ .Values.piper.argoWorkflows.server.token | b64enc | quote }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /helm-chart/templates/config.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.piper.workflowsConfig }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: piper-workflows-config 6 | labels: 7 | {{- include "piper.labels" . | nindent 4 }} 8 | data: 9 | {{- with .Values.piper.workflowsConfig }} 10 | {{- toYaml . | nindent 2 }} 11 | {{- end }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /helm-chart/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "piper.fullname" . }} 5 | labels: 6 | {{- include "piper.labels" . | nindent 4 }} 7 | spec: 8 | revisionHistoryLimit: 3 9 | {{- if not .Values.autoscaling.enabled }} 10 | replicas: {{ .Values.replicaCount }} 11 | {{- end }} 12 | selector: 13 | matchLabels: 14 | {{- include "piper.selectorLabels" . | nindent 6 }} 15 | template: 16 | metadata: 17 | {{- with .Values.podAnnotations }} 18 | annotations: 19 | {{- toYaml . | nindent 8 }} 20 | {{- end }} 21 | labels: 22 | {{- include "piper.selectorLabels" . | nindent 8 }} 23 | app: {{ .Chart.Name | trunc 63 | trimSuffix "-" }} 24 | version: {{ .Values.image.tag | default .Chart.AppVersion | trunc 63 | trimSuffix "-" }} 25 | spec: 26 | volumes: 27 | {{- if .Values.piper.workflowsConfig }} 28 | - name: piper-workflows-config 29 | configMap: 30 | name: piper-workflows-config 31 | {{- end }} 32 | {{- with .Values.volumes }} 33 | {{- toYaml . | nindent 8 }} 34 | {{- end }} 35 | {{- with .Values.imagePullSecrets }} 36 | imagePullSecrets: 37 | {{- toYaml . | nindent 8 }} 38 | {{- end }} 39 | serviceAccountName: {{ include "piper.serviceAccountName" . }} 40 | securityContext: 41 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 42 | containers: 43 | - name: {{ .Chart.Name }} 44 | volumeMounts: 45 | {{- if .Values.piper.workflowsConfig }} 46 | - mountPath: /piper-config 47 | name: piper-workflows-config 48 | readOnly: true 49 | {{- end }} 50 | {{- with .Values.volumeMounts }} 51 | {{- toYaml . | nindent 12 }} 52 | {{- end }} 53 | securityContext: 54 | {{- toYaml .Values.securityContext | nindent 12 }} 55 | {{- with .Values.lifecycle }} 56 | lifecycle: 57 | {{- toYaml . | nindent 10 }} 58 | {{- end }} 59 | image: {{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }} 60 | imagePullPolicy: {{ .Values.image.pullPolicy }} 61 | ports: 62 | - containerPort: 8080 63 | protocol: TCP 64 | livenessProbe: 65 | httpGet: 66 | path: /healthz 67 | port: 8080 68 | scheme: HTTP 69 | initialDelaySeconds: 10 70 | timeoutSeconds: 10 71 | periodSeconds: 60 72 | successThreshold: 1 73 | failureThreshold: 4 74 | readinessProbe: 75 | httpGet: 76 | path: /readyz 77 | port: 8080 78 | scheme: HTTP 79 | initialDelaySeconds: 2 80 | timeoutSeconds: 1 81 | periodSeconds: 4 82 | successThreshold: 1 83 | failureThreshold: 2 84 | resources: 85 | {{- toYaml .Values.resources | nindent 12 }} 86 | env: 87 | {{- if or .Values.rookout.token .Values.rookout.existingSecret }} 88 | - name: ROOKOUT_TOKEN 89 | valueFrom: 90 | secretKeyRef: 91 | name: {{ template "rookout.secretName" . }} 92 | key: token 93 | {{- end }} 94 | - name: GIT_PROVIDER 95 | value: {{ .Values.piper.gitProvider.name | quote }} 96 | - name: GIT_TOKEN 97 | valueFrom: 98 | secretKeyRef: 99 | name: {{ template "piper.gitProvider.tokenSecretName" . }} 100 | key: token 101 | - name: GIT_ORG_NAME 102 | value: {{ .Values.piper.gitProvider.organization.name | quote }} 103 | - name: GIT_URL 104 | value: {{ .Values.piper.gitProvider.url | quote }} 105 | - name: GIT_WEBHOOK_URL 106 | value: {{ .Values.piper.gitProvider.webhook.url | quote }} 107 | - name: GIT_WEBHOOK_SECRET 108 | valueFrom: 109 | secretKeyRef: 110 | name: {{ template "piper.gitProvider.webhook.secretName" . }} 111 | key: secret 112 | - name: GIT_ORG_LEVEL_WEBHOOK 113 | value: {{ .Values.piper.gitProvider.webhook.orgLevel | quote }} 114 | - name: GIT_WEBHOOK_REPO_LIST 115 | value: {{ join "," .Values.piper.gitProvider.webhook.repoList | quote }} 116 | {{- if or .Values.piper.argoWorkflows.server.token .Values.piper.argoWorkflows.server.existingSecret }} 117 | - name: ARGO_WORKFLOWS_TOKEN 118 | valueFrom: 119 | secretKeyRef: 120 | name: {{ template "piper.argoWorkflows.tokenSecretName" . }} 121 | key: token 122 | {{- end }} 123 | - name: ARGO_WORKFLOWS_NAMESPACE 124 | value: {{ .Values.piper.argoWorkflows.server.namespace | default .Release.Namespace | quote }} 125 | - name: ARGO_WORKFLOWS_ADDRESS 126 | value: {{ .Values.piper.argoWorkflows.server.address | quote }} 127 | - name: ARGO_WORKFLOWS_CREATE_CRD 128 | value: {{ .Values.piper.argoWorkflows.crdCreation | quote }} 129 | {{- with .Values.env }} 130 | {{- toYaml . | nindent 10 }} 131 | {{- end }} 132 | {{- with .Values.nodeSelector }} 133 | nodeSelector: 134 | {{- toYaml . | nindent 8 }} 135 | {{- end }} 136 | {{- with .Values.affinity }} 137 | affinity: 138 | {{- toYaml . | nindent 8 }} 139 | {{- end }} 140 | {{- with .Values.tolerations }} 141 | tolerations: 142 | {{- toYaml . | nindent 8 }} 143 | {{- end }} 144 | -------------------------------------------------------------------------------- /helm-chart/templates/git-token-secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.piper.gitProvider.token (not .Values.piper.gitProvider.existingSecret) }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "piper.gitProvider.tokenSecretName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | app.kubernetes.io/name: {{ .Chart.Name }} 9 | app.kubernetes.io/instance: {{ .Release.Name }} 10 | type: Opaque 11 | data: 12 | token: {{ .Values.piper.gitProvider.token | b64enc | quote }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /helm-chart/templates/hpa.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.autoscaling.enabled }} 2 | apiVersion: autoscaling/v2beta1 3 | kind: HorizontalPodAutoscaler 4 | metadata: 5 | name: {{ include "piper.fullname" . }} 6 | labels: 7 | {{- include "piper.labels" . | nindent 4 }} 8 | spec: 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: {{ include "piper.fullname" . }} 13 | minReplicas: {{ .Values.autoscaling.minReplicas }} 14 | maxReplicas: {{ .Values.autoscaling.maxReplicas }} 15 | metrics: 16 | {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} 17 | - type: Resource 18 | resource: 19 | name: cpu 20 | targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} 21 | {{- end }} 22 | {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} 23 | - type: Resource 24 | resource: 25 | name: memory 26 | targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} 27 | {{- end }} 28 | {{- end }} 29 | -------------------------------------------------------------------------------- /helm-chart/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "piper.fullname" . -}} 3 | {{- $svcPort := .Values.service.port -}} 4 | {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} 5 | {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} 6 | {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} 7 | {{- end }} 8 | {{- end }} 9 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} 10 | apiVersion: networking.k8s.io/v1 11 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 12 | apiVersion: networking.k8s.io/v1beta1 13 | {{- else -}} 14 | apiVersion: extensions/v1beta1 15 | {{- end }} 16 | kind: Ingress 17 | metadata: 18 | name: {{ $fullName }} 19 | labels: 20 | {{- include "piper.labels" . | nindent 4 }} 21 | {{- with .Values.ingress.annotations }} 22 | annotations: 23 | {{- toYaml . | nindent 4 }} 24 | {{- end }} 25 | spec: 26 | {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} 27 | ingressClassName: {{ .Values.ingress.className }} 28 | {{- end }} 29 | {{- if .Values.ingress.tls }} 30 | tls: 31 | {{- range .Values.ingress.tls }} 32 | - hosts: 33 | {{- range .hosts }} 34 | - {{ . | quote }} 35 | {{- end }} 36 | secretName: {{ .secretName }} 37 | {{- end }} 38 | {{- end }} 39 | rules: 40 | {{- range .Values.ingress.hosts }} 41 | - host: {{ .host | quote }} 42 | http: 43 | paths: 44 | {{- range .paths }} 45 | - path: {{ .path }} 46 | {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} 47 | pathType: {{ .pathType }} 48 | {{- end }} 49 | backend: 50 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} 51 | service: 52 | name: {{ $fullName }} 53 | port: 54 | number: {{ $svcPort }} 55 | {{- else }} 56 | serviceName: {{ $fullName }} 57 | servicePort: {{ $svcPort }} 58 | {{- end }} 59 | {{- end }} 60 | {{- end }} 61 | {{- end }} 62 | -------------------------------------------------------------------------------- /helm-chart/templates/rookout-token.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.rookout.token (not .Values.rookout.existingSecret) }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "rookout.secretName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | app.kubernetes.io/name: {{ .Chart.Name }} 9 | app.kubernetes.io/instance: {{ .Release.Name }} 10 | type: Opaque 11 | data: 12 | token: {{ .Values.rookout.token | b64enc | quote }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /helm-chart/templates/security.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: {{ include "piper.fullname" . }} 5 | rules: 6 | - apiGroups: 7 | - "" 8 | resources: 9 | - events 10 | verbs: 11 | - list 12 | - watch 13 | - create 14 | - patch 15 | - apiGroups: 16 | - argoproj.io 17 | resources: 18 | - workflows 19 | - workflows/finalizers 20 | - workflowtasksets 21 | - workflowtasksets/finalizers 22 | - workflowartifactgctasks 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - update 28 | - patch 29 | - delete 30 | - create 31 | --- 32 | apiVersion: rbac.authorization.k8s.io/v1 33 | kind: ClusterRoleBinding 34 | metadata: 35 | name: {{ include "piper.fullname" . }} 36 | subjects: 37 | - kind: ServiceAccount 38 | name: {{ include "piper.fullname" . }} 39 | namespace: {{ .Release.Namespace | quote }} 40 | roleRef: 41 | apiGroup: rbac.authorization.k8s.io 42 | kind: ClusterRole 43 | name: {{ include "piper.fullname" . }} 44 | -------------------------------------------------------------------------------- /helm-chart/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "piper.fullname" . }} 5 | labels: 6 | {{- include "piper.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.service.type }} 9 | ports: 10 | - port: {{ .Values.service.port }} 11 | targetPort: 8080 12 | protocol: TCP 13 | name: http 14 | selector: 15 | {{- include "piper.selectorLabels" . | nindent 4 }} -------------------------------------------------------------------------------- /helm-chart/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "piper.serviceAccountName" . }} 6 | labels: 7 | {{- include "piper.labels" . | nindent 4 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /helm-chart/templates/webhook-secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.piper.gitProvider.webhook.existingSecret }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "piper.gitProvider.webhook.secretName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | app.kubernetes.io/name: {{ .Chart.Name }} 9 | app.kubernetes.io/instance: {{ .Release.Name }} 10 | type: Opaque 11 | data: 12 | {{- if and .Values.piper.gitProvider.webhook.secret }} 13 | secret: {{ .Values.piper.gitProvider.webhook.secret | b64enc | quote }} 14 | {{- else }} 15 | secret: {{ randAlphaNum 30 | b64enc | quote }} 16 | {{- end }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /helm-chart/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for Piper. 2 | # For more information head to https:/github.com/quickube/piper 3 | 4 | # Map of Piper configurations. 5 | piper: 6 | gitProvider: 7 | # -- Name of your git provider (github/bitbucket/gitlab). 8 | name: github 9 | # -- The token for authentication with the Git provider. 10 | # -- This will create a secret named -git-token and with the key 'token' 11 | token: 12 | # -- The token for authentication with the Git provider. 13 | # -- Reference to existing token with 'token' key. 14 | # -- can be created with `kubectl create secret generic piper-git-token --from-literal=token=YOUR_TOKEN` 15 | existingSecret: #piper-git-token 16 | # -- git provider url 17 | # -- relevant when using gitlab self hosted 18 | url: "" 19 | # Map of organization configurations. 20 | organization: 21 | # -- Name of your Git Organization (GitHub) / Workspace (Bitbucket) or Group (Gitlab) 22 | name: "" 23 | # Map of webhook configurations. 24 | webhook: 25 | # -- The secret that will be used for webhook authentication 26 | # -- If not provided, will be generated 27 | # -- This will create a secret named -webhook-secret and with the key 'secret' 28 | secret: "" 29 | # -- The secret for webhook encryption 30 | # -- Reference to existing token with 'secret' key. 31 | # -- can be created with `kubectl create secret generic piper-webhook-secret --from-literal=secret=YOUR_TOKEN` 32 | existingSecret: #piper-webhook-secret 33 | # -- The url in which piper listens for webhook, the path should be /webhook 34 | url: "" #https://piper.example.local/webhook 35 | # -- Whether config webhook on org level (GitHub) or at workspace level (Bitbucket - not supported yet) 36 | orgLevel: false 37 | # -- (Github/Gitlab) Used of orgLevel=false, to configure webhook for each of the repos provided. 38 | repoList: [] 39 | 40 | # Map of Argo Workflows configurations. 41 | argoWorkflows: 42 | # Map of Argo Workflows server configurations. 43 | server: 44 | # -- The namespace in which the Workflow CRD will be created. 45 | namespace: "" 46 | # -- The DNS address of Argo Workflow server that Piper can address. 47 | address: "" 48 | # -- The token for authentication with Argo Workflows server. 49 | # -- This will create a secret named -token and with the key 'token' 50 | token: "" 51 | # -- The token for authentication with Argo Workflows server. 52 | # -- Reference to existing token with 'token' key. 53 | # -- can be created with `kubectl create secret generic piper-argo-token --from-literal=token=YOUR_TOKEN` 54 | existingSecret: #piper-argo-token 55 | # -- Whether create Workflow CRD or send direct commands to Argo Workflows server. 56 | crdCreation: true 57 | 58 | workflowsConfig: 59 | {} 60 | # default: | 61 | # spec: 62 | # volumes: 63 | # - name: shared-volume 64 | # emptyDir: {} 65 | # serviceAccountName: argo-wf 66 | # activeDeadlineSeconds: 7200 # (seconds) == 2 hours 67 | # ttlStrategy: 68 | # secondsAfterCompletion: 28800 # (seconds) == 8 hours 69 | # podGC: 70 | # strategy: OnPodSuccess 71 | # archiveLogs: true 72 | # artifactRepositoryRef: 73 | # configMap: artifact-repositories 74 | # nodeSelector: 75 | # node_pool: workflows 76 | # tolerations: 77 | # - effect: NoSchedule 78 | # key: node_pool 79 | # operator: Equal 80 | # value: workflows 81 | # onExit: # optinal, will be overwritten if specifc in .wokrflows/exit.yaml. 82 | # - name: github-status 83 | # template: exit-handler 84 | # arguments: 85 | # parameters: 86 | # - name: param1 87 | # value: "{{ workflow.labels.repo }}" 88 | 89 | rookout: 90 | # -- Rookout token for agent configuration and enablement. 91 | token: "" 92 | # -- The token for Rookout. 93 | # -- Reference to existing token with 'token' key. 94 | # -- can be created with `kubectl create secret generic piper-rookout-token --from-literal=token=YOUR_TOKEN` 95 | existingSecret: "" 96 | 97 | # -- Piper number of replicas 98 | replicaCount: 1 99 | 100 | image: 101 | # -- Piper image name 102 | name: piper 103 | # -- Piper public dockerhub repo 104 | repository: quickube 105 | # -- Piper image pull policy 106 | pullPolicy: IfNotPresent 107 | # -- Piper image tag 108 | tag: "" 109 | 110 | # -- secret to use for image pulling 111 | imagePullSecrets: [] 112 | 113 | # -- String to partially override "piper.fullname" template 114 | nameOverride: "" 115 | 116 | # -- String to fully override "piper.fullname" template 117 | fullnameOverride: "" 118 | 119 | serviceAccount: 120 | # -- Specifies whether a service account should be created 121 | create: true 122 | # -- Annotations to add to the service account 123 | annotations: {} 124 | # -- The name of the service account to use. 125 | # If not set and create is true, a name is generated using the fullname template 126 | name: "" 127 | 128 | # -- Annotations to be added to the Piper pods 129 | podAnnotations: {} 130 | 131 | # -- Security Context to set on the pod level 132 | podSecurityContext: 133 | fsGroup: 1001 134 | runAsUser: 1001 135 | runAsGroup: 1001 136 | 137 | # -- Security Context to set on the container level 138 | securityContext: 139 | runAsUser: 1001 140 | capabilities: 141 | drop: 142 | - ALL 143 | readOnlyRootFilesystem: true 144 | runAsNonRoot: true 145 | 146 | service: 147 | # -- Sets the type of the Service 148 | type: ClusterIP 149 | # -- Service port 150 | # For TLS mode change the port to 443 151 | port: 80 152 | #nodePort: 153 | # -- Piper service extra labels 154 | labels: {} 155 | # -- Piper service extra annotations 156 | annotations: {} 157 | 158 | ingress: 159 | # -- Enable Piper ingress support 160 | enabled: false 161 | # -- Piper ingress class name 162 | className: "" 163 | # -- Piper ingress annotations 164 | annotations: 165 | {} 166 | # kubernetes.io/ingress.class: nginx 167 | # kubernetes.io/tls-acme: "true" 168 | 169 | # -- Piper ingress hosts 170 | ## Hostnames must be provided if Ingress is enabled. 171 | hosts: 172 | - host: piper.example.local 173 | paths: 174 | - path: / 175 | pathType: ImplementationSpecific 176 | # -- Controller ingress tls 177 | tls: [] 178 | # - secretName: chart-example-tls 179 | # hosts: 180 | # - chart-example.local 181 | 182 | # -- Additional environment variables for Piper. A list of name/value maps. 183 | env: [] 184 | 185 | # -- Resource limits and requests for the pods. 186 | resources: 187 | requests: 188 | cpu: 200m 189 | memory: 512Mi 190 | 191 | # -- [Node selector] 192 | nodeSelector: {} 193 | 194 | # -- [Tolerations] for use with node taints 195 | tolerations: [] 196 | 197 | # -- Assign custom [affinity] rules to the deployment 198 | affinity: {} 199 | 200 | # -- Deployment and pods extra labels 201 | extraLabels: {} 202 | 203 | autoscaling: 204 | # -- Wheter to enable auto-scaling of piper. 205 | enabled: false 206 | # -- Minimum reoplicas of Piper. 207 | minReplicas: 1 208 | # -- Maximum reoplicas of Piper. 209 | maxReplicas: 5 210 | # -- CPU utilization percentage threshold. 211 | targetCPUUtilizationPercentage: 85 212 | # -- Memory utilization percentage threshold. 213 | targetMemoryUtilizationPercentage: 85 214 | 215 | # -- Volumes of Piper Pod. 216 | volumes: [] 217 | 218 | # -- Volumes to mount to Piper container. 219 | volumeMounts: [] 220 | 221 | # -- Specify postStart and preStop lifecycle hooks for Piper container 222 | lifecycle: {} 223 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/sh 2 | CLUSTER_DEPLOYED := $(shell kind get clusters -q | grep piper) 3 | 4 | .PHONY: ngrok 5 | ngrok: 6 | ngrok http 80 7 | 8 | .PHONY: local-build 9 | local-build: 10 | DOCKER_BUILDKIT=1 docker build -t localhost:5001/piper:latest . 11 | 12 | .PHONY: local-push 13 | local-push: 14 | docker push localhost:5001/piper:latest 15 | 16 | .PHONY: init-kind 17 | init-kind: 18 | ifndef CLUSTER_DEPLOYED 19 | sh ./scripts/init-kind.sh 20 | else 21 | $(info Kind piper cluster exists, skipping cluster installation) 22 | endif 23 | kubectl config set-context kind-piper 24 | 25 | .PHONY: init-nginx 26 | init-nginx: init-kind 27 | sh ./scripts/init-nginx.sh 28 | 29 | .PHONY: init-argo-workflows 30 | init-argo-workflows: init-kind 31 | sh ./scripts/init-argo-workflows.sh 32 | 33 | .PHONY: init-piper 34 | init-piper: init-kind local-build 35 | sh ./scripts/init-piper.sh 36 | 37 | .PHONY: init-gitlab 38 | init-gitlab: init-kind 39 | @sh ./scripts/init-gitlab.sh $(GITLAB_LICENSE) 40 | 41 | .PHONY: deploy 42 | deploy: init-kind init-nginx init-argo-workflows local-build local-push init-piper 43 | 44 | .PHONY: restart 45 | restart: local-build 46 | docker push localhost:5001/piper:latest 47 | kubectl rollout restart deployment piper 48 | 49 | .PHONY: clean 50 | clean: 51 | kind delete cluster --name piper 52 | docker stop kind-registry && docker rm kind-registry 53 | 54 | .PHONY: helm 55 | helm: 56 | helm lint ./helm-chart 57 | helm template ./helm-chart --debug > _lint.yaml 58 | helm-docs 59 | 60 | .PHONY: test 61 | test: 62 | go test -short ./pkg/... 63 | 64 | $(GOPATH)/bin/golangci-lint: 65 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b `go env GOPATH`/bin v1.52.2 66 | 67 | .PHONY: lint 68 | lint: $(GOPATH)/bin/golangci-lint 69 | $(GOPATH)/bin/golangci-lint run --fix --verbose -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Piper - Multibranch Pipeline for ArgoWorkflows 2 | site_description: 'Piper project for multibranch pipeline in Argo Workflows' 3 | site_author: 'George Dozoretz' 4 | docs_dir: docs/ 5 | repo_url: https://github.com/quickube/piper 6 | repo_name: quickube/piper 7 | theme: 8 | name: material 9 | icon: 10 | repo: fontawesome/brands/github 11 | palette: 12 | - scheme: default 13 | toggle: 14 | icon: material/weather-night 15 | name: Switch to dark mode 16 | - scheme: slate 17 | toggle: 18 | icon: material/weather-sunny 19 | name: Switch to light mode 20 | features: 21 | - content.code.annotate 22 | plugins: 23 | - mkdocs-video: 24 | is_video: True 25 | video_loop: True 26 | video_muted: True 27 | video_autoplay: True 28 | markdown_extensions: 29 | - pymdownx.highlight: 30 | anchor_linenums: true 31 | - pymdownx.inlinehilite 32 | - pymdownx.snippets 33 | - pymdownx.superfences 34 | 35 | nav: 36 | - Introduction: index.md 37 | - Getting Started: getting_started/installation.md 38 | - Configuration: 39 | - configuration/environment_variables.md 40 | - configuration/health_check.md 41 | - Use piper: 42 | - usage/workflows_folder.md 43 | - usage/global_variables.md 44 | - usage/workflows_config.md 45 | - Developers: CONTRIBUTING.md -------------------------------------------------------------------------------- /pkg/clients/types.go: -------------------------------------------------------------------------------- 1 | package clients 2 | 3 | import ( 4 | "github.com/quickube/piper/pkg/git_provider" 5 | "github.com/quickube/piper/pkg/workflow_handler" 6 | ) 7 | 8 | type Clients struct { 9 | GitProvider git_provider.Client 10 | Workflows workflow_handler.WorkflowsClient 11 | } 12 | -------------------------------------------------------------------------------- /pkg/common/types.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/quickube/piper/pkg/git_provider" 5 | ) 6 | 7 | type WorkflowsBatch struct { 8 | OnStart []*git_provider.CommitFile 9 | OnExit []*git_provider.CommitFile 10 | Templates []*git_provider.CommitFile 11 | Parameters *git_provider.CommitFile 12 | Config *string 13 | Payload *git_provider.WebhookPayload 14 | } 15 | -------------------------------------------------------------------------------- /pkg/conf/conf.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "fmt" 5 | "github.com/kelseyhightower/envconfig" 6 | ) 7 | 8 | type GlobalConfig struct { 9 | GitProviderConfig 10 | WorkflowServerConfig 11 | RookoutConfig 12 | WorkflowsConfig 13 | } 14 | 15 | func (cfg *GlobalConfig) Load() error { 16 | err := envconfig.Process("", cfg) 17 | if err != nil { 18 | return fmt.Errorf("failed to load the configuration, error: %v", err) 19 | } 20 | 21 | return nil 22 | } 23 | 24 | func LoadConfig() (*GlobalConfig, error) { 25 | cfg := new(GlobalConfig) 26 | 27 | err := cfg.Load() 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | return cfg, nil 33 | } 34 | -------------------------------------------------------------------------------- /pkg/conf/git_provider.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/kelseyhightower/envconfig" 7 | ) 8 | 9 | type GitProviderConfig struct { 10 | Provider string `envconfig:"GIT_PROVIDER" required:"true"` 11 | Token string `envconfig:"GIT_TOKEN" required:"true"` 12 | Url string `envconfig:"GIT_URL" required:"false"` 13 | OrgName string `envconfig:"GIT_ORG_NAME" required:"true"` 14 | OrgLevelWebhook bool `envconfig:"GIT_ORG_LEVEL_WEBHOOK" default:"false" required:"false"` 15 | RepoList string `envconfig:"GIT_WEBHOOK_REPO_LIST" required:"false"` 16 | WebhookURL string `envconfig:"GIT_WEBHOOK_URL" required:"false"` 17 | WebhookSecret string `envconfig:"GIT_WEBHOOK_SECRET" required:"false"` 18 | WebhookAutoCleanup bool `envconfig:"GIT_WEBHOOK_AUTO_CLEANUP" default:"false" required:"false"` 19 | EnforceOrgBelonging bool `envconfig:"GIT_ENFORCE_ORG_BELONGING" default:"false" required:"false"` 20 | OrgID int64 21 | FullHealthCheck bool `envconfig:"GIT_FULL_HEALTH_CHECK" default:"false" required:"false"` 22 | } 23 | 24 | func (cfg *GitProviderConfig) GitConfLoad() error { 25 | err := envconfig.Process("", cfg) 26 | if err != nil { 27 | return fmt.Errorf("failed to load the Git provider configuration, error: %v", err) 28 | } 29 | 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/conf/rookout.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/kelseyhightower/envconfig" 7 | ) 8 | 9 | type RookoutConfig struct { 10 | Token string `envconfig:"ROOKOUT_TOKEN" default:""` 11 | Labels string `envconfig:"ROOKOUT_LABELS" default:"service:piper"` 12 | RemoteOrigin string `envconfig:"ROOKOUT_REMOTE_ORIGIN" default:"https://github.com/quickube/piper.git"` 13 | } 14 | 15 | func (cfg *RookoutConfig) RookoutConfLoad() error { 16 | err := envconfig.Process("", cfg) 17 | if err != nil { 18 | return fmt.Errorf("failed to load the Rookout configuration, error: %v", err) 19 | } 20 | 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /pkg/conf/workflow_server.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "fmt" 5 | "github.com/kelseyhightower/envconfig" 6 | ) 7 | 8 | type WorkflowServerConfig struct { 9 | ArgoToken string `envconfig:"ARGO_WORKFLOWS_TOKEN" required:"false"` 10 | ArgoAddress string `envconfig:"ARGO_WORKFLOWS_ADDRESS" required:"false"` 11 | CreateCRD bool `envconfig:"ARGO_WORKFLOWS_CREATE_CRD" default:"true"` 12 | Namespace string `envconfig:"ARGO_WORKFLOWS_NAMESPACE" default:"default"` 13 | KubeConfig string `envconfig:"KUBE_CONFIG" default:""` 14 | } 15 | 16 | func (cfg *WorkflowServerConfig) ArgoConfLoad() error { 17 | err := envconfig.Process("", cfg) 18 | if err != nil { 19 | return fmt.Errorf("failed to load the Argo configuration, error: %v", err) 20 | } 21 | 22 | return nil 23 | } 24 | -------------------------------------------------------------------------------- /pkg/conf/workflows_config.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | 7 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 8 | "github.com/quickube/piper/pkg/utils" 9 | ) 10 | 11 | type WorkflowsConfig struct { 12 | Configs map[string]*ConfigInstance 13 | } 14 | 15 | type ConfigInstance struct { 16 | Spec v1alpha1.WorkflowSpec `yaml:"spec"` 17 | OnExit []v1alpha1.DAGTask `yaml:"onExit"` 18 | } 19 | 20 | func (wfc *WorkflowsConfig) WorkflowsSpecLoad(configPath string) error { 21 | var jsonBytes []byte 22 | wfc.Configs = make(map[string]*ConfigInstance) 23 | 24 | configs, err := utils.GetFilesData(configPath) 25 | if len(configs) == 0 { 26 | log.Printf("No config files to load at %s", configPath) 27 | return nil 28 | } 29 | if err != nil { 30 | return err 31 | } 32 | 33 | for key, config := range configs { 34 | tmp := new(ConfigInstance) 35 | jsonBytes, err = utils.ConvertYAMLToJSON(config) 36 | if err != nil { 37 | return err 38 | } 39 | err = json.Unmarshal(jsonBytes, &tmp) 40 | if err != nil { 41 | return err 42 | } 43 | wfc.Configs[key] = tmp 44 | } 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/event_handler/event_notifier.go: -------------------------------------------------------------------------------- 1 | package event_handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 7 | "github.com/quickube/piper/pkg/clients" 8 | "github.com/quickube/piper/pkg/conf" 9 | "github.com/quickube/piper/pkg/utils" 10 | ) 11 | 12 | type eventNotifier struct { 13 | cfg *conf.GlobalConfig 14 | clients *clients.Clients 15 | } 16 | 17 | func NewEventNotifier(cfg *conf.GlobalConfig, clients *clients.Clients) EventNotifier { 18 | return &eventNotifier{ 19 | cfg: cfg, 20 | clients: clients, 21 | } 22 | } 23 | 24 | func (en *eventNotifier) Notify(ctx context.Context, workflow *v1alpha1.Workflow) error { 25 | fmt.Printf("Notifing workflow, %s\n", workflow.GetName()) 26 | 27 | repo, ok := workflow.GetLabels()["repo"] 28 | if !ok { 29 | return fmt.Errorf("failed get repo label for workflow: %s", workflow.GetName()) 30 | } 31 | commit, ok := workflow.GetLabels()["commit"] 32 | if !ok { 33 | return fmt.Errorf("failed get commit label for workflow: %s", workflow.GetName()) 34 | } 35 | 36 | workflowLink := fmt.Sprintf("%s/workflows/%s/%s", en.cfg.WorkflowServerConfig.ArgoAddress, en.cfg.Namespace, workflow.GetName()) 37 | 38 | status, err := en.clients.GitProvider.GetCorrelatingEvent(ctx, &workflow.Status.Phase) 39 | if err != nil { 40 | return fmt.Errorf("failed to translate workflow status for phase: %s status: %s", string(workflow.Status.Phase), status) 41 | } 42 | 43 | message := utils.TrimString(workflow.Status.Message, 140) // Max length of message is 140 characters 44 | err = en.clients.GitProvider.SetStatus(ctx, &repo, &commit, &workflowLink, &status, &message) 45 | if err != nil { 46 | return fmt.Errorf("failed to set status for workflow %s: %s", workflow.GetName(), err) 47 | } 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/event_handler/event_notifier_test.go: -------------------------------------------------------------------------------- 1 | package event_handler 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "github.com/quickube/piper/pkg/git_provider" 7 | assertion "github.com/stretchr/testify/assert" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "net/http" 10 | "testing" 11 | 12 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 13 | "github.com/quickube/piper/pkg/clients" 14 | "github.com/quickube/piper/pkg/conf" 15 | ) 16 | 17 | type mockGitProvider struct{} 18 | 19 | func (m *mockGitProvider) GetFile(ctx context.Context, repo string, branch string, path string) (*git_provider.CommitFile, error) { 20 | return nil, nil 21 | } 22 | 23 | func (m *mockGitProvider) GetFiles(ctx context.Context, repo string, branch string, paths []string) ([]*git_provider.CommitFile, error) { 24 | return nil, nil 25 | } 26 | 27 | func (m *mockGitProvider) ListFiles(ctx context.Context, repo string, branch string, path string) ([]string, error) { 28 | return nil, nil 29 | } 30 | 31 | func (m *mockGitProvider) SetWebhook(ctx context.Context, repo *string) (*git_provider.HookWithStatus, error) { 32 | return nil, nil 33 | } 34 | 35 | func (m *mockGitProvider) UnsetWebhook(ctx context.Context, hook *git_provider.HookWithStatus) error { 36 | return nil 37 | } 38 | 39 | func (m *mockGitProvider) HandlePayload(ctx context.Context, request *http.Request, secret []byte) (*git_provider.WebhookPayload, error) { 40 | return nil, nil 41 | } 42 | 43 | func (m *mockGitProvider) SetStatus(ctx context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error { 44 | return nil 45 | } 46 | func (m *mockGitProvider) GetCorrelatingEvent(ctx context.Context, workflowEvent *v1alpha1.WorkflowPhase) (string, error) { 47 | return "", nil 48 | } 49 | func (m *mockGitProvider) PingHook(ctx context.Context, hook *git_provider.HookWithStatus) error { 50 | return nil 51 | } 52 | 53 | func (m *mockGitProvider) GetHooks() []*git_provider.HookWithStatus { 54 | return nil 55 | } 56 | 57 | func TestNotify(t *testing.T) { 58 | assert := assertion.New(t) 59 | ctx := context.Background() 60 | 61 | // Define test cases 62 | tests := []struct { 63 | name string 64 | workflow *v1alpha1.Workflow 65 | wantedError error 66 | }{ 67 | { 68 | name: "Succeeded workflow", 69 | workflow: &v1alpha1.Workflow{ 70 | ObjectMeta: metav1.ObjectMeta{ 71 | Name: "test-workflow", 72 | Labels: map[string]string{ 73 | "repo": "test-repo", 74 | "commit": "test-commit", 75 | }, 76 | }, 77 | Status: v1alpha1.WorkflowStatus{ 78 | Phase: v1alpha1.WorkflowSucceeded, 79 | Message: "", 80 | }, 81 | }, 82 | wantedError: nil, 83 | }, 84 | { 85 | name: "Failed workflow", 86 | workflow: &v1alpha1.Workflow{ 87 | ObjectMeta: metav1.ObjectMeta{ 88 | Name: "test-workflow", 89 | Labels: map[string]string{ 90 | "repo": "test-repo", 91 | "commit": "test-commit", 92 | }, 93 | }, 94 | Status: v1alpha1.WorkflowStatus{ 95 | Phase: v1alpha1.WorkflowFailed, 96 | Message: "something", 97 | }, 98 | }, 99 | wantedError: nil, 100 | }, 101 | { 102 | name: "Error workflow", 103 | workflow: &v1alpha1.Workflow{ 104 | ObjectMeta: metav1.ObjectMeta{ 105 | Name: "test-workflow", 106 | Labels: map[string]string{ 107 | "repo": "test-repo", 108 | "commit": "test-commit", 109 | }, 110 | }, 111 | Status: v1alpha1.WorkflowStatus{ 112 | Phase: v1alpha1.WorkflowError, 113 | Message: "something", 114 | }, 115 | }, 116 | wantedError: nil, 117 | }, 118 | { 119 | name: "Pending workflow", 120 | workflow: &v1alpha1.Workflow{ 121 | ObjectMeta: metav1.ObjectMeta{ 122 | Name: "test-workflow", 123 | Labels: map[string]string{ 124 | "repo": "test-repo", 125 | "commit": "test-commit", 126 | }, 127 | }, 128 | Status: v1alpha1.WorkflowStatus{ 129 | Phase: v1alpha1.WorkflowPending, 130 | Message: "something", 131 | }, 132 | }, 133 | wantedError: nil, 134 | }, 135 | { 136 | name: "Running workflow", 137 | workflow: &v1alpha1.Workflow{ 138 | ObjectMeta: metav1.ObjectMeta{ 139 | Name: "test-workflow", 140 | Labels: map[string]string{ 141 | "repo": "test-repo", 142 | "commit": "test-commit", 143 | }, 144 | }, 145 | Status: v1alpha1.WorkflowStatus{ 146 | Phase: v1alpha1.WorkflowRunning, 147 | Message: "something", 148 | }, 149 | }, 150 | wantedError: nil, 151 | }, 152 | { 153 | name: "Missing label repo", 154 | workflow: &v1alpha1.Workflow{ 155 | ObjectMeta: metav1.ObjectMeta{ 156 | Name: "test-workflow", 157 | Labels: map[string]string{ 158 | "commit": "test-commit", 159 | }, 160 | }, 161 | Status: v1alpha1.WorkflowStatus{ 162 | Phase: v1alpha1.WorkflowSucceeded, 163 | Message: "something", 164 | }, 165 | }, 166 | wantedError: errors.New("some error"), 167 | }, 168 | { 169 | name: "Missing label commit", 170 | workflow: &v1alpha1.Workflow{ 171 | ObjectMeta: metav1.ObjectMeta{ 172 | Name: "test-workflow", 173 | Labels: map[string]string{ 174 | "repo": "test-repo", 175 | }, 176 | }, 177 | Status: v1alpha1.WorkflowStatus{ 178 | Phase: v1alpha1.WorkflowSucceeded, 179 | Message: "something", 180 | }, 181 | }, 182 | wantedError: errors.New("some error"), 183 | }, 184 | } 185 | 186 | // Create a mock configuration and clients 187 | cfg := &conf.GlobalConfig{ 188 | GitProviderConfig: conf.GitProviderConfig{Provider: "github"}, 189 | WorkflowServerConfig: conf.WorkflowServerConfig{ 190 | ArgoAddress: "http://workflow-server", 191 | Namespace: "test-namespace", 192 | }, 193 | } 194 | globalClients := &clients.Clients{ 195 | GitProvider: &mockGitProvider{}, 196 | } 197 | 198 | // Create a new eventNotifier instance 199 | gn := NewEventNotifier(cfg, globalClients) 200 | 201 | // Call the Notify method 202 | 203 | // Run test cases 204 | for _, test := range tests { 205 | t.Run(test.name, func(t *testing.T) { 206 | // Call the function being tested 207 | err := gn.Notify(ctx, test.workflow) 208 | 209 | // Use assert to check the equality of the error 210 | if test.wantedError != nil { 211 | assert.Error(err) 212 | assert.NotNil(err) 213 | } else { 214 | assert.NoError(err) 215 | assert.Nil(err) 216 | } 217 | }) 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /pkg/event_handler/main.go: -------------------------------------------------------------------------------- 1 | package event_handler 2 | 3 | import ( 4 | "context" 5 | "github.com/quickube/piper/pkg/clients" 6 | "github.com/quickube/piper/pkg/conf" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "log" 9 | ) 10 | 11 | func Start(ctx context.Context, stop context.CancelFunc, cfg *conf.GlobalConfig, clients *clients.Clients) { 12 | labelSelector := &metav1.LabelSelector{ 13 | MatchExpressions: []metav1.LabelSelectorRequirement{ 14 | {Key: "piper.quickube.com/notified", 15 | Operator: metav1.LabelSelectorOpExists}, 16 | }, 17 | } 18 | watcher, err := clients.Workflows.Watch(ctx, labelSelector) 19 | if err != nil { 20 | log.Printf("[event handler] Failed to watch workflow error:%s", err) 21 | return 22 | } 23 | 24 | notifier := NewEventNotifier(cfg, clients) 25 | handler := &workflowEventHandler{ 26 | Clients: clients, 27 | Notifier: notifier, 28 | } 29 | go func() { 30 | 31 | for { 32 | select { 33 | case <-ctx.Done(): 34 | log.Print("[event handler] context canceled, exiting") 35 | watcher.Stop() 36 | return 37 | case event, ok := <-watcher.ResultChan(): 38 | if !ok { 39 | log.Print("[event handler] result channel closed") 40 | stop() 41 | return 42 | } 43 | if err2 := handler.Handle(ctx, &event); err2 != nil { 44 | log.Printf("[event handler] failed to handle workflow event: %v", err2) 45 | } 46 | } 47 | } 48 | }() 49 | } 50 | -------------------------------------------------------------------------------- /pkg/event_handler/types.go: -------------------------------------------------------------------------------- 1 | package event_handler 2 | 3 | import ( 4 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 5 | "golang.org/x/net/context" 6 | "k8s.io/apimachinery/pkg/watch" 7 | ) 8 | 9 | type EventHandler interface { 10 | Handle(ctx context.Context, event *watch.Event) error 11 | } 12 | 13 | type EventNotifier interface { 14 | Notify(ctx context.Context, workflow *v1alpha1.Workflow) error 15 | } 16 | -------------------------------------------------------------------------------- /pkg/event_handler/workflow_event_handler.go: -------------------------------------------------------------------------------- 1 | package event_handler 2 | 3 | import ( 4 | "fmt" 5 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 6 | "github.com/quickube/piper/pkg/clients" 7 | "golang.org/x/net/context" 8 | "k8s.io/apimachinery/pkg/watch" 9 | "log" 10 | ) 11 | 12 | type workflowEventHandler struct { 13 | Clients *clients.Clients 14 | Notifier EventNotifier 15 | } 16 | 17 | func (weh *workflowEventHandler) Handle(ctx context.Context, event *watch.Event) error { 18 | workflow, ok := event.Object.(*v1alpha1.Workflow) 19 | if !ok { 20 | return fmt.Errorf( 21 | "event object is not a Workflow object, got: %v\n", 22 | event.DeepCopy().Object, 23 | ) 24 | } 25 | 26 | currentPiperNotifyLabelStatus, ok := workflow.GetLabels()["piper.quickube.com/notified"] 27 | if !ok { 28 | return fmt.Errorf( 29 | "workflow %s missing piper.quickube.com/notified label\n", 30 | workflow.GetName(), 31 | ) 32 | } 33 | 34 | if currentPiperNotifyLabelStatus == string(workflow.Status.Phase) { 35 | log.Printf( 36 | "workflow %s already informed for %s status. skiping... \n", 37 | workflow.GetName(), 38 | workflow.Status.Phase, 39 | ) //INFO 40 | return nil 41 | } 42 | 43 | err := weh.Notifier.Notify(ctx, workflow) 44 | if err != nil { 45 | return fmt.Errorf("failed to Notify workflow to git provider, error:%s\n", err) 46 | } 47 | 48 | err = weh.Clients.Workflows.UpdatePiperWorkflowLabel(ctx, workflow.GetName(), "notified", string(workflow.Status.Phase)) 49 | if err != nil { 50 | return fmt.Errorf("error in workflow %s status patch: %s", workflow.GetName(), err) 51 | } 52 | log.Printf( 53 | "[event handler] done with event of type: %s for worklfow: %s phase: %s message: %s\n", 54 | event.Type, 55 | workflow.GetName(), 56 | workflow.Status.Phase, 57 | workflow.Status.Message) //INFO 58 | 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /pkg/git_provider/bitbucket.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "bytes" 5 | context2 "context" 6 | "encoding/json" 7 | "fmt" 8 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 9 | "github.com/ktrysmt/go-bitbucket" 10 | "github.com/quickube/piper/pkg/conf" 11 | "github.com/quickube/piper/pkg/utils" 12 | "github.com/tidwall/gjson" 13 | "io" 14 | "log" 15 | "net/http" 16 | "strings" 17 | ) 18 | 19 | type BitbucketClientImpl struct { 20 | client *bitbucket.Client 21 | cfg *conf.GlobalConfig 22 | HooksHashTable map[string]int64 23 | } 24 | 25 | func NewBitbucketServerClient(cfg *conf.GlobalConfig) (Client, error) { 26 | client := bitbucket.NewOAuthbearerToken(cfg.GitProviderConfig.Token) 27 | 28 | err := ValidateBitbucketPermissions(client, cfg) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | return &BitbucketClientImpl{ 34 | client: client, 35 | cfg: cfg, 36 | HooksHashTable: make(map[string]int64), 37 | }, err 38 | } 39 | 40 | func (b BitbucketClientImpl) ListFiles(ctx context2.Context, repo string, branch string, path string) ([]string, error) { 41 | var filesList []string 42 | fileOptions := bitbucket.RepositoryFilesOptions{ 43 | Owner: b.cfg.GitProviderConfig.OrgName, 44 | RepoSlug: repo, 45 | Ref: branch, 46 | Path: path, 47 | MaxDepth: 0, 48 | } 49 | files, err := b.client.Repositories.Repository.ListFiles(&fileOptions) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | for _, f := range files { 55 | fileWithoutPath := strings.ReplaceAll(f.Path, path+"/", "") 56 | filesList = append(filesList, fileWithoutPath) 57 | } 58 | 59 | return filesList, nil 60 | } 61 | 62 | func (b BitbucketClientImpl) GetFile(ctx context2.Context, repo string, branch string, path string) (*CommitFile, error) { 63 | fileOptions := bitbucket.RepositoryFilesOptions{ 64 | Owner: b.cfg.GitProviderConfig.OrgName, 65 | RepoSlug: repo, 66 | Ref: branch, 67 | Path: path, 68 | MaxDepth: 0, 69 | } 70 | fileContent, err := b.client.Repositories.Repository.GetFileContent(&fileOptions) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | stringContent := string(fileContent[:]) 76 | return &CommitFile{ 77 | Path: &path, 78 | Content: &stringContent, 79 | }, nil 80 | } 81 | 82 | func (b BitbucketClientImpl) GetFiles(ctx context2.Context, repo string, branch string, paths []string) ([]*CommitFile, error) { 83 | var commitFiles []*CommitFile 84 | for _, path := range paths { 85 | file, err := b.GetFile(ctx, repo, branch, path) 86 | if err != nil { 87 | return nil, err 88 | } 89 | if file == nil { 90 | log.Printf("file %s not found in repo %s branch %s", path, repo, branch) 91 | continue 92 | } 93 | commitFiles = append(commitFiles, file) 94 | } 95 | return commitFiles, nil 96 | } 97 | 98 | func (b BitbucketClientImpl) SetWebhook(ctx context2.Context, repo *string) (*HookWithStatus, error) { 99 | webhookOptions := &bitbucket.WebhooksOptions{ 100 | Owner: b.cfg.GitProviderConfig.OrgName, 101 | RepoSlug: *repo, 102 | Uuid: "", 103 | Description: "Piper", 104 | Url: b.cfg.GitProviderConfig.WebhookURL, 105 | Active: true, 106 | Events: []string{"repo:push", "pullrequest:created", "pullrequest:updated", "pullrequest:fulfilled", "pullrequest:approved"}, 107 | } 108 | 109 | hook, exists := b.isRepoWebhookExists(*repo) 110 | if exists { 111 | log.Printf("webhook already exists for repository %s, skipping creation... \n", *repo) 112 | addHookToHashTable(utils.RemoveBraces(hook.Uuid), b.HooksHashTable) 113 | hookID, err := getHookByUUID(utils.RemoveBraces(hook.Uuid), b.HooksHashTable) 114 | if err != nil { 115 | return nil, err 116 | } 117 | return &HookWithStatus{ 118 | HookID: hookID, 119 | HealthStatus: true, 120 | RepoName: repo, 121 | }, nil 122 | } 123 | 124 | hook, err := b.client.Repositories.Webhooks.Create(webhookOptions) 125 | if err != nil { 126 | return nil, err 127 | } 128 | log.Printf("created webhook for repository %s \n", *repo) 129 | 130 | addHookToHashTable(utils.RemoveBraces(hook.Uuid), b.HooksHashTable) 131 | hookID, err := getHookByUUID(utils.RemoveBraces(hook.Uuid), b.HooksHashTable) 132 | if err != nil { 133 | return nil, err 134 | } 135 | 136 | return &HookWithStatus{ 137 | HookID: hookID, 138 | HealthStatus: true, 139 | RepoName: repo, 140 | }, nil 141 | } 142 | 143 | func (b BitbucketClientImpl) UnsetWebhook(ctx context2.Context, hook *HookWithStatus) error { 144 | //TODO implement me 145 | panic("implement me") 146 | } 147 | 148 | func (b BitbucketClientImpl) HandlePayload(ctx context2.Context, request *http.Request, secret []byte) (*WebhookPayload, error) { 149 | var webhookPayload *WebhookPayload 150 | var buf bytes.Buffer 151 | 152 | // Used for authentication, hookID generated by bitbucket. 153 | hookID, err := getHookByUUID(request.Header.Get("X-Hook-UUID"), b.HooksHashTable) 154 | if err != nil { 155 | return nil, fmt.Errorf("failed to get hook by UUID, %s", err) 156 | } 157 | 158 | _, err = io.Copy(&buf, request.Body) 159 | if err != nil { 160 | return nil, fmt.Errorf("error reading response: %s", err) 161 | } 162 | 163 | var body map[string]interface{} 164 | err = json.Unmarshal(buf.Bytes(), &body) 165 | if err != nil { 166 | return nil, fmt.Errorf("failed to unmarshal response: %s", err) 167 | } 168 | 169 | // https://support.atlassian.com/bitbucket-cloud/docs/event-payloads 170 | switch request.Header.Get("X-Event-Key") { 171 | case "repo:push": 172 | if gjson.GetBytes(buf.Bytes(), "push.changes.0.new.type").Value().(string) == "tag" { 173 | webhookPayload = &WebhookPayload{ 174 | Event: "tag", 175 | Repo: utils.SanitizeString(gjson.GetBytes(buf.Bytes(), "repository.name").Value().(string)), 176 | Branch: gjson.GetBytes(buf.Bytes(), "push.changes.0.new.name").Value().(string), 177 | Commit: gjson.GetBytes(buf.Bytes(), "push.changes.0.new.name").Value().(string), 178 | User: gjson.GetBytes(buf.Bytes(), "actor.display_name").Value().(string), 179 | HookID: hookID, 180 | } 181 | } else { 182 | webhookPayload = &WebhookPayload{ 183 | Event: "push", 184 | Repo: utils.SanitizeString(gjson.GetBytes(buf.Bytes(), "repository.name").Value().(string)), 185 | Branch: gjson.GetBytes(buf.Bytes(), "push.changes.0.new.name").Value().(string), 186 | Commit: gjson.GetBytes(buf.Bytes(), "push.changes.0.commits.0.hash").Value().(string), 187 | UserEmail: utils.ExtractStringsBetweenTags(gjson.GetBytes(buf.Bytes(), "push.changes.0.commits.0.author.raw").Value().(string))[0], 188 | User: gjson.GetBytes(buf.Bytes(), "actor.display_name").Value().(string), 189 | HookID: hookID, 190 | } 191 | } 192 | 193 | case "pullrequest:created", "pullrequest:updated", "pullrequest:approved": 194 | webhookPayload = &WebhookPayload{ 195 | Event: "pull_request", 196 | Repo: utils.SanitizeString(gjson.GetBytes(buf.Bytes(), "repository.name").Value().(string)), 197 | Branch: gjson.GetBytes(buf.Bytes(), "pullrequest.source.branch.name").Value().(string), 198 | Commit: gjson.GetBytes(buf.Bytes(), "pullrequest.source.commit.hash").Value().(string), 199 | User: gjson.GetBytes(buf.Bytes(), "pullrequest.author.display_name").Value().(string), 200 | PullRequestURL: gjson.GetBytes(buf.Bytes(), "pullrequest.links.html.href").Value().(string), 201 | PullRequestTitle: gjson.GetBytes(buf.Bytes(), "pullrequest.title").Value().(string), 202 | DestBranch: gjson.GetBytes(buf.Bytes(), "pullrequest.destination.branch.name").Value().(string), 203 | HookID: hookID, 204 | } 205 | } 206 | return webhookPayload, nil 207 | } 208 | 209 | func (b BitbucketClientImpl) SetStatus(ctx context2.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error { 210 | commitOptions := bitbucket.CommitsOptions{ 211 | Owner: b.cfg.GitProviderConfig.OrgName, 212 | RepoSlug: *repo, 213 | Revision: *commit, 214 | } 215 | commitStatusOptions := bitbucket.CommitStatusOptions{ 216 | Key: "build", 217 | Url: *linkURL, 218 | State: *status, 219 | Description: *message, 220 | } 221 | _, err := b.client.Repositories.Commits.CreateCommitStatus(&commitOptions, &commitStatusOptions) 222 | if err != nil { 223 | return err 224 | } 225 | log.Printf("set status of commit %s in repo %s to %s", *commit, *repo, *status) 226 | return nil 227 | } 228 | 229 | func (b BitbucketClientImpl) GetCorrelatingEvent(ctx context2.Context, workflowEvent *v1alpha1.WorkflowPhase) (string, error) { 230 | var event string 231 | switch *workflowEvent { 232 | case v1alpha1.WorkflowUnknown: 233 | event = "INPROGRESS" 234 | case v1alpha1.WorkflowPending: 235 | event = "INPROGRESS" 236 | case v1alpha1.WorkflowRunning: 237 | 238 | event = "INPROGRESS" 239 | case v1alpha1.WorkflowSucceeded: 240 | event = "SUCCESSFUL" 241 | case v1alpha1.WorkflowFailed: 242 | event = "FAILED" 243 | case v1alpha1.WorkflowError: 244 | event = "STOPPED" 245 | default: 246 | return "", fmt.Errorf("unimplemented workflow event") 247 | } 248 | return event, nil 249 | } 250 | 251 | func (b BitbucketClientImpl) PingHook(ctx context2.Context, hook *HookWithStatus) error { 252 | //TODO implement me 253 | panic("implement me") 254 | } 255 | 256 | func (b BitbucketClientImpl) isRepoWebhookExists(repo string) (*bitbucket.Webhook, bool) { 257 | emptyHook := bitbucket.Webhook{} 258 | 259 | webhookOptions := bitbucket.WebhooksOptions{ 260 | Owner: b.cfg.GitProviderConfig.OrgName, 261 | RepoSlug: repo, 262 | } 263 | hooks, err := b.client.Repositories.Webhooks.List(&webhookOptions) 264 | 265 | if err != nil { 266 | log.Printf("failed to list existing hooks for repository %s. error:%s", repo, err) 267 | return &emptyHook, false 268 | } 269 | 270 | if len(hooks) == 0 { 271 | return &emptyHook, false 272 | } 273 | 274 | for _, hook := range hooks { 275 | if hook.Description == "Piper" && hook.Url == b.cfg.GitProviderConfig.WebhookURL { 276 | return &hook, true 277 | } 278 | } 279 | 280 | return &emptyHook, false 281 | } 282 | -------------------------------------------------------------------------------- /pkg/git_provider/bitbucket_test.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/ktrysmt/go-bitbucket" 8 | "github.com/quickube/piper/pkg/conf" 9 | "github.com/quickube/piper/pkg/utils" 10 | assertion "github.com/stretchr/testify/assert" 11 | "golang.org/x/net/context" 12 | "net/http" 13 | "testing" 14 | ) 15 | 16 | func TestBitbucketListFiles(t *testing.T) { 17 | // Prepare 18 | client, mux, _, teardown := setupBitbucket() 19 | defer teardown() 20 | 21 | repoContent := &bitbucket.RepositoryFile{ 22 | Type: "file", 23 | Path: ".workflows/exit.yaml", 24 | } 25 | 26 | repoContent2 := &bitbucket.RepositoryFile{ 27 | Type: "file", 28 | Path: ".workflows/main.yaml", 29 | } 30 | 31 | data := map[string]interface{}{"values": []bitbucket.RepositoryFile{*repoContent, *repoContent2}} 32 | jsonBytes, _ := json.Marshal(data) 33 | 34 | mux.HandleFunc("/repositories/test/test-repo1/src/branch1/.workflows/", func(w http.ResponseWriter, r *http.Request) { 35 | testMethod(t, r, "GET") 36 | //testFormValues(t, r, values{}) 37 | 38 | _, _ = fmt.Fprint(w, string(jsonBytes)) 39 | }) 40 | 41 | c := BitbucketClientImpl{ 42 | client: client, 43 | cfg: &conf.GlobalConfig{ 44 | GitProviderConfig: conf.GitProviderConfig{ 45 | OrgLevelWebhook: false, 46 | OrgName: "test", 47 | RepoList: "test-repo1", 48 | }, 49 | }, 50 | } 51 | ctx := context.Background() 52 | 53 | // Execute 54 | actualContent, err := c.ListFiles(ctx, "test-repo1", "branch1", ".workflows") 55 | expectedContent := []string{"exit.yaml", "main.yaml"} 56 | 57 | // Assert 58 | assert := assertion.New(t) 59 | assert.NotNil(t, err) 60 | assert.Equal(expectedContent, actualContent) 61 | 62 | } 63 | 64 | func TestBitbucketSetStatus(t *testing.T) { 65 | // Prepare 66 | ctx := context.Background() 67 | assert := assertion.New(t) 68 | client, mux, _, teardown := setupBitbucket() 69 | defer teardown() 70 | 71 | mux.HandleFunc("/repositories/test/test-repo1/commit/test-commit/statuses/build", func(w http.ResponseWriter, r *http.Request) { 72 | testMethod(t, r, "POST") 73 | testFormValues(t, r, values{}) 74 | 75 | w.WriteHeader(http.StatusCreated) 76 | jsonBytes := []byte(`{"status": "ok"}`) 77 | _, _ = fmt.Fprint(w, string(jsonBytes)) 78 | }) 79 | 80 | c := BitbucketClientImpl{ 81 | client: client, 82 | cfg: &conf.GlobalConfig{ 83 | GitProviderConfig: conf.GitProviderConfig{ 84 | Provider: "bitbucket", 85 | OrgLevelWebhook: false, 86 | OrgName: "test", 87 | RepoList: "test-repo1", 88 | }, 89 | }, 90 | } 91 | 92 | // Define test cases 93 | tests := []struct { 94 | name string 95 | repo *string 96 | commit *string 97 | linkURL *string 98 | status *string 99 | message *string 100 | wantedError error 101 | }{ 102 | { 103 | name: "Notify success", 104 | repo: utils.SPtr("test-repo1"), 105 | commit: utils.SPtr("test-commit"), 106 | linkURL: utils.SPtr("https://argo"), 107 | status: utils.SPtr("success"), 108 | message: utils.SPtr(""), 109 | wantedError: nil, 110 | }, 111 | { 112 | name: "Notify pending", 113 | repo: utils.SPtr("test-repo1"), 114 | commit: utils.SPtr("test-commit"), 115 | linkURL: utils.SPtr("https://argo"), 116 | status: utils.SPtr("pending"), 117 | message: utils.SPtr(""), 118 | wantedError: nil, 119 | }, 120 | { 121 | name: "Notify error", 122 | repo: utils.SPtr("test-repo1"), 123 | commit: utils.SPtr("test-commit"), 124 | linkURL: utils.SPtr("https://argo"), 125 | status: utils.SPtr("error"), 126 | message: utils.SPtr("some message"), 127 | wantedError: nil, 128 | }, 129 | { 130 | name: "Notify failure", 131 | repo: utils.SPtr("test-repo1"), 132 | commit: utils.SPtr("test-commit"), 133 | linkURL: utils.SPtr("https://argo"), 134 | status: utils.SPtr("failure"), 135 | message: utils.SPtr(""), 136 | wantedError: nil, 137 | }, 138 | { 139 | name: "Non managed repo", 140 | repo: utils.SPtr("non-existing-repo"), 141 | commit: utils.SPtr("test-commit"), 142 | linkURL: utils.SPtr("https://argo"), 143 | status: utils.SPtr("error"), 144 | message: utils.SPtr(""), 145 | wantedError: errors.New("some error"), 146 | }, 147 | { 148 | name: "Non existing commit", 149 | repo: utils.SPtr("test-repo1"), 150 | commit: utils.SPtr("not-exists"), 151 | linkURL: utils.SPtr("https://argo"), 152 | status: utils.SPtr("error"), 153 | message: utils.SPtr(""), 154 | wantedError: errors.New("some error"), 155 | }, 156 | } 157 | // Run test cases 158 | for _, test := range tests { 159 | t.Run(test.name, func(t *testing.T) { 160 | 161 | // Call the function being tested 162 | err := c.SetStatus(ctx, test.repo, test.commit, test.linkURL, test.status, test.message) 163 | 164 | // Use assert to check the equality of the error 165 | if test.wantedError != nil { 166 | assert.Error(err) 167 | assert.NotNil(err) 168 | } else { 169 | assert.NoError(err) 170 | assert.Nil(err) 171 | } 172 | }) 173 | } 174 | 175 | } 176 | -------------------------------------------------------------------------------- /pkg/git_provider/bitbucket_utils.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "fmt" 5 | bitbucket "github.com/ktrysmt/go-bitbucket" 6 | "github.com/quickube/piper/pkg/conf" 7 | "github.com/quickube/piper/pkg/utils" 8 | "log" 9 | "net/http" 10 | "strings" 11 | ) 12 | 13 | func ValidateBitbucketPermissions(client *bitbucket.Client, cfg *conf.GlobalConfig) error { 14 | 15 | repoAdminScopes := []string{"webhook", "repository:admin", "pullrequest:write"} 16 | repoGranularScopes := []string{"webhook", "repository", "pullrequest"} 17 | 18 | scopes, err := GetBitbucketTokenScopes(client, cfg) 19 | 20 | if err != nil { 21 | return fmt.Errorf("failed to get scopes: %v", err) 22 | } 23 | if len(scopes) == 0 { 24 | return fmt.Errorf("permissions error: no scopes found for the github client") 25 | } 26 | 27 | if utils.ListContains(repoAdminScopes, scopes) { 28 | return nil 29 | } 30 | if utils.ListContains(repoGranularScopes, scopes) { 31 | return nil 32 | } 33 | 34 | return fmt.Errorf("permissions error: %v is not a valid scopes", scopes) 35 | } 36 | 37 | func GetBitbucketTokenScopes(client *bitbucket.Client, cfg *conf.GlobalConfig) ([]string, error) { 38 | 39 | req, err := http.NewRequest("GET", fmt.Sprintf("%s/repositories/%s", client.GetApiBaseURL(), cfg.GitProviderConfig.OrgName), nil) 40 | if err != nil { 41 | log.Println("Error creating request:", err) 42 | return nil, err 43 | } 44 | req.Header.Set("Accept", "application/json") 45 | req.Header.Set("Authorization", "Bearer "+cfg.GitProviderConfig.Token) 46 | 47 | resp, err := client.HttpClient.Do(req) 48 | if err != nil { 49 | log.Println("Error making request:", err) 50 | return nil, err 51 | } 52 | defer resp.Body.Close() 53 | 54 | if resp.StatusCode != 200 { 55 | return nil, fmt.Errorf("token validation failed: %v", resp.Status) 56 | } 57 | 58 | // Check the "X-OAuth-Scopes" header to get the token scopes 59 | acceptedScopes := resp.Header.Get("X-Accepted-OAuth-Scopes") 60 | scopes := resp.Header.Get("X-OAuth-Scopes") 61 | log.Println("Bitbucket Token Scopes are:", scopes, acceptedScopes) 62 | 63 | scopes = strings.ReplaceAll(scopes, " ", "") 64 | return append(strings.Split(scopes, ","), acceptedScopes), nil 65 | 66 | } 67 | 68 | func addHookToHashTable(hookUuid string, hookHashTable map[string]int64) { 69 | hookHashTable[hookUuid] = utils.StringToInt64(hookUuid) 70 | } 71 | 72 | func getHookByUUID(hookUuid string, hookHashTable map[string]int64) (int64, error) { 73 | res, ok := hookHashTable[hookUuid] 74 | if !ok { 75 | return 0, fmt.Errorf("hookUuid %s not found", hookUuid) 76 | } 77 | return res, nil 78 | } 79 | -------------------------------------------------------------------------------- /pkg/git_provider/github_utils.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | 9 | "github.com/quickube/piper/pkg/utils" 10 | 11 | "github.com/google/go-github/v52/github" 12 | "github.com/quickube/piper/pkg/conf" 13 | ) 14 | 15 | func isOrgWebhookEnabled(ctx context.Context, c *GithubClientImpl) (*github.Hook, bool) { 16 | emptyHook := github.Hook{} 17 | hooks, resp, err := c.client.Organizations.ListHooks(ctx, c.cfg.GitProviderConfig.OrgName, &github.ListOptions{}) 18 | if err != nil { 19 | return &emptyHook, false 20 | } 21 | if resp.StatusCode != 200 { 22 | return &emptyHook, false 23 | } 24 | if len(hooks) == 0 { 25 | return &emptyHook, false 26 | } 27 | for _, hook := range hooks { 28 | if hook.GetActive() && hook.GetName() == "web" && hook.Config["url"] == c.cfg.GitProviderConfig.WebhookURL { 29 | return hook, true 30 | } 31 | } 32 | return &emptyHook, false 33 | } 34 | 35 | func isRepoWebhookEnabled(ctx context.Context, c *GithubClientImpl, repo string) (*github.Hook, bool) { 36 | emptyHook := github.Hook{} 37 | hooks, resp, err := c.client.Repositories.ListHooks(ctx, c.cfg.GitProviderConfig.OrgName, repo, &github.ListOptions{}) 38 | if err != nil { 39 | return &emptyHook, false 40 | } 41 | if resp.StatusCode != 200 { 42 | return &emptyHook, false 43 | } 44 | if len(hooks) == 0 { 45 | return &emptyHook, false 46 | } 47 | 48 | for _, hook := range hooks { 49 | if hook.GetActive() && hook.GetName() == "web" && hook.Config["url"] == c.cfg.GitProviderConfig.WebhookURL { 50 | return hook, true 51 | } 52 | } 53 | 54 | return &emptyHook, false 55 | } 56 | 57 | func GetScopes(ctx context.Context, client *github.Client) ([]string, error) { 58 | // Make a request to the "Get the authenticated user" endpoint 59 | req, err := http.NewRequest("GET", "https://api.github.com/user", nil) 60 | if err != nil { 61 | fmt.Println("Error creating request:", err) 62 | return nil, err 63 | } 64 | resp, err := client.Do(ctx, req, nil) 65 | if err != nil { 66 | fmt.Println("Error making request:", err) 67 | return nil, err 68 | } 69 | defer resp.Body.Close() 70 | 71 | // Check the "X-OAuth-Scopes" header to get the token scopes 72 | scopes := resp.Header.Get("X-OAuth-Scopes") 73 | fmt.Println("Github Token Scopes are:", scopes) 74 | 75 | scopes = strings.ReplaceAll(scopes, " ", "") 76 | return strings.Split(scopes, ","), nil 77 | 78 | } 79 | 80 | func ValidatePermissions(ctx context.Context, client *github.Client, cfg *conf.GlobalConfig) error { 81 | 82 | orgScopes := []string{"admin:org_hook"} 83 | repoAdminScopes := []string{"admin:repo_hook"} 84 | repoGranularScopes := []string{"write:repo_hook", "read:repo_hook"} 85 | 86 | scopes, err := GetScopes(ctx, client) 87 | 88 | if err != nil { 89 | return fmt.Errorf("failed to get scopes: %v", err) 90 | } 91 | if len(scopes) == 0 { 92 | return fmt.Errorf("permissions error: no scopes found for the github client") 93 | } 94 | 95 | if cfg.GitProviderConfig.OrgLevelWebhook { 96 | if utils.ListContains(orgScopes, scopes) { 97 | return nil 98 | } 99 | return fmt.Errorf("permissions error: %v is not a valid scope for the org level permissions", scopes) 100 | } 101 | 102 | if utils.ListContains(repoAdminScopes, scopes) { 103 | return nil 104 | } 105 | if utils.ListContains(repoGranularScopes, scopes) { 106 | return nil 107 | } 108 | 109 | return fmt.Errorf("permissions error: %v is not a valid scope for the repo level permissions", scopes) 110 | } 111 | -------------------------------------------------------------------------------- /pkg/git_provider/github_utils_test.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "testing" 9 | 10 | "github.com/google/go-github/v52/github" 11 | "github.com/quickube/piper/pkg/conf" 12 | "github.com/quickube/piper/pkg/utils" 13 | assertion "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestIsOrgWebhookEnabled(t *testing.T) { 17 | // 18 | // Prepare 19 | // 20 | client, mux, _, teardown := setup() 21 | defer teardown() 22 | 23 | config := make(map[string]interface{}) 24 | config["url"] = "https://bla.com" 25 | Hooks := github.Hook{ 26 | Active: utils.BPtr(true), 27 | Name: utils.SPtr("web"), 28 | Config: config, 29 | } 30 | jsonBytes, _ := json.Marshal(&[]github.Hook{Hooks}) 31 | 32 | mux.HandleFunc("/orgs/test/hooks", func(w http.ResponseWriter, r *http.Request) { 33 | testMethod(t, r, "GET") 34 | testFormValues(t, r, values{}) 35 | _, _ = fmt.Fprint(w, string(jsonBytes)) 36 | }) 37 | 38 | c := GithubClientImpl{ 39 | client: client, 40 | cfg: &conf.GlobalConfig{ 41 | GitProviderConfig: conf.GitProviderConfig{ 42 | OrgLevelWebhook: true, 43 | OrgName: "test", 44 | WebhookURL: "https://bla.com", 45 | }, 46 | }, 47 | } 48 | ctx := context.Background() 49 | 50 | // 51 | // Execute 52 | // 53 | hooks, isEnabled := isOrgWebhookEnabled(ctx, &c) 54 | 55 | // 56 | // Assert 57 | // 58 | assert := assertion.New(t) 59 | assert.True(isEnabled) 60 | assert.NotNil(t, hooks) 61 | } 62 | 63 | func TestIsRepoWebhookEnabled(t *testing.T) { 64 | // 65 | // Prepare 66 | // 67 | client, mux, _, teardown := setup() 68 | defer teardown() 69 | 70 | config := make(map[string]interface{}) 71 | config["url"] = "https://bla.com" 72 | Hooks := github.Hook{ 73 | Active: utils.BPtr(true), 74 | Name: utils.SPtr("web"), 75 | Config: config, 76 | } 77 | jsonBytes, _ := json.Marshal(&[]github.Hook{Hooks}) 78 | 79 | mux.HandleFunc("/repos/test/test-repo2/hooks", func(w http.ResponseWriter, r *http.Request) { 80 | testMethod(t, r, "GET") 81 | testFormValues(t, r, values{}) 82 | _, _ = fmt.Fprint(w, string(jsonBytes)) 83 | }) 84 | 85 | c := GithubClientImpl{ 86 | client: client, 87 | cfg: &conf.GlobalConfig{ 88 | GitProviderConfig: conf.GitProviderConfig{ 89 | OrgLevelWebhook: false, 90 | OrgName: "test", 91 | WebhookURL: "https://bla.com", 92 | RepoList: "test-repo1,test-repo2", 93 | }, 94 | }, 95 | } 96 | ctx := context.Background() 97 | 98 | // 99 | // Execute 100 | // 101 | hook, isEnabled := isRepoWebhookEnabled(ctx, &c, "test-repo2") 102 | 103 | // 104 | // Assert 105 | // 106 | assert := assertion.New(t) 107 | assert.True(isEnabled) 108 | assert.NotNil(t, hook) 109 | 110 | // 111 | // Execute 112 | // 113 | hook, isEnabled = isRepoWebhookEnabled(ctx, &c, "test-repo3") 114 | 115 | // 116 | // Assert 117 | // 118 | assert = assertion.New(t) 119 | assert.False(isEnabled) 120 | assert.NotNil(t, hook) 121 | } 122 | -------------------------------------------------------------------------------- /pkg/git_provider/gitlab_utils.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha256" 6 | "encoding/base64" 7 | "encoding/hex" 8 | "fmt" 9 | "io" 10 | "log" 11 | "net/http" 12 | "strings" 13 | 14 | "github.com/quickube/piper/pkg/conf" 15 | "github.com/quickube/piper/pkg/utils" 16 | "github.com/xanzy/go-gitlab" 17 | "golang.org/x/net/context" 18 | ) 19 | 20 | func ValidateGitlabPermissions(ctx context.Context, client *gitlab.Client, cfg *conf.GlobalConfig) error { 21 | 22 | repoAdminScopes := []string{"api"} 23 | repoGranularScopes := []string{"write_repository", "read_api"} 24 | 25 | token, _, err := client.PersonalAccessTokens.GetSinglePersonalAccessToken() 26 | if err != nil { 27 | return fmt.Errorf("failed to get scopes: %v", err) 28 | } 29 | scopes := token.Scopes 30 | 31 | if len(scopes) == 0 { 32 | return fmt.Errorf("permissions error: no scopes found for the gitlab client") 33 | } 34 | 35 | if utils.ListContains(repoAdminScopes, scopes) { 36 | return nil 37 | } 38 | if utils.ListContains(repoGranularScopes, scopes) { 39 | return nil 40 | } 41 | 42 | return fmt.Errorf("permissions error: %v is not a valid scope for the project level permissions", scopes) 43 | } 44 | 45 | func IsGroupWebhookEnabled(ctx context.Context, c *GitlabClientImpl) (*gitlab.GroupHook, bool) { 46 | emptyHook := gitlab.GroupHook{} 47 | hooks, resp, err := c.client.Groups.ListGroupHooks(c.cfg.GitProviderConfig.OrgName, nil, gitlab.WithContext(ctx)) 48 | 49 | if err != nil { 50 | return &emptyHook, false 51 | } 52 | if resp.StatusCode != 200 { 53 | return &emptyHook, false 54 | } 55 | if len(hooks) != 0 { 56 | for _, hook := range hooks { 57 | if hook.URL == c.cfg.GitProviderConfig.WebhookURL { 58 | return hook, true 59 | } 60 | } 61 | } 62 | return &emptyHook, false 63 | } 64 | 65 | func IsProjectWebhookEnabled(ctx context.Context, c *GitlabClientImpl, projectId int) (*gitlab.ProjectHook, bool) { 66 | emptyHook := gitlab.ProjectHook{} 67 | 68 | hooks, resp, err := c.client.Projects.ListProjectHooks(projectId, nil, gitlab.WithContext(ctx)) 69 | if err != nil { 70 | return &emptyHook, false 71 | } 72 | if resp.StatusCode != 200 { 73 | return &emptyHook, false 74 | } 75 | if len(hooks) == 0 { 76 | return &emptyHook, false 77 | } 78 | 79 | for _, hook := range hooks { 80 | if hook.URL == c.cfg.GitProviderConfig.WebhookURL { 81 | return hook, true 82 | } 83 | } 84 | 85 | return &emptyHook, false 86 | } 87 | 88 | func ExtractLabelsId(labels []*gitlab.EventLabel) []string { 89 | var returnLabelsList []string 90 | for _, label := range labels { 91 | returnLabelsList = append(returnLabelsList, fmt.Sprint(label.ID)) 92 | } 93 | return returnLabelsList 94 | } 95 | 96 | func GetProjectId(ctx context.Context, c *GitlabClientImpl, repo *string) (*int, error) { 97 | projectFullName := fmt.Sprintf("%s/%s", c.cfg.GitProviderConfig.OrgName, *repo) 98 | IProject, _, err := c.client.Projects.GetProject(projectFullName, nil, gitlab.WithContext(ctx)) 99 | if err != nil { 100 | log.Printf("Failed to get project (%s): %v", *repo, err) 101 | return nil, err 102 | } 103 | return &IProject.ID, nil 104 | } 105 | 106 | func ValidatePayload(r *http.Request, secret []byte) ([]byte, error) { 107 | payload, err := io.ReadAll(r.Body) 108 | if err != nil { 109 | return nil, fmt.Errorf("error reading request body: %v", err) 110 | } 111 | 112 | // Get GitLab signature from headers 113 | gitlabSignature := r.Header.Get("X-Gitlab-Token") 114 | if gitlabSignature == "" { 115 | return nil, fmt.Errorf("no GitLab signature found in headers") 116 | } 117 | 118 | h := hmac.New(sha256.New, secret) 119 | _, err = h.Write(payload) 120 | if err != nil { 121 | return nil, fmt.Errorf("error computing HMAC: %v", err) 122 | } 123 | expectedMAC := hex.EncodeToString(h.Sum(nil)) 124 | 125 | isEqual := hmac.Equal([]byte(gitlabSignature), []byte(expectedMAC)) 126 | if !isEqual { 127 | return nil, fmt.Errorf("secret not correct") 128 | } 129 | return payload, nil 130 | } 131 | 132 | func FixRepoNames(c *GitlabClientImpl) error { 133 | var formattedRepos []string 134 | for _, repo := range strings.Split(c.cfg.GitProviderConfig.RepoList, ",") { 135 | userRepo := fmt.Sprintf("%s/%s", c.cfg.GitProviderConfig.OrgName, repo) 136 | formattedRepos = append(formattedRepos, userRepo) 137 | } 138 | c.cfg.GitProviderConfig.RepoList = strings.Join(formattedRepos, ",") 139 | return nil 140 | } 141 | 142 | func DecodeBase64ToStringPtr(encoded string) (*string, error) { 143 | decoded, err := base64.StdEncoding.DecodeString(encoded) 144 | if err != nil { 145 | return nil, err 146 | } 147 | 148 | result := string(decoded) 149 | return &result, nil 150 | } 151 | -------------------------------------------------------------------------------- /pkg/git_provider/gitlab_utils_test.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "testing" 9 | 10 | "github.com/quickube/piper/pkg/conf" 11 | assertion "github.com/stretchr/testify/assert" 12 | "github.com/xanzy/go-gitlab" 13 | "golang.org/x/net/context" 14 | ) 15 | 16 | func mockHTTPResponse(t *testing.T, w io.Writer, response interface{}) { 17 | err := json.NewEncoder(w).Encode(response) 18 | if err != nil { 19 | fmt.Printf("error %s", err) 20 | } 21 | } 22 | 23 | func TestValidateGitlabPermissions(t *testing.T) { 24 | // 25 | // Prepare 26 | // 27 | type testData = struct { 28 | name string 29 | scopes []string 30 | raiseErr bool 31 | } 32 | var CurrentTest testData 33 | mux, client := setupGitlab(t) 34 | c := GitlabClientImpl{ 35 | client: client, 36 | cfg: &conf.GlobalConfig{ 37 | GitProviderConfig: conf.GitProviderConfig{ 38 | OrgLevelWebhook: false, 39 | OrgName: "test", 40 | RepoList: "test-repo1", 41 | }, 42 | }, 43 | } 44 | ctx := context.Background() 45 | 46 | mux.HandleFunc("/api/v4/personal_access_tokens/self", func(w http.ResponseWriter, r *http.Request) { 47 | testMethod(t, r, "GET") 48 | mockHTTPResponse(t, w, gitlab.PersonalAccessToken{Scopes: CurrentTest.scopes}) 49 | }) 50 | // 51 | // Execute 52 | // 53 | tests := []testData{ 54 | {name: "validScope", scopes: []string{"api"}, raiseErr: false}, 55 | {name: "invalidScope", scopes: []string{"invalid"}, raiseErr: true}, 56 | } 57 | for _, test := range tests { 58 | CurrentTest = test 59 | t.Run(test.name, func(t *testing.T) { 60 | err := ValidateGitlabPermissions(ctx, c.client, c.cfg) 61 | // 62 | // Assert 63 | // 64 | assert := assertion.New(t) 65 | if test.raiseErr { 66 | assert.NotNil(err) 67 | } else { 68 | assert.Nil(err) 69 | } 70 | }) 71 | } 72 | } 73 | 74 | func TestIsGroupWebhookEnabled(t *testing.T) { 75 | // Prepare 76 | ctx := context.Background() 77 | mux, client := setupGitlab(t) 78 | c := GitlabClientImpl{ 79 | client: client, 80 | cfg: &conf.GlobalConfig{ 81 | GitProviderConfig: conf.GitProviderConfig{ 82 | OrgLevelWebhook: true, 83 | OrgName: "group1", 84 | WebhookURL: "testing-url", 85 | }, 86 | }, 87 | } 88 | 89 | hook := []gitlab.GroupHook{{ 90 | ID: 1234, 91 | URL: c.cfg.GitProviderConfig.WebhookURL, 92 | }} 93 | url := fmt.Sprintf("/api/v4/groups/%s/hooks", c.cfg.GitProviderConfig.OrgName) 94 | mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) { 95 | testMethod(t, r, "GET") 96 | mockHTTPResponse(t, w, hook) 97 | }) 98 | // Execute 99 | groupHook, isEnabled := IsGroupWebhookEnabled(ctx, &c) 100 | // Assert 101 | assert := assertion.New(t) 102 | assert.Equal(isEnabled, true) 103 | assert.Equal(groupHook.URL, c.cfg.GitProviderConfig.WebhookURL) 104 | } 105 | 106 | func TestIsProjectWebhookEnabled(t *testing.T) { 107 | // 108 | // Prepare 109 | // 110 | ctx := context.Background() 111 | mux, client := setupGitlab(t) 112 | project := "test-repo1" 113 | projectId := 1 114 | c := GitlabClientImpl{ 115 | client: client, 116 | cfg: &conf.GlobalConfig{ 117 | GitProviderConfig: conf.GitProviderConfig{ 118 | OrgLevelWebhook: false, 119 | OrgName: "group1", 120 | WebhookURL: "testing-url", 121 | RepoList: project, 122 | }, 123 | }, 124 | } 125 | 126 | hook := []gitlab.ProjectHook{{ 127 | ID: 1234, 128 | URL: c.cfg.GitProviderConfig.WebhookURL, 129 | }} 130 | 131 | url := fmt.Sprintf("/api/v4/projects/%d/hooks", projectId) 132 | mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) { 133 | testMethod(t, r, "GET") 134 | mockHTTPResponse(t, w, hook) 135 | }) 136 | // 137 | // Execute 138 | // 139 | projectHook, isEnabled := IsProjectWebhookEnabled(ctx, &c, projectId) 140 | // 141 | // Assert 142 | // 143 | assert := assertion.New(t) 144 | assert.Equal(isEnabled, true) 145 | assert.Equal(projectHook.URL, c.cfg.GitProviderConfig.WebhookURL) 146 | } 147 | -------------------------------------------------------------------------------- /pkg/git_provider/main.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "fmt" 5 | "github.com/quickube/piper/pkg/conf" 6 | ) 7 | 8 | func NewGitProviderClient(cfg *conf.GlobalConfig) (Client, error) { 9 | 10 | switch cfg.GitProviderConfig.Provider { 11 | case "github": 12 | gitClient, err := NewGithubClient(cfg) 13 | if err != nil { 14 | return nil, err 15 | } 16 | return gitClient, nil 17 | case "bitbucket": 18 | gitClient, err := NewBitbucketServerClient(cfg) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return gitClient, nil 23 | case "gitlab": 24 | gitClient, err := NewGitlabClient(cfg) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return gitClient, nil 29 | } 30 | 31 | return nil, fmt.Errorf("didn't find matching git provider %s", cfg.GitProviderConfig.Provider) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/git_provider/test_utils.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "net/url" 8 | "os" 9 | "testing" 10 | "time" 11 | 12 | "github.com/google/go-cmp/cmp" 13 | "github.com/google/go-github/v52/github" 14 | "github.com/ktrysmt/go-bitbucket" 15 | "github.com/xanzy/go-gitlab" 16 | ) 17 | 18 | const ( 19 | // baseURLPath is a non-empty Client.BaseURL path to use during tests, 20 | // to ensure relative URLs are used for all endpoints. See issue #752. 21 | baseURLPath = "/api-v3" 22 | bitbucketBaseURLPath = "/2.0" 23 | ) 24 | 25 | func setup() (client *github.Client, mux *http.ServeMux, serverURL string, teardown func()) { 26 | mux = http.NewServeMux() 27 | 28 | apiHandler := http.NewServeMux() 29 | apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, mux)) 30 | apiHandler.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { 31 | fmt.Fprintln(os.Stderr, "FAIL: Client.BaseURL path prefix is not preserved in the request URL:") 32 | fmt.Fprintln(os.Stderr) 33 | fmt.Fprintln(os.Stderr, "\t"+req.URL.String()) 34 | fmt.Fprintln(os.Stderr) 35 | fmt.Fprintln(os.Stderr, "\tDid you accidentally use an absolute endpoint URL rather than relative?") 36 | fmt.Fprintln(os.Stderr, "\tSee https://github.com/google/go-github/issues/752 for information.") 37 | http.Error(w, "Client.BaseURL path prefix is not preserved in the request URL.", http.StatusInternalServerError) 38 | }) 39 | 40 | // server is a test HTTP server used to provide mock API responses. 41 | server := httptest.NewServer(apiHandler) 42 | 43 | // client is the GitHub client being tested and is 44 | // configured to use test server. 45 | client = github.NewClient(nil) 46 | url, _ := url.Parse(server.URL + baseURLPath + "/") 47 | client.BaseURL = url 48 | client.UploadURL = url 49 | 50 | return client, mux, server.URL, server.Close 51 | } 52 | 53 | func testMethod(t *testing.T, r *http.Request, want string) { 54 | t.Helper() 55 | if got := r.Method; got != want { 56 | t.Errorf("Request method: %v, want %v", got, want) 57 | } 58 | } 59 | 60 | type values map[string]string 61 | 62 | func testFormValues(t *testing.T, r *http.Request, values values) { 63 | t.Helper() 64 | want := url.Values{} 65 | for k, v := range values { 66 | want.Set(k, v) 67 | } 68 | 69 | err := r.ParseForm() 70 | if err != nil { 71 | t.Errorf("Go error parsing form: %v", err) 72 | } 73 | if got := r.Form; !cmp.Equal(got, want) { 74 | t.Errorf("Request parameters: %v, want %v", got, want) 75 | } 76 | } 77 | 78 | func setupBitbucket() (client *bitbucket.Client, mux *http.ServeMux, serverURL string, teardown func()) { 79 | mux = http.NewServeMux() 80 | 81 | apiHandler := http.NewServeMux() 82 | apiHandler.Handle(bitbucketBaseURLPath+"/", http.StripPrefix(bitbucketBaseURLPath, mux)) 83 | apiHandler.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { 84 | http.Error(w, "Client.BaseURL path prefix is not preserved in the request URL.", http.StatusInternalServerError) 85 | }) 86 | 87 | server := httptest.NewServer(apiHandler) 88 | url, _ := url.Parse(server.URL + bitbucketBaseURLPath) 89 | client = bitbucket.NewBasicAuth("username", "password") 90 | client.SetApiBaseURL(*url) 91 | 92 | return client, mux, server.URL, server.Close 93 | } 94 | func setupGitlab(t *testing.T) (*http.ServeMux, *gitlab.Client) { 95 | // mux is the HTTP request multiplexer used with the test server. 96 | mux := http.NewServeMux() 97 | 98 | // server is a test HTTP server used to provide mock API responses. 99 | server := httptest.NewServer(mux) 100 | t.Cleanup(server.Close) 101 | 102 | // client is the Gitlab client being tested. 103 | client, err := gitlab.NewClient("", 104 | gitlab.WithBaseURL(server.URL), 105 | // Disable backoff to speed up tests that expect errors. 106 | gitlab.WithCustomBackoff(func(_, _ time.Duration, _ int, _ *http.Response) time.Duration { 107 | return 0 108 | }), 109 | ) 110 | if err != nil { 111 | t.Fatalf("Failed to create client: %v", err) 112 | } 113 | 114 | return mux, client 115 | } -------------------------------------------------------------------------------- /pkg/git_provider/types.go: -------------------------------------------------------------------------------- 1 | package git_provider 2 | 3 | import ( 4 | "context" 5 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 6 | "net/http" 7 | ) 8 | 9 | type HookWithStatus struct { 10 | HookID int64 11 | Uuid string 12 | HealthStatus bool 13 | RepoName *string 14 | } 15 | 16 | type CommitFile struct { 17 | Path *string `json:"path"` 18 | Content *string `json:"content"` 19 | } 20 | 21 | type WebhookPayload struct { 22 | Event string `json:"event"` 23 | Action string `json:"action"` 24 | Repo string `json:"repoName"` 25 | Branch string `json:"branch"` 26 | Commit string `json:"commit"` 27 | User string `json:"user"` 28 | UserEmail string `json:"user_email"` 29 | PullRequestURL string `json:"pull_request_url"` 30 | PullRequestTitle string `json:"pull_request_title"` 31 | DestBranch string `json:"dest_branch"` 32 | Labels []string `json:"labels"` 33 | HookID int64 `json:"hookID"` 34 | OwnerID int64 `json:"ownerID"` 35 | } 36 | 37 | type Client interface { 38 | ListFiles(ctx context.Context, repo string, branch string, path string) ([]string, error) 39 | GetFile(ctx context.Context, repo string, branch string, path string) (*CommitFile, error) 40 | GetFiles(ctx context.Context, repo string, branch string, paths []string) ([]*CommitFile, error) 41 | SetWebhook(ctx context.Context, repo *string) (*HookWithStatus, error) 42 | UnsetWebhook(ctx context.Context, hook *HookWithStatus) error 43 | HandlePayload(ctx context.Context, request *http.Request, secret []byte) (*WebhookPayload, error) 44 | SetStatus(ctx context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error 45 | PingHook(ctx context.Context, hook *HookWithStatus) error 46 | GetCorrelatingEvent(ctx context.Context, workflowEvent *v1alpha1.WorkflowPhase) (string, error) 47 | } 48 | -------------------------------------------------------------------------------- /pkg/server/main.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/quickube/piper/pkg/clients" 5 | "github.com/quickube/piper/pkg/conf" 6 | "golang.org/x/net/context" 7 | "log" 8 | ) 9 | 10 | func Start(ctx context.Context, stop context.CancelFunc, cfg *conf.GlobalConfig, clients *clients.Clients) { 11 | 12 | srv := NewServer(cfg, clients) 13 | gracefulShutdownHandler := NewGracefulShutdown(ctx, stop) 14 | srv.Start(ctx) 15 | 16 | gracefulShutdownHandler.Shutdown(srv) 17 | 18 | log.Println("Server exiting") 19 | } 20 | -------------------------------------------------------------------------------- /pkg/server/routes/healthz.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "github.com/quickube/piper/pkg/conf" 5 | "github.com/quickube/piper/pkg/webhook_creator" 6 | "golang.org/x/net/context" 7 | "log" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/gin-gonic/gin" 12 | ) 13 | 14 | func AddHealthRoutes(rg *gin.RouterGroup, wc *webhook_creator.WebhookCreatorImpl, cfg *conf.GlobalConfig) { 15 | health := rg.Group("/healthz") 16 | 17 | health.GET("", func(c *gin.Context) { 18 | if cfg.GitProviderConfig.FullHealthCheck { 19 | ctx := c.Copy().Request.Context() 20 | ctx2, cancel := context.WithTimeout(ctx, 5*time.Second) 21 | defer cancel() 22 | err := wc.RunDiagnosis(ctx2) 23 | if err != nil { 24 | log.Printf("error from healthz endpoint:%s\n", err) 25 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) 26 | return 27 | } 28 | } 29 | c.JSON(http.StatusOK, "healthy") 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/server/routes/readyz.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gin-gonic/gin" 7 | ) 8 | 9 | func AddReadyRoutes(rg *gin.RouterGroup) { 10 | health := rg.Group("/readyz") 11 | 12 | health.GET("", func(c *gin.Context) { 13 | c.JSON(http.StatusOK, "ready") 14 | }) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/server/routes/webhook.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "github.com/quickube/piper/pkg/webhook_creator" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/gin-gonic/gin" 9 | "github.com/quickube/piper/pkg/clients" 10 | "github.com/quickube/piper/pkg/conf" 11 | webhookHandler "github.com/quickube/piper/pkg/webhook_handler" 12 | ) 13 | 14 | func AddWebhookRoutes(cfg *conf.GlobalConfig, clients *clients.Clients, rg *gin.RouterGroup, wc *webhook_creator.WebhookCreatorImpl) { 15 | webhook := rg.Group("/webhook") 16 | 17 | webhook.POST("", func(c *gin.Context) { 18 | ctx := c.Request.Context() 19 | webhookPayload, err := clients.GitProvider.HandlePayload(ctx, c.Request, []byte(cfg.GitProviderConfig.WebhookSecret)) 20 | if err != nil { 21 | log.Println(err) 22 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) 23 | return 24 | } 25 | if webhookPayload.Event == "ping" { 26 | if cfg.GitProviderConfig.FullHealthCheck { 27 | err = wc.SetWebhookHealth(webhookPayload.HookID, true) 28 | if err != nil { 29 | c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) 30 | return 31 | } 32 | } 33 | c.JSON(http.StatusOK, gin.H{"status": "ok"}) 34 | return 35 | } 36 | 37 | wh, err := webhookHandler.NewWebhookHandler(cfg, clients, webhookPayload) 38 | if err != nil { 39 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) 40 | log.Printf("failed to create webhook handler, error: %v", err) 41 | return 42 | } 43 | 44 | workflowsBatches, err := webhookHandler.HandleWebhook(ctx, wh) 45 | if err != nil { 46 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) 47 | log.Printf("failed to handle webhook, error: %v", err) 48 | return 49 | } 50 | 51 | for _, wf := range workflowsBatches { 52 | err = clients.Workflows.HandleWorkflowBatch(ctx, wf) 53 | if err != nil { 54 | c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) 55 | log.Printf("failed to handle workflow, error: %v", err) 56 | return 57 | } 58 | } 59 | 60 | c.JSON(http.StatusOK, gin.H{"status": "ok"}) 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "github.com/gin-gonic/gin" 6 | "github.com/quickube/piper/pkg/clients" 7 | "github.com/quickube/piper/pkg/conf" 8 | "github.com/quickube/piper/pkg/server/routes" 9 | "github.com/quickube/piper/pkg/webhook_creator" 10 | "log" 11 | "net/http" 12 | ) 13 | 14 | func NewServer(config *conf.GlobalConfig, clients *clients.Clients) *Server { 15 | srv := &Server{ 16 | router: gin.New(), 17 | config: config, 18 | clients: clients, 19 | webhookCreator: webhook_creator.NewWebhookCreator(config, clients), 20 | } 21 | 22 | return srv 23 | } 24 | 25 | func (s *Server) startServer() *http.Server { 26 | srv := &http.Server{ 27 | Addr: ":8080", 28 | Handler: s.router, 29 | } 30 | 31 | go func() { 32 | log.Printf("Server is listening on %s", s.httpServer.Addr) 33 | if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { 34 | log.Fatalf("listen: %s\n", err) 35 | } 36 | }() 37 | 38 | return srv 39 | } 40 | 41 | func (s *Server) registerMiddlewares() { 42 | s.router.Use( 43 | gin.LoggerWithConfig(gin.LoggerConfig{ 44 | SkipPaths: []string{"/healthz", "/readyz"}, 45 | }), 46 | gin.Recovery(), 47 | ) 48 | 49 | } 50 | 51 | func (s *Server) getRoutes() { 52 | v1 := s.router.Group("/") 53 | routes.AddReadyRoutes(v1) 54 | routes.AddHealthRoutes(v1, s.webhookCreator, s.config) 55 | routes.AddWebhookRoutes(s.config, s.clients, v1, s.webhookCreator) 56 | } 57 | 58 | func (s *Server) startServices(ctx context.Context) { 59 | s.webhookCreator.Start(ctx) 60 | } 61 | 62 | func (s *Server) Start(ctx context.Context) { 63 | 64 | s.registerMiddlewares() 65 | 66 | s.getRoutes() 67 | 68 | s.httpServer = s.startServer() 69 | 70 | s.startServices(ctx) 71 | 72 | } 73 | -------------------------------------------------------------------------------- /pkg/server/shutdown.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "golang.org/x/net/context" 5 | "log" 6 | "time" 7 | ) 8 | 9 | type GracefulShutdown struct { 10 | ctx context.Context 11 | stop context.CancelFunc 12 | } 13 | 14 | func NewGracefulShutdown(ctx context.Context, stop context.CancelFunc) *GracefulShutdown { 15 | return &GracefulShutdown{ 16 | ctx: ctx, 17 | stop: stop, 18 | } 19 | } 20 | 21 | func (s *GracefulShutdown) StopServices(ctx context.Context, server *Server) { 22 | server.webhookCreator.Stop(ctx) 23 | } 24 | 25 | func (s *GracefulShutdown) Shutdown(server *Server) { 26 | // Listen for the interrupt signal. 27 | <-s.ctx.Done() 28 | 29 | // Restore default behavior on the interrupt signal and notify user of shutdown. 30 | s.stop() 31 | 32 | log.Println("shutting down gracefully...") 33 | // The context is used to inform the server it has 10 seconds to finish 34 | // the request it is currently handling 35 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 36 | defer cancel() 37 | 38 | s.StopServices(ctx, server) 39 | 40 | err := server.httpServer.Shutdown(ctx) 41 | if err != nil { 42 | log.Fatal("Server forced to shutdown: ", err) 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /pkg/server/types.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/gin-gonic/gin" 5 | "github.com/quickube/piper/pkg/clients" 6 | "github.com/quickube/piper/pkg/conf" 7 | "github.com/quickube/piper/pkg/webhook_creator" 8 | "net/http" 9 | ) 10 | 11 | type Server struct { 12 | router *gin.Engine 13 | config *conf.GlobalConfig 14 | clients *clients.Clients 15 | webhookCreator *webhook_creator.WebhookCreatorImpl 16 | httpServer *http.Server 17 | } 18 | 19 | type Interface interface { 20 | startServer() *http.Server 21 | registerMiddlewares() 22 | getRoutes() 23 | Start() *http.Server 24 | } 25 | -------------------------------------------------------------------------------- /pkg/utils/common.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "hash/fnv" 7 | "regexp" 8 | "strings" 9 | 10 | "gopkg.in/yaml.v3" 11 | "k8s.io/client-go/rest" 12 | "k8s.io/client-go/tools/clientcmd" 13 | ) 14 | 15 | func ListContains(subList, list []string) bool { 16 | if len(subList) > len(list) { 17 | return false 18 | } 19 | for _, element := range subList { 20 | found := false 21 | for _, b := range list { 22 | if element == b { 23 | found = true 24 | break 25 | } 26 | } 27 | if !found { 28 | return false 29 | } 30 | } 31 | return true 32 | } 33 | 34 | func IsElementExists(list []string, element string) bool { 35 | for _, item := range list { 36 | if item == element { 37 | return true 38 | } 39 | } 40 | return false 41 | } 42 | 43 | func IsElementMatch(element string, elements []string) bool { 44 | if IsElementExists(elements, "*") { 45 | return true 46 | } 47 | 48 | return IsElementExists(elements, element) 49 | } 50 | 51 | func GetClientConfig(kubeConfig string) (*rest.Config, error) { 52 | if kubeConfig != "" { 53 | return clientcmd.BuildConfigFromFlags("", kubeConfig) 54 | } 55 | return rest.InClusterConfig() 56 | } 57 | 58 | func AddPrefixToList(list []string, prefix string) []string { 59 | result := make([]string, len(list)) 60 | 61 | for i, item := range list { 62 | result[i] = prefix + item 63 | } 64 | 65 | return result 66 | } 67 | 68 | func StringToMap(str string) map[string]string { 69 | pairs := strings.Split(str, ",") 70 | m := make(map[string]string) 71 | 72 | for _, pair := range pairs { 73 | keyValue := strings.Split(pair, ":") 74 | if len(keyValue) == 2 { 75 | key := strings.TrimSpace(keyValue[0]) 76 | value := strings.TrimSpace(keyValue[1]) 77 | m[key] = value 78 | } 79 | } 80 | 81 | return m 82 | } 83 | 84 | func ConvertYAMLListToJSONList(yamlString string) ([]byte, error) { 85 | // Unmarshal YAML into a map[string]interface{} 86 | yamlData := make([]map[string]interface{}, 0) 87 | err := yaml.Unmarshal([]byte(yamlString), &yamlData) 88 | if err != nil { 89 | return nil, fmt.Errorf("failed to unmarshal YAML: %v", err) 90 | } 91 | 92 | // Marshal the YAML data as JSON 93 | jsonBytes, err := json.Marshal(&yamlData) 94 | if err != nil { 95 | return nil, fmt.Errorf("failed to marshal JSON: %v", err) 96 | } 97 | 98 | return jsonBytes, nil 99 | } 100 | 101 | func ConvertYAMLToJSON(yamlString []byte) ([]byte, error) { 102 | // Unmarshal YAML into a map[string]interface{} 103 | yamlData := make(map[string]interface{}) 104 | err := yaml.Unmarshal(yamlString, &yamlData) 105 | if err != nil { 106 | return nil, fmt.Errorf("failed to unmarshal YAML: %v", err) 107 | } 108 | 109 | // Marshal the YAML data as JSON 110 | jsonBytes, err := json.Marshal(&yamlData) 111 | if err != nil { 112 | return nil, fmt.Errorf("failed to marshal JSON: %v", err) 113 | } 114 | 115 | return jsonBytes, nil 116 | } 117 | 118 | func SPtr(str string) *string { 119 | return &str 120 | } 121 | 122 | func BPtr(b bool) *bool { 123 | return &b 124 | } 125 | 126 | func IPtr(i int64) *int64 { 127 | return &i 128 | } 129 | 130 | func ValidateHTTPFormat(input string) bool { 131 | regex := `^(https?://)([\w-]+(\.[\w-]+)*)(:\d+)?(/[\w-./?%&=]*)?$` 132 | match, _ := regexp.MatchString(regex, input) 133 | return match 134 | } 135 | 136 | func TrimString(s string, maxLength int) string { 137 | if maxLength >= len(s) { 138 | return s 139 | } 140 | return s[:maxLength] 141 | } 142 | 143 | func StringToInt64(input string) int64 { 144 | h := fnv.New64a() 145 | h.Write([]byte(input)) 146 | hashValue := h.Sum64() 147 | 148 | // Convert the hash value to int64 149 | int64Value := int64(hashValue) 150 | // Make sure the value is positive (int64 can represent only non-negative values) 151 | if int64Value < 0 { 152 | int64Value = int64Value * -1 153 | } 154 | 155 | return int64Value 156 | } 157 | 158 | func RemoveBraces(input string) string { 159 | output := strings.ReplaceAll(input, "{", "") 160 | output = strings.ReplaceAll(output, "}", "") 161 | return output 162 | } 163 | 164 | func ExtractStringsBetweenTags(input string) []string { 165 | re := regexp.MustCompile(`<([^>]+)>`) 166 | matches := re.FindAllStringSubmatch(input, -1) 167 | 168 | var result []string 169 | for _, match := range matches { 170 | result = append(result, match[1]) 171 | } 172 | 173 | return result 174 | } 175 | 176 | func SanitizeString(input string) string { 177 | // Replace whitespace with "-" 178 | input = strings.ReplaceAll(input, " ", "-") 179 | 180 | // Define a regular expression pattern to match characters that are not a-z, A-Z, _, or . 181 | // Use a negated character class to allow a-z, A-Z, _, and . 182 | pattern := regexp.MustCompile(`[^a-zA-Z_1-9\.-]+`) 183 | 184 | // Remove characters that don't match the pattern 185 | sanitized := pattern.ReplaceAllString(input, "") 186 | 187 | return sanitized 188 | } 189 | -------------------------------------------------------------------------------- /pkg/utils/os.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | func GetFilesData(directory string) (map[string][]byte, error) { 9 | fileData := make(map[string][]byte) 10 | var data []byte 11 | 12 | path, dirList, err := GetFilesInLinkDirectory(directory) 13 | if err != nil { 14 | return nil, err 15 | } 16 | for _, dir := range dirList { 17 | data, err = GetFileData(*path + "/" + dir) 18 | if err != nil { 19 | return nil, err 20 | } 21 | fileData[dir] = data 22 | } 23 | 24 | return fileData, nil 25 | } 26 | 27 | func GetFileData(filePath string) ([]byte, error) { 28 | data, err := os.ReadFile(filePath) 29 | if err != nil { 30 | return nil, err 31 | } 32 | return data, nil 33 | } 34 | 35 | func GetFilesInLinkDirectory(linkPath string) (*string, []string, error) { 36 | realPath, err := filepath.EvalSymlinks(linkPath) 37 | if err != nil { 38 | return nil, nil, err 39 | } 40 | 41 | var fileNames []string 42 | err = filepath.WalkDir(realPath, func(path string, d os.DirEntry, err error) error { 43 | if err != nil { 44 | return err 45 | } 46 | if !d.IsDir() { 47 | info, err := d.Info() 48 | if err != nil { 49 | return err 50 | } 51 | if info.Mode()&os.ModeSymlink != 0 { 52 | return nil // Skip symbolic links 53 | } 54 | relPath, err := filepath.Rel(realPath, path) 55 | if err != nil { 56 | return err 57 | } 58 | fileNames = append(fileNames, relPath) 59 | } 60 | return nil 61 | }) 62 | 63 | if err != nil { 64 | return nil, nil, err 65 | } 66 | 67 | return &realPath, fileNames, nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/utils/os_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | ) 10 | 11 | func createFileWithContent(path string, content string) error { 12 | file, err := os.Create(path) 13 | if err != nil { 14 | return err 15 | } 16 | defer file.Close() 17 | 18 | _, err = io.WriteString(file, content) 19 | return err 20 | } 21 | 22 | func TestGetFilesData(t *testing.T) { 23 | // Create a temporary directory for testing 24 | tempDir := os.TempDir() 25 | testDir := filepath.Join(tempDir, "test") 26 | err := os.Mkdir(testDir, 0755) 27 | if err != nil { 28 | t.Fatalf("Error creating temporary directory: %v", err) 29 | } 30 | defer os.RemoveAll(testDir) // Clean up the temporary directory 31 | 32 | // Create some dummy files in the test directory 33 | file1Path := filepath.Join(testDir, "file1.txt") 34 | err = createFileWithContent(file1Path, "File 1 data") 35 | if err != nil { 36 | t.Fatalf("Error creating file1: %v", err) 37 | } 38 | 39 | file2Path := filepath.Join(testDir, "file2.txt") 40 | err = createFileWithContent(file2Path, "File 2 data") 41 | if err != nil { 42 | t.Fatalf("Error creating file2: %v", err) 43 | } 44 | 45 | // Call the function being tested 46 | fileData, err := GetFilesData(testDir) 47 | if err != nil { 48 | t.Fatalf("Error calling GetFilesData: %v", err) 49 | } 50 | 51 | // Verify the results 52 | expectedData := map[string][]byte{ 53 | "file1.txt": []byte("File 1 data"), 54 | "file2.txt": []byte("File 2 data"), 55 | } 56 | 57 | for fileName, expected := range expectedData { 58 | actual, ok := fileData[fileName] 59 | if !ok { 60 | t.Errorf("Missing file data for %s", fileName) 61 | } 62 | 63 | if !bytes.Equal(actual, expected) { 64 | t.Errorf("File data mismatch for %s: expected '%s', got '%s'", fileName, expected, actual) 65 | } 66 | } 67 | } 68 | 69 | func stringSlicesEqual(a, b []string) bool { 70 | if len(a) != len(b) { 71 | return false 72 | } 73 | for i := range a { 74 | if a[i] != b[i] { 75 | return false 76 | } 77 | } 78 | return true 79 | } 80 | 81 | func TestGetFilesInLinkDirectory(t *testing.T) { 82 | tempDir := t.TempDir() 83 | 84 | fileNames := []string{"file1.txt", "file2.txt", "file3.txt"} 85 | for _, name := range fileNames { 86 | filePath := filepath.Join(tempDir, name) 87 | _, err := os.Create(filePath) 88 | if err != nil { 89 | t.Fatalf("Failed to create test file: %v", err) 90 | } 91 | } 92 | 93 | linkPath := filepath.Join(tempDir, "symlink") 94 | err := os.Symlink(tempDir, linkPath) 95 | if err != nil { 96 | t.Fatalf("Failed to create symbolic link: %v", err) 97 | } 98 | 99 | realPath, result, err := GetFilesInLinkDirectory(linkPath) 100 | if err != nil { 101 | t.Fatalf("Unexpected error: %v", err) 102 | } 103 | 104 | expectedRealPath, err := filepath.EvalSymlinks(linkPath) 105 | if err != nil { 106 | t.Fatalf("Failed to evaluate symlink: %v", err) 107 | } 108 | if *realPath != expectedRealPath { 109 | t.Errorf("Unexpected real path. Expected: %s, got: %s", expectedRealPath, *realPath) 110 | } 111 | 112 | t.Logf("Result: %v", result) // Add this line for debugging 113 | 114 | if !stringSlicesEqual(result, fileNames) { 115 | t.Errorf("Unexpected file names. Expected: %v, got: %v", fileNames, result) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /pkg/webhook_creator/main.go: -------------------------------------------------------------------------------- 1 | package webhook_creator 2 | 3 | import ( 4 | "fmt" 5 | "github.com/emicklei/go-restful/v3/log" 6 | "github.com/quickube/piper/pkg/clients" 7 | "github.com/quickube/piper/pkg/conf" 8 | "github.com/quickube/piper/pkg/git_provider" 9 | "github.com/quickube/piper/pkg/utils" 10 | "golang.org/x/net/context" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | type WebhookCreatorImpl struct { 18 | clients *clients.Clients 19 | cfg *conf.GlobalConfig 20 | hooks map[int64]*git_provider.HookWithStatus 21 | mu sync.Mutex 22 | } 23 | 24 | func NewWebhookCreator(cfg *conf.GlobalConfig, clients *clients.Clients) *WebhookCreatorImpl { 25 | wr := &WebhookCreatorImpl{ 26 | clients: clients, 27 | cfg: cfg, 28 | hooks: make(map[int64]*git_provider.HookWithStatus, 0), 29 | } 30 | 31 | return wr 32 | } 33 | 34 | func (wc *WebhookCreatorImpl) GetHooks() *map[int64]*git_provider.HookWithStatus { 35 | return &wc.hooks 36 | } 37 | 38 | func (wc *WebhookCreatorImpl) Start(ctx context.Context) { 39 | 40 | err := wc.initWebhooks(ctx) 41 | if err != nil { 42 | log.Print(err) 43 | panic("failed in initializing webhooks") 44 | } 45 | } 46 | 47 | func (wc *WebhookCreatorImpl) setWebhook(hookID int64, healthStatus bool, repoName string) { 48 | wc.mu.Lock() 49 | defer wc.mu.Unlock() 50 | wc.hooks[hookID] = &git_provider.HookWithStatus{HookID: hookID, HealthStatus: healthStatus, RepoName: &repoName} 51 | } 52 | 53 | func (wc *WebhookCreatorImpl) getWebhook(hookID int64) *git_provider.HookWithStatus { 54 | wc.mu.Lock() 55 | defer wc.mu.Unlock() 56 | hook, ok := wc.hooks[hookID] 57 | if !ok { 58 | return nil 59 | } 60 | return hook 61 | } 62 | 63 | func (wc *WebhookCreatorImpl) deleteWebhook(hookID int64) { 64 | wc.mu.Lock() 65 | defer wc.mu.Unlock() 66 | 67 | delete(wc.hooks, hookID) 68 | } 69 | 70 | func (wc *WebhookCreatorImpl) SetWebhookHealth(hookID int64, status bool) error { 71 | 72 | hook, ok := wc.hooks[hookID] 73 | if !ok { 74 | return fmt.Errorf("unable to find hookID: %d in internal hooks map %v", hookID, wc.hooks) 75 | } 76 | wc.setWebhook(hookID, status, *hook.RepoName) 77 | log.Printf("set health status to %s for hook id: %d", strconv.FormatBool(status), hookID) 78 | return nil 79 | } 80 | 81 | func (wc *WebhookCreatorImpl) setAllHooksHealth(status bool) { 82 | for hookID, hook := range wc.hooks { 83 | wc.setWebhook(hookID, status, *hook.RepoName) 84 | } 85 | log.Printf("set all hooks health status for to %s", strconv.FormatBool(status)) 86 | } 87 | 88 | func (wc *WebhookCreatorImpl) initWebhooks(ctx context.Context) error { 89 | 90 | if wc.cfg.GitProviderConfig.OrgLevelWebhook && len(wc.cfg.GitProviderConfig.RepoList) != 0 { 91 | return fmt.Errorf("org level webhook wanted but provided repositories list") 92 | } else if !wc.cfg.GitProviderConfig.OrgLevelWebhook && len(wc.cfg.GitProviderConfig.RepoList) == 0 { 93 | return fmt.Errorf("either org level webhook or repos list must be provided") 94 | } 95 | for _, repo := range strings.Split(wc.cfg.GitProviderConfig.RepoList, ",") { 96 | if wc.cfg.GitProviderConfig.Provider == "bitbucket" { 97 | repo = utils.SanitizeString(repo) 98 | } 99 | hook, err := wc.clients.GitProvider.SetWebhook(ctx, &repo) 100 | if err != nil { 101 | return err 102 | } 103 | wc.setWebhook(hook.HookID, hook.HealthStatus, *hook.RepoName) 104 | } 105 | 106 | return nil 107 | } 108 | 109 | func (wc *WebhookCreatorImpl) Stop(ctx context.Context) { 110 | if wc.cfg.GitProviderConfig.WebhookAutoCleanup { 111 | err := wc.deleteWebhooks(ctx) 112 | if err != nil { 113 | log.Printf("Failed to delete webhooks, error: %v", err) 114 | } 115 | } 116 | } 117 | 118 | func (wc *WebhookCreatorImpl) deleteWebhooks(ctx context.Context) error { 119 | for hookID, hook := range wc.hooks { 120 | err := wc.clients.GitProvider.UnsetWebhook(ctx, hook) 121 | if err != nil { 122 | return err 123 | } 124 | wc.deleteWebhook(hookID) 125 | } 126 | 127 | return nil 128 | } 129 | 130 | func (wc *WebhookCreatorImpl) checkHooksHealth(timeoutSeconds time.Duration) bool { 131 | startTime := time.Now() 132 | 133 | for { 134 | allHealthy := true 135 | for _, hook := range wc.hooks { 136 | if !hook.HealthStatus { 137 | allHealthy = false 138 | break 139 | } 140 | } 141 | 142 | if allHealthy { 143 | return true 144 | } 145 | 146 | if time.Since(startTime) >= timeoutSeconds { 147 | break 148 | } 149 | 150 | time.Sleep(1 * time.Second) // Adjust the sleep duration as per your requirement 151 | } 152 | 153 | return false 154 | } 155 | 156 | func (wc *WebhookCreatorImpl) recoverHook(ctx context.Context, hookID int64) error { 157 | 158 | log.Printf("started recover of hook %d", hookID) 159 | hook := wc.getWebhook(hookID) 160 | if hook == nil { 161 | return fmt.Errorf("failed to recover hook, hookID %d not found", hookID) 162 | } 163 | newHook, err := wc.clients.GitProvider.SetWebhook(ctx, hook.RepoName) 164 | if err != nil { 165 | return err 166 | } 167 | wc.deleteWebhook(hookID) 168 | wc.setWebhook(newHook.HookID, newHook.HealthStatus, *newHook.RepoName) 169 | log.Printf("successful recover of hook %d", hookID) 170 | return nil 171 | 172 | } 173 | 174 | func (wc *WebhookCreatorImpl) pingHooks(ctx context.Context) error { 175 | for _, hook := range wc.hooks { 176 | err := wc.clients.GitProvider.PingHook(ctx, hook) 177 | if err != nil { 178 | return err 179 | } 180 | } 181 | return nil 182 | } 183 | 184 | func (wc *WebhookCreatorImpl) RunDiagnosis(ctx context.Context) error { 185 | log.Printf("Starting webhook diagnostics") 186 | wc.setAllHooksHealth(false) 187 | err := wc.pingHooks(ctx) 188 | if err != nil { 189 | return err 190 | } 191 | if !wc.checkHooksHealth(5 * time.Second) { 192 | for hookID, hook := range wc.hooks { 193 | if !hook.HealthStatus { 194 | return fmt.Errorf("hook %d is not healthy", hookID) 195 | } 196 | } 197 | } 198 | 199 | log.Print("Successful webhook diagnosis") 200 | return nil 201 | } 202 | -------------------------------------------------------------------------------- /pkg/webhook_creator/mocks.go: -------------------------------------------------------------------------------- 1 | package webhook_creator 2 | 3 | import ( 4 | context2 "context" 5 | "errors" 6 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 7 | "github.com/quickube/piper/pkg/git_provider" 8 | "golang.org/x/net/context" 9 | "net/http" 10 | ) 11 | 12 | type MockGitProviderClient struct { 13 | ListFilesFunc func(ctx context.Context, repo string, branch string, path string) ([]string, error) 14 | GetFileFunc func(ctx context.Context, repo string, branch string, path string) (*git_provider.CommitFile, error) 15 | GetFilesFunc func(ctx context.Context, repo string, branch string, paths []string) ([]*git_provider.CommitFile, error) 16 | SetWebhookFunc func(ctx context.Context, repo *string) (*git_provider.HookWithStatus, error) 17 | UnsetWebhookFunc func(ctx context.Context, hook *git_provider.HookWithStatus) error 18 | HandlePayloadFunc func(request *http.Request, secret []byte) (*git_provider.WebhookPayload, error) 19 | SetStatusFunc func(ctx context.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error 20 | PingHookFunc func(ctx context.Context, hook *git_provider.HookWithStatus) error 21 | GetCorrelatingEventFunc func(ctx context.Context, workflowEvent *v1alpha1.WorkflowPhase) (string, error) 22 | } 23 | 24 | func (m *MockGitProviderClient) ListFiles(ctx context2.Context, repo string, branch string, path string) ([]string, error) { 25 | if m.ListFilesFunc != nil { 26 | return m.ListFilesFunc(ctx, repo, branch, path) 27 | } 28 | return nil, errors.New("unimplemented") 29 | } 30 | 31 | func (m *MockGitProviderClient) GetFile(ctx context2.Context, repo string, branch string, path string) (*git_provider.CommitFile, error) { 32 | if m.GetFileFunc != nil { 33 | return m.GetFileFunc(ctx, repo, branch, path) 34 | } 35 | return nil, errors.New("unimplemented") 36 | } 37 | 38 | func (m *MockGitProviderClient) GetFiles(ctx context2.Context, repo string, branch string, paths []string) ([]*git_provider.CommitFile, error) { 39 | if m.GetFilesFunc != nil { 40 | return m.GetFilesFunc(ctx, repo, branch, paths) 41 | } 42 | return nil, errors.New("unimplemented") 43 | } 44 | 45 | func (m *MockGitProviderClient) SetWebhook(ctx context2.Context, repo *string) (*git_provider.HookWithStatus, error) { 46 | if m.SetWebhookFunc != nil { 47 | return m.SetWebhookFunc(ctx, repo) 48 | } 49 | return nil, errors.New("unimplemented") 50 | } 51 | 52 | func (m *MockGitProviderClient) UnsetWebhook(ctx context2.Context, hook *git_provider.HookWithStatus) error { 53 | if m.UnsetWebhookFunc != nil { 54 | return m.UnsetWebhookFunc(ctx, hook) 55 | } 56 | return errors.New("unimplemented") 57 | } 58 | 59 | func (m *MockGitProviderClient) HandlePayload(ctx context2.Context, request *http.Request, secret []byte) (*git_provider.WebhookPayload, error) { 60 | if m.HandlePayloadFunc != nil { 61 | return m.HandlePayloadFunc(request, secret) 62 | } 63 | return nil, errors.New("unimplemented") 64 | } 65 | 66 | func (m *MockGitProviderClient) SetStatus(ctx context2.Context, repo *string, commit *string, linkURL *string, status *string, message *string) error { 67 | if m.SetStatusFunc != nil { 68 | return m.SetStatusFunc(ctx, repo, commit, linkURL, status, message) 69 | } 70 | return errors.New("unimplemented") 71 | } 72 | func (m *MockGitProviderClient) GetCorrelatingEvent(ctx context.Context, workflowEvent *v1alpha1.WorkflowPhase) (string, error) { 73 | if m.GetCorrelatingEventFunc != nil { 74 | return m.GetCorrelatingEventFunc(ctx, workflowEvent) 75 | } 76 | return "", errors.New("unimplemented") 77 | } 78 | 79 | func (m *MockGitProviderClient) PingHook(ctx context2.Context, hook *git_provider.HookWithStatus) error { 80 | if m.PingHookFunc != nil { 81 | return m.PingHookFunc(ctx, hook) 82 | } 83 | return errors.New("unimplemented") 84 | } 85 | -------------------------------------------------------------------------------- /pkg/webhook_creator/types.go: -------------------------------------------------------------------------------- 1 | package webhook_creator 2 | 3 | import "golang.org/x/net/context" 4 | 5 | type WebhookCreator interface { 6 | Stop(ctx *context.Context) 7 | Start() 8 | SetWebhookHealth(status bool, hookID *int64) error 9 | RunDiagnosis(ctx *context.Context) error 10 | } 11 | -------------------------------------------------------------------------------- /pkg/webhook_handler/types.go: -------------------------------------------------------------------------------- 1 | package webhook_handler 2 | 3 | import ( 4 | "context" 5 | "github.com/quickube/piper/pkg/common" 6 | ) 7 | 8 | type Trigger struct { 9 | Events *[]string `yaml:"events"` 10 | Branches *[]string `yaml:"branches"` 11 | OnStart *[]string `yaml:"onStart"` 12 | Templates *[]string `yaml:"templates"` 13 | OnExit *[]string `yaml:"onExit"` 14 | Config string `yaml:"config" default:"default"` 15 | } 16 | 17 | type WebhookHandler interface { 18 | RegisterTriggers(ctx context.Context) error 19 | PrepareBatchForMatchingTriggers(ctx context.Context) ([]*common.WorkflowsBatch, error) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/webhook_handler/webhook_handler.go: -------------------------------------------------------------------------------- 1 | package webhook_handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/quickube/piper/pkg/clients" 7 | "github.com/quickube/piper/pkg/common" 8 | "github.com/quickube/piper/pkg/conf" 9 | "github.com/quickube/piper/pkg/git_provider" 10 | "github.com/quickube/piper/pkg/utils" 11 | "gopkg.in/yaml.v3" 12 | "log" 13 | ) 14 | 15 | type WebhookHandlerImpl struct { 16 | cfg *conf.GlobalConfig 17 | clients *clients.Clients 18 | Triggers *[]Trigger 19 | Payload *git_provider.WebhookPayload 20 | } 21 | 22 | func NewWebhookHandler(cfg *conf.GlobalConfig, clients *clients.Clients, payload *git_provider.WebhookPayload) (*WebhookHandlerImpl, error) { 23 | var err error 24 | 25 | return &WebhookHandlerImpl{ 26 | cfg: cfg, 27 | clients: clients, 28 | Triggers: &[]Trigger{}, 29 | Payload: payload, 30 | }, err 31 | } 32 | 33 | func (wh *WebhookHandlerImpl) RegisterTriggers(ctx context.Context) error { 34 | if !IsFileExists(ctx, wh, "", ".workflows") { 35 | return fmt.Errorf(".workflows folder does not exist in %s/%s", wh.Payload.Repo, wh.Payload.Branch) 36 | } 37 | 38 | if !IsFileExists(ctx, wh, ".workflows", "triggers.yaml") { 39 | return fmt.Errorf(".workflows/triggers.yaml file does not exist in %s/%s", wh.Payload.Repo, wh.Payload.Branch) 40 | } 41 | 42 | triggers, err := wh.clients.GitProvider.GetFile(ctx, wh.Payload.Repo, wh.Payload.Branch, ".workflows/triggers.yaml") 43 | if err != nil { 44 | return fmt.Errorf("failed to get triggers content: %v", err) 45 | } 46 | 47 | log.Printf("triggers content is: \n %s \n", *triggers.Content) // DEBUG 48 | 49 | err = yaml.Unmarshal([]byte(*triggers.Content), wh.Triggers) 50 | if err != nil { 51 | return fmt.Errorf("failed to unmarshal triggers content: %v", err) 52 | } 53 | return nil 54 | } 55 | 56 | func (wh *WebhookHandlerImpl) PrepareBatchForMatchingTriggers(ctx context.Context) ([]*common.WorkflowsBatch, error) { 57 | triggered := false 58 | var workflowBatches []*common.WorkflowsBatch 59 | for _, trigger := range *wh.Triggers { 60 | if trigger.Branches == nil { 61 | return nil, fmt.Errorf("trigger from repo %s branch %s missing branch field", wh.Payload.Repo, wh.Payload.Branch) 62 | } 63 | if trigger.Events == nil { 64 | return nil, fmt.Errorf("trigger from repo %s branch %s missing event field", wh.Payload.Repo, wh.Payload.Branch) 65 | } 66 | 67 | eventToCheck := wh.Payload.Event 68 | if wh.Payload.Action != "" { 69 | eventToCheck += "." + wh.Payload.Action 70 | } 71 | if utils.IsElementMatch(wh.Payload.Branch, *trigger.Branches) && utils.IsElementMatch(eventToCheck, *trigger.Events) { 72 | log.Printf( 73 | "Triggering event %s for repo %s branch %s are triggered.", 74 | wh.Payload.Event, 75 | wh.Payload.Repo, 76 | wh.Payload.Branch, 77 | ) 78 | triggered = true 79 | onStartFiles, err := wh.clients.GitProvider.GetFiles( 80 | ctx, 81 | wh.Payload.Repo, 82 | wh.Payload.Branch, 83 | utils.AddPrefixToList(*trigger.OnStart, ".workflows/"), 84 | ) 85 | if len(onStartFiles) == 0 { 86 | return nil, fmt.Errorf("one or more of onStart: %s files found in repo: %s branch %s", *trigger.OnStart, wh.Payload.Repo, wh.Payload.Branch) 87 | } 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | onExitFiles := make([]*git_provider.CommitFile, 0) 93 | if trigger.OnExit != nil { 94 | onExitFiles, err = wh.clients.GitProvider.GetFiles( 95 | ctx, 96 | wh.Payload.Repo, 97 | wh.Payload.Branch, 98 | utils.AddPrefixToList(*trigger.OnExit, ".workflows/"), 99 | ) 100 | if len(onExitFiles) == 0 { 101 | log.Printf("one or more of onExist: %s files not found in repo: %s branch %s", *trigger.OnExit, wh.Payload.Repo, wh.Payload.Branch) 102 | } 103 | if err != nil { 104 | return nil, err 105 | } 106 | } 107 | 108 | templatesFiles := make([]*git_provider.CommitFile, 0) 109 | if trigger.Templates != nil { 110 | templatesFiles, err = wh.clients.GitProvider.GetFiles( 111 | ctx, 112 | wh.Payload.Repo, 113 | wh.Payload.Branch, 114 | utils.AddPrefixToList(*trigger.Templates, ".workflows/"), 115 | ) 116 | if len(templatesFiles) == 0 { 117 | log.Printf("one or more of templates: %s files not found in repo: %s branch %s", *trigger.Templates, wh.Payload.Repo, wh.Payload.Branch) 118 | } 119 | if err != nil { 120 | return nil, err 121 | } 122 | } 123 | 124 | parameters := &git_provider.CommitFile{ 125 | Path: nil, 126 | Content: nil, 127 | } 128 | if IsFileExists(ctx, wh, ".workflows", "parameters.yaml") { 129 | parameters, err = wh.clients.GitProvider.GetFile( 130 | ctx, 131 | wh.Payload.Repo, 132 | wh.Payload.Branch, 133 | ".workflows/parameters.yaml", 134 | ) 135 | if err != nil { 136 | return nil, err 137 | } 138 | } else { 139 | log.Printf("parameters.yaml not found in repo: %s branch %s", wh.Payload.Repo, wh.Payload.Branch) 140 | } 141 | 142 | workflowBatches = append(workflowBatches, &common.WorkflowsBatch{ 143 | OnStart: onStartFiles, 144 | OnExit: onExitFiles, 145 | Templates: templatesFiles, 146 | Parameters: parameters, 147 | Config: &trigger.Config, 148 | Payload: wh.Payload, 149 | }) 150 | } 151 | } 152 | if !triggered { 153 | return nil, fmt.Errorf("no matching trigger found for event: %s action: %s in branch :%s", wh.Payload.Event, wh.Payload.Action, wh.Payload.Branch) 154 | } 155 | return workflowBatches, nil 156 | } 157 | 158 | func IsFileExists(ctx context.Context, wh *WebhookHandlerImpl, path string, file string) bool { 159 | files, err := wh.clients.GitProvider.ListFiles(ctx, wh.Payload.Repo, wh.Payload.Branch, path) 160 | if err != nil { 161 | log.Printf("Error listing files in repo: %s branch: %s. %v", wh.Payload.Repo, wh.Payload.Branch, err) 162 | return false 163 | } 164 | if len(files) == 0 { 165 | log.Printf("Empty list of files in repo: %s branch: %s", wh.Payload.Repo, wh.Payload.Branch) 166 | return false 167 | } 168 | 169 | if utils.IsElementExists(files, file) { 170 | return true 171 | } 172 | 173 | return false 174 | } 175 | 176 | func HandleWebhook(ctx context.Context, wh *WebhookHandlerImpl) ([]*common.WorkflowsBatch, error) { 177 | err := wh.RegisterTriggers(ctx) 178 | if err != nil { 179 | return nil, fmt.Errorf("failed to register triggers, error: %v", err) 180 | } else { 181 | log.Printf("successfully registered triggers for repo: %s branch: %s", wh.Payload.Repo, wh.Payload.Branch) 182 | } 183 | 184 | workflowsBatches, err := wh.PrepareBatchForMatchingTriggers(ctx) 185 | if err != nil { 186 | return nil, fmt.Errorf("failed to prepare matching triggers, error: %v", err) 187 | } 188 | 189 | if len(workflowsBatches) == 0 { 190 | log.Printf("no workflows to execute") 191 | return nil, fmt.Errorf("no workflows to execute for repo: %s branch: %s", 192 | wh.Payload.Repo, 193 | wh.Payload.Branch, 194 | ) 195 | } 196 | return workflowsBatches, nil 197 | } 198 | -------------------------------------------------------------------------------- /pkg/workflow_handler/types.go: -------------------------------------------------------------------------------- 1 | package workflow_handler 2 | 3 | import ( 4 | "context" 5 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 6 | "github.com/quickube/piper/pkg/common" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | "k8s.io/apimachinery/pkg/watch" 9 | ) 10 | 11 | type WorkflowsClient interface { 12 | ConstructTemplates(workflowsBatch *common.WorkflowsBatch, configName string) ([]v1alpha1.Template, error) 13 | ConstructSpec(templates []v1alpha1.Template, params []v1alpha1.Parameter, configName string) (*v1alpha1.WorkflowSpec, error) 14 | CreateWorkflow(spec *v1alpha1.WorkflowSpec, workflowsBatch *common.WorkflowsBatch) (*v1alpha1.Workflow, error) 15 | SelectConfig(workflowsBatch *common.WorkflowsBatch) (string, error) 16 | Lint(wf *v1alpha1.Workflow) error 17 | Submit(ctx context.Context, wf *v1alpha1.Workflow) error 18 | HandleWorkflowBatch(ctx context.Context, workflowsBatch *common.WorkflowsBatch) error 19 | Watch(ctx context.Context, labelSelector *metav1.LabelSelector) (watch.Interface, error) 20 | UpdatePiperWorkflowLabel(ctx context.Context, workflowName string, label string, value string) error 21 | } 22 | -------------------------------------------------------------------------------- /pkg/workflow_handler/workflows.go: -------------------------------------------------------------------------------- 1 | package workflow_handler 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 8 | wfClientSet "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" 9 | "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/types" 12 | "k8s.io/apimachinery/pkg/watch" 13 | "log" 14 | "strings" 15 | 16 | "github.com/quickube/piper/pkg/common" 17 | "github.com/quickube/piper/pkg/conf" 18 | "github.com/quickube/piper/pkg/utils" 19 | ) 20 | 21 | const ( 22 | ENTRYPOINT = "entryPoint" 23 | ONEXIT = "exitHandler" 24 | ) 25 | 26 | type WorkflowsClientImpl struct { 27 | clientSet *wfClientSet.Clientset 28 | cfg *conf.GlobalConfig 29 | } 30 | 31 | func NewWorkflowsClient(cfg *conf.GlobalConfig) (WorkflowsClient, error) { 32 | restClientConfig, err := utils.GetClientConfig(cfg.WorkflowServerConfig.KubeConfig) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | clientSet := wfClientSet.NewForConfigOrDie(restClientConfig) //.ArgoprojV1alpha1().Workflows(namespace) 38 | return &WorkflowsClientImpl{ 39 | clientSet: clientSet, 40 | cfg: cfg, 41 | }, nil 42 | } 43 | 44 | func (wfc *WorkflowsClientImpl) ConstructTemplates(workflowsBatch *common.WorkflowsBatch, configName string) ([]v1alpha1.Template, error) { 45 | finalTemplate := make([]v1alpha1.Template, 0) 46 | onStart, err := CreateDAGTemplate(workflowsBatch.OnStart, ENTRYPOINT) 47 | if err != nil { 48 | return nil, err 49 | } 50 | finalTemplate = append(finalTemplate, *onStart) 51 | 52 | onExit, err := CreateDAGTemplate(workflowsBatch.OnExit, ONEXIT) 53 | if err != nil { 54 | return nil, err 55 | } 56 | if onExit == nil || len(onExit.DAG.Tasks) == 0 { 57 | if IsConfigExists(&wfc.cfg.WorkflowsConfig, configName) && IsConfigsOnExitExists(&wfc.cfg.WorkflowsConfig, configName) { 58 | template := &v1alpha1.Template{ 59 | Name: ONEXIT, 60 | DAG: &v1alpha1.DAGTemplate{ 61 | Tasks: wfc.cfg.WorkflowsConfig.Configs[configName].OnExit, 62 | }, 63 | } 64 | 65 | finalTemplate = append(finalTemplate, *template) 66 | } 67 | } else { 68 | finalTemplate = append(finalTemplate, *onExit) 69 | } 70 | 71 | finalTemplate, err = AddFilesToTemplates(finalTemplate, workflowsBatch.Templates) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | return finalTemplate, nil 77 | } 78 | 79 | func (wfc *WorkflowsClientImpl) ConstructSpec(templates []v1alpha1.Template, params []v1alpha1.Parameter, configName string) (*v1alpha1.WorkflowSpec, error) { 80 | finalSpec := &v1alpha1.WorkflowSpec{} 81 | if IsConfigExists(&wfc.cfg.WorkflowsConfig, configName) { 82 | *finalSpec = wfc.cfg.WorkflowsConfig.Configs[configName].Spec 83 | if len(wfc.cfg.WorkflowsConfig.Configs[configName].OnExit) != 0 { 84 | finalSpec.OnExit = ONEXIT 85 | } 86 | } 87 | 88 | finalSpec.Entrypoint = ENTRYPOINT 89 | finalSpec.Templates = templates 90 | finalSpec.Arguments.Parameters = params 91 | 92 | return finalSpec, nil 93 | } 94 | 95 | func (wfc *WorkflowsClientImpl) CreateWorkflow(spec *v1alpha1.WorkflowSpec, workflowsBatch *common.WorkflowsBatch) (*v1alpha1.Workflow, error) { 96 | workflow := &v1alpha1.Workflow{ 97 | ObjectMeta: metav1.ObjectMeta{ 98 | GenerateName: ConvertToValidString(workflowsBatch.Payload.Repo + "-" + workflowsBatch.Payload.Branch + "-"), 99 | Namespace: wfc.cfg.Namespace, 100 | Labels: map[string]string{ 101 | "piper.quickube.com/notified": "false", 102 | "repo": ConvertToValidString(workflowsBatch.Payload.Repo), 103 | "branch": ConvertToValidString(workflowsBatch.Payload.Branch), 104 | "user": ConvertToValidString(workflowsBatch.Payload.User), 105 | "commit": ConvertToValidString(workflowsBatch.Payload.Commit), 106 | }, 107 | }, 108 | Spec: *spec, 109 | } 110 | 111 | return workflow, nil 112 | } 113 | 114 | func (wfc *WorkflowsClientImpl) SelectConfig(workflowsBatch *common.WorkflowsBatch) (string, error) { 115 | var configName string 116 | if IsConfigExists(&wfc.cfg.WorkflowsConfig, "default") { 117 | configName = "default" 118 | } 119 | 120 | if *workflowsBatch.Config != "" { 121 | if IsConfigExists(&wfc.cfg.WorkflowsConfig, *workflowsBatch.Config) { 122 | configName = *workflowsBatch.Config 123 | } else { 124 | return configName, fmt.Errorf( 125 | "error in selecting config, staying with default config for repo %s branch %s", 126 | workflowsBatch.Payload.Repo, 127 | workflowsBatch.Payload.Branch, 128 | ) 129 | } 130 | } 131 | 132 | log.Printf( 133 | "%s config selected for workflow in repo: %s branch %s", 134 | configName, 135 | workflowsBatch.Payload.Repo, 136 | workflowsBatch.Payload.Branch, 137 | ) // Info 138 | 139 | return configName, nil 140 | } 141 | 142 | func (wfc *WorkflowsClientImpl) Lint(wf *v1alpha1.Workflow) error { 143 | //TODO implement me 144 | panic("implement me") 145 | } 146 | 147 | func (wfc *WorkflowsClientImpl) Submit(ctx context.Context, wf *v1alpha1.Workflow) error { 148 | workflowsClient := wfc.clientSet.ArgoprojV1alpha1().Workflows(wfc.cfg.Namespace) 149 | _, err := workflowsClient.Create(ctx, wf, metav1.CreateOptions{}) 150 | if err != nil { 151 | return err 152 | } 153 | 154 | return nil 155 | } 156 | 157 | func (wfc *WorkflowsClientImpl) HandleWorkflowBatch(ctx context.Context, workflowsBatch *common.WorkflowsBatch) error { 158 | var params []v1alpha1.Parameter 159 | 160 | configName, err := wfc.SelectConfig(workflowsBatch) 161 | if err != nil { 162 | return err 163 | } 164 | 165 | templates, err := wfc.ConstructTemplates(workflowsBatch, configName) 166 | if err != nil { 167 | return err 168 | } 169 | 170 | if workflowsBatch.Parameters != nil { 171 | params, err = GetParameters(workflowsBatch.Parameters) 172 | if err != nil { 173 | return err 174 | } 175 | } 176 | 177 | globalParams := []v1alpha1.Parameter{ 178 | {Name: "event", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Event)}, 179 | {Name: "action", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Action)}, 180 | {Name: "repo", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Repo)}, 181 | {Name: "branch", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Branch)}, 182 | {Name: "commit", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.Commit)}, 183 | {Name: "user", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.User)}, 184 | {Name: "user_email", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.UserEmail)}, 185 | {Name: "pull_request_url", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.PullRequestURL)}, 186 | {Name: "pull_request_title", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.PullRequestTitle)}, 187 | {Name: "dest_branch", Value: v1alpha1.AnyStringPtr(workflowsBatch.Payload.DestBranch)}, 188 | {Name: "pull_request_labels", Value: v1alpha1.AnyStringPtr(strings.Join(workflowsBatch.Payload.Labels, ","))}, 189 | } 190 | 191 | params = append(params, globalParams...) 192 | 193 | spec, err := wfc.ConstructSpec(templates, params, configName) 194 | if err != nil { 195 | return err 196 | } 197 | 198 | workflow, err := wfc.CreateWorkflow(spec, workflowsBatch) 199 | if err != nil { 200 | return err 201 | } 202 | 203 | err = wfc.Submit(ctx, workflow) 204 | if err != nil { 205 | return fmt.Errorf("failed to submit workflow, error: %v", err) 206 | } 207 | 208 | log.Printf("submit workflow for branch %s repo %s commit %s", workflowsBatch.Payload.Branch, workflowsBatch.Payload.Repo, workflowsBatch.Payload.Commit) 209 | return nil 210 | } 211 | 212 | func (wfc *WorkflowsClientImpl) Watch(ctx context.Context, labelSelector *metav1.LabelSelector) (watch.Interface, error) { 213 | workflowsClient := wfc.clientSet.ArgoprojV1alpha1().Workflows(wfc.cfg.Namespace) 214 | opts := v1.ListOptions{ 215 | Watch: true, 216 | LabelSelector: metav1.FormatLabelSelector(labelSelector), 217 | } 218 | watcher, err := workflowsClient.Watch(ctx, opts) 219 | if err != nil { 220 | return nil, err 221 | } 222 | 223 | return watcher, nil 224 | } 225 | 226 | func (wfc *WorkflowsClientImpl) UpdatePiperWorkflowLabel(ctx context.Context, workflowName string, label string, value string) error { 227 | workflowsClient := wfc.clientSet.ArgoprojV1alpha1().Workflows(wfc.cfg.Namespace) 228 | 229 | patch, err := json.Marshal(map[string]interface{}{"metadata": metav1.ObjectMeta{ 230 | Labels: map[string]string{ 231 | fmt.Sprintf("piper.quickube.com/%s", label): value, 232 | }, 233 | }}) 234 | if err != nil { 235 | return err 236 | } 237 | _, err = workflowsClient.Patch(ctx, workflowName, types.MergePatchType, patch, v1.PatchOptions{}) 238 | if err != nil { 239 | return err 240 | } 241 | 242 | fmt.Printf("workflow %s labels piper.quickube.com/%s updated to %s\n", workflowName, label, value) 243 | return nil 244 | } 245 | -------------------------------------------------------------------------------- /pkg/workflow_handler/workflows_test.go: -------------------------------------------------------------------------------- 1 | package workflow_handler 2 | 3 | import ( 4 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 5 | "github.com/quickube/piper/pkg/common" 6 | "github.com/quickube/piper/pkg/conf" 7 | "github.com/quickube/piper/pkg/git_provider" 8 | assertion "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | func TestSelectConfig(t *testing.T) { 13 | var wfc *conf.WorkflowsConfig 14 | 15 | assert := assertion.New(t) 16 | // Create a sample WorkflowsBatch object for testing 17 | configName := "default" 18 | workflowsBatch := &common.WorkflowsBatch{ 19 | Config: &configName, // Set the desired config name here 20 | Payload: &git_provider.WebhookPayload{}, 21 | } 22 | 23 | // Create a mock WorkflowsClientImpl object with necessary dependencies 24 | wfc = &conf.WorkflowsConfig{Configs: map[string]*conf.ConfigInstance{ 25 | "default": {Spec: v1alpha1.WorkflowSpec{}, 26 | OnExit: []v1alpha1.DAGTask{}}, 27 | "config1": {Spec: v1alpha1.WorkflowSpec{}, 28 | OnExit: []v1alpha1.DAGTask{}}, 29 | }} 30 | 31 | wfcImpl := &WorkflowsClientImpl{ 32 | cfg: &conf.GlobalConfig{ 33 | WorkflowsConfig: *wfc, 34 | }, 35 | } 36 | 37 | // Call the SelectConfig function 38 | returnConfigName, err := wfcImpl.SelectConfig(workflowsBatch) 39 | 40 | // Assert the expected output 41 | assert.Equal("default", returnConfigName) 42 | assert.Nil(err) 43 | 44 | // Test case 2 45 | configName = "config1" 46 | workflowsBatch = &common.WorkflowsBatch{ 47 | Config: &configName, // Set the desired config name here 48 | Payload: &git_provider.WebhookPayload{}, 49 | } 50 | 51 | // Call the SelectConfig function 52 | returnConfigName, err = wfcImpl.SelectConfig(workflowsBatch) 53 | 54 | // Assert the expected output 55 | assert.Equal("config1", returnConfigName) 56 | assert.Nil(err) 57 | 58 | // Test case 3 - selection of non-existing config when default config exists 59 | configName = "notInConfigs" 60 | workflowsBatch = &common.WorkflowsBatch{ 61 | Config: &configName, // Set the desired config name here 62 | Payload: &git_provider.WebhookPayload{}, 63 | } 64 | 65 | // Call the SelectConfig function 66 | returnConfigName, err = wfcImpl.SelectConfig(workflowsBatch) 67 | 68 | // Assert the expected output 69 | assert.Equal("default", returnConfigName) 70 | assert.NotNil(err) 71 | 72 | // Test case 4 - selection of non-existing config when default config not exists 73 | configName = "notInConfig" 74 | workflowsBatch = &common.WorkflowsBatch{ 75 | Config: &configName, // Set the desired config name here 76 | Payload: &git_provider.WebhookPayload{}, 77 | } 78 | 79 | wfc4 := &conf.WorkflowsConfig{Configs: map[string]*conf.ConfigInstance{ 80 | "config1": {Spec: v1alpha1.WorkflowSpec{}, 81 | OnExit: []v1alpha1.DAGTask{}}, 82 | }} 83 | 84 | wfcImpl4 := &WorkflowsClientImpl{ 85 | cfg: &conf.GlobalConfig{ 86 | WorkflowsConfig: *wfc4, 87 | }, 88 | } 89 | 90 | // Call the SelectConfig function 91 | returnConfigName, err = wfcImpl4.SelectConfig(workflowsBatch) 92 | 93 | // Assert the expected output 94 | assert.NotNil(returnConfigName) 95 | assert.NotNil(err) 96 | } 97 | 98 | func TestCreateWorkflow(t *testing.T) { 99 | var wfc *conf.WorkflowsConfig 100 | var wfs *conf.WorkflowServerConfig 101 | 102 | // Create a WorkflowsClientImpl instance 103 | assert := assertion.New(t) 104 | // Create a mock WorkflowsClientImpl object with necessary dependencies 105 | wfc = &conf.WorkflowsConfig{Configs: map[string]*conf.ConfigInstance{ 106 | "default": {Spec: v1alpha1.WorkflowSpec{}, 107 | OnExit: []v1alpha1.DAGTask{}}, 108 | "config1": {Spec: v1alpha1.WorkflowSpec{}, 109 | OnExit: []v1alpha1.DAGTask{}}, 110 | }} 111 | 112 | wfs = &conf.WorkflowServerConfig{Namespace: "default"} 113 | 114 | wfcImpl := &WorkflowsClientImpl{ 115 | cfg: &conf.GlobalConfig{ 116 | WorkflowsConfig: *wfc, 117 | WorkflowServerConfig: *wfs, 118 | }, 119 | } 120 | 121 | // Create a sample WorkflowSpec 122 | spec := &v1alpha1.WorkflowSpec{ 123 | // Assign values to the fields of WorkflowSpec 124 | // ... 125 | 126 | // Example assignments: 127 | Entrypoint: "my-entrypoint", 128 | } 129 | 130 | // Create a sample WorkflowsBatch 131 | workflowsBatch := &common.WorkflowsBatch{ 132 | Payload: &git_provider.WebhookPayload{ 133 | Repo: "my-repo", 134 | Branch: "my-branch", 135 | User: "my-user", 136 | Commit: "my-commit", 137 | }, 138 | } 139 | 140 | // Call the CreateWorkflow method 141 | workflow, err := wfcImpl.CreateWorkflow(spec, workflowsBatch) 142 | 143 | // Assert that no error occurred 144 | assert.NoError(err) 145 | 146 | // Assert that the returned workflow is not nil 147 | assert.NotNil(workflow) 148 | 149 | // Assert that the workflow's GenerateName, Namespace, and Labels are assigned correctly 150 | assert.Equal("my-repo-my-branch-", workflow.ObjectMeta.GenerateName) 151 | assert.Equal(wfcImpl.cfg.Namespace, workflow.ObjectMeta.Namespace) 152 | assert.Equal(map[string]string{ 153 | "piper.quickube.com/notified": "false", 154 | "repo": "my-repo", 155 | "branch": "my-branch", 156 | "user": "my-user", 157 | "commit": "my-commit", 158 | }, workflow.ObjectMeta.Labels) 159 | 160 | // Assert that the workflow's Spec is assigned correctly 161 | assert.Equal(*spec, workflow.Spec) 162 | } 163 | -------------------------------------------------------------------------------- /pkg/workflow_handler/workflows_utils.go: -------------------------------------------------------------------------------- 1 | package workflow_handler 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 7 | "github.com/quickube/piper/pkg/conf" 8 | "github.com/quickube/piper/pkg/git_provider" 9 | "github.com/quickube/piper/pkg/utils" 10 | "gopkg.in/yaml.v3" 11 | "log" 12 | "regexp" 13 | "strings" 14 | ) 15 | 16 | func CreateDAGTemplate(fileList []*git_provider.CommitFile, name string) (*v1alpha1.Template, error) { 17 | if len(fileList) == 0 { 18 | log.Printf("empty file list for %s", name) 19 | return nil, nil 20 | } 21 | DAGs := make([]v1alpha1.DAGTask, 0) 22 | for _, file := range fileList { 23 | if file.Content == nil || file.Path == nil { 24 | return nil, fmt.Errorf("missing content or path for %s", name) 25 | } 26 | DAGTask := make([]v1alpha1.DAGTask, 0) 27 | jsonBytes, err := utils.ConvertYAMLListToJSONList(*file.Content) 28 | if err != nil { 29 | return nil, err 30 | } 31 | err = json.Unmarshal(jsonBytes, &DAGTask) 32 | if err != nil { 33 | return nil, err 34 | } 35 | err = ValidateDAGTasks(DAGTask) 36 | if err != nil { 37 | return nil, err 38 | } 39 | DAGs = append(DAGs, DAGTask...) 40 | } 41 | 42 | if len(DAGs) == 0 { 43 | return nil, fmt.Errorf("no tasks for %s", name) 44 | } 45 | 46 | template := &v1alpha1.Template{ 47 | Name: name, 48 | DAG: &v1alpha1.DAGTemplate{ 49 | Tasks: DAGs, 50 | }, 51 | } 52 | 53 | return template, nil 54 | } 55 | 56 | func AddFilesToTemplates(templates []v1alpha1.Template, files []*git_provider.CommitFile) ([]v1alpha1.Template, error) { 57 | for _, f := range files { 58 | t := make([]v1alpha1.Template, 0) 59 | jsonBytes, err := utils.ConvertYAMLListToJSONList(*f.Content) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | err = json.Unmarshal(jsonBytes, &t) 65 | if err != nil { 66 | return nil, err 67 | } 68 | templates = append(templates, t...) 69 | } 70 | return templates, nil 71 | } 72 | 73 | func GetParameters(paramsFile *git_provider.CommitFile) ([]v1alpha1.Parameter, error) { 74 | var params []v1alpha1.Parameter 75 | err := yaml.Unmarshal([]byte(*paramsFile.Content), ¶ms) 76 | if err != nil { 77 | return nil, err 78 | } 79 | return params, nil 80 | } 81 | 82 | func IsConfigExists(cfg *conf.WorkflowsConfig, config string) bool { 83 | _, ok := cfg.Configs[config] 84 | return ok 85 | } 86 | 87 | func IsConfigsOnExitExists(cfg *conf.WorkflowsConfig, config string) bool { 88 | return len(cfg.Configs[config].OnExit) != 0 89 | } 90 | 91 | func ValidateDAGTasks(tasks []v1alpha1.DAGTask) error { 92 | for _, task := range tasks { 93 | if task.Name == "" { 94 | return fmt.Errorf("task name cannot be empty: %+v\n", task) 95 | } 96 | 97 | if task.Template == "" && task.TemplateRef == nil { 98 | return fmt.Errorf("task template or templateRef cannot be empty: %+v\n", task) 99 | } 100 | 101 | } 102 | return nil 103 | } 104 | 105 | func ConvertToValidString(input string) string { 106 | // Convert to lowercase 107 | lowercase := strings.ToLower(input) 108 | 109 | // Replace underscores with hyphens 110 | converted := strings.ReplaceAll(lowercase, "_", "-") 111 | 112 | // Remove symbols except . and - 113 | pattern := `[^a-z0-9.\-]` 114 | re := regexp.MustCompile(pattern) 115 | validString := re.ReplaceAllString(converted, "") 116 | 117 | return validString 118 | } 119 | -------------------------------------------------------------------------------- /pkg/workflow_handler/workflows_utils_test.go: -------------------------------------------------------------------------------- 1 | package workflow_handler 2 | 3 | import ( 4 | "fmt" 5 | "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" 6 | "github.com/quickube/piper/pkg/git_provider" 7 | assertion "github.com/stretchr/testify/assert" 8 | "testing" 9 | ) 10 | 11 | func TestAddFilesToTemplates(t *testing.T) { 12 | assert := assertion.New(t) 13 | 14 | template := make([]v1alpha1.Template, 0) 15 | files := make([]*git_provider.CommitFile, 0) 16 | 17 | content := ` 18 | - name: local-step 19 | inputs: 20 | parameters: 21 | - name: message 22 | script: 23 | image: alpine 24 | command: [ sh ] 25 | source: | 26 | echo "wellcome to {{ workflow.parameters.global }} 27 | echo "{{ inputs.parameters.message }}" 28 | ` 29 | path := "path" 30 | files = append(files, &git_provider.CommitFile{ 31 | Content: &content, 32 | Path: &path, 33 | }) 34 | 35 | template, err := AddFilesToTemplates(template, files) 36 | 37 | assert.Nil(err) 38 | assert.Equal("alpine", template[0].Script.Container.Image) 39 | assert.Equal([]string{"sh"}, template[0].Script.Command) 40 | expectedScript := "echo \"wellcome to {{ workflow.parameters.global }}\necho \"{{ inputs.parameters.message }}\"\n" 41 | assert.Equal(expectedScript, template[0].Script.Source) 42 | } 43 | func TestValidateDAGTasks(t *testing.T) { 44 | assert := assertion.New(t) 45 | // Define test cases 46 | tests := []struct { 47 | name string 48 | tasks []v1alpha1.DAGTask 49 | want error 50 | }{ 51 | { 52 | name: "Valid tasks", 53 | tasks: []v1alpha1.DAGTask{ 54 | {Name: "Task1", Template: "Template1"}, 55 | {Name: "Task2", TemplateRef: &v1alpha1.TemplateRef{Name: "Template2"}}, 56 | }, 57 | want: nil, 58 | }, 59 | { 60 | name: "Empty task name", 61 | tasks: []v1alpha1.DAGTask{ 62 | {Name: "", Template: "Template1"}, 63 | }, 64 | want: fmt.Errorf("task name cannot be empty"), 65 | }, 66 | { 67 | name: "Empty template and templateRef", 68 | tasks: []v1alpha1.DAGTask{ 69 | {Name: "Task1"}, 70 | }, 71 | want: fmt.Errorf("task template or templateRef cannot be empty"), 72 | }, 73 | } 74 | 75 | // Run test cases 76 | for _, test := range tests { 77 | t.Run(test.name, func(t *testing.T) { 78 | // Call the function being tested 79 | got := ValidateDAGTasks(test.tasks) 80 | 81 | // Use assert to check the equality of the error 82 | if test.want != nil { 83 | assert.Error(got) 84 | assert.NotNil(got) 85 | } else { 86 | assert.NoError(got) 87 | assert.Nil(got) 88 | } 89 | }) 90 | } 91 | } 92 | 93 | func TestCreateDAGTemplate(t *testing.T) { 94 | assert := assertion.New(t) 95 | t.Run("Empty file list", func(t *testing.T) { 96 | fileList := []*git_provider.CommitFile{} 97 | name := "template1" 98 | template, err := CreateDAGTemplate(fileList, name) 99 | assert.Nil(template) 100 | assert.Nil(err) 101 | }) 102 | 103 | t.Run("Missing content or path", func(t *testing.T) { 104 | file := &git_provider.CommitFile{ 105 | Content: nil, 106 | Path: nil, 107 | } 108 | fileList := []*git_provider.CommitFile{file} 109 | name := "template2" 110 | template, err := CreateDAGTemplate(fileList, name) 111 | assert.Nil(template) 112 | assert.NotNil(err) 113 | }) 114 | 115 | t.Run("Valid file list", func(t *testing.T) { 116 | path3 := "some-path" 117 | content3 := `- name: local-step1 118 | template: local-step 119 | arguments: 120 | parameters: 121 | - name: message 122 | value: step-1 123 | - name: local-step2 124 | templateRef: 125 | name: common-toolkit 126 | template: versioning 127 | clusterScope: true 128 | arguments: 129 | parameters: 130 | - name: message 131 | value: step-2 132 | dependencies: 133 | - local-step1` 134 | file := &git_provider.CommitFile{ 135 | Content: &content3, 136 | Path: &path3, 137 | } 138 | fileList := []*git_provider.CommitFile{file} 139 | name := "template3" 140 | template, err := CreateDAGTemplate(fileList, name) 141 | 142 | assert.Nil(err) 143 | assert.NotNil(template) 144 | 145 | assert.Equal(name, template.Name) 146 | assert.NotNil(template.DAG) 147 | assert.Equal(2, len(template.DAG.Tasks)) 148 | 149 | assert.NotNil(template.DAG.Tasks[0]) 150 | assert.Equal("local-step1", template.DAG.Tasks[0].Name) 151 | assert.Equal("local-step", template.DAG.Tasks[0].Template) 152 | assert.Equal(1, len(template.DAG.Tasks[0].Arguments.Parameters)) 153 | assert.Equal("message", template.DAG.Tasks[0].Arguments.Parameters[0].Name) 154 | assert.Equal("step-1", template.DAG.Tasks[0].Arguments.Parameters[0].Value.String()) 155 | 156 | assert.NotNil(template.DAG.Tasks[1]) 157 | assert.Equal("local-step2", template.DAG.Tasks[1].Name) 158 | assert.Equal(1, len(template.DAG.Tasks[1].Dependencies)) 159 | assert.Equal("local-step1", template.DAG.Tasks[1].Dependencies[0]) 160 | assert.NotNil(template.DAG.Tasks[1].TemplateRef) 161 | assert.Equal("common-toolkit", template.DAG.Tasks[1].TemplateRef.Name) 162 | assert.Equal("versioning", template.DAG.Tasks[1].TemplateRef.Template) 163 | assert.True(template.DAG.Tasks[1].TemplateRef.ClusterScope) 164 | }) 165 | 166 | t.Run("Invalid configuration", func(t *testing.T) { 167 | path4 := "some-path" 168 | content4 := `- noName: local-step1 169 | wrongkey2: local-step 170 | - noName: local-step2 171 | wrongkey: something 172 | dependencies: 173 | - local-step1` 174 | file := &git_provider.CommitFile{ 175 | Content: &content4, 176 | Path: &path4, 177 | } 178 | fileList := []*git_provider.CommitFile{file} 179 | name := "template4" 180 | template, err := CreateDAGTemplate(fileList, name) 181 | 182 | assert.Nil(template) 183 | assert.NotNil(err) 184 | }) 185 | 186 | t.Run("YAML syntax error", func(t *testing.T) { 187 | path5 := "some-path" 188 | content5 := `- noName: local-step1 189 | wrongkey2: local-step 190 | error: should be list` 191 | file := &git_provider.CommitFile{ 192 | Content: &content5, 193 | Path: &path5, 194 | } 195 | fileList := []*git_provider.CommitFile{file} 196 | name := "template5" 197 | template, err := CreateDAGTemplate(fileList, name) 198 | 199 | assert.Nil(template) 200 | assert.NotNil(err) 201 | }) 202 | } 203 | 204 | func TestConvertToValidString(t *testing.T) { 205 | assert := assertion.New(t) 206 | 207 | tests := []struct { 208 | input string 209 | expected string 210 | }{ 211 | {"A@bC!-123.def", "abc-123.def"}, 212 | {"Hello World!", "helloworld"}, 213 | {"123$%^", "123"}, 214 | {"abc_123.xyz", "abc-123.xyz"}, // Underscore (_) should be converted to hyphen (-) 215 | {"..--..", "..--.."}, // Only dots (.) and hyphens (-) should remain 216 | } 217 | 218 | for _, test := range tests { 219 | t.Run(test.input, func(t *testing.T) { 220 | converted := ConvertToValidString(test.input) 221 | assert.Equal(converted, test.expected) 222 | }) 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /scripts/gitlab-setup.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: piper-setup 5 | data: 6 | piper-setup.rb: | 7 | #enable the Admin->Network->Outbound requests -> allow for webhooks 8 | ApplicationSetting.current.update!(allow_local_requests_from_web_hooks_and_services: true) 9 | 10 | #create new group 11 | Group.create!(name: "Pied Pipers", path: "pied-pipers") 12 | g = Group.find_by(name: "Pied Pipers") 13 | n = Namespace.find_by(name: "Pied Pipers") 14 | 15 | #create new user 16 | 17 | u = User.new(username: 'piper-user', email: 'piper@example.com', name: 'piper-user', password: 'Aa123456', password_confirmation: 'Aa123456') 18 | u.assign_personal_namespace(g.organization) 19 | u.skip_confirmation! # Use only if you want the user to be automatically confirmed. If you do not use this, the user receives a confirmation email. 20 | u.save! 21 | 22 | # create user token 23 | token = u.personal_access_tokens.create(scopes: [:read_api, :write_repository, :api], name: 'p-token', expires_at: 365.days.from_now) 24 | utoken = token.token 25 | token.save! 26 | 27 | #add user to group 28 | g.add_member(u, :owner) 29 | g.save! 30 | u.save! 31 | 32 | #create new project 33 | project = g.projects.create(name: "piper-e2e-test", path: "piper-e2e-test", creator:u, organization:g.organization, namespace: n, visibility_level: 20) 34 | project.create_repository 35 | project.add_member(u, :owner) 36 | project.save! 37 | g.save! 38 | 39 | #GROUP ACCESS TOKEN: 40 | # Create the group bot user. For further group access tokens, the username should be `group_{group_id}_bot_{random_string}` and email address `group_{group_id}_bot_{random_string}@noreply.{Gitlab.config.gitlab.host}`. 41 | admin = User.find(1) 42 | random_string = SecureRandom.hex(16) 43 | bot = Users::CreateService.new(admin, {name: 'g_token', username: "group_#{g.id}_bot_#{random_string}", email: "group_#{g.id}_bot_#{random_string}@noreply.gitlab.local", user_type: :project_bot }).execute 44 | bot.confirm 45 | 46 | # Add the bot to the group with the required role. 47 | g.add_member(bot, :owner) 48 | token = bot.personal_access_tokens.create(scopes:[:read_api, :write_repository, :api], name: 'g-token', expires_at: 365.days.from_now) 49 | 50 | # Get the token value. 51 | gtoken = token.token 52 | 53 | puts "GROUP_TOKEN #{gtoken} USER_TOKEN #{utoken}" 54 | 55 | -------------------------------------------------------------------------------- /scripts/init-argo-workflows.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -o errexit 3 | 4 | if [ -z "$(helm list -n workflows | grep argo-workflow)" ]; then 5 | # 7. Install argo workflows 6 | helm repo add argo https://argoproj.github.io/argo-helm 7 | helm upgrade --install argo-workflow argo/argo-workflows -n workflows --create-namespace -f workflows.values.yaml 8 | else 9 | echo "Workflows release exists, skipping installation" 10 | fi 11 | 12 | 13 | cat </dev/null || true)" != 'true' ]; then 8 | docker run \ 9 | -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \ 10 | registry:2 11 | fi 12 | 13 | # 2. Create kind cluster with containerd registry config dir enabled 14 | 15 | if [ $(kind get clusters | grep "piper") = ""]; then 16 | cat <" ]; do 83 | sleep 0.1; 84 | done; 85 | 86 | # waiting for core dns to up - to indicate that scheduling can happen 87 | echo "waiting core dns to up" 88 | until [ "`kubectl rollout status deployment --namespace kube-system | grep coredns`"=="deployment "coredns" successfully rolled out" ]; do 89 | sleep 0.1; 90 | done; 91 | 92 | kubectl wait --namespace kube-system \ 93 | --for=condition=ready pod \ 94 | --selector=k8s-app=kube-dns \ 95 | --timeout=30s 96 | -------------------------------------------------------------------------------- /scripts/init-nginx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -o errexit 3 | 4 | if [ -z "$(kubectl get pods --all-namespaces | grep ingress-nginx-controller)" ]; then 5 | # 6. Deploy of nginx ingress controller to the cluster 6 | kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml && \ 7 | kubectl wait --namespace ingress-nginx \ 8 | --for=condition=complete job/ingress-nginx-admission-create \ 9 | --timeout=180s && \ 10 | kubectl wait --namespace ingress-nginx \ 11 | --for=condition=ready pod \ 12 | --selector=app.kubernetes.io/component=controller \ 13 | --timeout=360s 14 | else 15 | echo "Nginx already exists, skipping installation" 16 | fi 17 | -------------------------------------------------------------------------------- /scripts/init-piper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -o errexit 3 | 4 | if [ -z "$(helm list | grep piper)" ]; then 5 | # 8. Install Piper 6 | helm upgrade --install piper ./helm-chart -f values.dev.yaml 7 | else 8 | echo "Piper release exists, skipping installation" 9 | fi 10 | -------------------------------------------------------------------------------- /workflows.values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | workflowNamespaces: 3 | - workflows 4 | workflowDefaults: 5 | spec: 6 | serviceAccountName: argo-workflows-sa 7 | server: 8 | baseHref: /argo/ 9 | serviceAccount: 10 | create: true 11 | extraArgs: 12 | - server 13 | - --auth-mode=server 14 | ingress: 15 | enabled: true 16 | annotations: 17 | nginx.ingress.kubernetes.io/rewrite-target: /$1 18 | nginx.ingress.kubernetes.io/backend-protocol: HTTP 19 | paths: 20 | - /argo/(.*) 21 | - /argo 22 | pathType: ImplementationSpecific --------------------------------------------------------------------------------