├── .dapper ├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ └── broken-link.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── renovate.json └── workflows │ ├── branch.yml │ ├── codeowners.yml │ ├── consuming.yml │ ├── dependent-issues.yml │ ├── linting.yml │ ├── periodic.yml │ ├── release.yml │ ├── report.yml │ ├── stale.yml │ ├── testing.yml │ └── upgrade-e2e.yml ├── .gitignore ├── .gitlint ├── .gitmodules ├── .golangci.yml ├── .grype.yaml ├── .markdownlinkcheck.json ├── .markdownlint.yml ├── .shellcheckrc ├── .shipyard.e2e.ovn.yml ├── .shipyard.e2e.yml ├── .shipyard.scale.yml ├── .tekton ├── nettest-devel-pull-request.yaml └── nettest-devel-push.yaml ├── .yamllint.yml ├── CODE-OF-CONDUCT.md ├── CODEOWNERS ├── CODEOWNERS.in ├── CONTRIBUTING.md ├── Dockerfile.dapper ├── Dockerfile.linting ├── LICENSE ├── Makefile ├── Makefile.clusters ├── Makefile.dapper ├── Makefile.images ├── Makefile.inc ├── Makefile.linting ├── Makefile.shipyard ├── Makefile.versions ├── README.md ├── gh-actions ├── cache-images │ └── action.yaml ├── commit-size │ └── action.yaml ├── e2e │ └── action.yaml ├── post-mortem │ └── action.yaml ├── release-images │ └── action.yaml ├── restore-images │ └── action.yaml └── upgrade-e2e │ └── action.yaml ├── go.mod ├── go.sum ├── package ├── Dockerfile.nettest ├── Dockerfile.shipyard-dapper-base ├── Dockerfile.shipyard-linting └── j2 ├── release-notes ├── 20220831-deploy-with-overrides.md ├── 20220911-air-gapped-kind.md ├── 20220919-load-balancer.md ├── 20230508-e2e-non-gw-nodes.md ├── force-subctl-verify ├── get-subctl ├── removed-shflags └── support-aws-ocp ├── scripts ├── nettest │ ├── metricsproxy │ ├── nc │ └── simpleserver └── shared │ ├── backport.sh │ ├── build_image.sh │ ├── check-non-release-versions.sh │ ├── cleanup.sh │ ├── cloud-prepare.sh │ ├── clusters.sh │ ├── compile.sh │ ├── deploy.sh │ ├── dnf_install │ ├── e2e.sh │ ├── entry │ ├── filter_dependabot.sh │ ├── gen-codeowners │ ├── get-subctl.sh │ ├── kubeps1.sh │ ├── lib │ ├── cleanup_acm │ ├── cleanup_kind │ ├── cleanup_ocp │ ├── clusters_acm │ ├── clusters_kind │ ├── clusters_ocp │ ├── debug_functions │ ├── deploy_bundle │ ├── deploy_funcs │ ├── deploy_helm │ ├── deploy_ocm │ ├── deploy_operator │ ├── kubecfg │ ├── ocp_utils │ ├── source_only │ └── utils │ ├── post_mortem.sh │ ├── release_images.sh │ ├── reload_images.sh │ ├── resources │ ├── .gitignore │ ├── acm-auto-import-secret.yml │ ├── acm-managed-cluster.yml │ ├── bundle │ │ ├── broker.yaml │ │ └── submariner.yaml │ ├── common │ │ ├── catalogSource.yaml │ │ ├── operatorGroup.yaml │ │ └── subscription.yaml │ ├── dummypod.yaml │ ├── kind-cluster-config.yaml │ ├── kind-cluster-dual-stack-config.yaml │ ├── kind-cluster-ipv6-stack-config.yaml │ ├── netshoot.yaml │ ├── nginx-demo.yaml │ ├── ocm │ │ ├── klusterlet.yaml │ │ ├── managedClusterAddOn.yaml │ │ ├── managedClusterSet.yaml │ │ └── submarinerConfig.yaml │ ├── plugins │ │ └── sample_hook │ ├── prometheus │ │ ├── bundle.yaml │ │ ├── clusterrole.yaml │ │ ├── clusterrolebinding.yaml │ │ ├── prometheus.yaml │ │ └── serviceaccount.yaml │ └── sm-global-cm.yaml │ ├── targets.sh │ └── unit_test.sh ├── staticcheck.conf ├── test ├── e2e │ ├── dataplane │ │ └── tcp_connectivity.go │ ├── e2e.go │ ├── e2e_test.go │ ├── framework │ │ ├── api_errors.go │ │ ├── cleanup.go │ │ ├── clusterglobalegressip.go │ │ ├── doc.go │ │ ├── docker.go │ │ ├── endpoints.go │ │ ├── exec.go │ │ ├── fips.go │ │ ├── framework.go │ │ ├── gateways.go │ │ ├── ginkgo_framework.go │ │ ├── ginkgowrapper │ │ │ └── wrapper.go │ │ ├── globalegressip.go │ │ ├── globalingressips.go │ │ ├── logging.go │ │ ├── network_pods.go │ │ ├── nodes.go │ │ ├── pods.go │ │ ├── service_exports.go │ │ ├── services.go │ │ ├── test_context.go │ │ └── util.go │ └── tcp │ │ └── connectivity.go └── scripts │ ├── cloud-prepare │ ├── no_gw_defined.yml │ ├── test.sh │ └── two_gw_defined.yml │ ├── compile │ ├── .gitignore │ ├── hello.go │ └── test.sh │ ├── post_mortem │ └── test.sh │ └── unit │ └── failing_test.go └── tools ├── go.mod ├── go.sum └── tools.go /.dapper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Pass credential file as read-only, so that the container can use it if necessary 4 | function pass_credentials() { 5 | local credentials=$1 6 | local mountpoint=${credentials/$HOME//root} 7 | mount_volume "$credentials" "$mountpoint" 8 | } 9 | 10 | function mount_volume() { 11 | local volume="$1" 12 | local mountpoint="$2" 13 | local suffix="$suffix" 14 | 15 | [ -z "${suffix}" ] && suffix=':ro' || suffix+=',ro' 16 | [ -r "$volume" ] || return 0 17 | dockerargs+=" -v $(realpath -s ${volume}):${mountpoint}${suffix}" 18 | } 19 | 20 | file=Dockerfile.dapper 21 | socket=false 22 | dockerargs= 23 | 24 | while true 25 | do 26 | case "$1" in 27 | --file|-f) 28 | file="$2" 29 | shift 2 30 | ;; 31 | --socket|-k) 32 | socket=true 33 | shift 34 | ;; 35 | --directory|-C) 36 | cd "$2" || exit 37 | shift 2 38 | ;; 39 | --shell|-s) 40 | command=bash 41 | DAPPER_ENV="${DAPPER_ENV} TERM" 42 | shift 43 | ;; 44 | --debug|-d) 45 | shift 46 | set -x 47 | ;; 48 | --version|-v) 49 | echo Shipyard Dapper 50 | exit 0 51 | ;; 52 | --mount-suffix|-S) 53 | suffix=":$2" 54 | shift 2 55 | ;; 56 | --) 57 | shift 58 | break 59 | ;; 60 | -*) 61 | echo "$0 doesn't support $1" >&2 62 | exit 1 63 | ;; 64 | *) 65 | break 66 | ;; 67 | esac 68 | done 69 | 70 | [ -n "$command" ] && set -- "$command" 71 | 72 | buildargs=(--build-arg "ORG=${ORG}" --build-arg "PROJECT=${PROJECT}") 73 | [ -n "${SHIPYARD_REPO}" ] && buildargs+=(--build-arg "SHIPYARD_REPO=${SHIPYARD_REPO}") 74 | [ -n "${SHIPYARD_TAG}" ] && buildargs+=(--build-arg "SHIPYARD_TAG=${SHIPYARD_TAG}") 75 | gitid="$(git symbolic-ref --short HEAD 2>/dev/null | tr / _ || :)" 76 | gitid="${gitid:-$(git show --format=%h -s)}" 77 | container="$(basename "$(pwd)"):${gitid}" 78 | docker build -t "${container}" -f "${file}" "${buildargs[@]}" . 79 | 80 | extract_var() { 81 | docker inspect "$1" | grep "$2" | sed -E "s/.*\"$2=(.*)\",?/\1/;q" 82 | } 83 | 84 | DAPPER_CP="$(extract_var "${container}" DAPPER_CP)" 85 | [ -z "${DAPPER_CP}" ] && DAPPER_CP="$(pwd)" 86 | DAPPER_ENV="${DAPPER_ENV} $(extract_var "${container}" DAPPER_ENV)" 87 | DAPPER_SOURCE="$(extract_var "${container}" DAPPER_SOURCE)" 88 | [ -z "${DAPPER_SOURCE}" ] && DAPPER_SOURCE="/source/" 89 | DAPPER_DOCKER_SOCKET="$(extract_var "${container}" DAPPER_DOCKER_SOCKET)" 90 | DAPPER_RUN_ARGS="$(extract_var "${container}" DAPPER_RUN_ARGS)" 91 | 92 | echo Extracted parameters: 93 | printf 'DAPPER_CP="%s"\n' "$DAPPER_CP" 94 | printf 'DAPPER_ENV="%s"\n' "$DAPPER_ENV" 95 | printf 'DAPPER_SOURCE="%s"\n' "$DAPPER_SOURCE" 96 | printf 'DAPPER_DOCKER_SOCKET="%s"\n' "$DAPPER_DOCKER_SOCKET" 97 | printf 'DAPPER_RUN_ARGS="%s"\n' "$DAPPER_RUN_ARGS" 98 | 99 | if [ "${socket}" = true ] || [ "${DAPPER_DOCKER_SOCKET}" = true ] 100 | then 101 | if [ -S /var/run/docker.sock ]; then 102 | # Docker 103 | dockerargs="${dockerargs} -v /var/run/docker.sock:/var/run/docker.sock${suffix}" 104 | else 105 | # Assume rootless Podman 106 | dockerargs="${dockerargs} -v /run/user/$(id -u)/podman/podman.sock:/var/run/docker.sock${suffix}" 107 | fi 108 | fi 109 | 110 | [ -t 1 ] && dockerargs="${dockerargs} -t" 111 | 112 | DAPPER_UID=$(id -u) 113 | DAPPER_GID=$(id -g) 114 | # If docker is provided by Podman, assume rootless and tell the container to use root internally 115 | if docker 2>&1 | grep -q podman; then 116 | DAPPER_UID=0 117 | DAPPER_GID=0 118 | fi 119 | 120 | # Pass through ~/.docker so that the container can get any authentication tokens in ~/.docker/config.json 121 | # We can't mount config.json specifically because "docker login" attempts to rename it, which fails 122 | if [ -d "${HOME}/.docker" ]; then 123 | dockerargs="${dockerargs} -v ${HOME}/.docker:/root/.docker${suffix}" 124 | fi 125 | 126 | # Pass through ~/.aws/credentials so that the container can operate on AWS (if needed). 127 | pass_credentials "${HOME}/.aws/credentials" 128 | 129 | # Pass through ~/.gcp/osServiceAccount.json so that the container can operate on GCP (if needed). 130 | pass_credentials "${HOME}/.gcp/osServiceAccount.json" 131 | 132 | # Attempt to mount any local replaces specified in `go.mod`. 133 | # We can't rely on having the right version (or any version) of go, so parse go.mod manually. 134 | while read -a replace; do 135 | mount_root="" 136 | [[ "${replace[1]}" =~ ^\. ]] && mount_root="$DAPPER_SOURCE/" 137 | mount_volume "${replace[1]}" "${mount_root}${replace[1]}" 138 | done < <(grep -o -E '=>[ ]*(\.\.?)?/[^ ]+' go.mod) 139 | 140 | docker run -i --rm \ 141 | $(printf -- " -e %s" $DAPPER_ENV) -e "DAPPER_UID=$DAPPER_UID" -e "DAPPER_GID=$DAPPER_GID" \ 142 | -v "${DAPPER_CP}:${DAPPER_SOURCE}${suffix}" \ 143 | ${dockerargs} \ 144 | ${DAPPER_RUN_ARGS} \ 145 | "${container}" "$@" 146 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.git 2 | **/vendor 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/broken-link.md: -------------------------------------------------------------------------------- 1 | Periodic link aliveness CI detected a broken link. Please see the [periodic job 2 | results](https://github.com/submariner-io/shipyard/actions?query=workflow%3APeriodic) for details. 3 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 14 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | updates: 4 | - package-ecosystem: github-actions 5 | directory: '/' 6 | schedule: 7 | interval: monthly 8 | groups: 9 | github-actions: 10 | patterns: 11 | - "*" 12 | - package-ecosystem: github-actions 13 | directory: '/' 14 | target-branch: "release-0.16" 15 | schedule: 16 | interval: monthly 17 | groups: 18 | github-actions: 19 | patterns: 20 | - "*" 21 | - package-ecosystem: github-actions 22 | directory: '/' 23 | target-branch: "release-0.17" 24 | schedule: 25 | interval: monthly 26 | groups: 27 | github-actions: 28 | patterns: 29 | - "*" 30 | - package-ecosystem: github-actions 31 | directory: '/' 32 | target-branch: "release-0.18" 33 | schedule: 34 | interval: monthly 35 | groups: 36 | github-actions: 37 | patterns: 38 | - "*" 39 | - package-ecosystem: github-actions 40 | directory: '/' 41 | target-branch: "release-0.19" 42 | schedule: 43 | interval: monthly 44 | groups: 45 | github-actions: 46 | patterns: 47 | - "*" 48 | - package-ecosystem: github-actions 49 | directory: '/' 50 | target-branch: "release-0.20" 51 | schedule: 52 | interval: monthly 53 | groups: 54 | github-actions: 55 | patterns: 56 | - "*" 57 | - package-ecosystem: gomod 58 | target-branch: "release-0.16" 59 | directory: "/" 60 | schedule: 61 | interval: weekly 62 | groups: 63 | gomod: 64 | patterns: 65 | - "*" 66 | allow: 67 | # Pick up k8s.io updates 68 | - dependency-name: k8s.io/client-go 69 | ignore: 70 | # 0.16 tracks the 0.27 branch 71 | - dependency-name: k8s.io/* 72 | versions: ">= 0.28.0-alpha.0" 73 | - package-ecosystem: gomod 74 | target-branch: "release-0.17" 75 | directory: "/" 76 | schedule: 77 | interval: weekly 78 | groups: 79 | gomod: 80 | patterns: 81 | - "*" 82 | allow: 83 | # Pick up k8s.io updates 84 | - dependency-name: k8s.io/client-go 85 | ignore: 86 | # 0.17 tracks the 0.29 branch 87 | - dependency-name: k8s.io/* 88 | versions: ">= 0.30.0-alpha.0" 89 | - package-ecosystem: gomod 90 | target-branch: "release-0.18" 91 | directory: "/" 92 | schedule: 93 | interval: weekly 94 | groups: 95 | gomod: 96 | patterns: 97 | - "*" 98 | allow: 99 | # Pick up k8s.io updates 100 | - dependency-name: k8s.io/client-go 101 | ignore: 102 | # 0.18 tracks the 0.30 branch 103 | - dependency-name: k8s.io/* 104 | versions: ">= 0.31.0-alpha.0" 105 | - package-ecosystem: gomod 106 | target-branch: "release-0.19" 107 | directory: "/" 108 | schedule: 109 | interval: weekly 110 | groups: 111 | gomod: 112 | patterns: 113 | - "*" 114 | allow: 115 | # Pick up k8s.io updates 116 | - dependency-name: k8s.io/client-go 117 | ignore: 118 | # 0.19 tracks the 0.31 branch 119 | - dependency-name: k8s.io/* 120 | versions: ">= 0.32.0-alpha.0" 121 | - package-ecosystem: gomod 122 | target-branch: "release-0.20" 123 | directory: "/" 124 | schedule: 125 | interval: weekly 126 | groups: 127 | gomod: 128 | patterns: 129 | - "*" 130 | allow: 131 | # Pick up k8s.io updates 132 | - dependency-name: k8s.io/client-go 133 | ignore: 134 | # 0.20 tracks the 0.32 branch 135 | - dependency-name: k8s.io/* 136 | versions: ">= 0.33.0-alpha.0" 137 | - package-ecosystem: gomod 138 | directory: "/" 139 | schedule: 140 | interval: weekly 141 | groups: 142 | gomod: 143 | patterns: 144 | - "*" 145 | ignore: 146 | # These are included by k8s.io/client-go 147 | - dependency-name: k8s.io/api 148 | - dependency-name: k8s.io/apimachinery 149 | - package-ecosystem: gomod 150 | directory: "/tools" 151 | schedule: 152 | interval: weekly 153 | groups: 154 | tools: 155 | patterns: 156 | - "*" 157 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": ["config:base"], 4 | "enabledManagers": ["tekton"] 5 | } 6 | -------------------------------------------------------------------------------- /.github/workflows/branch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Branch Checks 3 | 4 | on: 5 | pull_request: 6 | 7 | permissions: {} 8 | 9 | jobs: 10 | target_branch: 11 | name: PR targets branch 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Check that the PR targets devel 15 | if: ${{ github.base_ref != 'devel' }} 16 | run: exit 1 17 | -------------------------------------------------------------------------------- /.github/workflows/codeowners.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: CODEOWNERS 3 | 4 | on: 5 | pull_request: 6 | paths: 7 | - 'CODEOWNERS' 8 | - 'CODEOWNERS.in' 9 | 10 | permissions: {} 11 | 12 | jobs: 13 | updated: 14 | name: Up-to-date 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Check out the repository 18 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 19 | - name: Delete current CODEOWNERS file 20 | run: rm CODEOWNERS 21 | - name: Run gen-codeowners to rebuild CODEOWNERS file 22 | run: make CODEOWNERS 23 | - name: Validate new CODEOWNERS file is the same as tracked by Git 24 | run: git diff --exit-code 25 | -------------------------------------------------------------------------------- /.github/workflows/dependent-issues.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: PR Dependencies 3 | 4 | permissions: 5 | issues: write 6 | pull-requests: write 7 | statuses: write 8 | 9 | on: 10 | issues: 11 | types: 12 | - opened 13 | - edited 14 | - closed 15 | - reopened 16 | - synchronize 17 | pull_request_target: 18 | types: 19 | - opened 20 | - edited 21 | - closed 22 | - reopened 23 | - synchronize 24 | schedule: 25 | - cron: '0 0/6 * * *' # every 6 hours 26 | 27 | jobs: 28 | check: 29 | name: Check Dependencies 30 | if: github.repository_owner == 'submariner-io' 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: z0al/dependent-issues@950226e7ca8fc43dc209a7febf67c655af3bdb43 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | with: 37 | # The label to use to mark dependent issues 38 | label: dependent 39 | 40 | # Enable checking for dependencies in issues. 41 | check_issues: on 42 | 43 | # A comma-separated list of keywords to mark dependency. 44 | keywords: depends on, Depends on 45 | -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Linting 3 | 4 | on: 5 | pull_request: 6 | 7 | permissions: {} 8 | 9 | jobs: 10 | apply-suggestions-commits: 11 | name: 'No "Apply suggestions from code review" Commits' 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Get PR commits 15 | id: 'get-pr-commits' 16 | uses: tim-actions/get-pr-commits@198af03565609bb4ed924d1260247b4881f09e7d 17 | with: 18 | token: ${{ secrets.GITHUB_TOKEN }} 19 | 20 | - name: 'Verify no "Apply suggestions from code review" commits' 21 | uses: tim-actions/commit-message-checker-with-regex@094fc16ff83d04e2ec73edb5eaf6aa267db33791 22 | with: 23 | commits: ${{ steps.get-pr-commits.outputs.commits }} 24 | pattern: '^(?!.*(apply suggestions from code review))' 25 | flags: 'i' 26 | error: 'Commits addressing code review feedback should typically be squashed into the commits under review' 27 | 28 | - name: 'Verify no "fixup!" commits' 29 | uses: tim-actions/commit-message-checker-with-regex@094fc16ff83d04e2ec73edb5eaf6aa267db33791 30 | with: 31 | commits: ${{ steps.get-pr-commits.outputs.commits }} 32 | pattern: '^(?!fixup!)' 33 | flags: 'i' 34 | error: 'Fixup commits should be squashed into the commits under review' 35 | 36 | commit-size: 37 | name: Commit Size 38 | runs-on: ubuntu-latest 39 | steps: 40 | - name: Check out the repository 41 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 42 | - name: Make sure each commit in the PR is within reviewable size 43 | uses: ./gh-actions/commit-size 44 | with: 45 | size: 250 46 | 47 | gitlint: 48 | name: Commit Message(s) 49 | runs-on: ubuntu-latest 50 | steps: 51 | - name: Check out the repository 52 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 53 | with: 54 | fetch-depth: 0 55 | - name: Run gitlint 56 | run: make gitlint 57 | 58 | golangci-lint: 59 | name: Go 60 | runs-on: ubuntu-latest 61 | steps: 62 | - name: Check out the repository 63 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 64 | - name: Run golangci-lint 65 | run: make golangci-lint 66 | 67 | markdown-link-check: 68 | name: Markdown Links (modified files) 69 | runs-on: ubuntu-latest 70 | steps: 71 | - name: Check out the repository 72 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 73 | 74 | - name: Run markdown-link-check 75 | uses: gaurav-nelson/github-action-markdown-link-check@3c3b66f1f7d0900e37b71eca45b63ea9eedfce31 76 | with: 77 | config-file: ".markdownlinkcheck.json" 78 | check-modified-files-only: "yes" 79 | base-branch: ${{ github.base_ref }} 80 | 81 | markdownlint: 82 | name: Markdown 83 | runs-on: ubuntu-latest 84 | steps: 85 | - name: Check out the repository 86 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 87 | - name: Run markdownlint 88 | run: make markdownlint 89 | 90 | packagedoc-lint: 91 | name: Package Documentation 92 | runs-on: ubuntu-latest 93 | steps: 94 | - name: Check out the repository 95 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 96 | - name: Run packagedoc-lint 97 | run: make packagedoc-lint 98 | 99 | shellcheck: 100 | name: Shell 101 | runs-on: ubuntu-latest 102 | steps: 103 | - name: Check out the repository 104 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 105 | - name: Run shellcheck 106 | run: make shellcheck 107 | 108 | vulnerability-scan: 109 | name: Vulnerability Scanning 110 | runs-on: ubuntu-latest 111 | steps: 112 | - name: Check out the repository 113 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 114 | - name: Run Anchore vulnerability scanner 115 | uses: anchore/scan-action@2c901ab7378897c01b8efaa2d0c9bf519cc64b9e 116 | id: scan 117 | with: 118 | path: "." 119 | fail-build: true 120 | severity-cutoff: high 121 | - name: Show Anchore scan SARIF report 122 | if: always() 123 | run: cat ${{ steps.scan.outputs.sarif }} 124 | - name: Upload Anchore scan SARIF report 125 | if: always() 126 | uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f 127 | with: 128 | sarif_file: ${{ steps.scan.outputs.sarif }} 129 | 130 | yaml-lint: 131 | name: YAML 132 | runs-on: ubuntu-latest 133 | steps: 134 | - name: Check out the repository 135 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 136 | - name: Run yamllint 137 | run: make yamllint 138 | -------------------------------------------------------------------------------- /.github/workflows/periodic.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Periodic 3 | 4 | on: 5 | schedule: 6 | - cron: "0 0 * * 0" 7 | 8 | permissions: {} 9 | 10 | jobs: 11 | markdown-link-check-periodic: 12 | name: Markdown Links (all files) 13 | if: github.repository_owner == 'submariner-io' 14 | runs-on: ubuntu-latest 15 | permissions: 16 | issues: write 17 | steps: 18 | - name: Check out the repository 19 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 20 | 21 | - name: Run markdown-link-check 22 | uses: gaurav-nelson/github-action-markdown-link-check@3c3b66f1f7d0900e37b71eca45b63ea9eedfce31 23 | with: 24 | config-file: ".markdownlinkcheck.json" 25 | 26 | - name: Raise an Issue to report broken links 27 | if: ${{ failure() }} 28 | uses: peter-evans/create-issue-from-file@e8ef132d6df98ed982188e460ebb3b5d4ef3a9cd 29 | with: 30 | title: Broken link detected by CI 31 | content-filepath: .github/ISSUE_TEMPLATE/broken-link.md 32 | labels: automated, broken link 33 | 34 | go-mod-outdated-periodic: 35 | name: Outdated Dependencies 36 | if: github.repository_owner == 'submariner-io' 37 | runs-on: ubuntu-latest 38 | strategy: 39 | fail-fast: false 40 | matrix: 41 | project: [ 42 | 'admiral', 'cloud-prepare', 'lighthouse', 'shipyard', 43 | 'subctl', 'submariner-bot', 'submariner', 'submariner-operator' 44 | ] 45 | steps: 46 | - name: Check out the Shipyard repository 47 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 48 | 49 | - name: Build go-mod-outdated 50 | run: cd tools && go build -o ../bin/go-mod-outdated github.com/psampaz/go-mod-outdated 51 | 52 | - name: Check out the ${{ matrix.project }} repository 53 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 54 | with: 55 | repository: submariner-io/${{ matrix.project }} 56 | path: ${{ matrix.project }} 57 | 58 | - name: Check for updates 59 | run: (cd ${{ matrix.project }}; go list -u -m -json all) | bin/go-mod-outdated -direct -update 60 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Release Images 3 | 4 | on: 5 | workflow_dispatch: 6 | push: 7 | branches: 8 | - devel 9 | - release-* 10 | 11 | permissions: {} 12 | 13 | jobs: 14 | release: 15 | name: Release Images 16 | if: github.repository_owner == 'submariner-io' 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Check out the repository 20 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 21 | with: 22 | submodules: true 23 | 24 | - name: Build and release new images 25 | uses: ./gh-actions/release-images 26 | with: 27 | username: ${{ secrets.QUAY_USERNAME }} 28 | password: ${{ secrets.QUAY_PASSWORD }} 29 | -------------------------------------------------------------------------------- /.github/workflows/report.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Reporting 3 | 4 | on: 5 | push: 6 | branches: 7 | - devel 8 | - release-* 9 | 10 | permissions: {} 11 | 12 | jobs: 13 | vulnerability-scan: 14 | name: Vulnerability Scanning 15 | if: github.repository_owner == 'submariner-io' 16 | runs-on: ubuntu-latest 17 | permissions: 18 | security-events: write 19 | steps: 20 | - name: Check out the repository 21 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 22 | - name: Run Anchore vulnerability scanner 23 | uses: anchore/scan-action@2c901ab7378897c01b8efaa2d0c9bf519cc64b9e 24 | id: scan 25 | with: 26 | path: "." 27 | fail-build: false 28 | - name: Show Anchore scan SARIF report 29 | run: cat ${{ steps.scan.outputs.sarif }} 30 | - name: Upload Anchore scan SARIF report 31 | uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f 32 | with: 33 | sarif_file: ${{ steps.scan.outputs.sarif }} 34 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Stale 3 | 4 | on: 5 | schedule: 6 | - cron: "0 0 * * *" 7 | 8 | permissions: {} 9 | 10 | jobs: 11 | stale: 12 | name: Close Stale Issues and PRs 13 | if: github.repository_owner == 'submariner-io' 14 | runs-on: ubuntu-latest 15 | permissions: 16 | issues: write 17 | pull-requests: write 18 | steps: 19 | - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 20 | with: 21 | days-before-issue-stale: 120 22 | days-before-pr-stale: 14 23 | exempt-issue-labels: 'confirmed,security' 24 | exempt-pr-labels: 'confirmed,security' 25 | stale-issue-label: 'stale' 26 | stale-issue-message: | 27 | This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further 28 | activity occurs. Thank you for your contributions. 29 | stale-pr-label: 'stale' 30 | stale-pr-message: | 31 | This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further 32 | activity occurs. Thank you for your contributions. 33 | -------------------------------------------------------------------------------- /.github/workflows/upgrade-e2e.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Upgrade 3 | 4 | on: 5 | pull_request: 6 | branches: [devel] 7 | 8 | permissions: {} 9 | 10 | jobs: 11 | upgrade-e2e: 12 | name: Latest Release to Latest Version 13 | timeout-minutes: 30 14 | runs-on: ubuntu-latest 15 | strategy: 16 | fail-fast: false 17 | steps: 18 | - name: Check out the repository 19 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 20 | 21 | - name: Install an old cluster, upgrade it and check it 22 | uses: ./gh-actions/upgrade-e2e 23 | 24 | - name: Post Mortem 25 | if: failure() 26 | uses: ./gh-actions/post-mortem 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor 2 | .idea 3 | /bin 4 | /package/.image.* 5 | output 6 | dist 7 | junit.xml 8 | *.out 9 | *.coverprofile 10 | *~ 11 | /ovn-kubernetes 12 | -------------------------------------------------------------------------------- /.gitlint: -------------------------------------------------------------------------------- 1 | [general] 2 | # body-is-missing: Allow commit messages with only a title 3 | # body-min-length: Allow short body lines, like "Relates-to: #issue" 4 | ignore=body-is-missing,body-min-length 5 | # Our ignore-by-body regex is correct for re.search(). Required for suppressing a warning. See: 6 | # https://jorisroovers.com/gitlint/latest/configuration/general_options/#regex-style-search 7 | regex-style-search=true 8 | 9 | [ignore-by-body] 10 | # Dependabot doesn't follow our conventions, unfortunately 11 | regex=^Signed-off-by: dependabot\[bot\](.*) 12 | ignore=all 13 | 14 | [ignore-by-author-name] 15 | # Konflux doesn't follow our conventions, unfortunately 16 | regex=red-hat-konflux 17 | ignore=all 18 | 19 | [ignore-body-lines] 20 | # Allow long URLs in commit messages 21 | regex=^https?://[^ ]*$ 22 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/submariner-io/shipyard/97c0a8c94e2c83bc95bfa74b25fb8084d458e66e/.gitmodules -------------------------------------------------------------------------------- /.grype.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ignore: 3 | # False positive, CVE is actually about the C++ project protocolbuffers/protobuf 4 | # https://github.com/anchore/grype/issues/558 5 | - vulnerability: CVE-2015-5237 6 | package: 7 | name: google.golang.org/protobuf 8 | # False positive, CVE is actually about the C++ project protocolbuffers/protobuf 9 | # https://github.com/anchore/grype/issues/633 10 | - vulnerability: CVE-2021-22570 11 | package: 12 | name: google.golang.org/protobuf 13 | -------------------------------------------------------------------------------- /.markdownlinkcheck.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignorePatterns": [ 3 | { 4 | "pattern": "^https://github.com/\\S+/\\S+/(issues|pull)/[0-9]+" 5 | }, 6 | { 7 | "pattern": "^http://localhost:" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /.markdownlint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | line-length: 3 | line_length: 140 4 | -------------------------------------------------------------------------------- /.shellcheckrc: -------------------------------------------------------------------------------- 1 | # SC2154 is excluded to avoid false positives based on our use of global variables 2 | disable=SC2154 3 | -------------------------------------------------------------------------------- /.shipyard.e2e.ovn.yml: -------------------------------------------------------------------------------- 1 | --- 2 | cni: ovn 3 | submariner: true 4 | nodes: control-plane worker 5 | clusters: 6 | cluster1: 7 | cluster2: 8 | -------------------------------------------------------------------------------- /.shipyard.e2e.yml: -------------------------------------------------------------------------------- 1 | --- 2 | submariner: true 3 | nodes: control-plane worker 4 | clusters: 5 | cluster1: 6 | cluster2: 7 | -------------------------------------------------------------------------------- /.shipyard.scale.yml: -------------------------------------------------------------------------------- 1 | --- 2 | submariner: true 3 | nodes: control-plane 4 | cluster-count: 10 5 | -------------------------------------------------------------------------------- /.yamllint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | line-length: 6 | max: 140 7 | # Allow standard GHA syntax for "on: *" 8 | truthy: 9 | ignore: '.github/workflows/*.yml' 10 | 11 | ignore: | 12 | /vendor 13 | /scripts/shared/resources/cluster*-config.yaml 14 | /scripts/shared/resources/prometheus/bundle.yaml 15 | /.tekton 16 | -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Please see the [Code of Conduct docs on Submariner's website](https://submariner.io/community/code-of-conduct/). 4 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Auto-generated, do not edit; see CODEOWNERS.in 2 | * @Oats87 @skitt @sridhargaddam @tpantelis @vthapar 3 | *.md @dfarrell07 @Oats87 @skitt @sridhargaddam @tpantelis @vthapar 4 | /.github/workflows/ @mkolesnik @Oats87 @skitt @sridhargaddam @tpantelis @vthapar 5 | /scripts/ @mkolesnik @Oats87 @skitt @sridhargaddam @tpantelis @vthapar 6 | Dockerfile.dapper @mkolesnik @Oats87 @skitt @sridhargaddam @tpantelis @vthapar 7 | Makefile* @mkolesnik @Oats87 @skitt @sridhargaddam @tpantelis @vthapar 8 | package/ @mkolesnik @Oats87 @skitt @sridhargaddam @tpantelis @vthapar 9 | -------------------------------------------------------------------------------- /CODEOWNERS.in: -------------------------------------------------------------------------------- 1 | @dfarrell07 *.md 2 | @mkolesnik /.github/workflows/ /scripts/ Makefile* Dockerfile.dapper package/ 3 | @Oats87 * 4 | @skitt * 5 | @sridhargaddam * 6 | @tpantelis * 7 | @vthapar * 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Please see the [Development docs on Submariner's website](https://submariner.io/development/). 4 | -------------------------------------------------------------------------------- /Dockerfile.dapper: -------------------------------------------------------------------------------- 1 | ARG SHIPYARD_REPO=quay.io/submariner 2 | ARG SHIPYARD_TAG=devel 3 | FROM ${SHIPYARD_REPO}/shipyard-dapper-base:${SHIPYARD_TAG} 4 | 5 | ARG ORG 6 | ARG PROJECT 7 | ENV DAPPER_ENV="CI QUAY_USERNAME QUAY_PASSWORD MAKEFLAGS PLUGIN TEST_ARGS E2E_TESTDIR GITHUB_BASE_REF GITHUB_USER GITHUB_TOKEN USING" \ 8 | DAPPER_SOURCE=/go/src/github.com/${ORG}/${PROJECT} DAPPER_DOCKER_SOCKET=true \ 9 | DAPPER_RUN_ARGS="${DAPPER_RUN_ARGS} --sysctl net.ipv6.conf.all.forwarding=1" 10 | ENV OVN_DIR=${DAPPER_SOURCE}/ovn-kubernetes 11 | ENV DAPPER_OUTPUT=${DAPPER_SOURCE}/output 12 | 13 | WORKDIR ${DAPPER_SOURCE} 14 | 15 | RUN git config --global --add safe.directory ${DAPPER_SOURCE} 16 | RUN git config --global --add safe.directory ${OVN_DIR} 17 | 18 | ENTRYPOINT ["/opt/shipyard/scripts/entry"] 19 | CMD ["sh"] 20 | -------------------------------------------------------------------------------- /Dockerfile.linting: -------------------------------------------------------------------------------- 1 | ARG SHIPYARD_REPO=quay.io/submariner 2 | ARG SHIPYARD_TAG=devel 3 | FROM ${SHIPYARD_REPO}/shipyard-linting:${SHIPYARD_TAG} 4 | 5 | ENV DAPPER_ENV="CI GITHUB_SHA MAKEFLAGS" \ 6 | DAPPER_SOURCE=/opt/linting 7 | ENV DAPPER_OUTPUT=${DAPPER_SOURCE}/output 8 | 9 | WORKDIR ${DAPPER_SOURCE} 10 | 11 | RUN git config --global --add safe.directory ${DAPPER_SOURCE} 12 | 13 | ENTRYPOINT ["/opt/shipyard/scripts/entry"] 14 | CMD ["sh"] 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BASE_BRANCH ?= devel 2 | OCM_BASE_BRANCH ?= main 3 | IMAGES ?= shipyard-dapper-base shipyard-linting nettest 4 | LOCAL_COMPONENTS := submariner-metrics-proxy 5 | MULTIARCH_IMAGES ?= $(IMAGES) 6 | EXTRA_PRELOAD_IMAGES := $(PRELOAD_IMAGES) 7 | PLATFORMS ?= linux/amd64,linux/arm64 8 | NON_DAPPER_GOALS += images multiarch-images 9 | PLUGIN ?= 10 | 11 | export LOCAL_COMPONENTS 12 | 13 | export BASE_BRANCH OCM_BASE_BRANCH 14 | 15 | ifneq (,$(DAPPER_HOST_ARCH)) 16 | 17 | # Running in Dapper 18 | 19 | ifneq (,$(filter ovn%,$(USING))) 20 | SETTINGS ?= $(DAPPER_SOURCE)/.shipyard.e2e.ovn.yml 21 | else 22 | SETTINGS ?= $(DAPPER_SOURCE)/.shipyard.e2e.yml 23 | endif 24 | 25 | ifneq (,$(filter ovn-ic,$(USING))) 26 | export OVN_IC = true 27 | endif 28 | 29 | export LAZY_DEPLOY = false 30 | 31 | scale: SETTINGS = $(DAPPER_SOURCE)/.shipyard.scale.yml 32 | 33 | include Makefile.inc 34 | 35 | # In Shipyard we don't need to preload the dapper images, so override the default behavior. 36 | ifneq ($(AIR_GAPPED),true) 37 | override PRELOAD_IMAGES=nettest $(EXTRA_PRELOAD_IMAGES) 38 | endif 39 | 40 | # Prevent rebuilding images inside dapper since they're already built outside it in Shipyard's case 41 | package/.image.shipyard-dapper-base: ; 42 | 43 | # Project-specific targets go here 44 | deploy: package/.image.nettest 45 | 46 | e2e: clusters package/.image.nettest 47 | 48 | else 49 | 50 | # Not running in Dapper 51 | 52 | export SCRIPTS_DIR=./scripts/shared 53 | 54 | include Makefile.images 55 | include Makefile.versions 56 | 57 | # Shipyard-specific starts 58 | # We need to ensure images, including the Shipyard base image, are updated 59 | # before we start Dapper 60 | clean-clusters cleanup cloud-prepare clusters deploy deploy-latest e2e golangci-lint post-mortem packagedoc-lint print-version scale unit upgrade-e2e: package/.image.shipyard-dapper-base 61 | deploy deploy-latest e2e upgrade-e2e: package/.image.nettest 62 | 63 | .DEFAULT_GOAL := lint 64 | # Shipyard-specific ends 65 | 66 | include Makefile.dapper 67 | 68 | # Make sure linting goals have up-to-date linting image 69 | $(LINTING_GOALS): package/.image.shipyard-linting 70 | 71 | scale: scale-prereqs 72 | scale-prereqs: 73 | @echo "We need to change some system parameters in order to continue, these may have a lasting effect on your system." 74 | @read -p "Do you wish to continue [y/N]?" response; \ 75 | [[ "$${response,,}" =~ ^(yes|y)$$ ]] || exit 1 76 | # Increase general limits interfering with running multiple KIND clusters 77 | sudo sysctl -w fs.inotify.max_user_watches=1073741824 78 | sudo sysctl -w fs.inotify.max_user_instances=524288 79 | sudo sysctl -w kernel.pty.max=524288 80 | # Lower swappiness to avoid swapping unnecessarily, which would hurt the performance 81 | sudo sysctl -w vm.swappiness=5 82 | # Increase GC thresholds for IPv4 stack, otherwise we'll hit ARP table overflows 83 | sudo sysctl -w net.ipv4.neigh.default.gc_thresh1=2048 84 | sudo sysctl -w net.ipv4.neigh.default.gc_thresh2=4096 85 | sudo sysctl -w net.ipv4.neigh.default.gc_thresh3=8192 86 | # Increase open files limit. TODO: Find a way to make this change transient and not persistent 87 | echo "* - nofile 100000000" | sudo tee /etc/security/limits.d/shipyard.scale.conf 88 | 89 | script-test: .dapper images 90 | -docker network create -d bridge kind 91 | $(RUN_IN_DAPPER) $(SCRIPT_TEST_ARGS) 92 | 93 | .PHONY: script-test 94 | 95 | endif 96 | -------------------------------------------------------------------------------- /Makefile.clusters: -------------------------------------------------------------------------------- 1 | ### VARIABLES ### 2 | 3 | , := , 4 | USING = $(subst $(,), ,$(using)) 5 | _using = ${USING} 6 | 7 | # General make flags/variables 8 | PARALLEL ?= true 9 | TIMEOUT ?= 5m 10 | export PARALLEL SETTINGS TIMEOUT 11 | 12 | # Flags affecting `make clusters` 13 | K8S_VERSION ?= 1.33 14 | METALLB_VERSION ?= 0.13.5 15 | OLM_VERSION ?= v0.18.3 16 | PROVIDER ?= kind 17 | export AIR_GAPPED DUAL_STACK IPV6_STACK K8S_VERSION LOAD_BALANCER METALLB_VERSION NFTABLES OLM OLM_VERSION OVERLAPPING PROMETHEUS PROVIDER 18 | 19 | ### PROCESSING `using=` ### 20 | 21 | ifneq (,$(filter ocp,$(_using))) 22 | PROVIDER = ocp 23 | endif 24 | 25 | ifneq (,$(filter acm,$(_using))) 26 | PROVIDER = acm 27 | endif 28 | 29 | ifneq (,$(filter load-balancer,$(_using))) 30 | LOAD_BALANCER = true 31 | endif 32 | 33 | ifneq (,$(filter air-gap,$(_using))) 34 | AIR_GAPPED = true 35 | endif 36 | 37 | ifneq (,$(filter dual-stack,$(_using))) 38 | DUAL_STACK = true 39 | endif 40 | 41 | ifneq (,$(filter ipv6-stack,$(_using))) 42 | IPV6_STACK = true 43 | endif 44 | 45 | ifneq (,$(filter overlapping,$(_using))) 46 | OVERLAPPING = true 47 | endif 48 | 49 | ifneq (,$(filter prometheus,$(_using))) 50 | PROMETHEUS = true 51 | endif 52 | 53 | ifneq (,$(filter nftables,$(_using))) 54 | NFTABLES = true 55 | endif 56 | 57 | ### TARGETS ### 58 | 59 | # [clean-clusters] removes running clusters 60 | clean-clusters: 61 | $(SCRIPTS_DIR)/cleanup.sh 62 | cleanup: clean-clusters 63 | 64 | # [clusters] creates KIND clusters that can then be used to deploy Submariner 65 | clusters: 66 | $(SCRIPTS_DIR)/$@.sh 67 | 68 | .PHONY: clean-clusters cleanup clusters 69 | -------------------------------------------------------------------------------- /Makefile.dapper: -------------------------------------------------------------------------------- 1 | # This Makefile contains the rules required to set up our 2 | # Dapper-based build environment 3 | 4 | ORG ?= submariner-io 5 | PROJECT ?= $(notdir $(CURDIR)) 6 | SHIPYARD_TAG ?= $(BASE_BRANCH) 7 | SHIPYARD_URL ?= https://raw.githubusercontent.com/submariner-io/shipyard/$(BASE_BRANCH) 8 | export ORG PROJECT SHIPYARD_TAG SHIPYARD_URL 9 | 10 | Makefile.shipyard: 11 | @echo Downloading $@ 12 | @curl -sfLO $(SHIPYARD_URL)/$@ 13 | 14 | include Makefile.shipyard 15 | 16 | # [prune-images] removes all Submariner-provided images and all untagged images 17 | # Use this to ensure you use current images 18 | # Copied from Makefile to provide this everywhere (until we can share 19 | # non-Dapper goals across projects) 20 | prune-images: 21 | docker images | grep -E '(admiral|cloud-prepare|coastguard|lighthouse|nettest|shipyard|subctl|submariner|)' | while read image tag hash _; do \ 22 | if [ "$$tag" != "" ]; then \ 23 | docker rmi $$image:$$tag; \ 24 | else \ 25 | docker rmi $$hash; \ 26 | fi \ 27 | done 28 | 29 | .PHONY: prune-images 30 | -------------------------------------------------------------------------------- /Makefile.images: -------------------------------------------------------------------------------- 1 | # Shared target to build images. 2 | # To build an image, simply add a dependency on the target `package/.image.` 3 | # where corresponds to an existing `package/Dockerfile.` file. 4 | # The resulting image name will be ``. 5 | 6 | SHELL := /bin/bash 7 | REPO ?= quay.io/submariner 8 | 9 | ### Tunable variables for affecting make commands ### 10 | # Affecting multiple commands 11 | export REPO ?= quay.io/submariner 12 | 13 | # Specific to `build_images.sh` 14 | export USE_CACHE ?= true 15 | 16 | # Specific to `images` 17 | export OCIFILE PLATFORM 18 | 19 | # Specific to `preload-images` 20 | export PRELOAD_IMAGES 21 | 22 | # Automatically preload any images that the project builds, on top of any (if there were) requested by the caller 23 | ifdef IMAGES 24 | override PRELOAD_IMAGES += $(IMAGES) 25 | endif 26 | 27 | # Specific to `release-images` 28 | export TAG ?= $(CUTTING_EDGE) 29 | 30 | # Force rebuild an image if it's not in docker 31 | force_image_rebuild = $(if $(shell docker image history $(REPO)/$(1):$(BASE_BRANCH)),,FORCE_IMAGE) 32 | 33 | # Force rebuild dynamically by calling this target 34 | FORCE_IMAGE: ; 35 | 36 | # Dockerfile dependencies are the file and any file copied into it 37 | # We have to run it through a variable in order to expand any * that might be 38 | # in the COPY command; find is used to handle directories as dependencies 39 | # Files copied from another image are ignored 40 | docker_deps = $(shell files=($(1) $$(awk '/COPY/ && substr($$2, 1, 7) != "--from=" && $$2 != "." { $$1 = $$NF = ""; print }' $(1))) && find $${files[*]} -type f -o -type l) 41 | 42 | # Patterned recipe to use to build any image from any Dockerfile 43 | # An empty file is used for make to figure out if dependencies changed or not 44 | .SECONDEXPANSION: 45 | package/.image.%: $$(call docker_deps,package/Dockerfile.$$*) $$(call force_image_rebuild,$$*) 46 | $(SCRIPTS_DIR)/build_image.sh $* $< $@ 47 | 48 | # Build an OCI tarball from a Dockerfile 49 | .SECONDEXPANSION: 50 | package/%.tar: $$(call docker_deps,package/Dockerfile.$$*) $$(call force_image_rebuid,$$*) 51 | OCIFILE="$@" $(SCRIPTS_DIR)/build_image.sh $* $< package/.image.$* 52 | 53 | # [images] builds all the container images for the project 54 | # Default target to build images based on IMAGES variable 55 | # All these images are built for the default architecture; this enables 56 | # local consumption e.g. for deploy and e2e. Targets needing usable images 57 | # during a build must depend on this target. 58 | images: $(foreach image,$(IMAGES),package/.image.$(image)) 59 | 60 | # [multiarch-images] builds all multi-arch container images 61 | # Default target to build images based on the MULTIARCH_IMAGES variable 62 | # Default platforms based on the PLATFORMS variable 63 | # These images are not built in a locally-consumable form (OCI tarballs only) 64 | # and are only intended for publication. 65 | multiarch-images: export PLATFORM=$(PLATFORMS) 66 | multiarch-images: $(foreach image,$(MULTIARCH_IMAGES),package/$(image).tar) 67 | 68 | # [release-images] uploads the project's images to a public repository 69 | release-images: 70 | ifneq (,$(filter-out $(MULTIARCH_IMAGES),$(IMAGES))) 71 | $(SCRIPTS_DIR)/release_images.sh $(filter-out $(MULTIARCH_IMAGES),$(IMAGES)) 72 | endif 73 | ifneq (,$(MULTIARCH_IMAGES)) 74 | OCIDIR='package/' $(SCRIPTS_DIR)/release_images.sh $(MULTIARCH_IMAGES) 75 | endif 76 | 77 | # [preload-images] into the local registry for testing, based on PRELOAD_IMAGES variable 78 | preload-images: images 79 | set -e;\ 80 | . $(SCRIPTS_DIR)/lib/deploy_funcs;\ 81 | . $(SCRIPTS_DIR)/lib/debug_functions;\ 82 | for image in $(PRELOAD_IMAGES); do\ 83 | import_image $(REPO)/$${image};\ 84 | done 85 | 86 | # [reload-images] rebuilds and reloads the images, optionally restarting with `restart` variable 87 | reload-images: preload-images 88 | $(SCRIPTS_DIR)/reload_images.sh 89 | 90 | .PHONY: images multiarch-images preload-images release-images reload-images 91 | -------------------------------------------------------------------------------- /Makefile.linting: -------------------------------------------------------------------------------- 1 | ### VARIABLES ### 2 | 3 | SHELLCHECK_ARGS += $(shell [ ! -d scripts ] || find scripts -type f -exec awk 'FNR == 1 && /sh$$/ { print FILENAME }' {} +) 4 | export SHELLCHECK_ARGS 5 | 6 | ### TARGETS ### 7 | 8 | .PHONY: gitlint golangci-lint markdownlint packagedoc-lint shellcheck yamllint 9 | 10 | # [gitlint] validates the commits are valid 11 | gitlint: 12 | if [ -r .gitlint ]; then \ 13 | gitlint --commits origin/$(BASE_BRANCH)..HEAD; \ 14 | else \ 15 | gitlint --config $(SHIPYARD_DIR)/.gitlint --commits origin/$(BASE_BRANCH)..HEAD; \ 16 | fi 17 | 18 | # [golangci-lint] validates Go code in the project 19 | golangci-lint: 20 | ifneq (,$(shell find . -name '*.go')) 21 | golangci-lint version 22 | golangci-lint linters 23 | # Set up a workspace to include all modules, if necessary 24 | # If a workspace is already set up, it will be used instead 25 | find . -name go.mod -execdir git grep -qFl 'func ' -- '*.go' \; -printf '%h ' | xargs go work init ||: 26 | # Analyse all the modules containing function declarations 27 | golangci-lint run --output.text.print-issued-lines --output.text.colors --timeout 10m $$(find . -name go.mod -execdir git grep -qFl 'func ' -- '*.go' \; -printf '%h/...\n') 28 | else 29 | @echo 'There are no Go files to lint.' 30 | endif 31 | 32 | # [markdownlint] validates Markdown files in the project 33 | markdownlint: 34 | md_ignored=(); \ 35 | if [ -r .mdignore ]; then \ 36 | md_ignored+=($$(< .mdignore)); \ 37 | fi; \ 38 | markdownlint -c .markdownlint.yml $${md_ignored[@]/#/-i } . 39 | 40 | # [packagedoc-lint] checks that the package docs don’t include the SPDX header 41 | packagedoc-lint: 42 | result=0; \ 43 | for package in $$(find . -name vendor -prune -o -name \*.go -printf "%h\n" | sort -u); do \ 44 | if $(GO) doc $$package | grep -q SPDX; then \ 45 | echo $$package has an invalid package documentation; \ 46 | result=1; \ 47 | fi; \ 48 | done 2>/dev/null; \ 49 | exit $$result 50 | 51 | # [shellcheck] validates your shell files 52 | shellcheck: 53 | # Only run shellcheck if there are files to check 54 | ifneq (,$(SHELLCHECK_ARGS)) 55 | shellcheck -x -P $${SCRIPTS_DIR} $(SHELLCHECK_ARGS) 56 | # https://github.com/koalaman/shellcheck/issues/1659 57 | ! grep $$'\t' $(SHELLCHECK_ARGS) 58 | else 59 | @echo 'There are no shell scripts to check; if this is incorrect, specify them in SHELLCHECK_ARGS.' 60 | endif 61 | 62 | # [yamllint] validates YAML files in the project 63 | yamllint: 64 | yamllint --strict . 65 | 66 | 67 | -------------------------------------------------------------------------------- /Makefile.shipyard: -------------------------------------------------------------------------------- 1 | # This Makefile contains the basic goals to set up a Shipyard based environment, which runs inside a container 2 | 3 | , := , 4 | BASE_DAPPER := Dockerfile.dapper 5 | USING = $(subst $(,), ,$(using)) 6 | LINTING_DAPPER := Dockerfile.linting 7 | LINTING_GOALS := gitlint shellcheck yamllint markdownlint 8 | NON_DAPPER_GOALS += .dapper shell targets $(LINTING_GOALS) 9 | SHIPYARD_GOALS += cleanup clean-clusters clusters golangci-lint packagedoc-lint 10 | export MAKEFLAGS 11 | export USING 12 | 13 | # Define LOCAL_BUILD to build directly on the host and not inside a Dapper container 14 | ifdef LOCAL_BUILD 15 | DAPPER_HOST_ARCH ?= $(shell go env GOHOSTARCH) 16 | SHIPYARD_DIR ?= ../shipyard 17 | SCRIPTS_DIR ?= $(SHIPYARD_DIR)/scripts/shared 18 | 19 | export DAPPER_HOST_ARCH 20 | export SHIPYARD_DIR 21 | export SCRIPTS_DIR 22 | 23 | RUN_IN_DAPPER := 24 | 25 | .dapper: 26 | 27 | else 28 | 29 | .dapper: 30 | @echo Downloading dapper 31 | @curl -sfLO $(SHIPYARD_URL)/$@ 32 | @chmod +x .dapper 33 | @./.dapper -v 34 | 35 | SELINUX_CONTEXT := $(shell (selinuxenabled && echo -S z) 2>/dev/null) 36 | RUN_IN_DAPPER = ./.dapper $(DAPPER_ARGS) $(SELINUX_CONTEXT) -- 37 | 38 | endif 39 | 40 | ifeq (true,$(DEBUG_PRINT)) 41 | MAKE_DEBUG_FLAG = --debug=b 42 | endif 43 | 44 | USING = $(subst $(,), ,$(using)) 45 | _using = ${USING} 46 | 47 | ifneq (,$(filter dual-stack ipv6-stack,$(_using))) 48 | IPV6_FLAGS = --ipv6 --subnet fc00:1234:4444::/64 49 | endif 50 | 51 | # Run only Shipyard goals inside the container when requested, otherwise run any suitable goal inside the container. 52 | ifdef ONLY_SHIPYARD_GOALS 53 | $(SHIPYARD_GOALS): .dapper $(BASE_DAPPER) 54 | else 55 | $(filter-out .dapper prune-images shell targets $(NON_DAPPER_GOALS),$(MAKECMDGOALS)): .dapper $(BASE_DAPPER) 56 | endif 57 | @[ -z "$$CI" ] || echo "::group::Launching a container to run 'make $@'" 58 | -docker network create $(IPV6_FLAGS) -d bridge kind 59 | +$(RUN_IN_DAPPER) make $(MAKE_DEBUG_FLAG) $@ 60 | 61 | # The original dockerfiles will live in Shipyard and be downloaded by consuming projects. 62 | $(BASE_DAPPER) $(LINTING_DAPPER): 63 | @echo Downloading $@ 64 | @curl -sfLO $(SHIPYARD_URL)/$@ 65 | 66 | # Run silently as the commands are pretty straightforward and `make` hasn't a lot to do 67 | $(LINTING_GOALS): DAPPER_ARGS := -f $(LINTING_DAPPER) 68 | $(LINTING_GOALS): .dapper $(LINTING_DAPPER) 69 | @[ -z "$$CI" ] || echo "::group::Launching a container to run 'make $@'" 70 | @$(RUN_IN_DAPPER) make $@ 71 | 72 | shell: DAPPER_ARGS := -s 73 | shell: .dapper $(BASE_DAPPER) 74 | $(RUN_IN_DAPPER) 75 | 76 | # Run silently to just list the targets (hence we can't use the generic dapper wrapper recipe). 77 | # This only lists targets accessible inside dapper (which are 99% of targets we use) 78 | targets: DAPPER_ARGS := -f $(LINTING_DAPPER) 79 | targets: $(LINTING_DAPPER) 80 | @$(RUN_IN_DAPPER) eval "\$${SCRIPTS_DIR}/targets.sh" 81 | 82 | .PHONY: shell targets $(LINTING_GOALS) 83 | -------------------------------------------------------------------------------- /Makefile.versions: -------------------------------------------------------------------------------- 1 | # Calculate versions; these can be overridden 2 | CUTTING_EDGE := devel 3 | DEV_VERSION := dev 4 | override CALCULATED_VERSION := $(BASE_BRANCH)-$(shell git rev-parse --short=12 HEAD) 5 | VERSION ?= $(CALCULATED_VERSION) 6 | 7 | export CUTTING_EDGE DEV_VERSION VERSION 8 | 9 | print-version: 10 | @echo CALCULATED_VERSION=$(CALCULATED_VERSION) 11 | @echo VERSION=$(VERSION) 12 | 13 | .PHONY: print-version 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shipyard 2 | 3 | 4 | [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4865/badge)](https://bestpractices.coreinfrastructure.org/projects/4865) 5 | [![Release Images](https://github.com/submariner-io/shipyard/workflows/Release%20Images/badge.svg)](https://github.com/submariner-io/shipyard/actions?query=workflow%3A%22Release+Images%22) 6 | [![Periodic](https://github.com/submariner-io/shipyard/workflows/Periodic/badge.svg)](https://github.com/submariner-io/shipyard/actions?query=workflow%3APeriodic) 7 | 8 | 9 | The Shipyard project provides tooling for creating K8s clusters with [kind] and provides a Go framework for creating E2E tests. 10 | 11 | ## Prerequisites 12 | 13 | - [go 1.12] with [$GOPATH configured] 14 | - [docker] 15 | 16 | ## Usage 17 | 18 | To use Shipyard for your project, it's easiest to use Dapper and Make. 19 | To use Dapper, you'll need a specific Dockerfile that Dapper consumes to create a consistent environment based upon Shipyard's base image. 20 | To use Make, you'll need some commands to enable Dapper and also include the targets which ship in the base image. 21 | 22 | ### Dockerfile.dapper 23 | 24 | Shipyard provides this file automatically for you. You can also define it explicitly to be more tailored to the specific project. 25 | 26 | The Dockerfile should build upon `quay.io/submariner/shipyard-dapper-base`. 27 | 28 | For example, this very basic file allows E2E testing: 29 | 30 | ```Dockerfile 31 | FROM quay.io/submariner/shipyard-dapper-base:devel 32 | 33 | ENV DAPPER_SOURCE=/go/src/github.com/submariner-io/submariner DAPPER_DOCKER_SOCKET=true 34 | ENV DAPPER_OUTPUT=${DAPPER_SOURCE}/output 35 | 36 | WORKDIR ${DAPPER_SOURCE} 37 | 38 | ENTRYPOINT ["./scripts/entry"] 39 | CMD ["ci"] 40 | ``` 41 | 42 | You can also refer to the project's own [Dockerfile.dapper](Dockerfile.dapper) as an example. 43 | 44 | ### Makefile 45 | 46 | The Makefile should include targets to run everything in Dapper. 47 | They're defined in [Makefile.dapper](Makefile.dapper) and can be copied as-is and included, but it's best to download and import it. 48 | To use Shipyard's target, simply include the [Makefile.inc](Makefile.inc) file in your own Makefile. 49 | 50 | The simplest Makefile would look like this: 51 | 52 | ```Makefile 53 | BASE_BRANCH=devel 54 | PROJECT=shipyard 55 | SHIPYARD_URL=https://raw.githubusercontent.com/submariner-io/shipyard/$(BASE_BRANCH) 56 | export BASE_BRANCH PROJECT SHIPYARD_URL 57 | 58 | ifneq (,$(DAPPER_HOST_ARCH)) 59 | 60 | # Running in Dapper 61 | 62 | include $(SHIPYARD_DIR)/Makefile.inc 63 | 64 | else 65 | 66 | # Not running in Dapper 67 | 68 | Makefile.dapper: 69 | @echo Downloading $@ 70 | @curl -sfLO $(SHIPYARD_URL)/$@ 71 | 72 | include Makefile.dapper 73 | 74 | endif 75 | 76 | # Disable rebuilding Makefile 77 | Makefile Makefile.dapper Makefile.inc: ; 78 | ``` 79 | 80 | You can also refer to the project's own [Makefile](Makefile) as an example. 81 | 82 | ## Releases 83 | 84 | Get the latest version from the [Releases] page. 85 | 86 | 87 | [go 1.12]: https://blog.golang.org/go1.12 88 | [docker]: https://docs.docker.com/install/ 89 | [$GOPATH configured]: https://github.com/golang/go/wiki/SettingGOPATH 90 | [Releases]: https://github.com/submariner-io/shipyard/releases/ 91 | [kind]: https://github.com/kubernetes-sigs/kind 92 | -------------------------------------------------------------------------------- /gh-actions/cache-images/action.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Cache Images' 3 | description: 'Builds project images and caches them' 4 | inputs: 5 | cache: 6 | description: 'Location of the cache' 7 | required: false 8 | default: '~/image-cache' 9 | runs: 10 | using: "composite" 11 | steps: 12 | - name: Set up the cache 13 | id: image-cache 14 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 15 | with: 16 | path: ${{ inputs.cache }} 17 | key: image-cache-${{ github.sha }} 18 | 19 | - name: Build the images if necessary 20 | if: steps.image-cache.outputs.cache-hit != 'true' 21 | shell: bash 22 | run: | 23 | echo "::group::Building images and storing them" 24 | make images 25 | mkdir -p ${{ inputs.cache }} 26 | for image in package/.image.*; do \ 27 | docker save quay.io/submariner/${image#package/.image.} | \ 28 | gzip > ${{ inputs.cache }}/${image#package/.image.}.tar.gz; \ 29 | cp $image ${{ inputs.cache }}; \ 30 | done 31 | echo "::endgroup::" 32 | -------------------------------------------------------------------------------- /gh-actions/commit-size/action.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Commit Size' 3 | description: 'Make sure no commit in the pull request goes over a certain size' 4 | inputs: 5 | size: 6 | description: 'Total maximum size of commit (lines added+removed).' 7 | required: false 8 | default: '500' 9 | runs: 10 | using: "composite" 11 | steps: 12 | - name: Make sure the size of each commit doesn't pass the desired size 13 | shell: bash 14 | if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-size-check') }} 15 | run: | 16 | set -e 17 | # Fetch 'origin' to get all commits for size checking 18 | git fetch -q -n origin 19 | 20 | result=0 21 | commits_range=${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} 22 | sum_commit() { git diff --numstat ${1} ${1}^ | awk '{ sum += $1 + $2 } END { print sum }'; } 23 | for commit in $(git log --pretty="format:%H" ${commits_range}); do 24 | if [[ $(git cat-file -p "${commit}" | grep -c "^parent ") -gt 1 ]]; then 25 | echo "Ignoring merge commit ${commit}" 26 | continue 27 | fi 28 | 29 | size=$(sum_commit ${commit}) 30 | printf "Size of commit %s (%d) " "${commit}" "${size}" 31 | if [[ ${size} -gt ${{ inputs.size }} ]]; then 32 | echo "is larger than the allowed size ${{ inputs.size }}" 33 | result=1 34 | continue 35 | fi 36 | 37 | echo "is within allowed size ${{ inputs.size }}" 38 | done 39 | exit $result 40 | -------------------------------------------------------------------------------- /gh-actions/e2e/action.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'End to End' 3 | description: 'Runs end to end tests with multiple clusters' 4 | inputs: 5 | cache: 6 | description: 'Location of the cache' 7 | required: false 8 | default: '~/image-cache' 9 | k8s_version: 10 | description: 'Version of Kubernetes to use for clusters' 11 | required: false 12 | default: '1.33' 13 | using: 14 | description: 'Various options to pass via using="..."' 15 | required: false 16 | target: 17 | description: 'Target for make' 18 | required: false 19 | default: 'e2e' 20 | test_args: 21 | description: 'Extra arguments to pass to E2E tests' 22 | required: false 23 | default: '--ginkgo.fail-fast' 24 | testdir: 25 | description: 'Where to look for the E2E tests' 26 | required: false 27 | default: 'test/e2e' 28 | plugin: 29 | description: 'Path to the plugin that has pre/post hooks' 30 | required: false 31 | default: '' 32 | runs: 33 | using: "composite" 34 | steps: 35 | - shell: bash 36 | run: echo "DEBUG_PRINT=true" >> $GITHUB_ENV 37 | - shell: bash 38 | run: | 39 | echo "::group::Reclaiming free space" 40 | # Clean up tools we don't need for our CI to free up more space on the hosted runner 41 | rm -rf /usr/share/dotnet 42 | df -h 43 | echo "::endgroup::" 44 | 45 | - shell: bash 46 | run: | 47 | echo "::group::Disable swap" 48 | sudo swapoff -a 49 | echo "::endgroup::" 50 | 51 | - shell: bash 52 | run: | 53 | echo "::group::Report available RAM" 54 | free -h 55 | echo "::endgroup::" 56 | 57 | - shell: bash 58 | run: | 59 | echo "::group::Increase inotify settings" 60 | sudo sysctl -w fs.inotify.max_user_watches=524288 fs.inotify.max_user_instances=512 61 | echo "::endgroup::" 62 | 63 | - name: Install WireGuard specific modules 64 | shell: bash 65 | run: | 66 | [[ "${{ inputs.using }}" =~ "wireguard" ]] || exit 0 67 | echo "::group::Installing WireGuard modules" 68 | sudo apt install -y linux-headers-$(uname -r) wireguard 69 | sudo modprobe wireguard 70 | echo "::endgroup::" 71 | 72 | - name: Restore images from the cache 73 | uses: submariner-io/shipyard/gh-actions/restore-images@devel 74 | with: 75 | cache: ${{ inputs.cache }} 76 | working-directory: ${{ inputs.working-directory }} 77 | 78 | - name: Run E2E deployment and tests 79 | shell: bash 80 | run: | 81 | k8s_version=${{ inputs.k8s_version }} && 82 | make "${{ inputs.target }}" \ 83 | using="${{ inputs.using }}" \ 84 | ${k8s_version:+K8S_VERSION="$k8s_version"} \ 85 | PLUGIN="${{ inputs.plugin }}" \ 86 | TEST_ARGS="${{ inputs.test_args }}" \ 87 | E2E_TESTDIR="${{ inputs.testdir }}" 88 | -------------------------------------------------------------------------------- /gh-actions/post-mortem/action.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Post Mortem' 3 | description: 'Autopsy' 4 | runs: 5 | using: "composite" 6 | steps: 7 | - shell: bash 8 | run: | 9 | echo "::group::Report available disk space" 10 | df -h 11 | echo "::endgroup::" 12 | echo "::group::Report available RAM" 13 | free -h 14 | echo "::endgroup::" 15 | 16 | - shell: bash 17 | run: | 18 | echo "::group::Running post mortem" 19 | make post-mortem 20 | echo "::endgroup::" 21 | 22 | - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 23 | with: 24 | name: submariner-gather 25 | path: gather_output 26 | -------------------------------------------------------------------------------- /gh-actions/release-images/action.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Release Images' 3 | description: 'Builds images and uploads them to the public registry' 4 | inputs: 5 | username: 6 | description: 'User name for the registry' 7 | required: true 8 | password: 9 | description: 'Password for the registry' 10 | required: true 11 | runs: 12 | using: "composite" 13 | steps: 14 | - shell: bash 15 | run: echo "DEBUG_PRINT=true" >> $GITHUB_ENV 16 | - name: Set up QEMU (to support building on non-native architectures) 17 | uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 18 | - name: Set up buildx 19 | uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 20 | - name: Build new images 21 | # This needs to be kept separate so that the release stage runs using the new Shipyard base image 22 | shell: bash 23 | run: | 24 | echo "::group::Build new images" 25 | make images multiarch-images USE_CACHE=false 26 | echo "::endgroup::" 27 | - name: Release newly built images 28 | shell: bash 29 | env: 30 | QUAY_USERNAME: ${{ inputs.username }} 31 | QUAY_PASSWORD: ${{ inputs.password }} 32 | # Pass GITHUB_REF on the call, since when it's set in the `env` directive it doesn't get properly expanded 33 | run: make release-images TAG="${GITHUB_REF##*/}" 34 | -------------------------------------------------------------------------------- /gh-actions/restore-images/action.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Restore Images' 3 | description: 'Restores cached images' 4 | inputs: 5 | cache: 6 | description: 'Location of the cache' 7 | required: false 8 | default: '~/image-cache' 9 | working-directory: 10 | description: 'Working directory to run in' 11 | required: false 12 | default: '.' 13 | runs: 14 | using: "composite" 15 | steps: 16 | - name: Set up the cache 17 | id: image-cache 18 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 19 | with: 20 | path: ${{ inputs.cache }} 21 | key: image-cache-${{ github.sha }} 22 | 23 | - name: Restore images from cache 24 | if: steps.image-cache.outputs.cache-hit == 'true' 25 | shell: bash 26 | run: | 27 | for archive in ${{ inputs.cache }}/*.tar*; do docker load -i $archive; done 28 | if [ -d ${{ inputs.working-directory }}/package ]; then \ 29 | cp ${{ inputs.cache }}/.image.* ${{ inputs.working-directory }}/package/; \ 30 | fi 31 | -------------------------------------------------------------------------------- /gh-actions/upgrade-e2e/action.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Upgrades End to End' 3 | description: 'Runs an upgrade procedure, followed by end to end tests with multiple clusters' 4 | runs: 5 | using: "composite" 6 | steps: 7 | - shell: bash 8 | run: echo "DEBUG_PRINT=true" >> $GITHUB_ENV 9 | - shell: bash 10 | run: | 11 | echo "::group::Reclaiming free space" 12 | # Clean up tools we don't need for our CI to free up more space on the hosted runner 13 | rm -rf /usr/share/dotnet 14 | df -h 15 | echo "::endgroup::" 16 | 17 | - shell: bash 18 | run: | 19 | echo "::group::Disable swap" 20 | sudo swapoff -a 21 | echo "::endgroup::" 22 | 23 | - shell: bash 24 | run: | 25 | echo "::group::Report available RAM" 26 | free -h 27 | echo "::endgroup::" 28 | 29 | - shell: bash 30 | run: | 31 | echo "::group::Increase inotify settings" 32 | sudo sysctl -w fs.inotify.max_user_watches=524288 fs.inotify.max_user_instances=512 33 | echo "::endgroup::" 34 | 35 | - shell: bash 36 | run: | 37 | make upgrade-e2e 38 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/submariner-io/shipyard 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/onsi/ginkgo/v2 v2.23.4 7 | github.com/onsi/gomega v1.37.0 8 | github.com/pkg/errors v0.9.1 9 | k8s.io/api v0.32.5 10 | k8s.io/apimachinery v0.32.5 11 | k8s.io/client-go v0.32.5 12 | k8s.io/klog/v2 v2.130.1 13 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 14 | sigs.k8s.io/mcs-api v0.2.0 15 | ) 16 | 17 | require ( 18 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 19 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 20 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 21 | github.com/go-logr/logr v1.4.2 // indirect 22 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 23 | github.com/go-openapi/jsonreference v0.20.2 // indirect 24 | github.com/go-openapi/swag v0.23.0 // indirect 25 | github.com/go-task/slim-sprig/v3 v3.0.0 // indirect 26 | github.com/gogo/protobuf v1.3.2 // indirect 27 | github.com/golang/protobuf v1.5.4 // indirect 28 | github.com/google/gnostic-models v0.6.8 // indirect 29 | github.com/google/go-cmp v0.7.0 // indirect 30 | github.com/google/gofuzz v1.2.0 // indirect 31 | github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect 32 | github.com/google/uuid v1.6.0 // indirect 33 | github.com/gorilla/websocket v1.5.0 // indirect 34 | github.com/josharian/intern v1.0.0 // indirect 35 | github.com/json-iterator/go v1.1.12 // indirect 36 | github.com/mailru/easyjson v0.7.7 // indirect 37 | github.com/moby/spdystream v0.5.0 // indirect 38 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 39 | github.com/modern-go/reflect2 v1.0.2 // indirect 40 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 41 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect 42 | github.com/spf13/pflag v1.0.5 // indirect 43 | github.com/x448/float16 v0.8.4 // indirect 44 | go.uber.org/automaxprocs v1.6.0 // indirect 45 | golang.org/x/net v0.38.0 // indirect 46 | golang.org/x/oauth2 v0.27.0 // indirect 47 | golang.org/x/sys v0.32.0 // indirect 48 | golang.org/x/term v0.30.0 // indirect 49 | golang.org/x/text v0.23.0 // indirect 50 | golang.org/x/time v0.7.0 // indirect 51 | golang.org/x/tools v0.31.0 // indirect 52 | google.golang.org/protobuf v1.36.5 // indirect 53 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 54 | gopkg.in/inf.v0 v0.9.1 // indirect 55 | gopkg.in/yaml.v3 v3.0.1 // indirect 56 | k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect 57 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect 58 | sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect 59 | sigs.k8s.io/yaml v1.4.0 // indirect 60 | ) 61 | -------------------------------------------------------------------------------- /package/Dockerfile.nettest: -------------------------------------------------------------------------------- 1 | ARG FEDORA_VERSION=42 2 | 3 | FROM --platform=${BUILDPLATFORM} fedora:${FEDORA_VERSION} AS base 4 | ARG VERSION 5 | ARG FEDORA_VERSION 6 | ARG TARGETPLATFORM 7 | 8 | COPY scripts/shared/dnf_install / 9 | 10 | RUN /dnf_install -a ${TARGETPLATFORM} -v ${FEDORA_VERSION} -r /output/nettest \ 11 | glibc bash glibc-minimal-langpack coreutils-single libcurl-minimal \ 12 | bind-utils busybox curl-minimal iperf3 iproute iputils netperf nmap-ncat tcpdump 13 | 14 | FROM scratch 15 | ARG TARGETPLATFORM 16 | ARG VERSION 17 | ENV PATH="/app:$PATH" 18 | 19 | WORKDIR /app 20 | 21 | COPY --from=base /output/nettest / 22 | 23 | COPY scripts/nettest/* /app/ 24 | 25 | RUN echo ${VERSION} >> /app/version 26 | 27 | CMD ["/bin/bash","-l"] 28 | -------------------------------------------------------------------------------- /package/Dockerfile.shipyard-dapper-base: -------------------------------------------------------------------------------- 1 | FROM fedora:42 2 | 3 | # Unless specified otherwise, compress to a medium level which gives (from experemintation) a 4 | # good balance between compression time and resulting image size. 5 | ARG UPX_LEVEL=-5 6 | ENV DAPPER_HOST_ARCH=amd64 SHIPYARD_DIR=/opt/shipyard SHELL=/bin/bash \ 7 | DAPPER_RUN_ARGS="--net=kind" 8 | ENV HOST_ARCH=${DAPPER_HOST_ARCH} ARCH=${DAPPER_HOST_ARCH} PATH=/go/bin:/root/.local/bin:/usr/local/go/bin:$PATH \ 9 | GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm=armv6l GOLANG_ARCH=GOLANG_ARCH_${DAPPER_HOST_ARCH} \ 10 | GOPATH=/go GO111MODULE=on GOPROXY=https://proxy.golang.org \ 11 | SCRIPTS_DIR=${SHIPYARD_DIR}/scripts 12 | 13 | # Requirements: 14 | # Component | Usage 15 | # ------------------------------------------------------------- 16 | # curl | download other tools 17 | # docker-buildx | multi-arch image construction 18 | # file | file identification (in compile/test.sh) 19 | # findutils | make unit (find unit test dirs) 20 | # gcc | needed by `go test -race` (https://github.com/golang/go/issues/27089) 21 | # gh | backport, releases 22 | # git-core | find the workspace root, git clones 23 | # golang | build 24 | # helm | Helm package construction 25 | # jq | JSON processing (GitHub API) 26 | # kubernetes-client | kubectl, used in e2e tests 27 | # make | builds 28 | # moby-engine | Docker (for Dapper) 29 | # moreutils | sponge (for system tests) 30 | # pip | Python package installation 31 | # procps-ng | watch (for installing ACM) 32 | # protobuf-compiler | protobuf compilation 33 | # python3-jinja2-cli | Jinja2 template engine (used by OVN's kind setup script) 34 | # qemu-user-static-* | Emulation (for multiarch builds) 35 | # skopeo | container image manipulation 36 | # unzip | ZIP extraction 37 | # upx | binary compression 38 | # yq | YAML processing (OCM deploy tool) 39 | 40 | # This layer's versioning is handled by dnf, and isn't expected to be rebuilt much except in CI 41 | # Removals and UPX are done after all installations, since image layers are diffs. 42 | # We remove: 43 | # - DNF cache 44 | # - Any unnecessary packages and executables 45 | RUN dnf -y install --nodocs --setopt=install_weak_deps=False \ 46 | curl \ 47 | docker-buildx \ 48 | file \ 49 | findutils \ 50 | gcc \ 51 | gh \ 52 | git-core \ 53 | gitlint \ 54 | golang \ 55 | helm \ 56 | jq \ 57 | kubernetes-client \ 58 | make \ 59 | moby-engine \ 60 | moreutils \ 61 | pip \ 62 | procps-ng \ 63 | protobuf-compiler \ 64 | python3-jinja2-cli \ 65 | qemu-user-static-aarch64 qemu-user-static-x86 \ 66 | skopeo \ 67 | unzip \ 68 | upx \ 69 | yq && \ 70 | rpm -e --nodeps containerd && \ 71 | rpm -qa "selinux*" | xargs -r rpm -e --nodeps && \ 72 | dnf -y clean all && \ 73 | rm -f /usr/bin/{dockerd,lto-dump} \ 74 | /usr/libexec/gcc/x86_64-redhat-linux/10/lto1 && \ 75 | find /usr/bin /usr/lib/golang /usr/libexec -type f -executable -newercc /proc -size +1M ! -name hyperkube \( -execdir upx ${UPX_LEVEL} {} \; -o -true \) && \ 76 | ln -f /usr/bin/kubectl /usr/bin/hyperkube 77 | 78 | COPY tools/go.mod /tools.mod 79 | 80 | # This layer's versioning is determined by us, and thus could be rebuilt more frequently to test different versions 81 | RUN LINT_VERSION=$(awk '/golangci-lint/ { print $2 }' /tools.mod) && \ 82 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin -d ${LINT_VERSION} && \ 83 | KIND_VERSION=$(awk '/sigs.k8s.io.kind/ { print $2 }' /tools.mod) && \ 84 | curl -Lo /go/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-${ARCH}" && chmod a+x /go/bin/kind && \ 85 | mkdir -p /usr/local/libexec/docker/cli-plugins && \ 86 | curl -L https://raw.githubusercontent.com/jonmosco/kube-ps1/1b8fe913b25ba857b84a94c3b1dbf7bb34f7caef/kube-ps1.sh -o /etc/profile.d/kube-ps1.sh && \ 87 | find /go/bin /usr/local/libexec/docker/cli-plugins -type f -executable -newercc /proc -exec strip {} + && \ 88 | find /go/bin /usr/local/libexec/docker/cli-plugins -type f -executable -newercc /proc \( -execdir upx ${UPX_LEVEL} {} \; -o -true \) && \ 89 | go clean -cache -modcache && rm -f /tools.mod 90 | 91 | # Link get-subctl script so it can be easily run inside a shell 92 | RUN mkdir -p /root/.local/bin && ln -s $SCRIPTS_DIR/get-subctl.sh /root/.local/bin/subctl 93 | 94 | # Copy kubecfg to always run on the shell 95 | COPY scripts/shared/lib/kubecfg /etc/profile.d/kubecfg.sh 96 | 97 | # Print kube context on interactive shell 98 | COPY scripts/shared/kubeps1.sh /etc/profile.d/ 99 | 100 | # Copy shared files so that downstream projects can use them 101 | COPY Makefile.* .gitlint ${SHIPYARD_DIR}/ 102 | 103 | # Copy the global dapper file so that we can make sure consuming projects are up to date 104 | COPY Dockerfile.dapper ${SHIPYARD_DIR}/ 105 | 106 | # Copy CI deployment scripts into image to share with all projects 107 | WORKDIR $SCRIPTS_DIR 108 | COPY scripts/shared/ . 109 | 110 | # Copy our j2 wrapper for jinja2 111 | COPY package/j2 /usr/local/bin/ 112 | -------------------------------------------------------------------------------- /package/Dockerfile.shipyard-linting: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | ENV DAPPER_HOST_ARCH=amd64 SHELL=/bin/bash \ 4 | SHIPYARD_DIR=/opt/shipyard 5 | ENV HOST_ARCH=${DAPPER_HOST_ARCH} ARCH=${DAPPER_HOST_ARCH} \ 6 | SCRIPTS_DIR=${SHIPYARD_DIR}/scripts 7 | 8 | # Requirements: 9 | # Component | Usage 10 | # ------------------------------------------------------------------- 11 | # bash | Just your basic shell 12 | # findutils | Finding executables to compress 13 | # gitlint | Git commit message linting 14 | # grep | For listing targets 15 | # make | Running makefiles inside the container 16 | # markdownlint | Markdown linting 17 | # nodejs | Used by markdownlint 18 | # npm | Installing markdownlint (Removed afterwards) 19 | # py3-pip | Installing gitlint (Removed afterwards) 20 | # py3-six | Required by gitlint 21 | # shellcheck | Shell script linting 22 | # upx | Compressing executables to get a smaller image 23 | # yamllint | YAML linting 24 | # yq | YAML processing 25 | 26 | ENV MARKDOWNLINT_VERSION=0.39.0 \ 27 | GITLINT_VERSION=0.19.1 28 | 29 | RUN apk add --no-cache bash findutils git grep make nodejs py3-six shellcheck upx yamllint yq && \ 30 | apk add --no-cache --virtual installers npm py3-pip && \ 31 | npm install -g markdownlint-cli@${MARKDOWNLINT_VERSION} && \ 32 | pip install --break-system-packages gitlint==${GITLINT_VERSION} && \ 33 | find /usr/bin/ -type f -executable -newercc /proc -size +1M \( -execdir upx {} \; -o -true \) && \ 34 | find /usr/lib/ -name __pycache__ -type d -exec rm -rf {} + && \ 35 | apk del installers 36 | 37 | # Copy shared files so that downstream projects can use it 38 | COPY Makefile.* .gitlint ${SHIPYARD_DIR}/ 39 | 40 | # Copy the global dapper file so that we can make sure consuming projects are up to date 41 | COPY Dockerfile.linting ${SHIPYARD_DIR}/ 42 | 43 | # Copy shared scripts into image to share with all projects 44 | WORKDIR $SCRIPTS_DIR 45 | COPY scripts/shared/ . 46 | -------------------------------------------------------------------------------- /package/j2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | env | jinja2 --format=env "$@" 4 | -------------------------------------------------------------------------------- /release-notes/20220831-deploy-with-overrides.md: -------------------------------------------------------------------------------- 1 | 2 | Deploying using Shipyard now uses specific image overrides determined by the `PRELOAD_IMAGES` variable. 3 | For developers using `make deploy` (or `make e2e`) there won't be a noticeable change in behavior, but only those images specified by the 4 | `PRELOAD_IMAGES` will be preloaded and used. 5 | Developers can also specify the variable explicitly to further control which images get preloaded and used. 6 | -------------------------------------------------------------------------------- /release-notes/20220911-air-gapped-kind.md: -------------------------------------------------------------------------------- 1 | 2 | Support for simulated "air-gapped" environments has been added to kind clusters. 3 | The air-gap is simulated by blocking outgoing traffic to any public subnet on the cluster nodes, effectively isolating the host network. 4 | To use, deploy with `USING=air-gap` or `AIR_GAPPED=true`. 5 | -------------------------------------------------------------------------------- /release-notes/20220919-load-balancer.md: -------------------------------------------------------------------------------- 1 | 2 | Support was added in the Shipyard project to easily deploy Submariner with a LoadBalancer type Service in front. 3 | To use, simply specify the target (e.g. `deploy`) with `USING=load-balancer` or `LOAD_BALANCER=true`. 4 | For kind-based deployments, [MetalLB](https://metallb.universe.tf/) is deployed to provide the capability. 5 | The MetalLB version can be specified using `METALLB_VERSION=x.y.z`. 6 | -------------------------------------------------------------------------------- /release-notes/20230508-e2e-non-gw-nodes.md: -------------------------------------------------------------------------------- 1 | 2 | `subctl verify` has been enhanced to select nodes labeled with `test.submariner.io/non-gateway-node=true` as non-Gateway 3 | nodes while scheduling the test pods. If none of the nodes have the label `test.submariner.io/non-gateway-node=true`, the 4 | test framework falls back to the existing approach of randomly selecting a non-Gateway node. 5 | -------------------------------------------------------------------------------- /release-notes/force-subctl-verify: -------------------------------------------------------------------------------- 1 | Support was added to force running `subctl verify` when testing end-to-end, ignoring any local tests. 2 | To use this feature, run `make e2e using=subctl-verify`. 3 | Verifications can be now specified using the `SUBCTL_VERIFICATIONS` flag, instead of relying on the default behavior. 4 | e.g.: `make e2e using=subctl-verify SUBCTL_VERIFICATIONS=connectivity,service-discovery`. 5 | -------------------------------------------------------------------------------- /release-notes/get-subctl: -------------------------------------------------------------------------------- 1 | Added a centralized script for getting `subctl`. It now impersonates `subctl` so that users who run `make shell` will have it downloaded the first time they try to use it. 2 | To influence the version being installed, the `SUBCTL_VERSION` variable can be set. 3 | -------------------------------------------------------------------------------- /release-notes/removed-shflags: -------------------------------------------------------------------------------- 1 | All `shflags` usage has been removed from Shipyard and consuming projects. 2 | Users/Developers running commands with `using=` are not affected. 3 | Users/Developers running commands with `*_ARGS=--flag` should switch to using environment variables instead. 4 | For example, if you used to run `make deploy DEPLOY_ARGS='--settings ...'`, please run `make deploy SETTINGS='...'` instead. 5 | -------------------------------------------------------------------------------- /release-notes/support-aws-ocp: -------------------------------------------------------------------------------- 1 | Support was added in shipyard to easily create test clusters using OCP on top of AWS. 2 | This allows developers and users to easily stand up a test/PoC environment with Submariner using AWS+OCP. 3 | The deployed environment will use the latest published Submariner images, it's currently not possible to use locally built images. 4 | 5 | The following commands are supported: 6 | * `make clusters using=aws-ocp` 7 | * `make deploy using=aws-ocp` 8 | * `make e2e using=aws-ocp` 9 | * `make cleanup using=aws-ocp` 10 | -------------------------------------------------------------------------------- /scripts/nettest/metricsproxy: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Arguments: source-port target-IP target-port 4 | exec /usr/sbin/busybox nc -v -lk -p "$1" -e /usr/sbin/busybox nc "$2" "$3" 5 | -------------------------------------------------------------------------------- /scripts/nettest/nc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This wrapper converts Busybox syntax to nmap ncat: 4 | # * -e becomes -c 5 | # * -w and -i need a s suffix on their argument 6 | # * with -l, -p XX and -s YY become YY XX 7 | 8 | args=() 9 | listening= 10 | sourceport= 11 | sourceaddress= 12 | while [ -n "$1" ] && [ "$1" != "-e" ]; do 13 | case "$1" in 14 | -w|-i) 15 | args+=("$1" "$2"s) 16 | shift 2;; 17 | -l|-lk) 18 | listening=1 19 | args+=("$1") 20 | shift;; 21 | -p) 22 | sourceport="$2" 23 | shift 2;; 24 | -s) 25 | sourceaddress="$2" 26 | shift 2;; 27 | *) 28 | args+=("$1") 29 | shift;; 30 | esac 31 | done 32 | 33 | if [ -n "$listening" ]; then 34 | args+=(-l ${sourceaddress:+"$sourceaddress"} ${sourceport:+"$sourceport"}) 35 | else 36 | if [ -n "$sourceaddress" ]; then 37 | args+=(-s "$sourceaddress") 38 | fi 39 | if [ -n "$sourceport" ]; then 40 | args+=(-p "$sourceport") 41 | fi 42 | fi 43 | 44 | cmd="" 45 | if [ "$1" = "-e" ]; then 46 | shift 47 | cmd="$*" 48 | fi 49 | 50 | exec /usr/bin/ncat "${args[@]}" ${cmd:+"-c $cmd"} 51 | -------------------------------------------------------------------------------- /scripts/nettest/simpleserver: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # By default, ncat fails if it can't use an IPv4 connection; this tries an unspecified connection first 5 | # (listening on IPv4 and optionally IPv6), and if that fails, an IPv6-only connection 6 | while true 7 | do 8 | echo -e "HTTP/1.1 200 OK\r\n\r\nHello World" | (/usr/bin/ncat -l -p 8080 || /usr/bin/ncat -6 -l -p 8080) 9 | done 10 | -------------------------------------------------------------------------------- /scripts/shared/build_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | source "${SCRIPTS_DIR}/lib/utils" 5 | 6 | [[ $# == 3 ]] || exit_error 'You must specify exactly 3 arguments: The image name, the Dockerfile and a hash file to write to' 7 | [[ "${PLATFORM}" =~ , && -z "${OCIFILE}" ]] && exit_error 'Multi-arch builds require OCI output, please set OCIFILE' 8 | 9 | print_env OCIFILE PLATFORM REPO 10 | source "${SCRIPTS_DIR}/lib/debug_functions" 11 | 12 | ### Arguments ### 13 | 14 | image="$1" 15 | dockerfile="$2" 16 | hashfile="$3" 17 | 18 | ### Main ### 19 | 20 | local_image="${REPO}/${image}:${DEV_VERSION}" 21 | cache_image="${REPO}/${image}:${CUTTING_EDGE}" 22 | 23 | # When using cache pull latest image from the repo, so that its layers may be reused. 24 | declare -a cache_flags 25 | if [[ "${USE_CACHE}" = true ]]; then 26 | cache_flags+=(--cache-from "${cache_image}") 27 | if [[ -z "$(docker image ls -q "${cache_image}")" ]]; then 28 | docker pull "${cache_image}" || : 29 | fi 30 | # The shellcheck linting tool recommends piping to a while read loop, but that doesn't work for us 31 | # because the while loop ends up in a subshell 32 | # shellcheck disable=SC2013 33 | for parent in $(awk '/FROM/ { 34 | for (i = 2; i <= NF; i++) { 35 | if ($i == "AS") next; 36 | if (!($i ~ /^--platform/ || $i ~ /scratch/)) 37 | print gensub("\\${BASE_BRANCH}", ENVIRON["BASE_BRANCH"], "g", $i) 38 | } 39 | }' "${dockerfile}"); do 40 | cache_flags+=(--cache-from "${parent}") 41 | docker pull "${parent}" || : 42 | done 43 | fi 44 | 45 | output_flag=--load 46 | [[ -z "${OCIFILE}" ]] || output_flag="--output=type=oci,dest=${OCIFILE}" 47 | 48 | # Default to linux/amd64 (for CI); platforms match Go OS/arch 49 | if command -v "${GO:-go}" >/dev/null; then 50 | default_platform="$(${GO:-go} env GOOS)/$(${GO:-go} env GOARCH)" 51 | else 52 | echo Unable to determine default container image platform, assuming linux/amd64 53 | default_platform=linux/amd64 54 | fi 55 | [[ -n "$PLATFORM" ]] || PLATFORM="$default_platform" 56 | 57 | # Rebuild the image to update any changed layers and tag it back so it will be used. 58 | buildargs_flags=(--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg "BASE_BRANCH=${BASE_BRANCH}" --build-arg "VERSION=${VERSION}") 59 | if [[ "${PLATFORM}" != "${default_platform}" ]] && docker buildx version > /dev/null 2>&1; then 60 | docker buildx use buildx_builder || docker buildx create --name buildx_builder --use 61 | docker buildx build "${output_flag}" -t "${local_image}" "${cache_flags[@]}" -f "${dockerfile}" --iidfile "${hashfile}" --platform "${PLATFORM}" "${buildargs_flags[@]}" . 62 | else 63 | # Fall back to plain BuildKit 64 | if [[ "${PLATFORM}" != "${default_platform}" ]]; then 65 | echo "WARNING: buildx isn't available, cross-arch builds won't work as expected" 66 | fi 67 | DOCKER_BUILDKIT=1 docker build -t "${local_image}" "${cache_flags[@]}" -f "${dockerfile}" --iidfile "${hashfile}" "${buildargs_flags[@]}" . 68 | fi 69 | 70 | # We can only tag the image in non-OCI mode 71 | [[ -n "${OCIFILE}" ]] || docker tag "${local_image}" "${cache_image}" 72 | -------------------------------------------------------------------------------- /scripts/shared/check-non-release-versions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | tmpdir=$(mktemp -d) 4 | trap 'rm -rf $tmpdir' EXIT 5 | 6 | # List all submariner-io dependencies with a - in their version 7 | # We're looking for versions pointing to commits, of the form 8 | # vX.Y.Z-0.YYYYMMDDhhmmss-hash 9 | failed=0 10 | shopt -s lastpipe 11 | GOWORK=off go list -m -mod=mod -json all | 12 | jq -r 'select(.Path | contains("/submariner-io/")) | select(.Main != true) | select(.Version | contains ("-")) | select(.Version | length > 14) | "\(.Path) \(.Version)"' | 13 | while read -r project version; do 14 | cd "$tmpdir" || exit 1 15 | git clone "https://$project" 16 | cd "${project##*/}" || exit 1 17 | hash="${version##*-}" 18 | branch="${GITHUB_BASE_REF:-devel}" 19 | if ! git merge-base --is-ancestor "$hash" "origin/$branch"; then 20 | printf "This project depends on %s %s\n" "$project" "$version" 21 | printf "but %s branch %s does not contain commit %s\n" "$project" "$branch" "$hash" 22 | failed=1 23 | fi 24 | done 25 | 26 | exit $failed 27 | -------------------------------------------------------------------------------- /scripts/shared/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -em 4 | 5 | source "${SCRIPTS_DIR}/lib/utils" 6 | print_env PLUGIN 7 | source "${SCRIPTS_DIR}/lib/debug_functions" 8 | 9 | # Source plugin if the path is passed via plugin argument and the file exists 10 | # shellcheck disable=SC1090 11 | [[ -n "${PLUGIN}" ]] && [[ -f "${PLUGIN}" ]] && source "${PLUGIN}" 12 | 13 | ### Main ### 14 | 15 | load_library cleanup PROVIDER 16 | run_if_defined pre_cleanup 17 | provider_initialize 18 | 19 | run_all_clusters provider_delete_cluster 20 | 21 | # Remove any files inside the output directory, but not any directories as a provider might be using them. 22 | \rm -f "${OUTPUT_DIR:?}"/* 2> /dev/null || true 23 | 24 | provider_finalize 25 | run_if_defined post_cleanup 26 | -------------------------------------------------------------------------------- /scripts/shared/cloud-prepare.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -em -o pipefail 4 | 5 | source "${SCRIPTS_DIR}/lib/debug_functions" 6 | source "${SCRIPTS_DIR}/lib/utils" 7 | 8 | readonly GATEWAY_LABEL='submariner.io/gateway=true' 9 | 10 | ### Functions ### 11 | 12 | function cloud_prepare() { 13 | [[ ${cluster_subm[$cluster]} = "true" ]] || return 0 14 | ! check_gateway_exists || return 0 15 | 16 | case "${PROVIDER}" in 17 | kind|ocp) 18 | "prepare_${PROVIDER//-/_}" 19 | ;; 20 | *) 21 | echo "Unknown PROVIDER ${PROVIDER@Q}." 22 | return 1 23 | esac 24 | } 25 | 26 | function check_gateway_exists() { 27 | [[ $(kubectl get nodes -l "${GATEWAY_LABEL}" --no-headers | wc -l) -gt 0 ]] 28 | } 29 | 30 | function prepare_kind() { 31 | local gw_count="${cluster_gateways[$cluster]:-1}" 32 | 33 | readarray -t nodes < <(kubectl get nodes -o yaml | yq '.items[].metadata.name' | sort -r) 34 | 35 | for node in "${nodes[@]:0:$gw_count}"; do 36 | kubectl label node "$node" "$GATEWAY_LABEL" --overwrite 37 | 38 | [[ "$AIR_GAPPED" = true ]] || [[ "$DUAL_STACK" || "$IPV6_STACK" ]] || continue 39 | # annotate both IPv4 and IPv6 addresses 40 | ips=$(kubectl get node "$node" -o jsonpath="{.status.addresses[?(@.type!='Hostname')].address}") 41 | 42 | local ipv4="" 43 | local ipv6="" 44 | 45 | for ip in $ips; do 46 | if [[ $ip == *:* ]]; then 47 | ipv6=$ip 48 | else 49 | ipv4=$ip 50 | fi 51 | done 52 | 53 | local annotation="" 54 | if [[ -n $ipv4 && -n $ipv6 ]]; then 55 | annotation="ipv4:$ipv4,ipv6:$ipv6" 56 | elif [[ -n $ipv4 ]]; then 57 | annotation="ipv4:$ipv4" 58 | elif [[ -n $ipv6 ]]; then 59 | annotation="ipv6:$ipv6" 60 | fi 61 | 62 | kubectl annotate node "$node" gateway.submariner.io/public-ip="$annotation" 63 | done 64 | } 65 | 66 | function prepare_ocp() { 67 | source "${SCRIPTS_DIR}/lib/ocp_utils" 68 | local platform 69 | platform=$(determine_ocp_platform "$OCP_TEMPLATE_DIR") 70 | 71 | # In case of OpenStack, `cloud prepare` addresses it as `rhos`. 72 | [[ "$platform" != "openstack" ]] || platform=rhos 73 | 74 | subctl cloud prepare "$platform" --context "${cluster}" --ocp-metadata "${OUTPUT_DIR}/ocp-${cluster}/" 75 | with_retries 60 sleep_on_fail 5s check_gateway_exists 76 | } 77 | 78 | ### Main ### 79 | 80 | load_settings 81 | declare_kubeconfig 82 | [[ "${PROVIDER}" == "kind" ]] || "${SCRIPTS_DIR}/get-subctl.sh" 83 | 84 | # Run in subshell to check response, otherwise `set -e` is not honored 85 | ( run_all_clusters with_retries 3 cloud_prepare; ) & 86 | wait $! || exit_error "Failed to prepare cloud" 87 | 88 | -------------------------------------------------------------------------------- /scripts/shared/clusters.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -em -o pipefail 4 | 5 | source "${SCRIPTS_DIR}/lib/utils" 6 | print_env AIR_GAPPED K8S_VERSION NFTABLES OLM OLM_VERSION OVERLAPPING PARALLEL PROMETHEUS PROVIDER SETTINGS TIMEOUT 7 | source "${SCRIPTS_DIR}/lib/debug_functions" 8 | 9 | ### Functions ### 10 | 11 | function deploy_olm() { 12 | echo "Applying OLM CRDs..." 13 | kubectl apply -f "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml" --validate=false 14 | kubectl wait --for=condition=Established -f "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml" 15 | echo "Applying OLM resources..." 16 | kubectl apply -f "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml" 17 | 18 | echo "Waiting for olm-operator deployment to be ready..." 19 | kubectl rollout status deployment/olm-operator --namespace=olm --timeout="${TIMEOUT}" 20 | echo "Waiting for catalog-operator deployment to be ready..." 21 | kubectl rollout status deployment/catalog-operator --namespace=olm --timeout="${TIMEOUT}" 22 | echo "Waiting for packageserver deployment to be ready..." 23 | kubectl rollout status deployment/packageserver --namespace=olm --timeout="${TIMEOUT}" 24 | } 25 | 26 | function deploy_prometheus() { 27 | echo "Deploying Prometheus..." 28 | # TODO Install in a separate namespace 29 | kubectl create ns submariner-operator 30 | # Bundle from prometheus-operator, namespace changed to submariner-operator 31 | kubectl apply -f "${SCRIPTS_DIR}/resources/prometheus/bundle.yaml" 32 | kubectl apply -f "${SCRIPTS_DIR}/resources/prometheus/serviceaccount.yaml" 33 | kubectl apply -f "${SCRIPTS_DIR}/resources/prometheus/clusterrole.yaml" 34 | kubectl apply -f "${SCRIPTS_DIR}/resources/prometheus/clusterrolebinding.yaml" 35 | kubectl apply -f "${SCRIPTS_DIR}/resources/prometheus/prometheus.yaml" 36 | } 37 | 38 | function deploy_cluster_capabilities() { 39 | [[ "${OLM}" != "true" ]] || deploy_olm 40 | [[ "${PROMETHEUS}" != "true" ]] || deploy_prometheus 41 | } 42 | 43 | ### Main ### 44 | 45 | mkdir -p "${KUBECONFIGS_DIR}" 46 | 47 | load_settings 48 | declare_cidrs 49 | 50 | load_library clusters PROVIDER 51 | provider_prepare 52 | 53 | # Run in subshell to check response, otherwise `set -e` is not honored 54 | ( run_all_clusters with_retries 3 provider_create_cluster; ) & 55 | if ! wait $!; then 56 | run_if_defined provider_failed 57 | exit_error "Failed to create clusters using ${PROVIDER@Q}." 58 | fi 59 | 60 | declare_kubeconfig 61 | run_if_defined provider_succeeded 62 | run_all_clusters deploy_cluster_capabilities 63 | print_clusters_message 64 | -------------------------------------------------------------------------------- /scripts/shared/compile.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | source "${SCRIPTS_DIR}/lib/utils" 5 | 6 | [[ -n "${BUILD_UPX}" ]] || BUILD_UPX=true 7 | 8 | ## Process command line arguments ## 9 | 10 | [[ $# -eq 2 ]] || exit_error "The binary and source must be specified!" 11 | 12 | binary=$1 13 | source_file=$2 14 | 15 | set -e 16 | 17 | print_env BUILD_DEBUG BUILD_UPX LDFLAGS 18 | source "${SCRIPTS_DIR}/lib/debug_functions" 19 | 20 | ### Functions ### 21 | 22 | # Determine GOARCH based on the last component of the target directory, if any 23 | function determine_goarch() { 24 | GOARCH="$(dirname "${binary}")" 25 | [[ "${GOARCH}" != '.' ]] || { unset GOARCH && return 0; } 26 | 27 | # Convert from Docker arch to Go arch 28 | GOARCH="${GOARCH/arm\/v7/arm}" 29 | export GOARCH="${GOARCH##*/}" 30 | } 31 | 32 | ## Main ## 33 | 34 | [[ -n "${GOARCH}" ]] || determine_goarch 35 | mkdir -p "${binary%/*}" 36 | 37 | echo "Building ${binary@Q} (LDFLAGS: ${LDFLAGS@Q})" 38 | [[ "$BUILD_DEBUG" == "true" ]] || LDFLAGS="-s -w ${LDFLAGS}" 39 | 40 | CGO_ENABLED=0 ${GO:-go} build -trimpath -ldflags "${LDFLAGS}" -o "$binary" "$source_file" 41 | [[ "$BUILD_UPX" != "true" ]] || [[ "$BUILD_DEBUG" == "true" ]] || upx "$binary" 42 | -------------------------------------------------------------------------------- /scripts/shared/dnf_install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Installs packages using dnf to a named root: 4 | # -a arch - use arch instead of the native arch 5 | # -k - keep the package cache 6 | # -r root - install to the named root instead of /output/base 7 | # -v ver - use the given Fedora version (required) 8 | # 9 | # %arch in the package references will be replaced with the chosen arch 10 | 11 | set -e 12 | 13 | INSTALL_ROOT=/output/base 14 | 15 | # Limit the number of files so that dnf doesn't spend ages processing fds 16 | if [[ $(ulimit -n) -gt 1048576 ]]; then 17 | ulimit -n 1048576 18 | fi 19 | 20 | while getopts a:kr:v: o 21 | do 22 | case "$o" in 23 | a) 24 | ARCH="$OPTARG" 25 | ;; 26 | k) 27 | KEEP_CACHE=true 28 | ;; 29 | r) 30 | INSTALL_ROOT="$OPTARG" 31 | ;; 32 | v) 33 | FEDORA_VERSION="$OPTARG" 34 | ;; 35 | *) 36 | echo "$0 doesn't support $o" >&2 37 | exit 1 38 | ;; 39 | esac 40 | done 41 | shift $((OPTIND - 1)) 42 | 43 | arch_args=() 44 | 45 | if [[ -n "${ARCH}" ]]; then 46 | # Convert container arch to Fedora arch 47 | ARCH="${ARCH##*/}" 48 | case "${ARCH}" in 49 | amd64) ARCH=x86_64;; 50 | arm64) ARCH=aarch64;; 51 | esac 52 | arch_args=(--forcearch "${ARCH}") 53 | else 54 | # This will be used later, but we won't force 55 | ARCH="$(rpm -q --qf "%{arch}" rpm)" 56 | fi 57 | 58 | [[ -z "${FEDORA_VERSION}" ]] && echo I need to know which version of Fedora to install, specify it with -v >&2 && exit 1 59 | 60 | if [[ "${INSTALL_ROOT}" != /output/base ]] && [[ ! -d "${INSTALL_ROOT}" ]] && [[ -d /output/base ]]; then 61 | cp -a /output/base "${INSTALL_ROOT}" 62 | fi 63 | 64 | dnf -y --setopt=install_weak_deps=0 --nodocs --use-host-config "${arch_args[@]}" \ 65 | --installroot "${INSTALL_ROOT}" --releasever "${FEDORA_VERSION}" \ 66 | install "${@//\%arch/${ARCH}}" 67 | 68 | [[ "${KEEP_CACHE}" == true ]] || dnf -y "${arch_args[@]}" --installroot "${INSTALL_ROOT}" --releasever "${FEDORA_VERSION}" clean all 69 | -------------------------------------------------------------------------------- /scripts/shared/e2e.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -em -o pipefail 4 | source "${SCRIPTS_DIR}/lib/utils" 5 | 6 | print_env LAZY_DEPLOY SUBCTL_VERIFICATIONS TEST_ARGS TESTDIR 7 | source "${SCRIPTS_DIR}/lib/debug_functions" 8 | source "${SCRIPTS_DIR}/lib/deploy_funcs" 9 | 10 | ### Functions ### 11 | 12 | function deploy_env_once() { 13 | if with_context "${clusters[0]}" kubectl wait --for=condition=Ready pods -l app=submariner-gateway -n "${SUBM_NS}" --timeout=3s > /dev/null 2>&1; then 14 | echo "Submariner already deployed, skipping deployment..." 15 | return 16 | fi 17 | 18 | make deploy 19 | declare_kubeconfig 20 | } 21 | 22 | function join_by { local IFS="$1"; shift; echo "$*"; } 23 | 24 | function generate_kubecontexts() { 25 | join_by , "${clusters[@]}" 26 | } 27 | 28 | function test_with_e2e_tests { 29 | local extra_flags=() 30 | 31 | cd "${DAPPER_SOURCE}/${TESTDIR}" 32 | 33 | [[ "$AIR_GAPPED" = true ]] && extra_flags+=(-nettest-image "${SUBM_IMAGE_REPO}/nettest:${SUBM_IMAGE_TAG}") 34 | 35 | # shellcheck disable=SC2086 # TEST_ARGS is split on purpose 36 | "${GO:-go}" test -v -timeout 30m -args -test.timeout 15m \ 37 | -submariner-namespace "$SUBM_NS" "${clusters[@]/#/-dp-context=}" \ 38 | "${extra_flags[@]}" \ 39 | --ginkgo.v --ginkgo.randomize-all --ginkgo.trace \ 40 | --ginkgo.junit-report "${DAPPER_OUTPUT}/e2e-junit.xml" \ 41 | $TEST_ARGS 2>&1 | tee "${DAPPER_OUTPUT}/e2e-tests.log" 42 | } 43 | 44 | function test_with_subctl { 45 | subctl verify --only "${SUBCTL_VERIFICATIONS}" --context "${clusters[0]}" --tocontext "${clusters[1]}" 46 | } 47 | 48 | function count_nodes() { 49 | wc -w <<< "${cluster_nodes[${clusters[$1]}]}" 50 | } 51 | 52 | # Make sure the biggest cluster is always first, as some tests rely on having a big first cluster. 53 | function order_clusters { 54 | local biggest_cluster=0 55 | for i in "${!clusters[@]}"; do 56 | if [[ $(count_nodes "$i") -gt $(count_nodes "${biggest_cluster}") ]]; then 57 | biggest_cluster="$i" 58 | fi 59 | done 60 | 61 | local orig_cluster="${clusters[0]}" 62 | clusters[0]="${clusters[$biggest_cluster]}" 63 | clusters[biggest_cluster]="${orig_cluster}" 64 | } 65 | 66 | ### Main ### 67 | 68 | load_settings 69 | order_clusters 70 | declare_kubeconfig 71 | [[ "${LAZY_DEPLOY}" != "true" ]] || deploy_env_once 72 | 73 | if [ -d "${DAPPER_SOURCE}/${TESTDIR}" ]; then 74 | test_with_e2e_tests 75 | else 76 | test_with_subctl 77 | fi 78 | 79 | print_clusters_message 80 | -------------------------------------------------------------------------------- /scripts/shared/entry: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | source "${SCRIPTS_DIR}/lib/debug_functions" 5 | 6 | # Change ownership of files created inside the container 7 | trap 'find . -writable -newercc /proc -execdir chown $DAPPER_UID:$DAPPER_GID {} +' exit 8 | 9 | mkdir -p bin dist output 10 | 11 | if [ -e "./scripts/$1" ]; then 12 | command="./scripts/$1" 13 | else 14 | command="$1" 15 | fi 16 | 17 | # This ends the GHA group that started in Makefile.dapper 18 | [[ -z "$CI" ]] || echo "::endgroup::" 19 | shift 20 | "$command" "$@" 21 | -------------------------------------------------------------------------------- /scripts/shared/filter_dependabot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | # Copyright Contributors to the Submariner project. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | # Adds ignore statements to .github/dependabot.yml for direct dependencies 20 | # already present in parent Submariner projects (i.e. projects which the 21 | # processed project depends on). 22 | # 23 | # Existing entries are removed, starting with the first entry marked 24 | # "Our own dependencies are handled during releases" (if none, all 25 | # entries are removed). This tool assumes that all Submariner projects 26 | # are checked out alongside each other (i.e. ../shipyard contains 27 | # Shipyard, ../admiral contains Admiral etc.). 28 | # 29 | # By default, the default branch is processed; specify another branch 30 | # as argument to handle that instead. 31 | 32 | declare -a seendeps 33 | 34 | function depseen() { 35 | local dep 36 | for dep in "${seendeps[@]}"; do 37 | if [ "$dep" = "$1" ]; then 38 | return 0 39 | fi 40 | done 41 | return 1 42 | } 43 | 44 | base="$(pwd)" 45 | conffile="${base}/.github/dependabot.yml" 46 | 47 | branch="${1:-null}" 48 | 49 | gomodfilter=".updates[] | select(.package-ecosystem == \"gomod\")" 50 | 51 | for dir in $(yq "(${gomodfilter}).directory" "$conffile"); do 52 | 53 | dirfilter="${gomodfilter} | select(.directory == \"${dir}\")" 54 | 55 | # Looping over branch is pointless since dependencies need to track 56 | # (i.e. other repos need to be in the same branch) 57 | 58 | if [ "$branch" = "null" ]; then 59 | branchfilter="${dirfilter} | select(has(\"target-branch\") | not)" 60 | else 61 | branchfilter="${dirfilter} | select(.target-branch == \"$branch\")" 62 | fi 63 | 64 | (cd "${base}${dir}" || exit 65 | 66 | # Count the ignores 67 | ignores="$(yq "(${branchfilter}).ignore | length" "$conffile")" 68 | firstauto=0 69 | 70 | # Look for the start of automated ignores 71 | for (( i=0; i < ignores; i++ )); do 72 | if [ "$(yq "(${branchfilter}).ignore[$i].dependency-name" "$conffile")" = "github.com/submariner-io/*" ]; then 73 | firstauto=$i 74 | break 75 | fi 76 | done 77 | 78 | # Remove the existing automated ignores 79 | while [ "$(yq "(${branchfilter}).ignore | length" "$conffile")" -gt "$firstauto" ]; do 80 | yq -i -P "del(${branchfilter}.ignore[$firstauto])" "$conffile" 81 | done 82 | 83 | # "See" remaining ignores 84 | read -ar seendeps < <(yq "(${branchfilter}).ignore[].dependency-name" "$conffile") 85 | 86 | # Restore the submariner-io exclusion 87 | yq -i -P "(${branchfilter}).ignore[$firstauto].dependency-name = \"github.com/submariner-io/*\"" "$conffile" 88 | yq -i -P "(${branchfilter}).ignore[$firstauto] head_comment = \"Our own dependencies are handled during releases\"" "$conffile" 89 | 90 | # Ignore all parent dependencies 91 | for parent in $(GOWORK=off go list -m -mod=mod -json all | jq -r 'select(.Path | contains("/submariner-io/")) | select(.Main != true) .Path | gsub("github.com/submariner-io/"; "")'); do 92 | first=true 93 | for dep in $(GOWORK=off go list -m -mod=mod -json all | jq -r 'select(.Path | contains("/submariner-io") | not) | select(.Indirect != true) | select(.Main != true) .Path'); do 94 | if ! depseen "$dep"; then 95 | if grep -q "$dep" "$base/../$parent/go.mod" && ! grep -q "$dep .*// indirect" "$base/../$parent/go.mod"; then 96 | yq -i -P "(${branchfilter}).ignore += { \"dependency-name\": \"$dep\" }" "$conffile" 97 | if $first; then 98 | yq -i -P "with(${branchfilter}; .ignore[.ignore | length - 1] head_comment = \"Managed in $parent\")" "$conffile" 99 | first=false 100 | fi 101 | seendeps+=("$dep") 102 | fi 103 | fi 104 | done 105 | done) 106 | 107 | done 108 | -------------------------------------------------------------------------------- /scripts/shared/gen-codeowners: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from fnmatch import fnmatch 4 | 5 | with open('CODEOWNERS.in', 'r') as ins, open('CODEOWNERS', 'w') as out: 6 | owners_by_path = {} 7 | for line in ins: 8 | if not line.startswith('#'): 9 | comps = line.split() 10 | if len(comps) > 1: 11 | for path in comps[1:]: 12 | owners_by_path.setdefault(path, set()).add(comps[0]) 13 | print('# Auto-generated, do not edit; see CODEOWNERS.in', file = out) 14 | for path in sorted(owners_by_path, key = str.lower): 15 | owners = owners_by_path[path] 16 | for extra_path in owners_by_path.keys(): 17 | if extra_path != path and fnmatch(path, extra_path): 18 | owners |= owners_by_path[extra_path] 19 | print('{0} {1}'.format(path, ' '.join(sorted(owners, key = str.lower))), file = out) 20 | -------------------------------------------------------------------------------- /scripts/shared/get-subctl.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e -o pipefail 4 | source "${SCRIPTS_DIR}/lib/utils" 5 | 6 | # In case we're pretending to be `subctl` 7 | if [[ "${0##*/}" = subctl ]] && [[ -L "$0" ]]; then 8 | run_subctl=true 9 | 10 | # Delete ourselves to ensure we don't run into issues with the new subctl 11 | rm -f "$0" 12 | fi 13 | 14 | # Default to devel if we don't know what base branch were on 15 | with_retries 3 curl -Lsf https://get.submariner.io | VERSION="${SUBCTL_VERSION:-${BASE_BRANCH:-devel}}" bash 16 | 17 | # If we're pretending to be subctl, run subctl with any given arguments 18 | [[ -z "${run_subctl}" ]] || subctl "$@" 19 | -------------------------------------------------------------------------------- /scripts/shared/kubeps1.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | PS1='[\u@\h \W $(kube_ps1)]\$ ' 3 | -------------------------------------------------------------------------------- /scripts/shared/lib/cleanup_acm: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | . "${SCRIPTS_DIR}/lib/ocp_utils" 3 | 4 | ### Functions ### 5 | 6 | function provider_initialize() { 7 | readarray -t clusters < <( 8 | find "${OUTPUT_DIR}" -type f -name metadata.json -exec sh -c 'basename $(dirname "$1")' shell {} \; | grep -o '^acm-.*$') 9 | [[ "${#clusters[@]}" -gt 0 ]] || { echo "No ACM clusters found." && return 0; } 10 | ensure_openshift_install 11 | } 12 | 13 | function provider_delete_cluster() { 14 | "${OCP_INSTALLER}" destroy cluster --dir="${OUTPUT_DIR}/${cluster}" 15 | } 16 | 17 | function provider_finalize() { 18 | \rm -rf "${OUTPUT_DIR}"/acm-* "${KUBECONFIGS_DIR}"/acm-* 19 | } 20 | -------------------------------------------------------------------------------- /scripts/shared/lib/cleanup_kind: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | 3 | ### Functions ### 4 | 5 | function provider_initialize() { 6 | # shellcheck disable=SC2034 7 | readarray -t clusters < <(kind get clusters) 8 | 9 | # kind cleanup uses a lock but doesn't retry 10 | # shellcheck disable=SC2034 11 | PARALLEL=false 12 | } 13 | 14 | function provider_delete_cluster() { 15 | kind delete cluster --name="${cluster}" 16 | } 17 | 18 | function provider_finalize { 19 | if registry_running; then 20 | echo "Stopping local KIND registry..." 21 | docker stop "$KIND_REGISTRY" 22 | fi 23 | 24 | docker system prune --volumes -f 25 | rm -f "${KUBECONFIGS_DIR}"/kind-config-* 26 | } 27 | -------------------------------------------------------------------------------- /scripts/shared/lib/cleanup_ocp: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | . "${SCRIPTS_DIR}/lib/ocp_utils" 3 | 4 | ### Functions ### 5 | 6 | function provider_initialize() { 7 | readarray -t clusters < <( 8 | find "${OUTPUT_DIR}" -type f -name metadata.json -exec sh -c 'basename $(dirname "$1")' shell {} \; | grep -o '^ocp-.*$') 9 | [[ "${#clusters[@]}" -gt 0 ]] || { echo "No OCP clusters found." && return 0; } 10 | ensure_openshift_install 11 | } 12 | 13 | function provider_delete_cluster() { 14 | "${OCP_INSTALLER}" destroy cluster --dir="${OUTPUT_DIR}/${cluster}" 15 | } 16 | 17 | function provider_finalize() { 18 | \rm -rf "${OUTPUT_DIR}"/ocp-* "${KUBECONFIGS_DIR}"/ocp-* 19 | } 20 | -------------------------------------------------------------------------------- /scripts/shared/lib/clusters_acm: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | 3 | source "${SCRIPTS_DIR}/lib/clusters_ocp" 4 | 5 | ### Functions ### 6 | 7 | function provider_prepare() { 8 | ensure_openshift_install 9 | create_ocp_template "$ACM_TEMPLATE_DIR" ACM 10 | } 11 | 12 | function provider_create_cluster() { 13 | export KUBECONFIG="${KUBECONFIGS_DIR}/acm-${cluster}" 14 | 15 | if kubectl cluster-info > /dev/null 2>&1; then 16 | echo "ACM cluster already exists, skipping its creation..." 17 | return 0 18 | fi 19 | 20 | configure_ocp_cluster "$ACM_TEMPLATE_DIR" 21 | [[ "$cluster" != "$broker" ]] || adjust_hub_deploy 22 | deploy_ocp_cluster "$ACM_TEMPLATE_DIR" ACM 23 | } 24 | 25 | function adjust_hub_deploy() { 26 | local install_dir=${ACM_TEMPLATE_DIR/template/${cluster}} 27 | case $(determine_ocp_platform "$install_dir") in 28 | aws) 29 | yq -i ".compute[0].platform.aws.type=\"m5.4xlarge\"" "${install_dir}/install-config.yaml" 30 | ;; 31 | gcp) 32 | yq -i ".compute[0].platform.gcp.type=\"n1-standard-8\"" "${install_dir}/install-config.yaml" 33 | ;; 34 | esac 35 | } 36 | 37 | function provider_succeeded() { 38 | local cluster=$broker 39 | local install_dir=${template_dir/template/${cluster}} 40 | 41 | echo "Logging in to quay.io to get a token for deploying ACM, make sure your account has the necessary permissions." 42 | echo "(https://github.com/stolostron/deploy#prepare-to-deploy-open-cluster-management-instance-only-do-once)" 43 | docker login quay.io 44 | QUAY_TOKEN=$(jq -r '.auths |= {"quay.io"}' ~/.docker/config.json | base64 -w 0) 45 | export QUAY_TOKEN 46 | 47 | ensure_openshift_cli 48 | git clone https://github.com/stolostron/deploy /tmp/acm_deploy 49 | cd /tmp/acm_deploy || exit 1 50 | 51 | local args=(--watch) 52 | if [[ -n "$ACM_VERSION" ]]; then 53 | echo "$ACM_VERSION" > snapshot.ver 54 | args+=(--silent) 55 | fi 56 | 57 | # Make sure to set default kubeconfig context, as ACM scripts use `oc` which determines the context from these settings 58 | kubectl config use-context "$broker" 59 | if ! ./start.sh "${args[@]}"; then 60 | echo "Failed to deploy ACM, cleaning up" 61 | ./clean-clusters.sh <<< $'DESTROY\n' 62 | return 1 63 | fi 64 | 65 | PARALLEL=false run_all_clusters add_cluster_to_acm 66 | } 67 | 68 | function add_cluster_to_acm() { 69 | [[ "$cluster" != "$broker" ]] || return 0 70 | local managed_cluster=$cluster 71 | local cluster=$broker 72 | 73 | # Make sure to set default kubeconfig context, otherwise ACM will fail to auto-import the cluster 74 | kubectl config use-context "$managed_cluster" 75 | kubectl create namespace "$managed_cluster" 76 | kubectl apply -f - < <(yq ".metadata.name = \"${managed_cluster}\"" "$RESOURCES_DIR"/acm-managed-cluster.yml) 77 | kubectl apply -f - < <( 78 | o=$(cat "${KUBECONFIGS_DIR}/acm-${managed_cluster}") \ 79 | yq ".metadata.namespace = \"${managed_cluster}\" | .stringData.kubeconfig = strenv(o)" "$RESOURCES_DIR"/acm-auto-import-secret.yml) 80 | } 81 | -------------------------------------------------------------------------------- /scripts/shared/lib/clusters_ocp: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | 3 | . "${SCRIPTS_DIR}/lib/ocp_utils" 4 | 5 | ### Functions ### 6 | 7 | function provider_prepare() { 8 | ensure_openshift_install 9 | create_ocp_template "$OCP_TEMPLATE_DIR" OCP 10 | } 11 | 12 | # Creates template for installation, which can be used for deploying platforms on top of an OCP install. 13 | function create_ocp_template() { 14 | local directory=$1 15 | local type=$2 16 | 17 | if [[ -d "$directory" ]]; then 18 | echo "Reusing existing template for ${type} install." 19 | echo "If you'd like to reinstall, please run 'make clean using=${PROVIDER}' and try again." 20 | return 0 21 | fi 22 | 23 | rm -rf "$directory" 24 | echo "Creating an initial configuration template for ${type}." 25 | echo "Please fill out all the necessary details." 26 | echo "Note: The cluster name will be suffixed (eg with '-cluster1'), please fill out just the prefix." 27 | "$OCP_INSTALLER" create install-config --dir "$directory" 28 | } 29 | 30 | function provider_create_cluster() { 31 | export KUBECONFIG=${KUBECONFIGS_DIR}/ocp-${cluster} 32 | 33 | if kubectl cluster-info > /dev/null 2>&1; then 34 | echo "OCP cluster already exists, skipping its creation..." 35 | return 0 36 | fi 37 | 38 | configure_ocp_cluster "$OCP_TEMPLATE_DIR" 39 | deploy_ocp_cluster "$OCP_TEMPLATE_DIR" OCP 40 | } 41 | 42 | function configure_ocp_cluster() { 43 | local template_dir=$1 44 | local install_dir=${template_dir/template/${cluster}} 45 | local install_config=${install_dir}/install-config.yaml 46 | local control_replicas compute_replicas 47 | control_replicas=$(echo "${cluster_nodes[${cluster}]}" | tr ' ' '\n' | grep -wc 'control-plane') 48 | compute_replicas=$(echo "${cluster_nodes[${cluster}]}" | tr ' ' '\n' | grep -wc 'worker') 49 | 50 | # OCP needs at least 1 compute node to work properly 51 | [[ "$compute_replicas" -gt 0 ]] || compute_replicas=1 52 | 53 | rm -rf "$install_dir" 54 | cp -r "$template_dir" "$install_dir" 55 | yq -i ".metadata.name += \"-${cluster}\"" "$install_config" 56 | yq -i ".networking.clusterNetwork[0].cidr = \"${cluster_CIDRs[${cluster}]}\"" "$install_config" 57 | yq -i ".networking.serviceNetwork[0] = \"${service_CIDRs[${cluster}]}\"" "$install_config" 58 | yq -i ".controlPlane.replicas = ${control_replicas}" "$install_config" 59 | yq -i ".compute[0].replicas = ${compute_replicas}" "$install_config" 60 | 61 | echo "Updated install config:" 62 | yq "del(.pullSecret)" "$install_config" 63 | } 64 | 65 | function deploy_ocp_cluster() { 66 | local template_dir=$1 67 | local type=$2 68 | local install_dir=${template_dir/template/${cluster}} 69 | 70 | rm -f "$KUBECONFIG" 71 | echo "Creating ${type} cluster..." 72 | 73 | if ! "$OCP_INSTALLER" create cluster --dir "$install_dir"; then 74 | echo "Failed to create ${type} cluster, removing the cluster" 75 | "$OCP_INSTALLER" destroy cluster --dir "$install_dir" 76 | return 1 77 | fi 78 | 79 | local kubeconfig=${install_dir}/auth/kubeconfig 80 | yq -i "(.. | select(. == \"admin\")) = \"${cluster}\"" "$kubeconfig" 81 | cp "$kubeconfig" "$KUBECONFIG" 82 | } 83 | -------------------------------------------------------------------------------- /scripts/shared/lib/debug_functions: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | ### Constants ### 3 | 4 | export CYAN_COLOR NO_COLOR 5 | CYAN_COLOR=$(echo -e '\e[36m') 6 | NO_COLOR=$(echo -e '\e[0m') 7 | 8 | ### Functions ### 9 | 10 | # Function to print each bash command before it is executed. 11 | # Only outputs when $DEBUG_PRINT is set, to control printing precisely. 12 | # Skips common command (printing, control structures, etc). 13 | # Skips parsing `$()` as it causes the internal command to be executed twice. 14 | # Skips parsing `=(` as it causes an error for eval. 15 | # Skips printing function name twice (caveat - if we ever do a recursion, we won't see it). 16 | function trap_commands() { 17 | trap '[[ "${DEBUG_PRINT}" == true ]] && 18 | declare -g cmd="$BASH_COMMAND" cur_func="${FUNCNAME[0]}" && 19 | ! [[ "$cmd" =~ ^(echo|read|\[|while|for|local|printf) ]] && 20 | { [[ "$cmd" =~ =\$?\( ]] && cmd="${cmd@Q}" || cmd=$(eval echo "$cmd"); } && 21 | [[ -n "$cmd" && "${cmd%% *}" != "$cur_func" ]] && 22 | ctxt="[$(date +%R:%S.%3N)] [dir=${PWD##*/}${cluster:+; cl=${cluster}}${cur_func:+; fn=${cur_func}}]" && 23 | echo "${CYAN_COLOR}${ctxt}\$ ${cmd}${NO_COLOR}" >&2; true' DEBUG 24 | } 25 | 26 | ### Main ### 27 | 28 | set -T 29 | trap_commands 30 | -------------------------------------------------------------------------------- /scripts/shared/lib/deploy_bundle: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | # shellcheck source=scripts/shared/lib/source_only 3 | . "${BASH_SOURCE%/*}"/source_only 4 | 5 | ### Constants ### 6 | readonly SUBM_GROUP="submariner-group" 7 | readonly SUBM_SUB="submariner-subscription" 8 | readonly SUBM_OPERATOR_NAME="submariner" 9 | 10 | # Variables 11 | 12 | BROKER_K8S_API_SERVER="" 13 | BROKER_K8S_API_SERVER_TOKEN="" 14 | BROKER_K8S_CA="" 15 | 16 | ### Functions ### 17 | 18 | function deploytool_prereqs() { 19 | expect_env SUBM_NS 20 | expect_env SUBM_CS 21 | expect_env SUBM_INDEX_IMG 22 | 23 | # Create new namespace 24 | run_subm_clusters "create_namespace ${SUBM_NS}" 25 | 26 | # Create the custom catalog source 27 | run_subm_clusters "create_catalog_source ${SUBM_CS} ${SUBM_NS} ${SUBM_INDEX_IMG}" 28 | 29 | # Create the operator group 30 | run_subm_clusters "create_operator_group ${SUBM_GROUP} ${SUBM_NS}" 31 | 32 | # Install the submariner operator bundle 33 | run_subm_clusters "install_bundle ${SUBM_SUB} ${SUBM_CS} ${SUBM_NS} ${SUBM_OPERATOR_NAME}" 34 | } 35 | 36 | function setup_broker() { 37 | local brokerClientSecret 38 | 39 | if ! (timeout 5m bash -c "until kubectl --context=${cluster} get crds brokers.submariner.io > /dev/null 2>&1; do sleep 10; done"); then 40 | exit_error "Broker CRD was not found." 41 | fi 42 | 43 | # Create the broker Namespace & RBAC 44 | echo "[INFO] Deploy the broker" 45 | if [ -d "${DAPPER_SOURCE}"/config/broker ]; then 46 | kubectl apply -k "${DAPPER_SOURCE}"/config/broker -n "${BROKER_NAMESPACE}" 47 | else 48 | [ -d /tmp/submariner-operator ] && rm -Rf /tmp/submariner-operator 49 | git clone --depth 1 --single-branch --branch "${BASE_BRANCH}" https://github.com/submariner-io/submariner-operator /tmp/submariner-operator 50 | kubectl apply -k /tmp/submariner-operator/config/broker -n "${BROKER_NAMESPACE}" 51 | fi 52 | 53 | # Enable the service-discovery component if defined 54 | local components="connectivity" 55 | components+=$([[ "${LIGHTHOUSE}" == "true" ]] && echo ", service-discovery" || echo "") 56 | components+=$([[ "${OVERLAPPING}" == "true" ]] && echo ", globalnet" || echo "") 57 | 58 | ### Create the Broker instance 59 | render_template "${RESOURCES_DIR}"/bundle/broker.yaml | kubectl apply -f - 60 | 61 | echo "[INFO] Wait for the broker readiness..." 62 | if ! (timeout 5m bash -c "until kubectl --context=${cluster} get brokers.submariner.io submariner-broker -n ${SUBM_NS} > /dev/null 2>&1; do sleep 10; done"); then 63 | exit_error "Broker is not ready." 64 | fi 65 | 66 | brokerClientSecret=$(kubectl -n "${BROKER_NAMESPACE}" get secrets -o json | jq -r -c '[.items[] | select(.metadata.annotations."kubernetes.io/service-account.name"=="'"${BROKER_CLIENT_SA}"'") | select(.data.token != null) | select(.data."ca.crt" != null)] | .[0]') 67 | # shellcheck disable=SC2034 # this variable is used elsewhere 68 | BROKER_K8S_API_SERVER=$(kubectl get endpoints kubernetes -n default -o jsonpath="{.subsets[0].addresses[0].ip}:{.subsets[0].ports[?(@.name=='https')].port}") 69 | # shellcheck disable=SC2034 # this variable is used elsewhere 70 | BROKER_K8S_API_SERVER_TOKEN=$(echo "${brokerClientSecret}" | jq -r '.data.token' | base64 --decode) 71 | # shellcheck disable=SC2034 # this variable is used elsewhere 72 | BROKER_K8S_CA=$(echo "${brokerClientSecret}" | jq -r '.data."ca.crt"') 73 | 74 | } 75 | 76 | function install_subm() { 77 | if [[ ${cluster_subm[$cluster]} != "true" ]]; then 78 | echo "Skipping installation as requested in cluster settings" 79 | return 80 | fi 81 | 82 | if kubectl wait --for=condition=Ready pods -l app=submariner-operator -n "${SUBM_NS}" --timeout=60s > /dev/null 2>&1; then 83 | echo "[WARN](${cluster}) Submariner already installed, skipping installation..." 84 | return 85 | fi 86 | 87 | if ! (timeout 5m bash -c "until kubectl --context=${cluster} get crds submariners.submariner.io > /dev/null 2>&1; do sleep 10; done"); then 88 | exit_error "Submariner CRD was not found." 89 | fi 90 | 91 | # Create the Submariner instance 92 | echo "[INFO](${cluster}) Deploy Submariner" 93 | render_template "${RESOURCES_DIR}"/bundle/submariner.yaml | kubectl apply -f - 94 | 95 | echo "[INFO](${cluster}) Submariner deployed" 96 | } 97 | 98 | function install_subm_all_clusters() { 99 | run_subm_clusters install_subm 100 | } 101 | -------------------------------------------------------------------------------- /scripts/shared/lib/deploy_funcs: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | # shellcheck source=scripts/shared/lib/source_only 3 | . "${BASH_SOURCE%/*}"/source_only 4 | 5 | ### Global Variables ### 6 | 7 | # shellcheck disable=SC2034 8 | SUBM_IMAGE_REPO=localhost:5000 9 | # shellcheck disable=SC2034 10 | SUBM_IMAGE_TAG="${IMAGE_TAG}" 11 | 12 | ### Functions ### 13 | 14 | function import_image() { 15 | local orig_image=$1 16 | local versioned_image="$1:${DEV_VERSION}" 17 | local local_image="localhost:5000/${orig_image##*/}:${2:-local}" 18 | if ! docker tag "${versioned_image}" "${local_image}"; then 19 | # The project doesn't build this image, pull it 20 | docker pull "${orig_image}:${CUTTING_EDGE}" 21 | docker tag "${orig_image}:${CUTTING_EDGE}" "${versioned_image}" 22 | docker tag "${versioned_image}" "${local_image}" 23 | fi 24 | 25 | docker push "${local_image}" 26 | } 27 | 28 | function get_globalip() { 29 | local svc_name=$1 30 | local gip 31 | gip=$(kubectl get svc "$svc_name" -o jsonpath='{.metadata.annotations.submariner\.io/globalIp}') 32 | if [[ -z "${gip}" ]]; then 33 | gip=$(kubectl get giip "$svc_name" -o jsonpath='{.status.allocatedIP}') 34 | if [[ -z "${gip}" ]]; then 35 | sleep 1 36 | return 1 37 | fi 38 | fi 39 | 40 | echo "$gip" 41 | } 42 | 43 | function get_svc_ip() { 44 | local svc_name=$1 45 | local svc_ip 46 | 47 | if [[ "${OVERLAPPING}" = "true" ]]; then 48 | svc_ip=$(with_retries 30 get_globalip "${svc_name}") 49 | else 50 | svc_ip=$(kubectl --context="$cluster" get svc -l "app=${svc_name}" | awk 'FNR == 2 {print $3}') 51 | fi 52 | 53 | [[ -n "$svc_ip" ]] || exit_error "Failed to get ${svc_name} IP" 54 | echo "$svc_ip" 55 | } 56 | 57 | function test_connection() { 58 | local source_pod=$1 59 | local target_address=$2 60 | 61 | echo "Attempting connectivity between clusters - $source_pod --> $target_address" 62 | kubectl exec "${source_pod}" -- curl --output /dev/null -m 10 --silent --head --fail "${target_address}" 63 | echo "Connection test was successful!" 64 | } 65 | 66 | function deploy_demo() { 67 | local target_cluster 68 | target_cluster="$1" 69 | deploy_resource "${RESOURCES_DIR}/netshoot.yaml" 70 | with_context "$target_cluster" deploy_resource "${RESOURCES_DIR}/nginx-demo.yaml" 71 | } 72 | 73 | function clean_demo() { 74 | local target_cluster 75 | target_cluster="$1" 76 | remove_resource "${RESOURCES_DIR}/netshoot.yaml" 77 | with_context "$target_cluster" remove_resource "${RESOURCES_DIR}/nginx-demo.yaml" 78 | } 79 | 80 | function connectivity_tests() { 81 | target_cluster="$1" 82 | 83 | deploy_demo "$target_cluster" 84 | 85 | local netshoot_pod 86 | netshoot_pod=$(kubectl get pods -l app=netshoot | awk 'FNR == 2 {print $1}') 87 | local nginx_svc=nginx-demo.default.svc.clusterset.local 88 | [[ "$LIGHTHOUSE" = true ]] || nginx_svc=$(with_context "$target_cluster" get_svc_ip nginx-demo) 89 | 90 | with_retries 10 sleep_on_fail 10s test_connection "$netshoot_pod" "$nginx_svc" 91 | 92 | clean_demo "$target_cluster" 93 | } 94 | 95 | function verify_gw_status() { 96 | sleep_duration=8 97 | # helm doesn't use the operator yet, and connection status is based on the operator object 98 | if subctl show connections 2>&1 | grep "the server could not find the requested resource"; then 99 | return 0 100 | fi 101 | 102 | if ! subctl show connections | grep "connected"; then 103 | echo "iter: $iteration. Clusters not yet connected. sleeping for $sleep_duration secs" 104 | sleep $sleep_duration 105 | else 106 | return 0 107 | fi 108 | # Before returning, show the subctl output 109 | subctl show connections 110 | return 1 111 | } 112 | 113 | function deploy_resource() { 114 | local resource_file=$1 115 | local ns=${2:-default} 116 | local resource_name resource 117 | resource_name=$(basename "$resource_file" ".yaml") 118 | render_template "${resource_file}" | kubectl apply -f - 119 | 120 | for kind in Deployment DaemonSet; do 121 | resource=$(yq e ".metadata.name == \"${resource_name}\" | parent | parent | select(.kind == \"${kind}\")" "$resource_file") 122 | [[ -n "$resource" ]] || continue 123 | 124 | echo "Waiting for ${kind} ${resource_name} to be ready." 125 | kubectl rollout status -n "$ns" "${kind,,}/${resource_name}" --timeout="${TIMEOUT}" 126 | done 127 | } 128 | 129 | function remove_resource() { 130 | local resource_file=$1 131 | render_template "${resource_file}" | kubectl delete -f - 132 | } 133 | 134 | function find_submariner_namespace() { 135 | local namespace 136 | namespace="$(kubectl get pods --all-namespaces | awk '/submariner/{ print $1 }' | grep -v broker | head -n 1)" 137 | [[ -n "${namespace}" ]] || exit_error "Could not find a Submariner deployment namespace" 138 | echo "${namespace}" 139 | } 140 | 141 | function reload_pods() { 142 | local resource_type=$1 # the resource type can be deployment or daemonset 143 | local resource_name=$2 # the name of the resource 144 | local namespace 145 | namespace="$(find_submariner_namespace)" 146 | 147 | kubectl patch -n "${namespace}" "${resource_type}" "${resource_name}" \ 148 | --type='json' \ 149 | -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/imagePullPolicy", "value": "Always" },{"op": "replace", "path": "/spec/template/metadata/labels/modified", "value": "'"$(date +%s)"'"}]' 150 | } 151 | -------------------------------------------------------------------------------- /scripts/shared/lib/deploy_helm: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | # shellcheck source=scripts/shared/lib/source_only 3 | . "${BASH_SOURCE%/*}"/source_only 4 | 5 | # Allow overriding for using local charts from a directory 6 | HELM_REPO_LOCATION="${HELM_REPO_LOCATION:-submariner-latest}" 7 | 8 | ### Functions ### 9 | 10 | function deploytool_prereqs() { 11 | helm version 12 | helm repo add submariner-latest https://submariner-io.github.io/submariner-charts/charts 13 | } 14 | 15 | function setup_broker() { 16 | if kubectl get crd clusters.submariner.io > /dev/null 2>&1; then 17 | echo "Submariner CRDs already exist, skipping broker creation..." 18 | else 19 | echo "Installing submariner broker..." 20 | # shellcheck disable=SC2086 # Split on purpose 21 | helm install --debug --devel "${BROKER_NAMESPACE}" \ 22 | "${HELM_REPO_LOCATION}"/submariner-k8s-broker \ 23 | --create-namespace \ 24 | --kube-context "${cluster}" \ 25 | --namespace "${BROKER_NAMESPACE}" 26 | fi 27 | 28 | submariner_broker_url=$(kubectl -n default get endpoints kubernetes -o jsonpath="{.subsets[0].addresses[0].ip}:{.subsets[0].ports[?(@.name=='https')].port}") 29 | submariner_broker_ca=$(kubectl -n "${BROKER_NAMESPACE}" get secrets "${BROKER_CLIENT_SA}-token" -o jsonpath="{.data['ca\.crt']}") 30 | submariner_broker_token=$(kubectl -n "${BROKER_NAMESPACE}" get secrets "${BROKER_CLIENT_SA}-token" -o jsonpath="{.data.token}"|base64 --decode) 31 | } 32 | 33 | function helm_install_subm() { 34 | local crd_create=false 35 | [[ "${cluster}" = "${broker}" ]] || crd_create=true 36 | 37 | if kubectl wait --for=condition=Ready pods -l app=submariner-operator -n "${SUBM_NS}" --timeout=60s > /dev/null 2>&1; then 38 | echo "Submariner already installed, skipping installation..." 39 | return 40 | fi 41 | 42 | local extra_flags=() 43 | for image in ${PRELOAD_IMAGES}; do 44 | if [[ "${image}" = submariner-operator ]]; then 45 | extra_flags+=(--set operator.image.repository="${SUBM_IMAGE_REPO}/submariner-operator" \ 46 | --set operator.image.tag="${SUBM_IMAGE_TAG}" \ 47 | --set operator.image.pullPolicy="IfNotPresent") 48 | continue 49 | fi 50 | 51 | extra_flags+=(--set "images.${image}=${SUBM_IMAGE_REPO}/${image}:${SUBM_IMAGE_TAG}") 52 | done 53 | 54 | [[ "$LOAD_BALANCER" = true ]] && extra_flags+=(--set submariner.loadBalancerEnabled='true') 55 | 56 | # Set repo/ver combo for air gapped environment, to make sure all Submariner images are only taken from local repo. 57 | if [[ "$AIR_GAPPED" = true ]]; then 58 | extra_flags+=(--set submariner.images.repository="${SUBM_IMAGE_REPO}" \ 59 | --set submariner.images.tag="${SUBM_IMAGE_TAG}") 60 | fi 61 | 62 | echo "Installing Submariner..." 63 | # shellcheck disable=SC2086 # Split on purpose 64 | helm --kube-context "${cluster}" install --debug --devel submariner-operator \ 65 | "${HELM_REPO_LOCATION}"/submariner-operator \ 66 | --create-namespace \ 67 | --namespace "${SUBM_NS}" \ 68 | --set ipsec.psk="${IPSEC_PSK}" \ 69 | --set broker.server="${submariner_broker_url}" \ 70 | --set broker.token="${submariner_broker_token}" \ 71 | --set broker.namespace="${BROKER_NAMESPACE}" \ 72 | --set broker.ca="${submariner_broker_ca}" \ 73 | --set broker.globalnet="${OVERLAPPING}" \ 74 | --set submariner.serviceDiscovery="${LIGHTHOUSE}" \ 75 | --set submariner.cableDriver="${CABLE_DRIVER}" \ 76 | --set submariner.clusterId="${cluster}" \ 77 | --set submariner.clusterCidr="${cluster_CIDRs[$cluster]}" \ 78 | --set submariner.serviceCidr="${service_CIDRs[$cluster]}" \ 79 | --set submariner.globalCidr="${global_CIDRs[$cluster]}" \ 80 | --set submariner.clustersetIpCidr="${clusterset_ip_CIDRs[$cluster]}" \ 81 | --set submariner.clustersetIpEnabled="${USE_CLUSTERSET_IP}" \ 82 | --set serviceAccounts.globalnet.create="${OVERLAPPING}" \ 83 | --set serviceAccounts.lighthouseAgent.create="${LIGHTHOUSE}" \ 84 | --set serviceAccounts.lighthouseCoreDns.create="${LIGHTHOUSE}" \ 85 | --set submariner.natEnabled="false" \ 86 | "${extra_flags[@]}" \ 87 | --set brokercrds.create="${crd_create}" 88 | } 89 | 90 | function install_subm_all_clusters() { 91 | run_subm_clusters helm_install_subm 92 | } 93 | -------------------------------------------------------------------------------- /scripts/shared/lib/deploy_operator: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | # shellcheck source=scripts/shared/lib/source_only 3 | . "${BASH_SOURCE%/*}"/source_only 4 | 5 | ### Constants ### 6 | 7 | # shellcheck disable=SC2034 # this variable is used elsewhere 8 | # Some projects rely on the default subctl being determined by the PATH 9 | readonly SUBCTL=${SUBCTL:-subctl} 10 | 11 | ### Variables ### 12 | 13 | declare -gA component_by_image 14 | component_by_image['submariner-gateway']=submariner-gateway 15 | component_by_image['submariner-globalnet']=submariner-globalnet 16 | component_by_image['submariner-networkplugin-syncer']=submariner-networkplugin-syncer 17 | component_by_image['submariner-operator']=submariner-operator 18 | component_by_image['submariner-route-agent']=submariner-routeagent 19 | component_by_image['lighthouse-agent']=submariner-lighthouse-agent 20 | component_by_image['lighthouse-coredns']=submariner-lighthouse-coredns 21 | component_by_image['nettest']="submariner-metrics-proxy submariner-nettest" 22 | 23 | ### Functions ### 24 | 25 | function deploytool_prereqs() { 26 | command -v subctl > /dev/null 2>&1 27 | "${SUBCTL}" version 28 | } 29 | 30 | function setup_broker() { 31 | local extra_flags=() 32 | [[ "${OVERLAPPING}" = true ]] && extra_flags+=(--globalnet) 33 | [[ "${USE_CLUSTERSET_IP}" = true ]] && extra_flags+=(--enable-clusterset-ip) 34 | if [[ "${LIGHTHOUSE}" == true ]]; then 35 | extra_flags+=(--components 'service-discovery,connectivity') 36 | else 37 | extra_flags+=(--components connectivity) 38 | fi 39 | echo "Installing broker..." 40 | 41 | # We use the "subctl" image_tag to indicate that we want to let 42 | # subctl use its default repository and version 43 | if [[ "${SUBM_IMAGE_TAG}" != "subctl" ]] && [[ "$AIR_GAPPED" = true || "${PRELOAD_IMAGES}" =~ submariner-operator ]]; then 44 | extra_flags+=(--repository "${SUBM_IMAGE_REPO}" --version "${SUBM_IMAGE_TAG}") 45 | fi 46 | 47 | ( 48 | # The subctl invocation here has to work with the previous release 49 | # so that the upgrade tests can run 50 | cd "${OUTPUT_DIR}" && 51 | "${SUBCTL}" deploy-broker \ 52 | --context "${cluster}" \ 53 | "${extra_flags[@]}" 54 | ) 55 | } 56 | 57 | function subctl_install_subm() { 58 | local extra_flags=() 59 | if [[ ${cluster_subm[$cluster]} != "true" ]]; then 60 | echo "Skipping installation as requested in cluster settings" 61 | return 62 | fi 63 | 64 | # We use the "subctl" image_tag to indicate that we want to let 65 | # subctl use its default repository and version 66 | if [ "${SUBM_IMAGE_TAG}" != "subctl" ]; then 67 | # Set repo/ver combo for air gapped environment, to make sure all Submariner images are only taken from local repo. 68 | [[ "$AIR_GAPPED" = true ]] && extra_flags+=(--repository "${SUBM_IMAGE_REPO}" --version "${SUBM_IMAGE_TAG}") 69 | 70 | for image in ${PRELOAD_IMAGES}; do 71 | local image_keys="${component_by_image[$image]}" 72 | [[ -n "${image_keys}" ]] || continue 73 | for key in ${image_keys}; do 74 | extra_flags+=(--image-override "${key}=${SUBM_IMAGE_REPO}/${image}:${SUBM_IMAGE_TAG}") 75 | done 76 | done 77 | fi 78 | 79 | [[ "$LOAD_BALANCER" = true ]] && extra_flags+=(--load-balancer) 80 | [[ "$AIR_GAPPED" = true ]] && extra_flags+=(--air-gapped) 81 | if [ "${PASS_CIDR_ARGS}" == "true" ];then 82 | extra_flags+=(--clustercidr "${cluster_CIDRs[${cluster}]}" --servicecidr "${service_CIDRs[${cluster}]}") 83 | fi 84 | 85 | # The subctl invocation here has to work with the previous release 86 | # so that the upgrade tests can run 87 | "${SUBCTL}" join --context "${cluster}" \ 88 | --clusterid "${cluster}" \ 89 | --nattport "${CE_IPSEC_NATTPORT}" \ 90 | --globalnet-cidr "${global_CIDRs[$cluster]}" \ 91 | --natt=false \ 92 | --cable-driver "${CABLE_DRIVER}" \ 93 | "${extra_flags[@]}" \ 94 | "${OUTPUT_DIR}"/broker-info.subm 95 | } 96 | 97 | function install_subm_all_clusters() { 98 | run_subm_clusters subctl_install_subm 99 | } 100 | -------------------------------------------------------------------------------- /scripts/shared/lib/kubecfg: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | export KUBECONFIG 3 | KUBECONFIG=$(find "${DAPPER_OUTPUT:-output}"/kubeconfigs -type f -printf %p: || :) 4 | -------------------------------------------------------------------------------- /scripts/shared/lib/ocp_utils: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | 3 | declare -gr OCP_TOOLS_URL="https://mirror.openshift.com/pub/openshift-v4/clients" 4 | declare -gr OCP_CLIENT="${HOME}/.local/bin/oc" 5 | declare -gr OCP_INSTALLER="${HOME}/.local/bin/openshift-install" 6 | declare -gA k8s_ocp_version 7 | k8s_ocp_version[1.20]=latest-4.7 8 | k8s_ocp_version[1.21]=latest-4.8 9 | k8s_ocp_version[1.22]=latest-4.9 10 | k8s_ocp_version[1.23]=latest-4.10 11 | k8s_ocp_version[1.24]=latest-4.11 12 | k8s_ocp_version[1.25]=candidate-4.12 13 | 14 | function determine_ocp_platform() { 15 | local directory=$1 16 | yq '.platform | keys | join("")' "${directory}/install-config.yaml" 17 | } 18 | 19 | function ensure_openshift_cli() { 20 | ensure_openshift_tool "$OCP_CLIENT" openshift-client 21 | } 22 | 23 | function ensure_openshift_install() { 24 | ensure_openshift_tool "$OCP_INSTALLER" openshift-install 25 | } 26 | 27 | # Downloads the OCP tool when we don't have it installed, or what we have isn't the requested version 28 | function ensure_openshift_tool() { 29 | local target_binary=$1 30 | local download_file=$2 31 | local requested_version=$OCP_VERSION 32 | 33 | # Allow using shorthand versions, e.g. `4.10`, by mapping to the full version from our versions map 34 | if [[ -n "$requested_version" ]] && grep -qw "$requested_version" <<< "${k8s_ocp_version[@]}"; then 35 | requested_version=$(grep -Eow "[a-z]+-${requested_version}" <<< "${k8s_ocp_version[@]}") 36 | fi 37 | 38 | local tool_version="${requested_version:-${k8s_ocp_version[$K8S_VERSION]}}" 39 | 40 | # Check if we already have the version installed, and if so skip re-downloading it 41 | ! grep -qw "${tool_version#[a-z]*-}" <("$target_binary" version 2>/dev/null) || return 0 42 | 43 | mkdir -p "${target_binary%/*}" 44 | 45 | # Try to get a GA tool first, but if it's not available fall back to dev previews 46 | local download_url="${OCP_TOOLS_URL}/ocp/${tool_version}" 47 | if ! curl -o /dev/null -f -Ls "$download_url"; then 48 | download_url="${OCP_TOOLS_URL}/ocp-dev-preview/${tool_version}" 49 | fi 50 | 51 | local filename_version 52 | [[ "$tool_version" =~ ^[a-z]+- ]] || filename_version="-${tool_version}" 53 | 54 | curl -Ls "${download_url}/${download_file}-linux${filename_version}.tar.gz" \ 55 | | tar -xzf - -C "${target_binary%/*}" "${target_binary##*/}" 56 | } 57 | -------------------------------------------------------------------------------- /scripts/shared/lib/source_only: -------------------------------------------------------------------------------- 1 | # shellcheck shell=bash 2 | # Make sure this script and the requested script name are sourced 3 | 4 | # $BASH_SOURCE is an array, the element at 1 is the script that sourced this one 5 | script_name="${BASH_SOURCE[1]##*/}" 6 | 7 | # $0 is the name of the script being executed (or the shell if it's a source operation) 8 | exec_name="${0##*/}" 9 | if [[ $script_name = "$exec_name" ]]; then 10 | echo "Don't run me, source me" >&2 11 | exit 1 12 | fi 13 | -------------------------------------------------------------------------------- /scripts/shared/post_mortem.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source "${SCRIPTS_DIR}/lib/debug_functions" 4 | source "${SCRIPTS_DIR}/lib/utils" 5 | 6 | ### Functions ### 7 | 8 | function print_section() { 9 | echo "====================================================================" 10 | echo "::endgroup::" 11 | echo "::group::$*" 12 | echo "======================= $* =======================" 13 | } 14 | 15 | function print_pods_logs() { 16 | local namespace=$1 17 | local selector=$2 18 | 19 | print_section "** Pods logs for NS $namespace using selector '$selector' **" 20 | for pod in $(kubectl get pods --selector="$selector" -n "$namespace" -o jsonpath='{.items[*].metadata.name}'); do 21 | if [ "$(kubectl get pods -n "$namespace" "$pod" -o jsonpath='{.status.containerStatuses[*].ready}')" != true ]; then 22 | print_section "*** $pod (terminated) ***" 23 | kubectl -n "$namespace" logs -p "$pod" 24 | else 25 | print_section "*** $pod ***" 26 | kubectl -n "$namespace" logs "$pod" 27 | fi 28 | done 29 | } 30 | 31 | function post_analyze() { 32 | print_section "* Kubernetes client and server versions in $cluster *" 33 | kubectl version || true 34 | 35 | print_section "* Overview of all resources in $cluster *" 36 | for resource in $(kubectl api-resources --verbs=list -o name); do 37 | print_section "** Resource: $resource" 38 | kubectl get --all-namespaces --show-kind -o wide --ignore-not-found "$resource" 39 | done 40 | 41 | print_section "* Details of pods with statuses other than Running in $cluster *" 42 | for pod in $(kubectl get pods -A | tail -n +2 | grep -v Running | sed 's/ */;/g'); do 43 | ns=$(echo "$pod" | cut -f1 -d';') 44 | name=$(echo "$pod" | cut -f2 -d';') 45 | print_section "** NS: $ns; Pod: $name **" 46 | kubectl -n "$ns" describe pod "$name" 47 | kubectl -n "$ns" logs "$name" 48 | done 49 | 50 | print_section "* Kube-controller-manager pod logs for $cluster *" 51 | print_pods_logs "kube-system" "component=kube-controller-manager" 52 | 53 | print_section "* Submariner-operator pod logs for $cluster *" 54 | print_pods_logs "submariner-operator" 55 | 56 | print_section "* Output of 'subctl show all' in $cluster *" 57 | subctl show all --context "$cluster" 58 | 59 | print_section "* Output of 'subctl diagnose all' in $cluster *" 60 | subctl diagnose all --context "$cluster" 61 | 62 | print_section "* Collecting 'subctl gather' in $cluster *" 63 | subctl gather --context "$cluster" --dir gather_output 64 | 65 | return 0 66 | } 67 | 68 | ### Main ### 69 | 70 | declare_kubeconfig 71 | "${SCRIPTS_DIR}/get-subctl.sh" 72 | for cluster in $(kind get clusters); do 73 | post_analyze 74 | done 75 | -------------------------------------------------------------------------------- /scripts/shared/release_images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | source "${SCRIPTS_DIR}/lib/utils" 5 | 6 | [[ $# -gt 0 ]] || exit_error "At least one image to release must be specified!" 7 | 8 | print_env REPO TAG 9 | source "${SCRIPTS_DIR}/lib/debug_functions" 10 | 11 | function release_image() { 12 | for target_tag in $VERSION $TAG; do 13 | local target_image="${image}:${target_tag#v}" 14 | if [[ -z "${OCIDIR}" ]]; then 15 | # Single-arch 16 | skopeo copy "docker-daemon:${REPO}/${image}:${DEV_VERSION}" "docker://${REPO}/${target_image}" 17 | else 18 | skopeo copy --all "oci-archive:${OCIDIR}/${image}.tar" "docker://${REPO}/${target_image}" 19 | fi 20 | done 21 | } 22 | 23 | echo "$QUAY_PASSWORD" | skopeo login quay.io -u "$QUAY_USERNAME" --password-stdin 24 | 25 | for image; do 26 | release_image 27 | done 28 | 29 | -------------------------------------------------------------------------------- /scripts/shared/reload_images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | source "${SCRIPTS_DIR}/lib/utils" 6 | print_env RESTART 7 | source "${SCRIPTS_DIR}/lib/debug_functions" 8 | source "${SCRIPTS_DIR}/lib/deploy_funcs" 9 | 10 | function find_resources() { 11 | local resource_type=$1 12 | kubectl -n "$(find_submariner_namespace)" get "${resource_type}" -o jsonpath="{range .items[*]}{.metadata.name}{'\n'}" 13 | } 14 | 15 | load_settings 16 | declare_kubeconfig 17 | 18 | case "${RESTART}" in 19 | none) 20 | ;; 21 | all) 22 | for resource in $(find_resources deployments); do 23 | run_subm_clusters reload_pods deployment "${resource}" 24 | done 25 | 26 | for resource in $(find_resources daemonsets); do 27 | run_subm_clusters reload_pods daemonset "${resource}" 28 | done 29 | ;; 30 | *) 31 | run_subm_clusters reload_pods deployment "submariner-${restart}" || \ 32 | run_subm_clusters reload_pods daemonset "submariner-${restart}" || : 33 | ;; 34 | esac 35 | 36 | -------------------------------------------------------------------------------- /scripts/shared/resources/.gitignore: -------------------------------------------------------------------------------- 1 | cluster*-config.yaml 2 | -------------------------------------------------------------------------------- /scripts/shared/resources/acm-auto-import-secret.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: auto-import-secret 6 | stringData: 7 | # the following value to specify the retry times when your cluster failed to import 8 | autoImportRetry: "5" 9 | # If you are using the kubeconfig file, add the following value for the kubeconfig file 10 | # that has the current context set to the cluster to import: 11 | kubeconfig: placeholder 12 | type: Opaque 13 | -------------------------------------------------------------------------------- /scripts/shared/resources/acm-managed-cluster.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: cluster.open-cluster-management.io/v1 3 | kind: ManagedCluster 4 | metadata: 5 | name: placeholder 6 | labels: 7 | cloud: auto-detect 8 | vendor: auto-detect 9 | spec: 10 | hubAcceptsClient: true 11 | -------------------------------------------------------------------------------- /scripts/shared/resources/bundle/broker.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: submariner.io/v1alpha1 3 | kind: Broker 4 | metadata: 5 | name: submariner-broker 6 | namespace: ${SUBM_NS} 7 | spec: 8 | components: ["${components}"] 9 | defaultGlobalnetClusterSize: 8192 10 | globalnetEnabled: ${OVERLAPPING} 11 | globalnetCIDRRange: 169.254.0.0/16 12 | clustersetIPEnabled: ${USE_CLUSTERSET_IP} 13 | -------------------------------------------------------------------------------- /scripts/shared/resources/bundle/submariner.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: submariner.io/v1alpha1 3 | kind: Submariner 4 | metadata: 5 | name: submariner 6 | namespace: ${SUBM_NS} 7 | spec: 8 | serviceCIDR: "${service_CIDRs[$cluster]}" 9 | clusterCIDR: "${cluster_CIDRs[$cluster]}" 10 | globalCIDR: "${global_CIDRs[$cluster]}" 11 | clustersetIPEnabled: ${USE_CLUSTERSET_IP} 12 | clustersetIPCIDR: "${clusterset_ip_CIDRs[$cluster]}" 13 | clusterID: "${cluster}" 14 | debug: false 15 | natEnabled: false 16 | serviceDiscoveryEnabled: ${LIGHTHOUSE} 17 | broker: "k8s" 18 | brokerK8sApiServer: "${BROKER_K8S_API_SERVER}" 19 | brokerK8sApiServerToken: "${BROKER_K8S_API_SERVER_TOKEN}" 20 | brokerK8sRemoteNamespace: "${BROKER_NAMESPACE}" 21 | brokerK8sCA: "${BROKER_K8S_CA}" 22 | cableDriver: "libreswan" 23 | ceIPSecPSK: "${IPSEC_PSK}" 24 | ceIPSecDebug: false 25 | ceIPSecIKEPort: ${CE_IPSEC_IKEPORT} 26 | ceIPSecNATTPort: ${CE_IPSEC_NATTPORT} 27 | namespace: "${SUBM_NS}" 28 | repository: "${SUBM_IMAGE_REPO}" 29 | version: "${SUBM_IMAGE_TAG}" 30 | connectionHealthCheck: 31 | enabled: true 32 | intervalSeconds: 1 33 | maxPacketLossCount: 5 34 | -------------------------------------------------------------------------------- /scripts/shared/resources/common/catalogSource.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: operators.coreos.com/v1alpha1 3 | kind: CatalogSource 4 | metadata: 5 | name: ${cs} 6 | namespace: ${MARKETPLACE_NAMESPACE} 7 | spec: 8 | sourceType: grpc 9 | image: ${iib} 10 | displayName: ${cs} 11 | publisher: Submariner.io (Test) 12 | updateStrategy: 13 | registryPoll: 14 | interval: 5m 15 | -------------------------------------------------------------------------------- /scripts/shared/resources/common/operatorGroup.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: operators.coreos.com/v1 3 | kind: OperatorGroup 4 | metadata: 5 | name: ${og} 6 | namespace: ${ns} 7 | spec: 8 | targetNamespaces: 9 | - ${ns} 10 | -------------------------------------------------------------------------------- /scripts/shared/resources/common/subscription.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: operators.coreos.com/v1alpha1 3 | kind: Subscription 4 | metadata: 5 | name: ${sub} 6 | namespace: ${ns} 7 | spec: 8 | installPlanApproval: Manual 9 | name: ${bundle} 10 | source: ${cs} 11 | sourceNamespace: ${MARKETPLACE_NAMESPACE} 12 | -------------------------------------------------------------------------------- /scripts/shared/resources/dummypod.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: DaemonSet 4 | metadata: 5 | name: dummypod 6 | namespace: ${ns} 7 | spec: 8 | selector: 9 | matchLabels: 10 | app: dummypod 11 | template: 12 | metadata: 13 | labels: 14 | app: dummypod 15 | spec: 16 | tolerations: 17 | - key: node-role.kubernetes.io/control-plane 18 | operator: Exists 19 | effect: NoSchedule 20 | - key: node-role.kubernetes.io/master 21 | operator: Exists 22 | effect: NoSchedule 23 | containers: 24 | - name: dummypod 25 | image: ${SUBM_IMAGE_REPO}/nettest:${SUBM_IMAGE_TAG} 26 | imagePullPolicy: IfNotPresent 27 | command: 28 | - sleep 29 | - infinity 30 | restartPolicy: Always 31 | -------------------------------------------------------------------------------- /scripts/shared/resources/kind-cluster-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Cluster 3 | apiVersion: kind.x-k8s.io/v1alpha4 4 | networking: 5 | disableDefaultCNI: ${disable_cni} 6 | podSubnet: ${pod_cidr} 7 | serviceSubnet: ${service_cidr} 8 | kubeProxyMode: ${kube_proxy_mode} 9 | containerdConfigPatches: 10 | - |- 11 | [plugins.\"io.containerd.grpc.v1.cri\".registry] 12 | config_path = \"/etc/containerd/certs.d\" 13 | kubeadmConfigPatches: 14 | - | 15 | apiVersion: kubeadm.k8s.io/v1beta2 16 | kind: ClusterConfiguration 17 | metadata: 18 | name: config 19 | networking: 20 | podSubnet: ${pod_cidr} 21 | serviceSubnet: ${service_cidr} 22 | dnsDomain: ${dns_domain} 23 | nodes: ${nodes} 24 | -------------------------------------------------------------------------------- /scripts/shared/resources/kind-cluster-dual-stack-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Cluster 3 | apiVersion: kind.x-k8s.io/v1alpha4 4 | networking: 5 | disableDefaultCNI: ${disable_cni} 6 | podSubnet: ${pod_cidr},${pod_cidr_ipv6} 7 | serviceSubnet: ${service_cidr},${service_cidr_ipv6} 8 | kubeProxyMode: ${kube_proxy_mode} 9 | ipFamily: dual 10 | containerdConfigPatches: 11 | - |- 12 | [plugins.\"io.containerd.grpc.v1.cri\".registry] 13 | config_path = \"/etc/containerd/certs.d\" 14 | kubeadmConfigPatches: 15 | - | 16 | apiVersion: kubeadm.k8s.io/v1beta2 17 | kind: ClusterConfiguration 18 | metadata: 19 | name: config 20 | networking: 21 | podSubnet: ${pod_cidr},${pod_cidr_ipv6} 22 | serviceSubnet: ${service_cidr},${service_cidr_ipv6} 23 | dnsDomain: ${dns_domain} 24 | nodes: ${nodes} 25 | -------------------------------------------------------------------------------- /scripts/shared/resources/kind-cluster-ipv6-stack-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Cluster 3 | apiVersion: kind.x-k8s.io/v1alpha4 4 | networking: 5 | disableDefaultCNI: ${disable_cni} 6 | podSubnet: ${pod_cidr_ipv6} 7 | serviceSubnet: ${service_cidr_ipv6} 8 | kubeProxyMode: ${kube_proxy_mode} 9 | ipFamily: ipv6 10 | containerdConfigPatches: 11 | - |- 12 | [plugins.\"io.containerd.grpc.v1.cri\".registry] 13 | config_path = \"/etc/containerd/certs.d\" 14 | kubeadmConfigPatches: 15 | - | 16 | apiVersion: kubeadm.k8s.io/v1beta2 17 | kind: ClusterConfiguration 18 | metadata: 19 | name: config 20 | networking: 21 | podSubnet: ${pod_cidr_ipv6} 22 | serviceSubnet: ${service_cidr_ipv6} 23 | dnsDomain: ${dns_domain} 24 | nodes: ${nodes} 25 | -------------------------------------------------------------------------------- /scripts/shared/resources/netshoot.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: netshoot 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: netshoot 10 | replicas: 1 11 | template: 12 | metadata: 13 | labels: 14 | app: netshoot 15 | spec: 16 | containers: 17 | - name: netshoot 18 | image: ${SUBM_IMAGE_REPO}/nettest:${SUBM_IMAGE_TAG} 19 | imagePullPolicy: IfNotPresent 20 | securityContext: 21 | allowPrivilegeEscalation: false 22 | capabilities: 23 | drop: 24 | - ALL 25 | command: 26 | - sleep 27 | - 1h 28 | restartPolicy: Always 29 | -------------------------------------------------------------------------------- /scripts/shared/resources/nginx-demo.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: nginx-demo 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: nginx-demo 10 | replicas: 2 11 | template: 12 | metadata: 13 | labels: 14 | app: nginx-demo 15 | spec: 16 | containers: 17 | - name: nginx-demo 18 | image: ${SUBM_IMAGE_REPO}/nettest:${SUBM_IMAGE_TAG} 19 | command: ["/app/simpleserver"] 20 | ports: 21 | - containerPort: 8080 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | name: nginx-demo 27 | labels: 28 | app: nginx-demo 29 | spec: 30 | type: ClusterIP 31 | ports: 32 | - protocol: TCP 33 | port: 80 34 | targetPort: 8080 35 | selector: 36 | app: nginx-demo 37 | --- 38 | apiVersion: multicluster.x-k8s.io/v1alpha1 39 | kind: ServiceExport 40 | metadata: 41 | name: nginx-demo 42 | -------------------------------------------------------------------------------- /scripts/shared/resources/ocm/klusterlet.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: operator.open-cluster-management.io/v1 3 | kind: Klusterlet 4 | metadata: 5 | name: klusterlet 6 | spec: 7 | registrationImagePullSpec: quay.io/open-cluster-management/registration 8 | workImagePullSpec: quay.io/open-cluster-management/work 9 | clusterName: ${cluster} 10 | namespace: open-cluster-management-agent 11 | externalServerURLs: 12 | - url: https://${master_ip} 13 | -------------------------------------------------------------------------------- /scripts/shared/resources/ocm/managedClusterAddOn.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: addon.open-cluster-management.io/v1alpha1 3 | kind: ManagedClusterAddOn 4 | metadata: 5 | name: submariner 6 | namespace: ${mc} 7 | spec: 8 | installNamespace: ${SUBM_NS} 9 | -------------------------------------------------------------------------------- /scripts/shared/resources/ocm/managedClusterSet.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: cluster.open-cluster-management.io/v1beta1 3 | kind: ManagedClusterSet 4 | metadata: 5 | name: submariner 6 | -------------------------------------------------------------------------------- /scripts/shared/resources/ocm/submarinerConfig.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: submarineraddon.open-cluster-management.io/v1alpha1 3 | kind: SubmarinerConfig 4 | metadata: 5 | name: submariner 6 | namespace: ${mc} 7 | spec: 8 | IPSecIKEPort: ${CE_IPSEC_IKEPORT} 9 | IPSecNATTPort: ${CE_IPSEC_NATTPORT} 10 | NATTEnable: false 11 | cableDriver: libreswan 12 | gatewayConfig: 13 | gateways: 1 14 | imagePullSpecs: 15 | submarinerImagePullSpec: ${SUBM_IMAGE_REPO}/submariner-gateway:${SUBM_IMAGE_TAG} 16 | submarinerRouteAgentImagePullSpec: ${SUBM_IMAGE_REPO}/submariner-route-agent:${SUBM_IMAGE_TAG} 17 | lighthouseAgentImagePullSpec: ${SUBM_IMAGE_REPO}/lighthouse-agent:${SUBM_IMAGE_TAG} 18 | lighthouseCoreDNSImagePullSpec: ${SUBM_IMAGE_REPO}/lighthouse-coredns:${SUBM_IMAGE_TAG} 19 | subscriptionConfig: 20 | source: ${SUBM_CS} 21 | sourceNamespace: ${MARKETPLACE_NAMESPACE} 22 | -------------------------------------------------------------------------------- /scripts/shared/resources/plugins/sample_hook: -------------------------------------------------------------------------------- 1 | # This file contains sample hook to be called from 2 | # deploy.sh and cleanup.sh. 3 | # These hook will be called if path to this file is 4 | # passed to the script via --hook argument. 5 | 6 | function pre_deploy() { 7 | echo "pre_deploy hook is called from script with below flags set." 8 | for flag in ${!FLAGS_*};do echo "${flag}: ${!flag}";done 9 | } 10 | 11 | function post_deploy() { 12 | echo "post_deploy hook is called." 13 | } 14 | 15 | function pre_cleanup() { 16 | echo "pre_cleanup hook is called from script with below flags set." 17 | for flag in ${!FLAGS_*};do echo "${flag}: ${!flag}";done 18 | } 19 | 20 | function post_cleanup() { 21 | echo "post_cleanup hook is called." 22 | } 23 | -------------------------------------------------------------------------------- /scripts/shared/resources/prometheus/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: prometheus 6 | rules: 7 | - apiGroups: [""] 8 | resources: 9 | - nodes 10 | - nodes/metrics 11 | - services 12 | - endpoints 13 | - pods 14 | verbs: ["get", "list", "watch"] 15 | - apiGroups: [""] 16 | resources: 17 | - configmaps 18 | verbs: ["get"] 19 | - nonResourceURLs: ["/metrics"] 20 | verbs: ["get"] 21 | -------------------------------------------------------------------------------- /scripts/shared/resources/prometheus/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: prometheus 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: prometheus 10 | subjects: 11 | - kind: ServiceAccount 12 | name: prometheus 13 | namespace: default 14 | - kind: ServiceAccount 15 | name: prometheus 16 | namespace: submariner-operator 17 | -------------------------------------------------------------------------------- /scripts/shared/resources/prometheus/prometheus.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: Prometheus 4 | metadata: 5 | name: prometheus 6 | labels: 7 | prometheus: prometheus 8 | spec: 9 | replicas: 1 10 | serviceAccountName: prometheus 11 | serviceMonitorNamespaceSelector: {} 12 | serviceMonitorSelector: {} 13 | -------------------------------------------------------------------------------- /scripts/shared/resources/prometheus/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: prometheus 6 | -------------------------------------------------------------------------------- /scripts/shared/resources/sm-global-cm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | data: 4 | use-nftables: \"${NFTABLES}\" 5 | kind: ConfigMap 6 | metadata: 7 | name: submariner-global 8 | namespace: ${ns} 9 | -------------------------------------------------------------------------------- /scripts/shared/targets.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function print_indent() { 4 | printf "%-24s%s\n" "$1" "$2" 5 | } 6 | 7 | print_indent Target Description | tee >(tr '[:alnum:]' '-') 8 | 9 | readarray -t make_targets < <(make -pRrq : 2>/dev/null |\ 10 | grep -oP '^(?!Makefile.*)[-[:alnum:]]*(?=:)' | sort -u) 11 | 12 | for target in "${make_targets[@]}"; do 13 | description=$(grep -hoP -m1 "(?<=\[${target}\] ).*" Makefile* "${SHIPYARD_DIR}"/Makefile* | head -1) 14 | print_indent "${target}" "${description}" 15 | done 16 | -------------------------------------------------------------------------------- /scripts/shared/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | source "${SCRIPTS_DIR}/lib/debug_functions" 6 | 7 | function _find() { 8 | declare -a excludes 9 | for entry in .git $(git ls-files -o -i --exclude-from=.gitignore --directory); do 10 | test -f "$entry" || excludes+=(-path "./${entry/%\/}" -prune -o) 11 | done 12 | 13 | find . "${excludes[@]}" "$@" -printf "%h\0" | sort -z -u 14 | } 15 | 16 | result=0 17 | echo "Looking for packages to test" 18 | readarray -d '' modules < <(_find -name go.mod) 19 | 20 | for module in "${modules[@]}"; do 21 | exclude_args=() 22 | echo "Looking for tests in module ${module}" 23 | 24 | # Exclude any sub-modules 25 | for exc_module in "${modules[@]}"; do 26 | if [ "$exc_module" != "$module" ] && [ "$exc_module" != "." ]; then 27 | exclude_args+=(-path "$exc_module" -prune -o) 28 | fi 29 | done 30 | 31 | # Run in subshell to return to base directory even if the tests fail 32 | ( 33 | cd "$module" 34 | 35 | # Exclude any directories containing e2e tests 36 | for dir in $(git grep -w -l e2e | grep _test.go | sed 's#\(.*/.*\)/.*$#\1#' | sort -u); do 37 | exclude_args+=(-path "./${dir}" -prune -o) 38 | done 39 | 40 | readarray -d '' packages < <(_find "${exclude_args[@]}" -path "./*/*_test.go") 41 | [ "${#packages[@]}" -gt 0 ] || exit 0 42 | 43 | echo "Running tests in ${packages[*]}" 44 | [ "${ARCH}" == "amd64" ] && race=-race 45 | # It's important that the `go test` command's exit status is reported from this () block. 46 | # Can't be one command (with -cover). Need detailed -coverprofile for Sonar and summary to console. 47 | # shellcheck disable=SC2086 # Split `$TEST_ARGS` on purpose 48 | "${GO:-go}" test -v ${race} -coverprofile unit.coverprofile "${packages[@]}" \ 49 | --ginkgo.v --ginkgo.trace --ginkgo.junit-report junit.xml $TEST_ARGS && \ 50 | go tool cover -func unit.coverprofile 51 | ) || result=1 52 | done 53 | 54 | exit $result 55 | -------------------------------------------------------------------------------- /staticcheck.conf: -------------------------------------------------------------------------------- 1 | initialisms = ["inherit", "CNI", "IPAM", "IPSec", "NAT", "NATT"] -------------------------------------------------------------------------------- /test/e2e/dataplane/tcp_connectivity.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Package dataplane runs the TCP/IP connectivity test. 20 | package dataplane 21 | 22 | import ( 23 | . "github.com/onsi/ginkgo/v2" 24 | "github.com/submariner-io/shipyard/test/e2e/framework" 25 | "github.com/submariner-io/shipyard/test/e2e/tcp" 26 | ) 27 | 28 | var _ = Describe("[dataplane] Basic TCP connectivity test", func() { 29 | f := framework.NewFramework("dataplane") 30 | 31 | When("a pod connects to another pod via TCP in the same cluster", func() { 32 | It("should send the expected data to the other pod", func() { 33 | tcp.RunConnectivityTest(&tcp.ConnectivityTestParams{ 34 | Framework: f, 35 | ToEndpointType: tcp.PodIP, 36 | Networking: framework.HostNetworking, 37 | FromCluster: framework.ClusterA, 38 | FromClusterScheduling: framework.NonGatewayNode, 39 | ToCluster: framework.ClusterA, 40 | ToClusterScheduling: framework.NonGatewayNode, 41 | }) 42 | }) 43 | }) 44 | }) 45 | -------------------------------------------------------------------------------- /test/e2e/e2e.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package e2e 20 | 21 | import ( 22 | "fmt" 23 | "testing" 24 | 25 | . "github.com/onsi/ginkgo/v2" 26 | "github.com/onsi/gomega" 27 | "github.com/submariner-io/shipyard/test/e2e/framework" 28 | "k8s.io/client-go/rest" 29 | "k8s.io/klog/v2" 30 | ) 31 | 32 | // There are certain operations we only want to run once per overall test invocation 33 | // (such as deleting old namespaces, or verifying that all system pods are running. 34 | // Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite 35 | // to ensure that these operations only run on the first parallel Ginkgo node. 36 | // 37 | // This function takes two parameters: one function which runs on only the first Ginkgo node, 38 | // returning an opaque byte array, and then a second function which runs on all Ginkgo nodes, 39 | // accepting the byte array. 40 | var _ = SynchronizedBeforeSuite(func() []byte { 41 | // Run only on Ginkgo node 1 42 | 43 | framework.BeforeSuite() 44 | return nil 45 | }, func(_ []byte) { 46 | // Run on all Ginkgo nodes 47 | }) 48 | 49 | // Similar to SynchornizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs). 50 | // Here, the order of functions is reversed; first, the function which runs everywhere, 51 | // and then the function that only runs on the first Ginkgo node. 52 | 53 | var _ = SynchronizedAfterSuite(func() { 54 | // Run on all Ginkgo nodes 55 | 56 | // framework.Logf("Running AfterSuite actions on all node") 57 | framework.RunCleanupActions() 58 | }, func() { 59 | // Run only Ginkgo on node 1 60 | }) 61 | 62 | func init() { 63 | klog.InitFlags(nil) 64 | } 65 | 66 | func RunE2ETests(t *testing.T) bool { 67 | framework.SetStatusFunction(By) 68 | 69 | framework.SetFailFunction(Fail) 70 | 71 | framework.SetUserAgentFunction(func() string { 72 | return fmt.Sprintf("%v -- %v", rest.DefaultKubernetesUserAgent(), CurrentSpecReport().FullText()) 73 | }) 74 | 75 | framework.ValidateFlags(framework.TestContext) 76 | gomega.RegisterFailHandler(Fail) 77 | 78 | suiteConfig, reporterConfig := GinkgoConfiguration() 79 | 80 | if framework.TestContext.SuiteConfig != nil { 81 | suiteConfig = *framework.TestContext.SuiteConfig 82 | } 83 | 84 | if framework.TestContext.ReporterConfig != nil { 85 | reporterConfig = *framework.TestContext.ReporterConfig 86 | } 87 | 88 | return RunSpecs(t, "Submariner E2E suite", suiteConfig, reporterConfig) 89 | } 90 | -------------------------------------------------------------------------------- /test/e2e/e2e_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | package e2e_test 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/submariner-io/shipyard/test/e2e" 24 | _ "github.com/submariner-io/shipyard/test/e2e/dataplane" 25 | ) 26 | 27 | func TestE2E(t *testing.T) { 28 | e2e.RunE2ETests(t) 29 | } 30 | -------------------------------------------------------------------------------- /test/e2e/framework/api_errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "k8s.io/apimachinery/pkg/api/errors" 23 | ) 24 | 25 | // identify API errors which could be considered transient/recoverable 26 | // due to server state. 27 | func IsTransientError(err error, opMsg string) bool { 28 | if errors.IsInternalError(err) || 29 | errors.IsServerTimeout(err) || 30 | errors.IsTimeout(err) || 31 | errors.IsServiceUnavailable(err) || 32 | errors.IsUnexpectedServerError(err) || 33 | errors.IsTooManyRequests(err) { 34 | Logf("Transient failure when attempting to %s: %v", opMsg, err) 35 | return true 36 | } 37 | 38 | return false 39 | } 40 | -------------------------------------------------------------------------------- /test/e2e/framework/cleanup.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import "sync" 22 | 23 | // CleanupActionHandle is an integer pointer type for handling cleanup action. 24 | type CleanupActionHandle *int 25 | 26 | var ( 27 | cleanupActionsLock sync.Mutex 28 | cleanupActions = map[CleanupActionHandle]func(){} 29 | ) 30 | 31 | // AddCleanupAction installs a function that will be called in the event of the 32 | // whole test being terminated. This allows arbitrary pieces of the overall 33 | // test to hook into SynchronizedAfterSuite(). 34 | func AddCleanupAction(fn func()) CleanupActionHandle { 35 | p := CleanupActionHandle(new(int)) 36 | 37 | cleanupActionsLock.Lock() 38 | defer cleanupActionsLock.Unlock() 39 | 40 | cleanupActions[p] = fn 41 | 42 | return p 43 | } 44 | 45 | // RemoveCleanupAction removes a function that was installed by 46 | // AddCleanupAction. 47 | func RemoveCleanupAction(p CleanupActionHandle) { 48 | cleanupActionsLock.Lock() 49 | defer cleanupActionsLock.Unlock() 50 | delete(cleanupActions, p) 51 | } 52 | 53 | // RunCleanupActions runs all functions installed by AddCleanupAction. It does 54 | // not remove them (see RemoveCleanupAction) but it does run unlocked, so they 55 | // may remove themselves. 56 | func RunCleanupActions() { 57 | list := []func(){} 58 | 59 | func() { 60 | cleanupActionsLock.Lock() 61 | defer cleanupActionsLock.Unlock() 62 | 63 | for _, fn := range cleanupActions { 64 | list = append(list, fn) 65 | } 66 | }() 67 | 68 | // Run unlocked. 69 | for _, fn := range list { 70 | fn() 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /test/e2e/framework/clusterglobalegressip.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | apierrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 29 | "k8s.io/apimachinery/pkg/runtime/schema" 30 | "k8s.io/client-go/dynamic" 31 | ) 32 | 33 | var clusterGlobalEgressIPGVR = &schema.GroupVersionResource{ 34 | Group: "submariner.io", 35 | Version: "v1", 36 | Resource: "clusterglobalegressips", 37 | } 38 | 39 | func (f *Framework) AwaitClusterGlobalEgressIPs(cluster ClusterIndex, name string) []string { 40 | gipClient := clusterGlobalEgressIPClient(cluster) 41 | 42 | return AwaitAllocatedEgressIPs(gipClient, name) 43 | } 44 | 45 | func AwaitAllocatedEgressIPs(client dynamic.ResourceInterface, name string) []string { 46 | obj := AwaitUntil("await allocated egress IPs for "+name, 47 | func() (interface{}, error) { 48 | resGip, err := client.Get(context.TODO(), name, metav1.GetOptions{}) 49 | if apierrors.IsNotFound(err) { 50 | return nil, nil //nolint:nilnil // We want to repeat but let the checker known that nothing was found. 51 | } 52 | 53 | return resGip, err 54 | }, 55 | func(result interface{}) (bool, string, error) { 56 | if result == nil { 57 | return false, fmt.Sprintf("Egress IP resource %q not found yet", name), nil 58 | } 59 | 60 | globalIPs := getGlobalIPs(result.(*unstructured.Unstructured)) 61 | if len(globalIPs) == 0 { 62 | return false, fmt.Sprintf("Egress IP resource %q exists but allocatedIPs not available yet", name), nil 63 | } 64 | 65 | return true, "", nil 66 | }) 67 | 68 | return getGlobalIPs(obj.(*unstructured.Unstructured)) 69 | } 70 | 71 | func clusterGlobalEgressIPClient(cluster ClusterIndex) dynamic.ResourceInterface { 72 | return DynClients[cluster].Resource(*clusterGlobalEgressIPGVR).Namespace(corev1.NamespaceAll) 73 | } 74 | 75 | func getGlobalIPs(obj *unstructured.Unstructured) []string { 76 | if obj != nil { 77 | globalIPs, _, _ := unstructured.NestedStringSlice(obj.Object, "status", "allocatedIPs") 78 | return globalIPs 79 | } 80 | 81 | return []string{} 82 | } 83 | -------------------------------------------------------------------------------- /test/e2e/framework/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Package framework contains the shared test framework for Submariner projects. 20 | package framework 21 | -------------------------------------------------------------------------------- /test/e2e/framework/docker.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "os/exec" 25 | "strings" 26 | "time" 27 | 28 | . "github.com/onsi/gomega" 29 | ) 30 | 31 | type Docker struct { 32 | Name string 33 | } 34 | 35 | func New(name string) *Docker { 36 | return &Docker{Name: name} 37 | } 38 | 39 | func (d *Docker) GetIP(networkName string) string { 40 | var stdout bytes.Buffer 41 | 42 | cmdargs := []string{ 43 | "inspect", d.Name, "-f", 44 | fmt.Sprintf("{{(index .NetworkSettings.Networks %q).IPAddress}}", networkName), 45 | } 46 | cmd := exec.Command("docker", cmdargs...) 47 | cmd.Stdout = &stdout 48 | err := cmd.Run() 49 | Expect(err).NotTo(HaveOccurred()) 50 | 51 | // output has trailing "\n", so it needs to be trimed 52 | return strings.TrimSuffix(stdout.String(), "\n") 53 | } 54 | 55 | func (d *Docker) GetLog() (string, string) { 56 | var stdout, stderr bytes.Buffer 57 | 58 | // get stdout and stderr of `docker log {d.Name}` command 59 | // #nosec G204 -- the caller-controlled value is only used as the logs argument 60 | cmd := exec.Command("docker", "logs", d.Name) 61 | cmd.Stdout = &stdout 62 | cmd.Stderr = &stderr 63 | 64 | err := cmd.Run() 65 | Expect(err).NotTo(HaveOccurred()) 66 | 67 | return stdout.String(), stderr.String() 68 | } 69 | 70 | func (d *Docker) RunCommand(command ...string) (string, string) { 71 | stdout, stderr, err := d.runCommand(command...) 72 | Expect(err).NotTo(HaveOccurred()) 73 | 74 | return stdout, stderr 75 | } 76 | 77 | func (d *Docker) RunCommandUntil(command ...string) (string, string) { 78 | var stdout, stderr string 79 | 80 | Eventually(func() error { 81 | var err error 82 | 83 | stdout, stderr, err = d.runCommand(command...) 84 | return err 85 | }, TestContext.OperationTimeoutToDuration(), 5*time.Second).Should(Succeed(), 86 | "Error attempting to run %v", append([]string{}, command...)) 87 | 88 | return stdout, stderr 89 | } 90 | 91 | func (d *Docker) runCommand(command ...string) (string, string, error) { 92 | var stdout, stderr bytes.Buffer 93 | 94 | cmdargs := []string{"exec", "-i", d.Name} 95 | cmdargs = append(cmdargs, command...) 96 | cmd := exec.Command("docker", cmdargs...) 97 | cmd.Stdout = &stdout 98 | cmd.Stderr = &stderr 99 | 100 | err := cmd.Run() 101 | 102 | return stdout.String(), stderr.String(), err 103 | } 104 | -------------------------------------------------------------------------------- /test/e2e/framework/endpoints.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" 29 | ) 30 | 31 | func (f *Framework) CreateTCPEndpoints(cluster ClusterIndex, epName, portName, address string, port int32) *corev1.Endpoints { 32 | endpointsSpec := corev1.Endpoints{ 33 | ObjectMeta: metav1.ObjectMeta{ 34 | Name: epName, 35 | }, 36 | Subsets: []corev1.EndpointSubset{ 37 | { 38 | Addresses: []corev1.EndpointAddress{ 39 | {IP: address}, 40 | }, 41 | Ports: []corev1.EndpointPort{ 42 | { 43 | Name: portName, 44 | Port: port, 45 | Protocol: corev1.ProtocolTCP, 46 | }, 47 | }, 48 | }, 49 | }, 50 | } 51 | 52 | ec := KubeClients[cluster].CoreV1().Endpoints(f.Namespace) 53 | 54 | return createEndpoints(ec, &endpointsSpec) 55 | } 56 | 57 | func createEndpoints(ec typedv1.EndpointsInterface, endpointsSpec *corev1.Endpoints) *corev1.Endpoints { 58 | return AwaitUntil("create endpoints", func() (interface{}, error) { 59 | ep, err := ec.Create(context.TODO(), endpointsSpec, metav1.CreateOptions{}) 60 | if errors.IsAlreadyExists(err) { 61 | err = ec.Delete(context.TODO(), endpointsSpec.Name, metav1.DeleteOptions{}) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | ep, err = ec.Create(context.TODO(), endpointsSpec, metav1.CreateOptions{}) 67 | } 68 | 69 | return ep, err 70 | }, NoopCheckResult).(*corev1.Endpoints) 71 | } 72 | 73 | func (f *Framework) DeleteEndpoints(cluster ClusterIndex, endpointsName string) { 74 | By(fmt.Sprintf("Deleting endpoints %q on %q", endpointsName, TestContext.ClusterIDs[cluster])) 75 | AwaitUntil("delete endpoints", func() (interface{}, error) { 76 | return nil, KubeClients[cluster].CoreV1().Endpoints(f.Namespace).Delete(context.TODO(), endpointsName, metav1.DeleteOptions{}) 77 | }, NoopCheckResult) 78 | } 79 | -------------------------------------------------------------------------------- /test/e2e/framework/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "bytes" 23 | "context" 24 | "io" 25 | "net/url" 26 | "strings" 27 | "time" 28 | 29 | v1 "k8s.io/api/core/v1" 30 | "k8s.io/client-go/kubernetes/scheme" 31 | restclient "k8s.io/client-go/rest" 32 | "k8s.io/client-go/tools/remotecommand" 33 | ) 34 | 35 | // ExecOptions passed to ExecWithOptions. 36 | type ExecOptions struct { 37 | Command []string 38 | 39 | Namespace string 40 | PodName string 41 | ContainerName string 42 | 43 | Stdin io.Reader 44 | CaptureStdout bool 45 | CaptureStderr bool 46 | // If false, whitespace in std{err,out} will be removed. 47 | PreserveWhitespace bool 48 | } 49 | 50 | // ExecWithOptions executes a command in the specified container, 51 | // returning stdout, stderr and error. `options` allowed for 52 | // additional parameters to be passed. 53 | func (f *Framework) ExecWithOptions(ctx context.Context, options *ExecOptions, index ClusterIndex) (string, string, error) { 54 | Logf("ExecWithOptions %+v", options) 55 | 56 | var err error 57 | 58 | const tty = false 59 | req := KubeClients[index].CoreV1().RESTClient().Post(). 60 | Resource("pods"). 61 | Name(options.PodName). 62 | Namespace(options.Namespace). 63 | SubResource("exec"). 64 | Param("container", options.ContainerName) 65 | 66 | req.VersionedParams(&v1.PodExecOptions{ 67 | Container: options.ContainerName, 68 | Command: options.Command, 69 | Stdin: options.Stdin != nil, 70 | Stdout: options.CaptureStdout, 71 | Stderr: options.CaptureStderr, 72 | TTY: tty, 73 | }, scheme.ParameterCodec) 74 | 75 | err = req.Error() 76 | if err != nil { 77 | return "", "", err 78 | } 79 | 80 | var stdout, stderr bytes.Buffer 81 | 82 | for attempts := 5; attempts > 0; attempts-- { 83 | err = execute(ctx, "POST", req.URL(), RestConfigs[index], options.Stdin, &stdout, &stderr, tty) 84 | if err == nil { 85 | break 86 | } 87 | 88 | time.Sleep(time.Millisecond * 5000) 89 | Logf("Retrying due to error %+v", err) 90 | } 91 | 92 | if options.PreserveWhitespace { 93 | return stdout.String(), stderr.String(), err 94 | } 95 | 96 | return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err 97 | } 98 | 99 | func execute(ctx context.Context, 100 | method string, reqURL *url.URL, config *restclient.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool, 101 | ) error { 102 | exec, err := remotecommand.NewSPDYExecutor(config, method, reqURL) 103 | if err != nil { 104 | return err 105 | } 106 | 107 | return exec.StreamWithContext(ctx, remotecommand.StreamOptions{ 108 | Stdin: stdin, 109 | Stdout: stdout, 110 | Stderr: stderr, 111 | Tty: tty, 112 | }) 113 | } 114 | -------------------------------------------------------------------------------- /test/e2e/framework/fips.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | "strings" 25 | 26 | . "github.com/onsi/gomega" 27 | apierrors "k8s.io/apimachinery/pkg/api/errors" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | ) 30 | 31 | const ( 32 | fipsNamespace = "kube-system" 33 | fipsConfigMapName = "cluster-config-v1" 34 | ) 35 | 36 | func DetectFIPSConfig(cluster ClusterIndex) (bool, error) { 37 | configMap, err := KubeClients[cluster].CoreV1().ConfigMaps(fipsNamespace).Get(context.TODO(), fipsConfigMapName, metav1.GetOptions{}) 38 | if err != nil { 39 | if apierrors.IsNotFound(err) { 40 | return false, nil 41 | } 42 | 43 | return false, err 44 | } 45 | 46 | return strings.Contains(configMap.Data["install-config"], "fips: true"), nil 47 | } 48 | 49 | func (f *Framework) FindFIPSEnabledCluster() ClusterIndex { 50 | for idx := range TestContext.ClusterIDs { 51 | fipsEnabled, err := DetectFIPSConfig(ClusterIndex(idx)) 52 | Expect(err).NotTo(HaveOccurred()) 53 | 54 | if fipsEnabled { 55 | return ClusterIndex(idx) 56 | } 57 | } 58 | 59 | return -1 60 | } 61 | 62 | func verifyFIPSOutput(data string) bool { 63 | return strings.Contains(strings.ToLower(data), "fips mode: yes") && 64 | strings.Contains(strings.ToLower(data), "fips mode enabled for pluto daemon") 65 | } 66 | 67 | func (f *Framework) TestGatewayNodeFIPSMode(cluster ClusterIndex, gwPod string) { 68 | By(fmt.Sprintf("Verify FIPS mode is enabled on gateway pod %q", gwPod)) 69 | 70 | ctx := context.TODO() 71 | cmd := []string{"ipsec", "pluto", "--selftest"} 72 | 73 | stdOut, stdErr, err := f.ExecWithOptions(ctx, &ExecOptions{ 74 | Command: cmd, 75 | Namespace: TestContext.SubmarinerNamespace, 76 | PodName: gwPod, 77 | ContainerName: SubmarinerGateway, 78 | CaptureStdout: true, 79 | CaptureStderr: true, 80 | }, cluster) 81 | Expect(err).To(Succeed()) 82 | 83 | if stdOut == "" && stdErr == "" { 84 | Fail(fmt.Sprintf("No output received from command %q", cmd)) 85 | } 86 | 87 | // The output of the "ipsec pluto --selftest" command could be written to stdout or stderr. 88 | // Checking both outputs for the expected strings. 89 | fipsStdOutResult := verifyFIPSOutput(stdOut) 90 | fipsStdErrResult := verifyFIPSOutput(stdErr) 91 | 92 | if fipsStdOutResult || fipsStdErrResult { 93 | By(fmt.Sprintf("FIPS mode is enabled on gateway pod %q", gwPod)) 94 | return 95 | } 96 | 97 | Fail(fmt.Sprintf("FIPS mode is not enabled on gateway pod %q", gwPod)) 98 | } 99 | -------------------------------------------------------------------------------- /test/e2e/framework/ginkgo_framework.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import "github.com/onsi/ginkgo/v2" 22 | 23 | // NewFramework creates a test framework, under ginkgo. 24 | func NewFramework(baseName string) *Framework { 25 | f := NewBareFramework(baseName) 26 | 27 | ginkgo.BeforeEach(f.BeforeEach) 28 | ginkgo.AfterEach(f.AfterEach) 29 | 30 | AddCleanupAction(f.GatewayCleanup) 31 | 32 | return f 33 | } 34 | -------------------------------------------------------------------------------- /test/e2e/framework/ginkgowrapper/wrapper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package ginkgowrapper wraps Ginkgo Fail and Skip functions to panic 18 | // with structured data instead of a constant string. 19 | package ginkgowrapper 20 | 21 | import ( 22 | "bufio" 23 | "bytes" 24 | "regexp" 25 | "runtime" 26 | "runtime/debug" 27 | "strings" 28 | 29 | "github.com/onsi/ginkgo/v2" 30 | ) 31 | 32 | // FailurePanic is the value that will be panicked from Fail. 33 | type FailurePanic struct { 34 | Message string // The failure message passed to Fail 35 | Filename string // The filename that is the source of the failure 36 | Line int // The line number of the filename that is the source of the failure 37 | FullStackTrace string // A full stack trace starting at the source of the failure 38 | } 39 | 40 | const ginkgoFailurePanic = ` 41 | Your test failed. 42 | Ginkgo panics to prevent subsequent assertions from running. 43 | Normally Ginkgo rescues this panic so you shouldn't see it. 44 | But, if you make an assertion in a goroutine, Ginkgo can't capture the panic. 45 | To circumvent this, you should call 46 | defer GinkgoRecover() 47 | at the top of the goroutine that caused this panic. 48 | ` 49 | 50 | // String makes FailurePanic look like the old Ginkgo panic when printed. 51 | func (FailurePanic) String() string { return ginkgoFailurePanic } 52 | 53 | // Fail wraps ginkgo.Fail so that it panics with more useful 54 | // information about the failure. This function will panic with a 55 | // FailurePanic. 56 | func Fail(message string, callerSkip ...int) { 57 | skip := 1 58 | if len(callerSkip) > 0 { 59 | skip += callerSkip[0] 60 | } 61 | 62 | _, file, line, _ := runtime.Caller(skip) 63 | fp := FailurePanic{ 64 | Message: message, 65 | Filename: file, 66 | Line: line, 67 | FullStackTrace: pruneStack(skip), 68 | } 69 | 70 | defer func() { 71 | e := recover() 72 | if e != nil { 73 | panic(fp) 74 | } 75 | }() 76 | 77 | ginkgo.Fail(message, skip) 78 | } 79 | 80 | // SkipPanic is the value that will be panicked from Skip. 81 | type SkipPanic struct { 82 | Message string // The failure message passed to Fail 83 | Filename string // The filename that is the source of the failure 84 | Line int // The line number of the filename that is the source of the failure 85 | FullStackTrace string // A full stack trace starting at the source of the failure 86 | } 87 | 88 | const ginkgoSkipPanic = ` 89 | Your test was skipped. 90 | Ginkgo panics to prevent subsequent assertions from running. 91 | Normally Ginkgo rescues this panic so you shouldn't see it. 92 | But, if you make an assertion in a goroutine, Ginkgo can't capture the panic. 93 | To circumvent this, you should call 94 | defer GinkgoRecover() 95 | at the top of the goroutine that caused this panic. 96 | ` 97 | 98 | // String makes SkipPanic look like the old Ginkgo panic when printed. 99 | func (SkipPanic) String() string { return ginkgoSkipPanic } 100 | 101 | // Skip wraps ginkgo.Skip so that it panics with more useful 102 | // information about why the test is being skipped. This function will 103 | // panic with a SkipPanic. 104 | func Skip(message string, callerSkip ...int) { 105 | skip := 1 106 | if len(callerSkip) > 0 { 107 | skip += callerSkip[0] 108 | } 109 | 110 | _, file, line, _ := runtime.Caller(skip) 111 | sp := SkipPanic{ 112 | Message: message, 113 | Filename: file, 114 | Line: line, 115 | FullStackTrace: pruneStack(skip), 116 | } 117 | 118 | defer func() { 119 | e := recover() 120 | if e != nil { 121 | panic(sp) 122 | } 123 | }() 124 | 125 | ginkgo.Skip(message, skip) 126 | } 127 | 128 | // ginkgo adds a lot of test running infrastructure to the stack, so 129 | // we filter those out. 130 | var stackSkipPattern = regexp.MustCompile(`onsi/ginkgo`) 131 | 132 | func pruneStack(skip int) string { 133 | skip += 2 // one for pruneStack and one for debug.Stack 134 | stack := debug.Stack() 135 | scanner := bufio.NewScanner(bytes.NewBuffer(stack)) 136 | var prunedStack []string 137 | 138 | // skip the top of the stack 139 | for range 2*skip + 1 { 140 | scanner.Scan() 141 | } 142 | 143 | for scanner.Scan() { 144 | if stackSkipPattern.Match(scanner.Bytes()) { 145 | scanner.Scan() // these come in pairs 146 | } else { 147 | prunedStack = append(prunedStack, scanner.Text()) 148 | scanner.Scan() // these come in pairs 149 | prunedStack = append(prunedStack, scanner.Text()) 150 | } 151 | } 152 | 153 | return strings.Join(prunedStack, "\n") 154 | } 155 | -------------------------------------------------------------------------------- /test/e2e/framework/globalegressip.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "context" 23 | 24 | apierrors "k8s.io/apimachinery/pkg/api/errors" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 27 | "k8s.io/apimachinery/pkg/runtime/schema" 28 | "k8s.io/client-go/dynamic" 29 | ) 30 | 31 | var globalEgressIPGVR = &schema.GroupVersionResource{ 32 | Group: "submariner.io", 33 | Version: "v1", 34 | Resource: "globalegressips", 35 | } 36 | 37 | func globalEgressIPClient(cluster ClusterIndex, namespace string) dynamic.ResourceInterface { 38 | return DynClients[cluster].Resource(*globalEgressIPGVR).Namespace(namespace) 39 | } 40 | 41 | func CreateGlobalEgressIP(cluster ClusterIndex, obj *unstructured.Unstructured) error { 42 | geipClient := globalEgressIPClient(cluster, obj.GetNamespace()) 43 | 44 | AwaitUntil("create GlobalEgressIP", func() (interface{}, error) { 45 | egressIP, err := geipClient.Create(context.TODO(), obj, metav1.CreateOptions{}) 46 | if apierrors.IsAlreadyExists(err) { 47 | err = nil 48 | } 49 | 50 | return egressIP, err 51 | }, NoopCheckResult) 52 | 53 | return nil 54 | } 55 | 56 | func AwaitGlobalEgressIPs(cluster ClusterIndex, name, namespace string) []string { 57 | gipClient := globalEgressIPClient(cluster, namespace) 58 | 59 | return AwaitAllocatedEgressIPs(gipClient, name) 60 | } 61 | -------------------------------------------------------------------------------- /test/e2e/framework/globalingressips.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | 25 | apierrors "k8s.io/apimachinery/pkg/api/errors" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 28 | "k8s.io/apimachinery/pkg/runtime/schema" 29 | "k8s.io/client-go/dynamic" 30 | ) 31 | 32 | var globalIngressIPGVR = &schema.GroupVersionResource{ 33 | Group: "submariner.io", 34 | Version: "v1", 35 | Resource: "globalingressips", 36 | } 37 | 38 | func (f *Framework) AwaitGlobalIngressIP(cluster ClusterIndex, name, namespace string) string { 39 | if TestContext.GlobalnetEnabled { 40 | gipClient := globalIngressIPClient(cluster, namespace) 41 | obj := AwaitUntil(fmt.Sprintf("await GlobalIngressIP %s/%s", namespace, name), 42 | func() (interface{}, error) { 43 | resGip, err := gipClient.Get(context.TODO(), name, metav1.GetOptions{}) 44 | if apierrors.IsNotFound(err) { 45 | return nil, nil //nolint:nilnil // We want to repeat but let the checker known that nothing was found. 46 | } 47 | 48 | return resGip, err 49 | }, 50 | func(result interface{}) (bool, string, error) { 51 | if result == nil { 52 | return false, fmt.Sprintf("GlobalEgressIP %s not found yet", name), nil 53 | } 54 | 55 | globalIP := getGlobalIP(result.(*unstructured.Unstructured)) 56 | if globalIP == "" { 57 | return false, fmt.Sprintf("GlobalIngress %q exists but allocatedIP not available yet", 58 | name), nil 59 | } 60 | 61 | return true, "", nil 62 | }) 63 | 64 | return getGlobalIP(obj.(*unstructured.Unstructured)) 65 | } 66 | 67 | return "" 68 | } 69 | 70 | func (f *Framework) AwaitGlobalIngressIPRemoved(cluster ClusterIndex, name, namespace string) { 71 | gipClient := globalIngressIPClient(cluster, namespace) 72 | AwaitUntil(fmt.Sprintf("await GlobalIngressIP %s/%s removed", namespace, name), 73 | func() (interface{}, error) { 74 | _, err := gipClient.Get(context.TODO(), name, metav1.GetOptions{}) 75 | if apierrors.IsNotFound(err) { 76 | return true, nil 77 | } 78 | 79 | return false, err 80 | }, 81 | func(result interface{}) (bool, string, error) { 82 | gone := result.(bool) 83 | return gone, "", nil 84 | }) 85 | } 86 | 87 | func globalIngressIPClient(cluster ClusterIndex, namespace string) dynamic.ResourceInterface { 88 | return DynClients[cluster].Resource(*globalIngressIPGVR).Namespace(namespace) 89 | } 90 | 91 | func getGlobalIP(obj *unstructured.Unstructured) string { 92 | if obj != nil { 93 | globalIP, _, _ := unstructured.NestedString(obj.Object, "status", "allocatedIP") 94 | return globalIP 95 | } 96 | 97 | return "" 98 | } 99 | -------------------------------------------------------------------------------- /test/e2e/framework/logging.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "fmt" 23 | "time" 24 | 25 | "github.com/submariner-io/shipyard/test/e2e/framework/ginkgowrapper" 26 | ) 27 | 28 | func nowStamp() string { 29 | return time.Now().Format(time.StampMilli) 30 | } 31 | 32 | func log(level, format string, args ...interface{}) { 33 | By(fmt.Sprintf(nowStamp()+": "+level+": "+format+"\n", args...)) 34 | } 35 | 36 | func Errorf(format string, args ...interface{}) { 37 | log("ERROR", format, args...) 38 | } 39 | 40 | func Logf(format string, args ...interface{}) { 41 | log("INFO", format, args...) 42 | } 43 | 44 | func Failf(format string, args ...interface{}) { 45 | FailfWithOffset(1, format, args...) 46 | } 47 | 48 | // FailfWithOffset calls "Fail" and logs the error at "offset" levels above its caller 49 | // (for example, for call chain f -> g -> FailfWithOffset(1, ...) error would be logged for "f"). 50 | func FailfWithOffset(offset int, format string, args ...interface{}) { 51 | Fail(nowStamp()+": "+fmt.Sprintf(format, args...), 1+offset) 52 | } 53 | 54 | func Skipf(format string, args ...interface{}) { 55 | msg := fmt.Sprintf(format, args...) 56 | log("INFO", msg) 57 | ginkgowrapper.Skip(nowStamp() + ": " + msg) 58 | } 59 | -------------------------------------------------------------------------------- /test/e2e/framework/nodes.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "context" 23 | "strconv" 24 | "strings" 25 | 26 | . "github.com/onsi/gomega" 27 | v1 "k8s.io/api/core/v1" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | "k8s.io/apimachinery/pkg/labels" 30 | "k8s.io/apimachinery/pkg/selection" 31 | "k8s.io/apimachinery/pkg/types" 32 | ) 33 | 34 | const ( 35 | gatewayStatusLabel = "gateway.submariner.io/status" 36 | gatewayStatusActive = "active" 37 | ) 38 | 39 | // FindGatewayNodes finds nodes in a given cluster by matching 'submariner.io/gateway' value. 40 | func FindGatewayNodes(cluster ClusterIndex) []v1.Node { 41 | nodes, err := KubeClients[cluster].CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ 42 | LabelSelector: labels.Set{GatewayLabel: "true"}.String(), 43 | }) 44 | Expect(err).NotTo(HaveOccurred()) 45 | 46 | return nodes.Items 47 | } 48 | 49 | // FindNonGatewayNodes finds nodes in a given cluster that doesn't match 'submariner.io/gateway' value. 50 | func FindNonGatewayNodes(cluster ClusterIndex) []v1.Node { 51 | nodes, err := KubeClients[cluster].CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ 52 | LabelSelector: labels.NewSelector().Add( 53 | NewRequirement(GatewayLabel, selection.NotEquals, []string{"true"})).String(), 54 | }) 55 | Expect(err).NotTo(HaveOccurred()) 56 | 57 | return nodes.Items 58 | } 59 | 60 | // FindClusterWithMultipleGateways finds the cluster with multiple GW nodes. 61 | // Returns cluster index. 62 | func (f *Framework) FindClusterWithMultipleGateways() int { 63 | for idx := range TestContext.ClusterIDs { 64 | gatewayNodes := FindGatewayNodes(ClusterIndex(idx)) 65 | if len(gatewayNodes) >= 2 { 66 | return idx 67 | } 68 | } 69 | 70 | return -1 71 | } 72 | 73 | // SetGatewayLabelOnNode sets the 'submariner.io/gateway' value for a node to the specified value. 74 | func (f *Framework) SetGatewayLabelOnNode(ctx context.Context, cluster ClusterIndex, nodeName string, isGateway bool) { 75 | // Escape the '/' char in the label name with the special sequence "~1" so it isn't treated as part of the path 76 | //nolint:contextcheck // Function `PatchString->doPatchOperation->AwaitUntil->AwaitResultOrError` should pass the context parameter. 77 | PatchString("/metadata/labels/"+strings.ReplaceAll(GatewayLabel, "/", "~1"), strconv.FormatBool(isGateway), 78 | func(pt types.PatchType, payload []byte) error { 79 | _, err := KubeClients[cluster].CoreV1().Nodes().Patch(ctx, nodeName, pt, payload, metav1.PatchOptions{}) 80 | if err != nil && f.stopped { 81 | Errorf("Error setting gateway label on node %q: %v", nodeName, err) 82 | err = nil 83 | } 84 | 85 | return err 86 | }) 87 | } 88 | -------------------------------------------------------------------------------- /test/e2e/framework/service_exports.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "context" 23 | 24 | "k8s.io/apimachinery/pkg/api/errors" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 27 | "k8s.io/apimachinery/pkg/runtime/schema" 28 | ) 29 | 30 | var gvr = schema.GroupVersionResource{ 31 | Group: "multicluster.x-k8s.io", 32 | Version: "v1alpha1", 33 | Resource: "serviceexports", 34 | } 35 | 36 | func (f *Framework) CreateServiceExport(cluster ClusterIndex, name string) { 37 | resourceServiceExport := &unstructured.Unstructured{} 38 | resourceServiceExport.SetName(name) 39 | resourceServiceExport.SetNamespace(f.Namespace) 40 | resourceServiceExport.SetKind("ServiceExport") 41 | resourceServiceExport.SetAPIVersion("multicluster.x-k8s.io/v1alpha1") 42 | 43 | svcExs := DynClients[cluster].Resource(gvr).Namespace(f.Namespace) 44 | 45 | _ = AwaitUntil("create service export", func() (interface{}, error) { 46 | result, err := svcExs.Create(context.TODO(), resourceServiceExport, metav1.CreateOptions{}) 47 | if errors.IsAlreadyExists(err) { 48 | err = nil 49 | } 50 | return result, err 51 | }, NoopCheckResult).(*unstructured.Unstructured) 52 | } 53 | 54 | func (f *Framework) DeleteServiceExport(cluster ClusterIndex, name string) { 55 | AwaitUntil("delete service export", func() (interface{}, error) { 56 | return nil, DynClients[cluster].Resource(gvr).Namespace(f.Namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) 57 | }, NoopCheckResult) 58 | } 59 | -------------------------------------------------------------------------------- /test/e2e/framework/services.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | apierrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/util/intstr" 29 | typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" 30 | ) 31 | 32 | const ( 33 | TestAppLabel = "test-app" 34 | ) 35 | 36 | func (f *Framework) NewService(name, portName string, port int32, protocol corev1.Protocol, selector map[string]string, 37 | isHeadless bool, ipFamily *corev1.IPFamily, 38 | ) *corev1.Service { 39 | service := corev1.Service{ 40 | ObjectMeta: metav1.ObjectMeta{ 41 | Name: name, 42 | }, 43 | Spec: corev1.ServiceSpec{ 44 | Ports: []corev1.ServicePort{{ 45 | Port: port, 46 | Name: portName, 47 | TargetPort: intstr.FromInt32(port), 48 | Protocol: protocol, 49 | }}, 50 | }, 51 | } 52 | 53 | if ipFamily != nil { 54 | service.Spec.IPFamilies = []corev1.IPFamily{*ipFamily} 55 | } 56 | 57 | if selector != nil { 58 | service.Spec.Selector = selector 59 | } 60 | 61 | if isHeadless { 62 | service.Spec.Type = corev1.ServiceTypeClusterIP 63 | service.Spec.ClusterIP = corev1.ClusterIPNone 64 | } 65 | 66 | return &service 67 | } 68 | 69 | func (f *Framework) CreateTCPServiceWithIPFamily( 70 | cluster ClusterIndex, 71 | selectorName string, 72 | port int32, 73 | ipFamily *corev1.IPFamily, 74 | ) *corev1.Service { 75 | tcpService := f.NewService("test-svc-"+selectorName, "tcp", port, corev1.ProtocolTCP, 76 | map[string]string{TestAppLabel: selectorName}, false, ipFamily) 77 | sc := KubeClients[cluster].CoreV1().Services(f.Namespace) 78 | 79 | return f.CreateService(sc, tcpService) 80 | } 81 | 82 | func (f *Framework) CreateHeadlessTCPService(cluster ClusterIndex, selectorName string, port int32) *corev1.Service { 83 | tcpService := f.NewService("test-svc"+selectorName, "tcp", port, corev1.ProtocolTCP, 84 | map[string]string{TestAppLabel: selectorName}, true, nil) 85 | sc := KubeClients[cluster].CoreV1().Services(f.Namespace) 86 | 87 | return f.CreateService(sc, tcpService) 88 | } 89 | 90 | func (f *Framework) NewNginxServiceWithIPFamilyPolicy(cluster ClusterIndex, ipFamilyPolicy *corev1.IPFamilyPolicy) *corev1.Service { 91 | nginxService := corev1.Service{ 92 | ObjectMeta: metav1.ObjectMeta{ 93 | Name: "nginx-demo", 94 | Labels: map[string]string{ 95 | "app": "nginx-demo", 96 | }, 97 | }, 98 | Spec: corev1.ServiceSpec{ 99 | Type: corev1.ServiceTypeClusterIP, 100 | IPFamilyPolicy: ipFamilyPolicy, 101 | Ports: []corev1.ServicePort{ 102 | { 103 | Port: 80, 104 | Name: "http", 105 | Protocol: corev1.ProtocolTCP, 106 | TargetPort: intstr.IntOrString{ 107 | Type: intstr.Int, 108 | IntVal: 8080, 109 | }, 110 | }, 111 | { 112 | Port: 8183, 113 | Name: "metrics", 114 | Protocol: corev1.ProtocolTCP, 115 | TargetPort: intstr.IntOrString{ 116 | Type: intstr.Int, 117 | IntVal: 8082, 118 | }, 119 | }, 120 | }, 121 | Selector: map[string]string{ 122 | "app": "nginx-demo", 123 | }, 124 | }, 125 | } 126 | 127 | sc := KubeClients[cluster].CoreV1().Services(f.Namespace) 128 | 129 | return f.CreateService(sc, &nginxService) 130 | } 131 | 132 | func (f *Framework) NewNginxService(cluster ClusterIndex) *corev1.Service { 133 | return f.NewNginxServiceWithIPFamilyPolicy(cluster, nil) 134 | } 135 | 136 | func (f *Framework) CreateTCPServiceWithoutSelector(cluster ClusterIndex, svcName, portName string, port int32) *corev1.Service { 137 | serviceSpec := f.NewService(svcName, portName, port, corev1.ProtocolTCP, nil, false, nil) 138 | sc := KubeClients[cluster].CoreV1().Services(f.Namespace) 139 | 140 | return f.CreateService(sc, serviceSpec) 141 | } 142 | 143 | func (f *Framework) CreateService(sc typedv1.ServiceInterface, serviceSpec *corev1.Service) *corev1.Service { 144 | return AwaitUntil("create service", func() (interface{}, error) { 145 | service, err := sc.Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) 146 | if apierrors.IsAlreadyExists(err) { 147 | err = sc.Delete(context.TODO(), serviceSpec.Name, metav1.DeleteOptions{}) 148 | if err != nil { 149 | return nil, err 150 | } 151 | 152 | service, err = sc.Create(context.TODO(), serviceSpec, metav1.CreateOptions{}) 153 | } 154 | 155 | return service, err 156 | }, NoopCheckResult).(*corev1.Service) 157 | } 158 | 159 | func (f *Framework) DeleteService(cluster ClusterIndex, serviceName string) { 160 | By(fmt.Sprintf("Deleting service %q on %q", serviceName, TestContext.ClusterIDs[cluster])) 161 | AwaitUntil("delete service", func() (interface{}, error) { 162 | return nil, KubeClients[cluster].CoreV1().Services(f.Namespace).Delete(context.TODO(), serviceName, metav1.DeleteOptions{}) 163 | }, NoopCheckResult) 164 | } 165 | -------------------------------------------------------------------------------- /test/e2e/framework/test_context.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "flag" 23 | "os" 24 | "strings" 25 | "time" 26 | 27 | "github.com/onsi/ginkgo/v2/types" 28 | "k8s.io/apimachinery/pkg/runtime/schema" 29 | "k8s.io/klog/v2" 30 | ) 31 | 32 | type contextArray []string 33 | 34 | type TestContextType struct { 35 | ReporterConfig *types.ReporterConfig 36 | SuiteConfig *types.SuiteConfig 37 | KubeConfigs []string // KubeConfigs provides an alternative to KubeConfig + KubeContexts 38 | KubeConfig string 39 | KubeContexts contextArray 40 | ClusterIDs []string 41 | NumNodesInCluster map[ClusterIndex]int 42 | SubmarinerNamespace string 43 | ConnectionTimeout uint 44 | ConnectionAttempts uint 45 | OperationTimeout uint 46 | PacketSize uint 47 | SkipConnectorSrcIPCheck bool 48 | GlobalnetEnabled bool 49 | ClientQPS float32 50 | ClientBurst int 51 | GroupVersion *schema.GroupVersion 52 | NettestImageURL string 53 | } 54 | 55 | func (contexts *contextArray) String() string { 56 | return strings.Join(*contexts, ",") 57 | } 58 | 59 | func (contexts *contextArray) Set(value string) error { 60 | *contexts = append(*contexts, value) 61 | return nil 62 | } 63 | 64 | var TestContext = &TestContextType{ 65 | ClientQPS: 20, 66 | ClientBurst: 50, 67 | NettestImageURL: "quay.io/submariner/nettest:devel", 68 | } 69 | 70 | func init() { 71 | flag.StringVar(&TestContext.KubeConfig, "kubeconfig", os.Getenv("KUBECONFIG"), 72 | "Path to kubeconfig containing embedded authinfo.") 73 | flag.Var(&TestContext.KubeContexts, "dp-context", "kubeconfig context for dataplane clusters (use several times).") 74 | flag.StringVar(&TestContext.SubmarinerNamespace, "submariner-namespace", "submariner", 75 | "Namespace in which the submariner components are deployed.") 76 | flag.UintVar(&TestContext.ConnectionTimeout, "connection-timeout", 18, 77 | "The timeout in seconds per connection attempt when verifying communication between clusters.") 78 | flag.UintVar(&TestContext.ConnectionAttempts, "connection-attempts", 7, 79 | "The number of connection attempts when verifying communication between clusters.") 80 | flag.UintVar(&TestContext.OperationTimeout, "operation-timeout", 190, "The general operation timeout in seconds.") 81 | flag.StringVar(&TestContext.NettestImageURL, "nettest-image", TestContext.NettestImageURL, 82 | "URL of the nettest image.") 83 | } 84 | 85 | func ValidateFlags(t *TestContextType) { 86 | if t.KubeConfig == "" && len(t.KubeConfigs) == 0 { 87 | klog.Fatalf("kubeconfig parameter or KUBECONFIG environment variable is required") 88 | } 89 | 90 | if len(t.KubeContexts) < 1 && len(t.KubeConfigs) < 1 { 91 | klog.Fatalf("at least one kubernetes context must be specified.") 92 | } 93 | } 94 | 95 | func (t *TestContextType) OperationTimeoutToDuration() time.Duration { 96 | //nolint:gosec // Ignore G115: integer overflow conversion uint -> int64 97 | return time.Duration(TestContext.OperationTimeout) * time.Second 98 | } 99 | -------------------------------------------------------------------------------- /test/e2e/framework/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package framework 20 | 21 | import ( 22 | "strings" 23 | 24 | . "github.com/onsi/gomega" 25 | "k8s.io/apimachinery/pkg/labels" 26 | "k8s.io/apimachinery/pkg/selection" 27 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // We need GCP authentication. 28 | restclient "k8s.io/client-go/rest" 29 | "k8s.io/client-go/tools/clientcmd" 30 | ) 31 | 32 | func loadConfig(configPath, context string) (*restclient.Config, error) { 33 | rules := clientcmd.NewDefaultClientConfigLoadingRules() 34 | rules.Precedence = strings.Split(configPath, ":") 35 | 36 | rules.DefaultClientConfig = &clientcmd.DefaultClientConfig 37 | overrides := &clientcmd.ConfigOverrides{ClusterDefaults: clientcmd.ClusterDefaults} 38 | 39 | if context != "" { 40 | overrides.CurrentContext = context 41 | } 42 | 43 | return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig() 44 | } 45 | 46 | func ExpectNoError(err error, explain ...interface{}) { 47 | ExpectNoErrorWithOffset(1, err, explain...) 48 | } 49 | 50 | // ExpectNoErrorWithOffset checks if "err" is set, and if so, fails assertion while logging the error at "offset" levels above its caller 51 | // (for example, for call chain f -> g -> ExpectNoErrorWithOffset(1, ...) error would be logged for "f"). 52 | func ExpectNoErrorWithOffset(offset int, err error, explain ...interface{}) { 53 | if err != nil { 54 | Logf("Unexpected error occurred: %v", err) 55 | } 56 | 57 | ExpectWithOffset(1+offset, err).NotTo(HaveOccurred(), explain...) 58 | } 59 | 60 | func NewRequirement(key string, op selection.Operator, vals []string) labels.Requirement { 61 | r, err := labels.NewRequirement(key, op, vals) 62 | Expect(err).To(Succeed()) 63 | 64 | return *r 65 | } 66 | 67 | // FindOtherClusterIndex looks within the environment for another cluster 68 | // besides the one provided and returns its index or -1 if none other is found. 69 | func FindOtherClusterIndex(mainCluster int) int { 70 | for idx := range TestContext.ClusterIDs { 71 | if idx != mainCluster { 72 | return idx 73 | } 74 | } 75 | 76 | return -1 77 | } 78 | -------------------------------------------------------------------------------- /test/scripts/cloud-prepare/no_gw_defined.yml: -------------------------------------------------------------------------------- 1 | --- 2 | submariner: true 3 | nodes: control-plane worker worker 4 | clusters: 5 | cluster-test: 6 | -------------------------------------------------------------------------------- /test/scripts/cloud-prepare/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e -o pipefail 4 | 5 | source ${SCRIPTS_DIR}/lib/debug_functions 6 | source ${SCRIPTS_DIR}/lib/utils 7 | 8 | function verify_gw_topology() { 9 | local expected_gateways="${1}" 10 | local gateways_count 11 | 12 | echo "Verifying ${expected_gateways} gateway nodes" 13 | gateways_count=$(kubectl get nodes -l=submariner.io/gateway=true \ 14 | -o yaml | yq '.items | length' -) 15 | 16 | if [[ "$expected_gateways" -ne "$gateways_count" ]]; then 17 | echo "Expected ${expected_gateways} gateways nodes but detected ${gateways_count}" 18 | return 1 19 | fi 20 | 21 | echo "Found expected number of gateways - ${expected_gateways}" 22 | } 23 | 24 | function remove_gw_labels() { 25 | local gw_nodes 26 | 27 | echo "Reset gateway nodes (unlabel)" 28 | readarray -t gw_nodes < <(kubectl get nodes -l=submariner.io/gateway=true \ 29 | -o yaml | yq '.items[].metadata.name' -) 30 | 31 | for node in "${gw_nodes[@]}"; do 32 | kubectl label --overwrite nodes "$node" submariner.io/gateway- 33 | done 34 | } 35 | 36 | function test_gateways() { 37 | local scenario="$1" 38 | local expected_gateways="$2" 39 | 40 | echo "Set gateway nodes according to ${scenario} scenario" 41 | export SETTINGS="$(dirname $0)/${scenario}" 42 | make cloud-prepare 43 | 44 | load_settings 45 | run_all_clusters verify_gw_topology "$expected_gateways" 46 | run_all_clusters remove_gw_labels 47 | } 48 | 49 | echo "Prepare cluster" 50 | make clusters SETTINGS="$(dirname $0)/no_gw_defined.yml" 51 | declare_kubeconfig 52 | 53 | test_gateways no_gw_defined.yml 1 54 | test_gateways two_gw_defined.yml 2 55 | echo "Gateways scenarios have been verified" 56 | -------------------------------------------------------------------------------- /test/scripts/cloud-prepare/two_gw_defined.yml: -------------------------------------------------------------------------------- 1 | --- 2 | submariner: true 3 | nodes: control-plane worker worker 4 | gateways: 2 5 | clusters: 6 | cluster-test: 7 | -------------------------------------------------------------------------------- /test/scripts/compile/.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | -------------------------------------------------------------------------------- /test/scripts/compile/hello.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package main 20 | 21 | import "fmt" 22 | 23 | var MYVAR = "nobody" 24 | 25 | func main() { 26 | fmt.Println("hello " + MYVAR) 27 | } 28 | -------------------------------------------------------------------------------- /test/scripts/compile/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | source "${SCRIPTS_DIR}"/lib/utils 5 | 6 | function validate_ldflags() { 7 | expected=$1 8 | actual=$($binary) 9 | [[ "$expected" == "$actual" ]] || exit_error "Expected ${expected@Q}, but got ${actual@Q}" 10 | } 11 | 12 | function test_compile_arch() { 13 | local binary=$1 14 | local arch=$2 15 | ${SCRIPTS_DIR}/compile.sh $binary hello.go 16 | if ! file $binary | grep -q $arch; then 17 | exit_error "Should have compiled ${arch@Q} but got $(file $binary)." 18 | fi 19 | } 20 | 21 | cd $(dirname $0) 22 | binary=bin/linux/amd64/hello 23 | export BUILD_UPX=false 24 | ${SCRIPTS_DIR}/compile.sh $binary hello.go 25 | validate_ldflags "hello nobody" 26 | 27 | LDFLAGS="-X main.MYVAR=somebody" ${SCRIPTS_DIR}/compile.sh $binary hello.go 28 | validate_ldflags "hello somebody" 29 | 30 | BUILD_DEBUG=true ${SCRIPTS_DIR}/compile.sh $binary hello.go 31 | file $binary | grep "not stripped" > /dev/null || exit_error "Debug information got stripped, even when requested!" 32 | 33 | BUILD_UPX=true ${SCRIPTS_DIR}/compile.sh $binary hello.go 34 | upx $binary > /dev/null 2>&1 && exit_error "Binary wasn't UPX'd although requested" 35 | 36 | test_compile_arch bin/linux/arm/v7/hello ARM 37 | 38 | GOARCH=arm test_compile_arch bin/hello ARM 39 | -------------------------------------------------------------------------------- /test/scripts/post_mortem/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | source ${SCRIPTS_DIR}/lib/debug_functions 6 | source ${SCRIPTS_DIR}/lib/utils 7 | 8 | function image_name() { 9 | echo "norepo/${cluster}_image" 10 | } 11 | 12 | function deploy_deadshoot() { 13 | filename="/tmp/netshoot_${cluster}" 14 | cp ${RESOURCES_DIR}/netshoot.yaml "${filename}" 15 | 16 | sed -i "s#image: .*#image: $(image_name)#" "${filename}" 17 | 18 | kubectl apply -f "${filename}" 19 | } 20 | 21 | declare_kubeconfig 22 | clusters=($(kind get clusters)) 23 | run_parallel "${clusters[*]}" deploy_deadshoot 24 | 25 | post_mortem=$(make post-mortem) 26 | echo "$post_mortem" 27 | 28 | for cluster in "${clusters[@]}"; do 29 | img=$(image_name) 30 | [[ $post_mortem =~ $(image_name) ]] || exit_error "Post mortem failed, didn't find failure for ${cluster}" 31 | done 32 | -------------------------------------------------------------------------------- /test/scripts/unit/failing_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | SPDX-License-Identifier: Apache-2.0 3 | 4 | Copyright Contributors to the Submariner project. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | package unit_test 19 | 20 | import ( 21 | "flag" 22 | "testing" 23 | 24 | . "github.com/onsi/ginkgo/v2" 25 | . "github.com/onsi/gomega" 26 | ) 27 | 28 | var testFailing bool 29 | 30 | func init() { 31 | flag.BoolVar(&testFailing, "fail", false, "Set to make test fail") 32 | } 33 | 34 | func TestUnit(t *testing.T) { 35 | RegisterFailHandler(Fail) 36 | RunSpecs(t, "Unit Test Suite") 37 | } 38 | 39 | var _ = Describe("Unit test", func() { 40 | It("should run", func() { 41 | if testFailing { 42 | Fail("") 43 | } 44 | }) 45 | }) 46 | -------------------------------------------------------------------------------- /tools/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | 3 | /* 4 | SPDX-License-Identifier: Apache-2.0 5 | 6 | Copyright Contributors to the Submariner project. 7 | 8 | Licensed under the Apache License, Version 2.0 (the "License"); 9 | you may not use this file except in compliance with the License. 10 | You may obtain a copy of the License at 11 | 12 | http://www.apache.org/licenses/LICENSE-2.0 13 | 14 | Unless required by applicable law or agreed to in writing, software 15 | distributed under the License is distributed on an "AS IS" BASIS, 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | See the License for the specific language governing permissions and 18 | limitations under the License. 19 | */ 20 | 21 | // Place any runtime dependencies as imports in this file. 22 | // Go modules will be forced to download and install them. 23 | package tools 24 | 25 | import ( 26 | _ "github.com/golangci/golangci-lint/v2/cmd/golangci-lint" 27 | _ "github.com/psampaz/go-mod-outdated" 28 | _ "sigs.k8s.io/kind/cmd/kind" 29 | ) 30 | --------------------------------------------------------------------------------