├── artifacts ├── manifests │ ├── .gitignore │ ├── pod.yaml │ ├── secret.yaml │ ├── storage-class.yaml │ ├── service.yaml │ ├── rs.yaml │ ├── pvc.yaml │ ├── cronjob.yaml │ └── podspec.yaml └── scripts │ ├── push.sh │ └── construct.sh ├── .gitignore ├── .gitattributes ├── .editorconfig ├── hack ├── update-deps.sh ├── verify-golangci-lint.sh ├── repos.sh ├── create-repos.sh └── fetch-all-latest-and-push.sh ├── configs ├── kubernetes-token ├── kubernetes-nightly-token ├── kubernetes-rules-configmap.yaml ├── kubernetes-nightly-rules-configmap.yaml ├── kubernetes-nightly ├── kubernetes ├── kubernetes-configmap.yaml ├── kubernetes-nightly-configmap.yaml ├── example ├── example-rules-configmap.yaml └── example-configmap.yaml ├── code-of-conduct.md ├── .git-crypt ├── .gitattributes └── keys │ └── default │ └── 0 │ ├── 15476E78262C1E3CB37095DE4C68E0F19F95EC33.gpg │ ├── 2583D03F747BF75DA761DAC4565C11F0FFCFA4EC.gpg │ ├── 79BD02F25334F112CF8B5CC9BA44BA8341537357.gpg │ └── A67E5FD880EB089F2317796780D83A796103BF59.gpg ├── OWNERS ├── cmd ├── update-rules │ ├── testdata │ │ └── invalid_rules.yaml │ ├── README.md │ ├── main_test.go │ └── main.go ├── sync-tags │ ├── main_test.go │ └── gomod.go ├── publishing-bot │ ├── github_test.go │ ├── publisher_test.go │ ├── publisher_logger_test.go │ ├── log_builder_test.go │ ├── config │ │ ├── config.go │ │ ├── rules_test.go │ │ └── rules.go │ ├── server.go │ ├── log_builder.go │ ├── github.go │ ├── publisher_logger.go │ ├── main.go │ └── publisher.go ├── validate-rules │ ├── main.go │ └── staging │ │ ├── github_utils.go │ │ └── validate.go ├── gomod-zip │ └── zip.go ├── collapsed-kube-commit-mapper │ └── main.go └── init-repo │ └── main.go ├── .github ├── dependabot.yml └── workflows │ └── golangci-lint.yml ├── CONTRIBUTING.md ├── SECURITY_CONTACTS ├── OWNERS_ALIASES ├── SECURITY.md ├── pkg ├── cache │ └── cache.go ├── git │ ├── kube.go │ ├── mapping.go │ └── mainline.go └── golang │ └── install.go ├── cloudbuild.yaml ├── go.mod ├── Dockerfile ├── test ├── run-bot-local.sh └── k8s-gen-bot-config.sh ├── Makefile ├── production.md ├── .golangci.yml ├── README.md ├── go.sum └── LICENSE /artifacts/manifests/.gitignore: -------------------------------------------------------------------------------- 1 | local.* -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _output 2 | /coverage.out 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | configs/*-token filter=git-crypt diff=git-crypt 2 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.sh] 2 | indent_style = space 3 | indent_size = 4 4 | -------------------------------------------------------------------------------- /hack/update-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | glide update --strip-vendor "$@" 4 | -------------------------------------------------------------------------------- /configs/kubernetes-token: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes/publishing-bot/HEAD/configs/kubernetes-token -------------------------------------------------------------------------------- /configs/kubernetes-nightly-token: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes/publishing-bot/HEAD/configs/kubernetes-nightly-token -------------------------------------------------------------------------------- /artifacts/manifests/pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: publisher 5 | spec: 6 | restartPolicy: Never 7 | -------------------------------------------------------------------------------- /configs/kubernetes-rules-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: publisher-rules 5 | data: 6 | config: | 7 | -------------------------------------------------------------------------------- /artifacts/manifests/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: github-token 5 | data: 6 | token: "TOKEN" 7 | type: Opaque 8 | -------------------------------------------------------------------------------- /configs/kubernetes-nightly-rules-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: publisher-rules 5 | data: 6 | config: | 7 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /.git-crypt/.gitattributes: -------------------------------------------------------------------------------- 1 | # Do not edit this file. To specify the files to encrypt, create your own 2 | # .gitattributes file in the directory where your files are. 3 | * !filter !diff 4 | -------------------------------------------------------------------------------- /artifacts/manifests/storage-class.yaml: -------------------------------------------------------------------------------- 1 | kind: StorageClass 2 | apiVersion: storage.k8s.io/v1 3 | metadata: 4 | name: ssd 5 | provisioner: kubernetes.io/gce-pd 6 | parameters: 7 | type: pd-ssd 8 | -------------------------------------------------------------------------------- /.git-crypt/keys/default/0/15476E78262C1E3CB37095DE4C68E0F19F95EC33.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes/publishing-bot/HEAD/.git-crypt/keys/default/0/15476E78262C1E3CB37095DE4C68E0F19F95EC33.gpg -------------------------------------------------------------------------------- /.git-crypt/keys/default/0/2583D03F747BF75DA761DAC4565C11F0FFCFA4EC.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes/publishing-bot/HEAD/.git-crypt/keys/default/0/2583D03F747BF75DA761DAC4565C11F0FFCFA4EC.gpg -------------------------------------------------------------------------------- /.git-crypt/keys/default/0/79BD02F25334F112CF8B5CC9BA44BA8341537357.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes/publishing-bot/HEAD/.git-crypt/keys/default/0/79BD02F25334F112CF8B5CC9BA44BA8341537357.gpg -------------------------------------------------------------------------------- /.git-crypt/keys/default/0/A67E5FD880EB089F2317796780D83A796103BF59.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kubernetes/publishing-bot/HEAD/.git-crypt/keys/default/0/A67E5FD880EB089F2317796780D83A796103BF59.gpg -------------------------------------------------------------------------------- /configs/kubernetes-nightly: -------------------------------------------------------------------------------- 1 | DOCKER_REPO = gcr.io/k8s-staging-publishing-bot/k8s-publishing-bot-nightly 2 | NAMESPACE = k8s-publishing-bot-nightly 3 | SCHEDULE = * */4 * * * 4 | INTERVAL = 14400 5 | MEMORY_REQUESTS = 2Gi 6 | MEMORY_LIMITS = 3Gi 7 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - sig-release-leads 3 | - release-engineering-approvers 4 | - dims 5 | - nikhita 6 | - sttts 7 | reviewers: 8 | - release-engineering-reviewers 9 | - akhilerm 10 | emeritus_approvers: 11 | - caesarxuchao 12 | -------------------------------------------------------------------------------- /configs/kubernetes: -------------------------------------------------------------------------------- 1 | DOCKER_REPO = gcr.io/k8s-staging-publishing-bot/k8s-publishing-bot 2 | NAMESPACE = publishing-bot 3 | SCHEDULE = * */4 * * * 4 | INTERVAL = 14400 5 | CPU_LIMITS = 2 6 | CPU_REQUESTS = 300m 7 | MEMORY_REQUESTS = 2Gi 8 | MEMORY_LIMITS = 2Gi 9 | -------------------------------------------------------------------------------- /artifacts/manifests/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: health 5 | spec: 6 | selector: 7 | name: publisher 8 | ports: 9 | - name: http 10 | protocol: TCP 11 | port: 80 12 | targetPort: 8080 13 | -------------------------------------------------------------------------------- /artifacts/manifests/rs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: ReplicaSet 3 | metadata: 4 | name: publisher 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | name: publisher 10 | template: 11 | metadata: 12 | labels: 13 | name: publisher 14 | spec: 15 | -------------------------------------------------------------------------------- /artifacts/manifests/pvc.yaml: -------------------------------------------------------------------------------- 1 | kind: PersistentVolumeClaim 2 | apiVersion: v1 3 | metadata: 4 | labels: 5 | app: publisher 6 | name: publisher-gopath 7 | spec: 8 | accessModes: 9 | - ReadWriteOnce 10 | resources: 11 | requests: 12 | storage: 100Gi 13 | storageClassName: ssd 14 | -------------------------------------------------------------------------------- /cmd/update-rules/testdata/invalid_rules.yaml: -------------------------------------------------------------------------------- 1 | rules: 2 | - destination: code-generator 3 | branches: 4 | - source: 5 | branch: master 6 | dir: staging/src/k8s.io/code-generator 7 | name: master 8 | - source: 9 | branch: release-1.19 10 | dir: staging/src/k8s.io/code-generator 11 | name: release-1.19 12 | go: 1.15.0 # invalid go version 13 | -------------------------------------------------------------------------------- /configs/kubernetes-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: publisher-config 5 | data: 6 | config: | 7 | source-org: kubernetes 8 | source-repo: kubernetes 9 | target-org: kubernetes 10 | rules-file: https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml 11 | github-issue: 56876 12 | dry-run: false 13 | -------------------------------------------------------------------------------- /configs/kubernetes-nightly-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: publisher-config 5 | data: 6 | config: | 7 | source-org: kubernetes 8 | source-repo: kubernetes 9 | target-org: kubernetes-nightly 10 | rules-file: https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml 11 | github-issue: 1 12 | dry-run: false 13 | -------------------------------------------------------------------------------- /artifacts/manifests/cronjob.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1beta1 2 | kind: CronJob 3 | metadata: 4 | name: publisher 5 | spec: 6 | schedule: "SCHEDULE" 7 | startingDeadlineSeconds: 36000 8 | concurrencyPolicy: Forbid 9 | successfulJobsHistoryLimit: 1 10 | failedJobsHistoryLimit: 14 11 | jobTemplate: 12 | metadata: 13 | labels: 14 | app: publisher 15 | template: 16 | spec: 17 | restartPolicy: Never 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | labels: 8 | - "area/dependency" 9 | - "release-note-none" 10 | - "ok-to-test" 11 | open-pull-requests-limit: 10 12 | 13 | - package-ecosystem: github-actions 14 | directory: "/" 15 | schedule: 16 | interval: daily 17 | labels: 18 | - "area/dependency" 19 | - "release-note-none" 20 | - "ok-to-test" 21 | open-pull-requests-limit: 10 22 | -------------------------------------------------------------------------------- /configs/example: -------------------------------------------------------------------------------- 1 | # the Docker repo for the bot image. Should be writable for you. 2 | DOCKER_REPO = /k8s-publishing-bot 3 | 4 | # the Kubernetes namespace to deploy all bot manifest to. You can leave this empty 5 | # to use the current context namespace. 6 | NAMESPACE = 7 | 8 | # time in seconds between publisher runs in seconds 9 | INTERVAL = 86400 10 | 11 | # requested resources for the main container 12 | # MEMORY_REQUESTS = 1Gi 13 | 14 | # requested limits for the main container 15 | # MEMORY_LIMITS = 2Gi 16 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thanks for taking the time to join our community and start contributing! 4 | 5 | The [Contributor Guide](https://github.com/kubernetes/community/blob/master/contributors/guide/README.md) 6 | provides detailed instructions on how to get your ideas and bug fixes seen and accepted. 7 | 8 | Please remember to sign the [CNCF CLA](https://github.com/kubernetes/community/blob/master/CLA.md) and 9 | read and observe the [Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 10 | 11 | Instructions for testing and deploying the publishing bot exist [here](README.md#testing-and-deploying-the-robot). -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | - master 9 | 10 | jobs: 11 | golangci-lint: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 15 | - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 16 | with: 17 | go-version: '1.24' 18 | check-latest: true 19 | - name: golangci-lint 20 | uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0 21 | with: 22 | version: v2.1 23 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Product Security Committee to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | cpanato 14 | dims 15 | jeremyrickard 16 | justaugustus 17 | nikhita 18 | puerco 19 | saschagrunert 20 | sttts 21 | -------------------------------------------------------------------------------- /configs/example-rules-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: publisher-rules 5 | data: 6 | config: | 7 | # Specify branches you want to skip 8 | skip-source-branches: 9 | # - release-1.7 10 | # ls-files pattern like: */BUILD *.ext pkg/foo.go Makefile 11 | recursive-delete-patterns: 12 | # - BUILD 13 | # - "*/BUILD" 14 | 15 | # Skip update/fix gomod 16 | # skip-gomod: true 17 | 18 | # Skip sync tags 19 | # skip-tags: true 20 | 21 | # a valid go version string like 1.10.2 or 1.10 22 | # if the go version is not specified in rules, 23 | # default-go-version is used. 24 | # default-go-version: 1.14 25 | 26 | rules: 27 | - destination: # eg. "client-go" 28 | branches: 29 | - name: # eg. "master" 30 | source: 31 | branch: # eg. "master" 32 | dir: # eg. "staging/src/k8s.io/client-go" 33 | publish-script: # eg. /publish.sh 34 | -------------------------------------------------------------------------------- /OWNERS_ALIASES: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners#owners_aliases 2 | 3 | aliases: 4 | sig-release-leads: 5 | - cpanato # SIG Technical Lead 6 | - jeremyrickard # SIG Chair 7 | - justaugustus # SIG Chair 8 | - puerco # SIG Technical Lead 9 | - saschagrunert # SIG Chair 10 | - Verolop # SIG Technical Lead 11 | release-engineering-approvers: 12 | - cpanato # subproject owner / Release Manager 13 | - jeremyrickard # subproject owner / Release Manager 14 | - justaugustus # subproject owner / Release Manager 15 | - palnabarun # Release Manager 16 | - puerco # subproject owner / Release Manager 17 | - saschagrunert # subproject owner / Release Manager 18 | - xmudrii # Release Manager 19 | - Verolop # subproject owner / Release Manager 20 | release-engineering-reviewers: 21 | - ameukam # Release Manager Associate 22 | - cici37 # Release Manager Associate 23 | - jimangel # Release Manager Associate 24 | - jrsapi # Release Manager Associate 25 | - salaxander # Release Manager Associate 26 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Security Announcements 4 | 5 | Join the [kubernetes-security-announce] group for security and vulnerability announcements. 6 | 7 | You can also subscribe to an RSS feed of the above using [this link][kubernetes-security-announce-rss]. 8 | 9 | ## Reporting a Vulnerability 10 | 11 | Instructions for reporting a vulnerability can be found on the 12 | [Kubernetes Security and Disclosure Information] page. 13 | 14 | ## Supported Versions 15 | 16 | Information about supported Kubernetes versions can be found on the 17 | [Kubernetes version and version skew support policy] page on the Kubernetes website. 18 | 19 | [kubernetes-security-announce]: https://groups.google.com/forum/#!forum/kubernetes-security-announce 20 | [kubernetes-security-announce-rss]: https://groups.google.com/forum/feed/kubernetes-security-announce/msgs/rss_v2_0.xml?num=50 21 | [Kubernetes version and version skew support policy]: https://kubernetes.io/docs/setup/release/version-skew-policy/#supported-versions 22 | [Kubernetes Security and Disclosure Information]: https://kubernetes.io/docs/reference/issues-security/security/#report-a-vulnerability 23 | -------------------------------------------------------------------------------- /hack/verify-golangci-lint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | VERSION=v1.55.2 22 | URL_BASE=https://raw.githubusercontent.com/golangci/golangci-lint 23 | URL=$URL_BASE/$VERSION/install.sh 24 | 25 | if [[ ! -f .golangci.yml ]]; then 26 | echo 'ERROR: missing .golangci.yml in repo root' >&2 27 | exit 1 28 | fi 29 | 30 | if ! command -v golangci-lint; then 31 | curl -sfL $URL | sh -s $VERSION 32 | PATH=$PATH:bin 33 | fi 34 | 35 | golangci-lint version 36 | golangci-lint linters 37 | golangci-lint run "$@" 38 | -------------------------------------------------------------------------------- /pkg/cache/cache.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cache 18 | 19 | import ( 20 | gogit "github.com/go-git/go-git/v5" 21 | "github.com/go-git/go-git/v5/plumbing" 22 | "github.com/go-git/go-git/v5/plumbing/object" 23 | ) 24 | 25 | var globalCommitCache = map[plumbing.Hash]*object.Commit{} 26 | 27 | func CommitObject(r *gogit.Repository, hash plumbing.Hash) (*object.Commit, error) { 28 | if c, found := globalCommitCache[hash]; found { 29 | if c == nil { 30 | return nil, plumbing.ErrObjectNotFound 31 | } 32 | return c, nil 33 | } 34 | 35 | c, err := r.CommitObject(hash) 36 | globalCommitCache[hash] = c 37 | return c, err 38 | } 39 | -------------------------------------------------------------------------------- /cmd/sync-tags/main_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import "testing" 20 | 21 | func Test_mappingOutputFileName(t *testing.T) { 22 | type args struct { 23 | fnameTpl string 24 | branch string 25 | tag string 26 | } 27 | tests := []struct { 28 | name string 29 | args args 30 | want string 31 | }{ 32 | {"tag and branch", args{"foo-{{.Branch}}-{{.Tag}}-bar", "a", "b"}, "foo-a-b-bar"}, 33 | } 34 | for _, tt := range tests { 35 | t.Run(tt.name, func(t *testing.T) { 36 | if got := mappingOutputFileName(tt.args.fnameTpl, tt.args.branch, tt.args.tag); got != tt.want { 37 | t.Errorf("mappingOutputFileName() = %v, want %v", got, tt.want) 38 | } 39 | }) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /cmd/publishing-bot/github_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package main 17 | 18 | import "testing" 19 | 20 | func TestGithubLogTransform(t *testing.T) { 21 | //nolint:dupword // this is intentional for the test 22 | originLog := `111111111111 23 | 222222 24 | + hello 25 | hello foo 26 | hello bar 27 | + holla 28 | holla foo 29 | holla bar 30 | + hi 31 | hi foo 32 | hi bar 33 | ` 34 | //nolint:dupword // this is intentional for the test 35 | expected := "```" + ` 36 | + hello 37 | hello foo 38 | hello bar 39 | + holla 40 | holla foo 41 | holla bar 42 | + hi 43 | hi foo 44 | hi bar` + "```\n" 45 | actual := transfromLogToGithubFormat(originLog, 3) 46 | if actual != expected { 47 | t.Errorf("log mismatched: expected(%q) actual(%q)", expected, actual) 48 | t.Fail() 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /cmd/publishing-bot/publisher_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | ) 23 | 24 | func Test_updateEnv(t *testing.T) { 25 | tests := []struct { 26 | name string 27 | env []string 28 | want []string 29 | }{ // TODO: Fix or remove 30 | /* 31 | {"no PATH", []string{"FOO=42"}, []string{"FOO=42", "PATH=/usr/local/go"}}, 32 | */ 33 | {"no PATH", []string{"FOO=42", "PATH=/bin", "BAR=1"}, []string{"FOO=42", "PATH=/usr/local/go:/bin", "BAR=1"}}, 34 | } 35 | for _, tt := range tests { 36 | t.Run(tt.name, func(t *testing.T) { 37 | if got := updateEnv(tt.env, "PATH", prependPath("/usr/local/go"), "/usr/local/go"); !reflect.DeepEqual(got, tt.want) { 38 | t.Errorf("updateEnv() = %v, want %v", got, tt.want) 39 | } 40 | }) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /artifacts/scripts/push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script sets up the .netrc file with the supplied token, then pushes to 18 | # the remote repo. 19 | # The script assumes that the working directory is the root of the repo. 20 | 21 | set -o errexit 22 | set -o nounset 23 | set -o pipefail 24 | 25 | if [ ! $# -eq 2 ]; then 26 | echo "usage: $0 token branch" 27 | exit 1 28 | fi 29 | 30 | TOKEN="$(cat ${1})" 31 | BRANCH="${2}" 32 | readonly TOKEN BRANCH 33 | 34 | # set up github token in /netrc/.netrc 35 | echo "machine github.com login ${TOKEN}" > /netrc/.netrc 36 | cleanup_github_token() { 37 | rm -rf /netrc/.netrc 38 | } 39 | trap cleanup_github_token EXIT SIGINT 40 | 41 | HOME=/netrc git push origin "${BRANCH}" --no-tags 42 | HOME=/netrc ../push-tags-$(basename "${PWD}")-${BRANCH/\//_}.sh 43 | -------------------------------------------------------------------------------- /pkg/git/kube.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package git 18 | 19 | import ( 20 | "strings" 21 | 22 | "github.com/go-git/go-git/v5/plumbing" 23 | "github.com/go-git/go-git/v5/plumbing/object" 24 | ) 25 | 26 | // SourceHash extracts kube commit from commit message 27 | // The baseRepoName default to "kubernetes". 28 | // TODO: Refactor so we take the commitMsgTag as argument and don't need to 29 | // construct the ancientSyncCommitSubjectPrefix or sourceCommitPrefix. 30 | func SourceHash(c *object.Commit, tag string) plumbing.Hash { 31 | lines := strings.Split(c.Message, "\n") 32 | sourceCommitPrefix := tag + ": " 33 | for _, line := range lines { 34 | if strings.HasPrefix(line, sourceCommitPrefix) { 35 | return plumbing.NewHash(strings.TrimSpace(line[len(sourceCommitPrefix):])) 36 | } 37 | } 38 | 39 | return plumbing.ZeroHash 40 | } 41 | -------------------------------------------------------------------------------- /configs/example-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: publisher-config 5 | data: 6 | config: | 7 | # this specifies the source repository coordinates (org/name) 8 | source-org: 9 | source-repo: 10 | # the github org or user to publish the new repos to 11 | target-org: 12 | 13 | # file path or URL 14 | rules-file: /etc/publisher-rules/config 15 | # rules-file: https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml 16 | 17 | # the github issue number in the source repo to publish logs to on errors. You must be 18 | # able to write to that issue. So you probably should create it with the bot user. 19 | # REMEMBER: do not run the bot with a user that can close arbitrary issues in the 20 | # source repo as that will trigger unwanted close events on push. 21 | # github-issue: 56916 22 | 23 | # if true, no push will be done. The bot will stop just before. 24 | dry-run: true 25 | 26 | # the github application token to use 27 | # CAUTION: do not check this into Github. You can also and probably should pass that as 28 | # TOKEN= to the "make deploy" command. 29 | # token: 30 | 31 | # the base path where the bot will look for a publish scripts in the source 32 | # repository. Default value is "./publish_scripts". 33 | # base-publish-script-path: 34 | -------------------------------------------------------------------------------- /hack/repos.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2021 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | # shellcheck disable=SC2034 22 | repos=( 23 | api 24 | apiextensions-apiserver 25 | apimachinery 26 | apiserver 27 | cli-runtime 28 | client-go 29 | cloud-provider 30 | cluster-bootstrap 31 | code-generator 32 | component-base 33 | component-helpers 34 | controller-manager 35 | cri-api 36 | cri-client 37 | csi-translation-lib 38 | dynamic-resource-allocation 39 | externaljwt 40 | endpointslice 41 | kms 42 | kube-aggregator 43 | kube-controller-manager 44 | kube-proxy 45 | kube-scheduler 46 | kubectl 47 | kubelet 48 | legacy-cloud-providers 49 | metrics 50 | mount-utils 51 | pod-security-admission 52 | sample-apiserver 53 | sample-cli-plugin 54 | sample-controller 55 | ) 56 | -------------------------------------------------------------------------------- /cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Google Cloud Build configuration: https://cloud.google.com/cloud-build/docs/build-config 2 | # Image building process: https://git.k8s.io/test-infra/config/jobs/image-pushing/README.md 3 | 4 | # this must be specified in seconds. If omitted, defaults to 600s (10 mins) 5 | timeout: 1200s 6 | 7 | options: 8 | substitution_option: ALLOW_LOOSE 9 | # TODO(image): Consider a smaller machine type 10 | machineType: E2_HIGHCPU_32 11 | 12 | steps: 13 | - name: 'gcr.io/k8s-staging-releng/k8s-ci-builder:latest-default' 14 | entrypoint: make 15 | env: 16 | - GIT_TAG=$_GIT_TAG 17 | - PULL_BASE_REF=$_PULL_BASE_REF 18 | - IMG_REGISTRY=gcr.io/$PROJECT_ID 19 | - IMG_VERSION=$_IMG_VERSION 20 | args: 21 | - build-and-push-image 22 | 23 | # TODO(image): Consider adding a container structure test 24 | #- name: 'gcr.io/gcp-runtimes/container-structure-test' 25 | # id: structure-test 26 | # args: 27 | # - test 28 | # - --image=gcr.io/$PROJECT_ID/publishing-bot:$_GIT_TAG 29 | # - --config=container-structure.yaml 30 | 31 | substitutions: 32 | # _GIT_TAG will be filled with a git-based tag for the image, of the form 33 | # vYYYYMMDD-hash, and can be used as a substitution 34 | _GIT_TAG: '12345' 35 | _PULL_BASE_REF: 'dev' 36 | _IMG_REGISTRY: 'null-registry' 37 | _IMG_VERSION: 'v0.0.0-1' 38 | 39 | tags: 40 | - 'publishing-bot' 41 | - ${_GIT_TAG} 42 | - ${_PULL_BASE_REF} 43 | - ${_IMG_REGISTRY} 44 | - ${_IMG_VERSION} 45 | 46 | images: 47 | - 'gcr.io/$PROJECT_ID/k8s-publishing-bot:$_GIT_TAG' 48 | - 'gcr.io/$PROJECT_ID/k8s-publishing-bot:$_IMG_VERSION' 49 | - 'gcr.io/$PROJECT_ID/k8s-publishing-bot:latest' 50 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module k8s.io/publishing-bot 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/blang/semver/v4 v4.0.0 7 | github.com/go-git/go-git/v5 v5.16.3 8 | github.com/golang/glog v1.2.5 9 | github.com/google/go-github v17.0.0+incompatible 10 | github.com/lithammer/dedent v1.1.0 11 | github.com/shurcooL/go v0.0.0-20171108033853-004faa6b0118 12 | golang.org/x/mod v0.31.0 13 | golang.org/x/oauth2 v0.34.0 14 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 15 | gopkg.in/yaml.v2 v2.4.0 16 | ) 17 | 18 | require ( 19 | dario.cat/mergo v1.0.0 // indirect 20 | github.com/Microsoft/go-winio v0.6.2 // indirect 21 | github.com/ProtonMail/go-crypto v1.1.6 // indirect 22 | github.com/cloudflare/circl v1.6.1 // indirect 23 | github.com/cyphar/filepath-securejoin v0.4.1 // indirect 24 | github.com/emirpasic/gods v1.18.1 // indirect 25 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect 26 | github.com/go-git/go-billy/v5 v5.6.2 // indirect 27 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect 28 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect 29 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 30 | github.com/kevinburke/ssh_config v1.2.0 // indirect 31 | github.com/pjbgf/sha1cd v0.3.2 // indirect 32 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect 33 | github.com/skeema/knownhosts v1.3.1 // indirect 34 | github.com/xanzy/ssh-agent v0.3.3 // indirect 35 | golang.org/x/crypto v0.45.0 // indirect 36 | golang.org/x/net v0.47.0 // indirect 37 | golang.org/x/sys v0.38.0 // indirect 38 | gopkg.in/warnings.v0 v0.1.2 // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /cmd/validate-rules/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | 23 | "github.com/golang/glog" 24 | "k8s.io/publishing-bot/cmd/publishing-bot/config" 25 | "k8s.io/publishing-bot/cmd/validate-rules/staging" 26 | ) 27 | 28 | const baseRefEnvKey = "PULL_BASE_REF" 29 | 30 | func main() { 31 | flag.Parse() 32 | err := flag.Set("alsologtostderr", "true") 33 | if err != nil { 34 | glog.Fatalf("attempting to log to stderr: %v", err) 35 | } 36 | 37 | for _, f := range flag.Args() { 38 | rules, err := config.LoadRules(f) 39 | if err != nil { 40 | glog.Fatalf("Cannot load rules file %q: %v", f, err) 41 | } 42 | if err := config.Validate(rules); err != nil { 43 | glog.Fatalf("Invalid rules file %q: %v", f, err) 44 | } 45 | errors := staging.EnsureStagingDirectoriesExist(rules, os.Getenv(baseRefEnvKey)) 46 | if errors != nil { 47 | for _, err := range errors { 48 | glog.Errorf("Error: %s", err) 49 | } 50 | glog.Fatalf("Invalid rules file %q", f) 51 | } 52 | glog.Infof("validation successful") 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM debian:bookworm 16 | MAINTAINER Stefan Schimanski 17 | RUN apt-get update \ 18 | && apt-get install -y -qq git=1:2.39.5-0+deb12u2 \ 19 | && apt-get install -y -qq mercurial \ 20 | && apt-get install -y -qq ca-certificates curl wget jq vim tmux bsdmainutils tig gcc zip \ 21 | && rm -rf /var/lib/apt/lists/* 22 | 23 | ENV GOPATH="/go-workspace" 24 | ENV GOROOT="/go-workspace/go" 25 | ENV PATH="${GOPATH}/bin:/go-workspace/go/bin:${PATH}" 26 | ENV GIT_COMMITTER_NAME="Kubernetes Publisher" 27 | ENV GIT_COMMITTER_EMAIL="k8s-publishing-bot@users.noreply.github.com" 28 | ENV TERM=xterm 29 | ENV PS1='\h:\w\$' 30 | ENV SHELL=/bin/bash 31 | 32 | WORKDIR "/" 33 | 34 | ADD _output/publishing-bot /publishing-bot 35 | ADD _output/collapsed-kube-commit-mapper /collapsed-kube-commit-mapper 36 | ADD _output/sync-tags /sync-tags 37 | ADD _output/init-repo /init-repo 38 | ADD _output/update-rules /update-rules 39 | 40 | ADD _output/gomod-zip /gomod-zip 41 | ADD artifacts/scripts/ /publish_scripts 42 | 43 | CMD ["/publishing-bot", "--dry-run", "--token-file=/token"] 44 | -------------------------------------------------------------------------------- /cmd/publishing-bot/publisher_logger_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "bytes" 21 | "sync" 22 | "testing" 23 | ) 24 | 25 | func TestLogLineWriter(t *testing.T) { 26 | buf := new(bytes.Buffer) 27 | fakeLogWriter := newSyncWriter( 28 | muxWriter{buf}, 29 | ) 30 | 31 | content1 := "XXXXXXXXXXXXXX" 32 | content2 := "YYYYYYYYYYYYYY" 33 | content3 := "ZZZZZZZZZZZZZZ" 34 | 35 | contents := []string{ 36 | content1, content2, content3, 37 | } 38 | 39 | wg := &sync.WaitGroup{} 40 | for _, content := range contents { 41 | w := newLineWriter(fakeLogWriter) 42 | content := content 43 | wg.Add(1) 44 | go func() { 45 | for i := 0; i < 99999; i++ { 46 | //nolint:errcheck // TODO(lint): Should we be checking errors here? 47 | w.Write([]byte(content + "\n")) 48 | } 49 | wg.Done() 50 | }() 51 | } 52 | wg.Wait() 53 | 54 | finalContent := buf.String() 55 | uniqueLines := make(map[string]struct{}) 56 | 57 | newLogBuilderWithMaxBytes(0, finalContent). 58 | Trim("\n"). 59 | Split("\n"). 60 | Filter(func(line string) bool { 61 | uniqueLines[line] = struct{}{} 62 | return true 63 | }).Log() 64 | 65 | for line := range uniqueLines { 66 | if line != content1 && line != content2 && line != content3 { 67 | t.Errorf("malformed log: %s", line) 68 | t.Fail() 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /artifacts/manifests/podspec.yaml: -------------------------------------------------------------------------------- 1 | initContainers: 2 | - name: initialize-repos 3 | command: 4 | - /init-repo 5 | - --alsologtostderr 6 | - --config=/etc/munge-config/config 7 | - --rules-file=/etc/publisher-rules/config 8 | - 2>&1 9 | image: DOCKER_IMAGE 10 | imagePullPolicy: Always 11 | resources: 12 | requests: 13 | cpu: 300m 14 | memory: 200Mi 15 | limits: 16 | cpu: 2 17 | memory: 1639Mi 18 | volumeMounts: 19 | - mountPath: /etc/munge-config 20 | name: munge-config 21 | - mountPath: /go-workspace 22 | name: publisher-gopath 23 | - mountPath: /etc/publisher-rules 24 | name: publisher-rules 25 | - mountPath: /.cache 26 | name: cache 27 | containers: 28 | - name: publisher 29 | command: 30 | - /publishing-bot 31 | - --alsologtostderr 32 | - --config=/etc/munge-config/config 33 | - --token-file=/etc/secret-volume/token 34 | - --interval=0 35 | - --server-port=8080 36 | - 2>&1 37 | image: DOCKER_IMAGE 38 | imagePullPolicy: Always 39 | resources: 40 | requests: 41 | cpu: CPU_REQUESTS 42 | memory: MEMORY_REQUESTS 43 | limits: 44 | cpu: CPU_LIMITS 45 | memory: MEMORY_LIMITS 46 | volumeMounts: 47 | - mountPath: /etc/munge-config 48 | name: munge-config 49 | - mountPath: /etc/publisher-rules 50 | name: publisher-rules 51 | - mountPath: /etc/secret-volume 52 | name: secret-volume 53 | - mountPath: /netrc 54 | name: netrc 55 | - mountPath: /gitrepos 56 | name: repo 57 | - mountPath: /go-workspace 58 | name: publisher-gopath 59 | - mountPath: /.cache 60 | name: cache 61 | volumes: 62 | - name: munge-config 63 | configMap: 64 | name: publisher-config 65 | - name: publisher-rules 66 | configMap: 67 | name: publisher-rules 68 | - name: secret-volume 69 | secret: 70 | secretName: publishing-bot-github-token 71 | - name: repo 72 | emptyDir: {} 73 | - name: cache 74 | emptyDir: {} 75 | - name: netrc 76 | emptyDir: 77 | medium: Memory 78 | - name: publisher-gopath 79 | persistentVolumeClaim: 80 | claimName: publisher-gopath 81 | -------------------------------------------------------------------------------- /test/run-bot-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2023 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | # This script expects a config and rules file in the BOT_CONFIG_DIRECTORY which will be used for running 23 | # the publishing bot 24 | # The image gcr.io/k8s-staging-publishing-bot/k8s-publishing-bot:latest should be available locally 25 | # in the docker daemon 26 | 27 | BOT_CONFIG_DIRECTORY="${1:-bot-configs}" 28 | 29 | # create the docker volumes 30 | docker volume create local-go-workspace && docker volume create cache 31 | 32 | docker run --rm \ 33 | --pull=never \ 34 | -v local-go-workspace:/go-workspace \ 35 | -v cache:/.cache \ 36 | -v "${PWD}/${BOT_CONFIG_DIRECTORY}":/etc/bot-configs \ 37 | gcr.io/k8s-staging-publishing-bot/k8s-publishing-bot:latest \ 38 | /init-repo \ 39 | --alsologtostderr \ 40 | --config=/etc/bot-configs/config \ 41 | --rules-file=/etc/bot-configs/rules 42 | 43 | docker run --rm \ 44 | --pull=never \ 45 | -v local-go-workspace:/go-workspace \ 46 | -v cache:/.cache \ 47 | -v "${PWD}/${BOT_CONFIG_DIRECTORY}":/etc/bot-configs \ 48 | gcr.io/k8s-staging-publishing-bot/k8s-publishing-bot:latest \ 49 | /publishing-bot \ 50 | --alsologtostderr \ 51 | --config=/etc/bot-configs/config \ 52 | --rules-file=/etc/bot-configs/rules \ 53 | --dry-run=true 54 | 55 | # cleanup the docker volumes 56 | docker volume rm local-go-workspace && docker volume rm cache 57 | -------------------------------------------------------------------------------- /cmd/publishing-bot/log_builder_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package main 17 | 18 | import ( 19 | "strings" 20 | "testing" 21 | ) 22 | 23 | func TestNewGithubLogBuilder(t *testing.T) { 24 | testCases := []struct { 25 | rawLog string 26 | maxBytes int 27 | expectedLog string 28 | }{ 29 | { 30 | rawLog: "abcdefg", 31 | maxBytes: 5, 32 | expectedLog: "...fg", 33 | }, 34 | { 35 | rawLog: "abcdefg", 36 | maxBytes: 1000, 37 | expectedLog: "abcdefg", 38 | }, 39 | { 40 | rawLog: "abcdefg", 41 | maxBytes: -1, 42 | expectedLog: "abcdefg", 43 | }, 44 | { 45 | rawLog: "", 46 | maxBytes: 100, 47 | expectedLog: "", 48 | }, 49 | } 50 | for _, c := range testCases { 51 | builder := newLogBuilderWithMaxBytes(c.maxBytes, c.rawLog) 52 | if l := builder.Log(); l != c.expectedLog { 53 | t.Errorf("log mismatched: expected(%q) actual(%q)", c.expectedLog, l) 54 | t.Fail() 55 | } 56 | } 57 | } 58 | 59 | func TestGithubLogBuilderManipulation(t *testing.T) { 60 | testLog := `foohello 61 | fooworld 62 | foobaz 63 | foobarfoo 64 | barbarbar 65 | ` 66 | expected := `**** 67 | foobarfoo 68 | foobaz 69 | fooworld` 70 | actual := newLogBuilderWithMaxBytes(0, testLog). 71 | AddHeading("****"). 72 | Trim("\n"). 73 | Split("\n"). 74 | Filter(func(line string) bool { 75 | return strings.HasPrefix(line, "foo") 76 | }). 77 | Tail(3). 78 | Reverse(). 79 | Join("\n"). 80 | Log() 81 | if expected != actual { 82 | t.Errorf("log mismatched: expected(%q) actual(%q)", expected, actual) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /cmd/publishing-bot/config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | // Config is how we are configured to talk to github. 20 | type Config struct { 21 | // GithubHost is the address for github. 22 | // Defaults to github.com 23 | GithubHost string `yaml:"github-host"` 24 | 25 | // BasePackage is the base package name for this repo. 26 | // Defaults to k8s.io when SourceOrg is kubernetes, otherwise, defaults 27 | // to ${GithubHost}/${TargetOrg} 28 | BasePackage string `yaml:"base-package"` 29 | 30 | // the organization to publish into, e.g. k8s-publishing-bot or kubernetes-nightly 31 | TargetOrg string `yaml:"target-org"` 32 | 33 | // the source repo name, e.g. "kubernetes" 34 | SourceRepo string `yaml:"source-repo"` 35 | 36 | // the source repo org name, e.g. "kubernetes" 37 | SourceOrg string `yaml:"source-org"` 38 | 39 | // the file with the clear-text github token 40 | TokenFile string `yaml:"token-file,omitempty"` 41 | 42 | // the file that contain the repository rules 43 | RulesFile string `yaml:"rules-file"` 44 | 45 | // If true, don't make any mutating API calls 46 | DryRun bool 47 | 48 | // A github issue number to report errors 49 | GithubIssue int `yaml:"github-issue,omitempty"` 50 | 51 | // BasePublishScriptPath determine the base path where we will look for a 52 | // publishing scripts in the source repo. It defaults to ./publishing_scripts'. 53 | BasePublishScriptPath string `yaml:"base-publish-script-path,omitempty"` 54 | 55 | // name of the default git branch in the repo. defaults to master 56 | GitDefaultBranch string `yaml:"git-default-branch,omitempty"` 57 | } 58 | -------------------------------------------------------------------------------- /cmd/publishing-bot/config/rules_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import "testing" 20 | 21 | func TestValidateGoVersion(t *testing.T) { 22 | tests := []struct { 23 | ver string 24 | isValid bool 25 | }{ 26 | {"0.0", true}, 27 | {"0.0rc1", true}, 28 | {"0.9", true}, 29 | {"0.9.0", false}, 30 | {"1.9", true}, 31 | {"0.0.1", true}, 32 | {"1.9.0", false}, 33 | {"1.9.1", true}, 34 | {"1.15", true}, 35 | {"1.15.0", false}, 36 | {"1.15.1", true}, 37 | {"1.15beta1", true}, 38 | {"1.15.0-beta.1", false}, 39 | {"1.15rc1", true}, 40 | {"1.15.0-rc.1", false}, 41 | {"1.15.10", true}, 42 | {"1.15.11", true}, 43 | {"1.15.20", true}, 44 | {"2.0", false}, 45 | {"2.0alpha1", true}, 46 | {"2.0-alpha1", false}, 47 | {"12.12.100", true}, 48 | {"999.999.9999", true}, 49 | {"999.999.9999-beta.1", false}, 50 | {"999.999.9999beta1", true}, 51 | {"1.21.0", true}, 52 | {"1.21", false}, 53 | {"1.21rc1", true}, 54 | {"1.21.0rc1", true}, 55 | {"1.20.0", false}, 56 | {"1.20rc1", true}, 57 | {"1.22.0", true}, 58 | {"1.22rc1", true}, 59 | {"1.122.0", true}, 60 | {"1.122", false}, 61 | {"1.30.0", true}, 62 | {"1.20rc.20rc2", false}, 63 | {"2.20", false}, 64 | {"2.21", false}, 65 | } 66 | 67 | for _, test := range tests { 68 | err := ensureValidGoVersion(test.ver) 69 | if err != nil { 70 | // got error, but the version is valid 71 | if test.isValid { 72 | t.Errorf("go version check failed for valid version '%s''", test.ver) 73 | } 74 | } else { 75 | // got no error, but the version is invalid 76 | if !test.isValid { 77 | t.Errorf("go version '%s' is invalid, but got no error", test.ver) 78 | } 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /test/k8s-gen-bot-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2023 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | # This script generates the config and rules required for testing the master branch of k/k 23 | # with publishing bot 24 | 25 | BOT_CONFIG_DIRECTORY="${1:-bot-configs}" 26 | 27 | mkdir "${BOT_CONFIG_DIRECTORY}" 28 | 29 | ## generate the required config 30 | # use the content from configmap in the data section 31 | sed -e '1,/config: |/d' configs/kubernetes-configmap.yaml > "${BOT_CONFIG_DIRECTORY}"/config 32 | # The additional .tmp extension is used after -i to make it portable across *BSD and GNU. 33 | # Ref: https://unix.stackexchange.com/a/92907 34 | # Also \t is not recognized in non GNU sed implementation. Therefore a tab is used as is. 35 | # remove leading white spaces from the generated file 36 | sed -i.tmp 's/^[ ]*//' "${BOT_CONFIG_DIRECTORY}"/config 37 | # remove the github-issue key 38 | sed -i.tmp '/github-issue/d' "${BOT_CONFIG_DIRECTORY}"/config 39 | # set dry run to true 40 | sed -i.tmp -e 's/dry-run: false/dry-run: true/g' "${BOT_CONFIG_DIRECTORY}"/config 41 | 42 | ## generate the required rules 43 | # get the rules file from the k/k repo 44 | wget https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml -O "${BOT_CONFIG_DIRECTORY}"/rules 45 | # change permission so that yq container can make changes to the rules file 46 | chmod 666 "${BOT_CONFIG_DIRECTORY}"/rules 47 | # only work on master branch 48 | # yq is used to remove non master branch related rules 49 | docker run \ 50 | --rm \ 51 | -v "${PWD}/${BOT_CONFIG_DIRECTORY}":/workdir \ 52 | mikefarah/yq:4.32.2 -i 'del( .rules.[].branches.[] | select (.name != "master"))' rules 53 | -------------------------------------------------------------------------------- /hack/create-repos.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2021 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}") 22 | source "${SCRIPT_ROOT}/repos.sh" 23 | 24 | if [ "$#" = 0 ] || [ "$#" -gt 2 ]; then 25 | echo "usage: $0 [source-github-user-name] dest-github-user-name" 26 | echo 27 | echo "This connects to git@github.com:/. Set GITHUB_HOST to access git@:/ instead." 28 | exit 1 29 | fi 30 | 31 | FROM="kubernetes" 32 | TO="${1}" 33 | if [ "$#" -ge 2 ]; then 34 | FROM="${TO}" 35 | TO="${2}" 36 | fi 37 | GITHUB_HOST=${GITHUB_HOST:-github.com} 38 | 39 | repo_count=${#repos[@]} 40 | 41 | # safety check 42 | if [ "${TO}" = "kubernetes" ]; then 43 | echo "Cannot operate on kubernetes directly" 1>&2 44 | exit 1 45 | fi 46 | 47 | destination_repos=( $(curl -ks https://api.github.com/orgs/${TO}/repos | jq ".[].name" | tr -d '"') ) 48 | destination_repo_count=${#destination_repos[@]} 49 | 50 | if ! command -v gh > /dev/null; then 51 | echo "Can't find 'gh' tool in PATH, please install from https://github.com/cli/cli" 52 | exit 1 53 | fi 54 | 55 | # Checks if you are logged in. Will error/bail if you are not. 56 | gh auth status 57 | 58 | echo "=======================" 59 | echo " create repos if needed" 60 | echo "=======================" 61 | for (( i=0; i<${repo_count}; i++ )); do 62 | found=0 63 | for (( j=0; j<${destination_repo_count}; j++ )); do 64 | if [[ "${repos[i]}" == ${destination_repos[j]} ]]; then 65 | found=1 66 | fi 67 | done 68 | if [[ $found -eq 1 ]]; then 69 | echo "repository found: ${repos[i]}" 70 | else 71 | echo "repository not found: ${repos[i]}" 72 | gh repo fork "kubernetes/${repos[i]}" --org "${TO}" --remote --clone=false 73 | fi 74 | done 75 | -------------------------------------------------------------------------------- /cmd/validate-rules/staging/github_utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package staging 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "errors" 23 | "io" 24 | "net/http" 25 | "time" 26 | ) 27 | 28 | const defaultTimeout = 5 * time.Minute 29 | 30 | type File struct { 31 | Name string `json:"name"` 32 | Path string `json:"path"` 33 | Type string `json:"type"` 34 | } 35 | 36 | // fetchKubernetesStagingDirectoryFiles uses the GH API to get the contents 37 | // of the contents/staging/src/k8s.io directory in a specified branch of kubernetes. 38 | func fetchKubernetesStagingDirectoryFiles(branch string) ([]File, error) { 39 | url := "https://api.github.com/repos/kubernetes/kubernetes/contents/staging/src/k8s.io?ref=" + branch 40 | 41 | ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) 42 | defer cancel() 43 | 44 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | var res *http.Response 50 | count := 0 51 | spaceClient := http.Client{} 52 | for { 53 | res, err = spaceClient.Do(req) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | if res.StatusCode == http.StatusForbidden { 59 | res.Body.Close() 60 | // try after some time as we hit GH API limit 61 | time.Sleep(5 * time.Second) 62 | count++ 63 | } else { 64 | // try for 10 mins then give up! 65 | if count == 120 { 66 | res.Body.Close() 67 | return nil, errors.New("hitting github API limits, bailing out") 68 | } 69 | 70 | break 71 | } 72 | } 73 | defer res.Body.Close() 74 | 75 | body, readErr := io.ReadAll(res.Body) 76 | if readErr != nil { 77 | return nil, readErr 78 | } 79 | 80 | var result []File 81 | jsonErr := json.Unmarshal(body, &result) 82 | if jsonErr != nil { 83 | return nil, jsonErr 84 | } 85 | 86 | return result, nil 87 | } 88 | -------------------------------------------------------------------------------- /hack/fetch-all-latest-and-push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | if [ "$#" = 0 ] || [ "$#" -gt 2 ]; then 22 | echo "usage: $0 [source-github-user-name] dest-github-user-name" 23 | echo 24 | echo "This connects to git@github.com:/. Set GITHUB_HOST to access git@:/ instead." 25 | exit 1 26 | fi 27 | 28 | FROM="kubernetes" 29 | TO="${1}" 30 | if [ "$#" -ge 2 ]; then 31 | FROM="${TO}" 32 | TO="${2}" 33 | fi 34 | GITHUB_HOST=${GITHUB_HOST:-github.com} 35 | 36 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}") 37 | source "${SCRIPT_ROOT}/repos.sh" 38 | 39 | repo_count=${#repos[@]} 40 | 41 | TMPDIR=$(mktemp -d) 42 | function delete() { 43 | echo "Deleting ${TMPDIR}..." 44 | rm -rf "${TMPDIR}" 45 | } 46 | trap delete EXIT INT 47 | 48 | # safety check 49 | if [ "${TO}" = "kubernetes" ]; then 50 | echo "Cannot operate on kubernetes directly" 1>&2 51 | exit 1 52 | fi 53 | 54 | echo "===================" 55 | echo " sync with upstream" 56 | echo "===================" 57 | for (( i=0; i<${repo_count}; i++ )); do 58 | git clone git@${GITHUB_HOST}:"${TO}/${repos[i]}".git ${TMPDIR}/"${repos[i]}" 59 | pushd ${TMPDIR}/"${repos[i]}" 60 | git remote add upstream git@${GITHUB_HOST}:"${FROM}/${repos[i]}".git 61 | 62 | # delete all tags and branches in origin 63 | rm -f .git/refs/tags/* 64 | branches=$(git branch -r | grep "^ *origin" | sed 's,^ *origin/,,' | grep -v HEAD | grep -v '^master' || true) 65 | tags=$(git tag | sed 's,^,refs/tags/,') 66 | if [ -n "${branches}${tags}" ]; then 67 | git push --atomic --delete origin ${branches} ${tags} 68 | fi 69 | 70 | # push all upstream tags and branches to origin 71 | git tag | xargs git tag -d 72 | git fetch upstream --tags --prune -q 73 | branches=$(git branch -r | grep "^ *upstream" | sed 's,^ *upstream/,,' | grep -v HEAD || true) 74 | branches_arg="" 75 | for branch in ${branches}; do 76 | branches_arg+=" upstream/${branch}:refs/heads/${branch}" 77 | done 78 | git push --atomic --tags -f origin ${branches_arg} 79 | 80 | popd 81 | done 82 | -------------------------------------------------------------------------------- /pkg/git/mapping.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package git 18 | 19 | import ( 20 | "fmt" 21 | 22 | gogit "github.com/go-git/go-git/v5" 23 | "github.com/go-git/go-git/v5/plumbing" 24 | "github.com/go-git/go-git/v5/plumbing/object" 25 | "github.com/golang/glog" 26 | ) 27 | 28 | // SourceCommitToDstCommits returns a mapping from all kube mainline commits 29 | // to the corresponding dst commits after collapsing using "git filter-branch --sub-directory-filter": 30 | // 31 | // dst upstream 32 | // 33 | // | | 34 | // F'<--F 35 | // z | 36 | // y | 37 | // E'<--E 38 | // x ,D 39 | // | / | 40 | // C'<--C 41 | // w | 42 | // v<-, | 43 | // |-B 44 | // `A - initial commit 45 | func SourceCommitToDstCommits(r *gogit.Repository, commitMsgTag string, dstFirstParents, srcFirstParents []*object.Commit) (map[plumbing.Hash]plumbing.Hash, error) { 46 | // compute merge point table 47 | kubeMergePoints, err := MergePoints(r, srcFirstParents) 48 | if err != nil { 49 | return nil, fmt.Errorf("failed to build merge point table: %w", err) 50 | } 51 | 52 | // convert dstFirstParents to HashesWithKubeHashes 53 | directKubeHashToDstMainLineHash := map[plumbing.Hash]plumbing.Hash{} 54 | firstDstCommit := plumbing.ZeroHash 55 | for _, c := range dstFirstParents { 56 | firstDstCommit = c.Hash 57 | 58 | // kh might be a non-mainline-merge (because we had used branch commits as kube hashes long ago) 59 | kh := SourceHash(c, commitMsgTag) 60 | if kh == plumbing.ZeroHash { 61 | continue 62 | } 63 | merge := kubeMergePoints[kh] 64 | if merge == nil { 65 | continue 66 | } 67 | // do not override, because we might have seen the actual merge before 68 | if _, found := directKubeHashToDstMainLineHash[merge.Hash]; !found { 69 | directKubeHashToDstMainLineHash[merge.Hash] = c.Hash 70 | } 71 | } 72 | 73 | // fill up mainlineKubeHashes in dstMainlineCommits with collapsed kube commits 74 | dst := firstDstCommit 75 | kubeHashToDstMainLineHash := map[plumbing.Hash]plumbing.Hash{} 76 | for i := len(srcFirstParents) - 1; i >= 0; i-- { 77 | kc := srcFirstParents[i] 78 | if dh, found := directKubeHashToDstMainLineHash[kc.Hash]; found { 79 | dst = dh 80 | } 81 | if dst != plumbing.ZeroHash { 82 | kubeHashToDstMainLineHash[kc.Hash] = dst 83 | } 84 | } 85 | if dst == firstDstCommit { 86 | glog.Warningf("no upstream mainline commit found on branch") 87 | } 88 | 89 | return kubeHashToDstMainLineHash, nil 90 | } 91 | -------------------------------------------------------------------------------- /cmd/publishing-bot/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "net/http" 23 | "sync" 24 | "time" 25 | 26 | "github.com/golang/glog" 27 | "k8s.io/publishing-bot/cmd/publishing-bot/config" 28 | ) 29 | 30 | type Server struct { 31 | Issue int 32 | RunChan chan bool 33 | 34 | mutex sync.RWMutex 35 | response HealthResponse 36 | config config.Config 37 | } 38 | 39 | type HealthResponse struct { 40 | Successful *bool `json:"successful,omitempty"` 41 | Time *time.Time `json:"time,omitempty"` 42 | UpstreamHash string `json:"upstreamHash,omitempty"` 43 | 44 | LastSuccessfulTime *time.Time `json:"lastSuccessfulTime,omitempty"` 45 | LastFailureTime *time.Time `json:"lastFailureTime,omitempty"` 46 | LastSuccessfulUpstreamHash string `json:"lastSuccessfulUpstreamHash,omitempty"` 47 | 48 | Issue string `json:"issue,omitempty"` 49 | } 50 | 51 | func (h *Server) SetHealth(healthy bool, hash string) { 52 | h.mutex.Lock() 53 | defer h.mutex.Unlock() 54 | 55 | h.response.Successful = &healthy 56 | now := time.Now() 57 | h.response.Time = &now 58 | h.response.UpstreamHash = hash 59 | 60 | if healthy { 61 | h.response.LastSuccessfulTime = h.response.Time 62 | h.response.LastSuccessfulUpstreamHash = h.response.UpstreamHash 63 | } else { 64 | h.response.LastFailureTime = h.response.Time 65 | } 66 | } 67 | 68 | func (h *Server) Run(port int) { 69 | mux := http.NewServeMux() 70 | mux.HandleFunc("/healthz", h.healthzHandler) 71 | mux.HandleFunc("/run", h.runHandler) 72 | addr := fmt.Sprintf("0.0.0.0:%d", port) 73 | glog.Infof("Listening on %v", addr) 74 | go func() { 75 | err := http.ListenAndServe(addr, mux) 76 | glog.Fatalf("Failed ListenAndServer: %v", err) 77 | }() 78 | } 79 | 80 | func (h *Server) runHandler(w http.ResponseWriter, _ *http.Request) { 81 | if h.RunChan == nil { 82 | http.Error(w, "run channel is closed", http.StatusInternalServerError) 83 | return 84 | } 85 | select { 86 | case h.RunChan <- true: 87 | default: 88 | } 89 | 90 | //nolint:errcheck // TODO(lint): Should we be checking errors here? 91 | w.Write([]byte("OK")) 92 | } 93 | 94 | func (h *Server) healthzHandler(w http.ResponseWriter, _ *http.Request) { 95 | h.mutex.RLock() 96 | resp := h.response 97 | if h.Issue != 0 { 98 | // We chose target org so the issue can be opened in different org than 99 | // a source repository. 100 | resp.Issue = fmt.Sprintf("https://%s/%s/%s/issues/%d", h.config.GithubHost, h.config.TargetOrg, h.config.SourceRepo, h.Issue) 101 | } 102 | h.mutex.RUnlock() 103 | 104 | bytes, err := json.MarshalIndent(resp, "", "\t") 105 | if err != nil { 106 | http.Error(w, err.Error(), http.StatusInternalServerError) 107 | return 108 | } 109 | 110 | //nolint:errcheck // TODO(lint): Should we be checking errors here? 111 | w.Write(bytes) 112 | } 113 | -------------------------------------------------------------------------------- /cmd/publishing-bot/log_builder.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package main 17 | 18 | import ( 19 | "bytes" 20 | "strings" 21 | ) 22 | 23 | type logBuilder struct { 24 | logs []string 25 | headings []string 26 | tailings []string 27 | } 28 | 29 | func newLogBuilderWithMaxBytes(maxBytes int, rawLogs ...string) *logBuilder { 30 | ignoreBytesLimits := maxBytes <= 0 31 | size := 0 32 | logBuilder := &logBuilder{} 33 | for i := len(rawLogs) - 1; i >= 0; i-- { 34 | if curSize := size + len(rawLogs[i]); !ignoreBytesLimits && curSize > maxBytes { 35 | rawLogs[i] = rawLogs[i][curSize-maxBytes:] 36 | rawLogs[i] = "..." + rawLogs[i][3:] 37 | logBuilder.logs = append(logBuilder.logs, rawLogs[i]) 38 | break 39 | } 40 | logBuilder.logs = append(logBuilder.logs, rawLogs[i]) 41 | size += len(rawLogs) 42 | } 43 | return logBuilder 44 | } 45 | 46 | func (builder *logBuilder) AddHeading(lines ...string) *logBuilder { 47 | builder.headings = append(builder.headings, lines...) 48 | return builder 49 | } 50 | 51 | func (builder *logBuilder) AddTailing(lines ...string) *logBuilder { 52 | builder.tailings = append(builder.tailings, lines...) 53 | return builder 54 | } 55 | 56 | func (builder *logBuilder) Split(sep string) *logBuilder { 57 | var splittedLogs []string 58 | for _, log := range builder.logs { 59 | splittedLogs = append(splittedLogs, strings.Split(log, sep)...) 60 | } 61 | builder.logs = splittedLogs 62 | return builder 63 | } 64 | 65 | func (builder *logBuilder) Trim(cutset string) *logBuilder { 66 | for idx := range builder.logs { 67 | builder.logs[idx] = strings.Trim(builder.logs[idx], cutset) 68 | } 69 | return builder 70 | } 71 | 72 | func (builder *logBuilder) Reverse() *logBuilder { 73 | s := builder.logs 74 | for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { 75 | s[i], s[j] = s[j], s[i] 76 | } 77 | return builder 78 | } 79 | 80 | func (builder *logBuilder) Filter(predicate func(line string) bool) *logBuilder { 81 | var filteredLogs []string 82 | for i := 0; i < len(builder.logs); i++ { 83 | if predicate(builder.logs[i]) { 84 | filteredLogs = append(filteredLogs, builder.logs[i]) 85 | } 86 | } 87 | builder.logs = filteredLogs 88 | return builder 89 | } 90 | 91 | func (builder *logBuilder) Tail(n int) *logBuilder { 92 | if len(builder.logs) <= n { 93 | return builder 94 | } 95 | builder.logs = builder.logs[len(builder.logs)-n:] 96 | return builder 97 | } 98 | 99 | func (builder *logBuilder) Join(sep string) *logBuilder { 100 | builder.logs = []string{strings.Join(builder.logs, sep)} 101 | return builder 102 | } 103 | 104 | func (builder *logBuilder) Log() string { 105 | buf := new(bytes.Buffer) 106 | for _, heading := range builder.headings { 107 | buf.WriteString(heading + "\n") 108 | } 109 | for _, log := range builder.logs { 110 | buf.WriteString(log) 111 | } 112 | for _, tailing := range builder.tailings { 113 | buf.WriteString(tailing + "\n") 114 | } 115 | return buf.String() 116 | } 117 | -------------------------------------------------------------------------------- /pkg/golang/install.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package golang 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "os/exec" 23 | "path/filepath" 24 | "strings" 25 | 26 | "github.com/golang/glog" 27 | "k8s.io/publishing-bot/cmd/publishing-bot/config" 28 | ) 29 | 30 | // deprecatedDefaultGoVersion hardcodes the (old) default go version. 31 | // The right way to define the default go version today is to specify in rules. 32 | // TODO(nikhita): remove deprecatedDefaultGoVersion when go 1.16 is released. 33 | var deprecatedDefaultGoVersion = "1.14.4" 34 | 35 | // InstallGoVersions download and unpacks the specified Golang versions to $GOPATH/ 36 | // If the DefaultGoVersion is not specfied in rules, it defaults to 1.14.4. 37 | func InstallGoVersions(rules *config.RepositoryRules) error { 38 | if rules == nil { 39 | return nil 40 | } 41 | 42 | defaultGoVersion := deprecatedDefaultGoVersion 43 | if rules.DefaultGoVersion != nil { 44 | defaultGoVersion = *rules.DefaultGoVersion 45 | } 46 | glog.Infof("Using %s as the default go version", defaultGoVersion) 47 | 48 | goVersions := []string{defaultGoVersion} 49 | for _, rule := range rules.Rules { 50 | for i := range rule.Branches { 51 | branch := rule.Branches[i] 52 | if branch.GoVersion != "" { 53 | found := false 54 | for _, v := range goVersions { 55 | if v == branch.GoVersion { 56 | found = true 57 | } 58 | } 59 | if !found { 60 | goVersions = append(goVersions, branch.GoVersion) 61 | } 62 | } 63 | } 64 | } 65 | systemGoPath := os.Getenv("GOPATH") 66 | for _, v := range goVersions { 67 | if err := installGoVersion(v, filepath.Join(systemGoPath, "go-"+v)); err != nil { 68 | return err 69 | } 70 | } 71 | goLink, target := filepath.Join(systemGoPath, "go"), filepath.Join(systemGoPath, "go-"+defaultGoVersion) 72 | os.Remove(goLink) 73 | if err := os.Symlink(target, goLink); err != nil { 74 | return fmt.Errorf("failed to link %s to %s: %w", goLink, target, err) 75 | } 76 | 77 | return nil 78 | } 79 | 80 | func installGoVersion(v, pth string) error { 81 | if s, err := os.Stat(pth); err != nil && !os.IsNotExist(err) { 82 | return err 83 | } else if err == nil { 84 | if s.IsDir() { 85 | glog.Infof("Found existing go %s at %s", v, pth) 86 | return nil 87 | } 88 | return fmt.Errorf("expected %s to be a directory", pth) 89 | } 90 | 91 | glog.Infof("Installing go %s to %s", v, pth) 92 | tmpPath, err := os.MkdirTemp(os.Getenv("GOPATH"), "go-tmp-") 93 | if err != nil { 94 | return err 95 | } 96 | defer os.RemoveAll(tmpPath) 97 | 98 | cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("curl -SLf https://dl.google.com/go/go%s.linux-amd64.tar.gz | tar -xz --strip 1 -C %s", v, tmpPath)) 99 | cmd.Dir = tmpPath 100 | cmd.Stdout = os.Stdout 101 | cmd.Stderr = os.Stderr 102 | if err := cmd.Run(); err != nil { 103 | return fmt.Errorf("command %q failed: %w", strings.Join(cmd.Args, " "), err) 104 | } 105 | 106 | return os.Rename(tmpPath, pth) 107 | } 108 | -------------------------------------------------------------------------------- /cmd/update-rules/README.md: -------------------------------------------------------------------------------- 1 | Update branch rules 2 | =================== 3 | 4 | Command line tooling to manage following operations for branch rules of existing destination repo: 5 | - add a new branch rule 6 | - with go version 7 | - without go version (sets blank "" go version) 8 | - update an existing branch rule with given go version 9 | 10 | For new branch rule, it refers the 'master' branch rule for each destination repository and appends the new 11 | branch rule to configured branches. 12 | 13 | For existing branch rule, it updates the given go version for each destination repository. 14 | 15 | ### Build: 16 | 17 | To build the `update-rules` CLI binary, run: 18 | 19 | ##### Linux: 20 | 21 | ``` 22 | make build 23 | ``` 24 | 25 | ##### macOS: 26 | 27 | ``` 28 | GOOS=darwin make build 29 | ``` 30 | 31 | The generated binary will be located at `_output/update-rules`. 32 | 33 | ##### Container Image: 34 | 35 | To build the container image, run: 36 | 37 | ``` 38 | make build-image 39 | ``` 40 | 41 | `update-rules` binary will be available at the root `/` in the image and can be invoked as: 42 | 43 | ``` 44 | docker run -t gcr.io/k8s-staging-publishing-bot/publishing-bot:latest /update-rules 45 | ``` 46 | 47 | ### Usage: 48 | 49 | Run the command line as: 50 | ``` 51 | update-rules -h 52 | 53 | Usage: update-rules --branch BRANCH --rules PATHorURL [--go VERSION | -o PATH | --delete] 54 | 55 | Examples: 56 | # Update rules for branch release-1.23 with go version 1.16.4 57 | update-rules -branch release-1.23 -go 1.16.4 -rules /go/src/k8s.io/kubernetes/staging/publishing/rules.yaml 58 | 59 | # Update rules using URL to input rules file 60 | update-rules -branch release-1.23 -go 1.16.4 -rules https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml 61 | 62 | # Update rules and export to /tmp/rules.yaml 63 | update-rules -branch release-1.24 -go 1.17.1 -o /tmp/rules.yaml -rules /go/src/k8s.io/kubernetes/staging/publishing/rules.yaml 64 | 65 | # Delete rules and export to /tmp/rules.yaml 66 | update-rules -branch release-1.24 -delete -o /tmp/rules.yaml -rules /go/src/k8s.io/kubernetes/staging/publishing/rules.yaml 67 | 68 | -alsologtostderr 69 | log to standard error as well as files 70 | -branch string 71 | [required] Branch to update rules for, e.g. --branch release-x.yy 72 | -go string 73 | Golang version to pin for this branch, e.g. --go 1.16.1 74 | -log_backtrace_at value 75 | when logging hits line file:N, emit a stack trace 76 | -log_dir string 77 | If non-empty, write log files in this directory 78 | -logtostderr 79 | log to standard error instead of files 80 | -o string 81 | Path to export the updated rules to, e.g. -o /tmp/rules.yaml 82 | -delete 83 | Delete old rules of deprecated branch 84 | -rules string 85 | [required] URL or Path of the rules file to update rules for, e.g. --rules path/or/url/to/rules/file.yaml 86 | -stderrthreshold value 87 | logs at or above this threshold go to stderr 88 | -v value 89 | log level for V logs 90 | -vmodule value 91 | comma-separated list of pattern=N settings for file-filtered logging 92 | ``` 93 | 94 | #### Required flags: 95 | 96 | - `-rules` flag with value is required for processing input rules file 97 | - `-branch` flag with value is required for adding/updating rules for all destination repos 98 | 99 | #### Optional flags: 100 | 101 | - `-go` flag refers to golang version which should be pinned for given branch, if not given an empty string is set 102 | - `-delete` flag refers to removing the branch from rules, if not set defaults to false 103 | - `-o` flag refers to output file where the processed rules should be exported, otherwise rules are printed on stdout 104 | -------------------------------------------------------------------------------- /cmd/validate-rules/staging/validate.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package staging 18 | 19 | import ( 20 | "fmt" 21 | "path/filepath" 22 | 23 | "github.com/golang/glog" 24 | "k8s.io/publishing-bot/cmd/publishing-bot/config" 25 | ) 26 | 27 | // globalMapBranchDirectories is a cache to avoid hitting GH limits 28 | // key is the branch (`master` or `release-1.23`) and the value 29 | // is the list of files/directories fetched using GH api in the 30 | // correct directory. 31 | var globalMapBranchDirectories = make(map[string][]File) 32 | 33 | const defaultBranch = "master" 34 | 35 | // EnsureStagingDirectoriesExist walks through the repository rules and checks 36 | // if the specified directories are present in the specific kubernetes branch. 37 | func EnsureStagingDirectoriesExist(rules *config.RepositoryRules, baseBranch string) []error { 38 | glog.Infof("Validating directories exist in the Kubernetes branches") 39 | 40 | if baseBranch == "" { 41 | baseBranch = defaultBranch 42 | } 43 | 44 | if baseBranch == defaultBranch { 45 | glog.Infof("Testing all rules because the base branch is: %s", baseBranch) 46 | } else { 47 | glog.Infof("Testing only rules matching the base branch: %s", baseBranch) 48 | } 49 | 50 | var errors []error 51 | for _, rule := range rules.Rules { 52 | for i := range rule.Branches { 53 | branchRule := rule.Branches[i] 54 | // ensure all the mentioned directories exist 55 | for _, dir := range branchRule.Source.Dirs { 56 | _, directory := filepath.Split(dir) 57 | if baseBranch != defaultBranch && baseBranch != branchRule.Source.Branch { 58 | glog.Infof("Skipping branch %q for repository %q", branchRule.Source.Branch, directory) 59 | continue 60 | } 61 | err := checkDirectoryExistsInBranch(directory, branchRule.Source.Branch) 62 | if err != nil { 63 | errors = append(errors, err) 64 | } 65 | } 66 | 67 | for _, dependency := range branchRule.Dependencies { 68 | if baseBranch != defaultBranch && baseBranch != dependency.Branch { 69 | glog.Infof("Skipping branch %q for dependency %q", dependency.Branch, dependency.Repository) 70 | continue 71 | } 72 | err := checkDirectoryExistsInBranch(dependency.Repository, dependency.Branch) 73 | if err != nil { 74 | errors = append(errors, err) 75 | } 76 | } 77 | } 78 | } 79 | return errors 80 | } 81 | 82 | func checkDirectoryExistsInBranch(directory, branch string) error { 83 | glog.Infof("Check if directory %q exists in branch %q", directory, branch) 84 | 85 | // Look in the cache first 86 | files, ok := globalMapBranchDirectories[branch] 87 | if !ok { 88 | var err error 89 | files, err = fetchKubernetesStagingDirectoryFiles(branch) 90 | if err != nil { 91 | globalMapBranchDirectories[branch] = []File{} 92 | return fmt.Errorf("error fetching directories from branch %s: %w", branch, err) 93 | } 94 | globalMapBranchDirectories[branch] = files 95 | } 96 | 97 | for _, file := range files { 98 | // check the name and that it is a directory! 99 | if file.Name == directory && file.Type == "dir" { 100 | return nil 101 | } 102 | } 103 | return fmt.Errorf("%s not found in branch %s", directory, branch) 104 | } 105 | -------------------------------------------------------------------------------- /cmd/gomod-zip/zip.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "bytes" 21 | "errors" 22 | "flag" 23 | "fmt" 24 | "io" 25 | "os" 26 | "path/filepath" 27 | 28 | "github.com/golang/glog" 29 | "golang.org/x/mod/modfile" 30 | modzip "golang.org/x/mod/zip" 31 | ) 32 | 33 | func Usage() { 34 | fmt.Fprintf(os.Stderr, `Creates a zip file at 35 | $GOPATH/pkg/mod/cache/download//@v/.zip. 36 | The zip file has the same hash as if it were created by go mod download. 37 | This tool can be used to package modules which haven't been uploaded anywhere 38 | yet and are only available locally. 39 | 40 | This tool assumes that the package is already checked out at the commit 41 | pointed by the pseudo-version. 42 | 43 | package-name should be equal to the import path of the package. 44 | 45 | Usage: %s --package-name --pseudo-version 46 | `, os.Args[0]) 47 | flag.PrintDefaults() 48 | } 49 | 50 | func main() { 51 | packageName := flag.String("package-name", "", "package to zip") 52 | pseudoVersion := flag.String("pseudo-version", "", "pseudoVersion to zip at") 53 | 54 | flag.Usage = Usage 55 | flag.Parse() 56 | 57 | if *packageName == "" { 58 | glog.Fatalf("package-name cannot be empty") 59 | } 60 | 61 | if *pseudoVersion == "" { 62 | glog.Fatalf("pseudo-version cannot be empty") 63 | } 64 | 65 | packagePath := fmt.Sprintf("%s/src/%s", os.Getenv("GOPATH"), *packageName) 66 | cacheDir := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v", os.Getenv("GOPATH"), *packageName) 67 | 68 | moduleFile, err := getModuleFile(packagePath, *pseudoVersion) 69 | if err != nil { 70 | glog.Fatalf("error getting module file: %v", err) 71 | } 72 | 73 | if err := createZipArchive(packagePath, moduleFile, cacheDir); err != nil { 74 | glog.Fatalf("error creating zip archive: %v", err) 75 | } 76 | } 77 | 78 | func getModuleFile(packagePath, version string) (*modfile.File, error) { 79 | goModPath := filepath.Join(packagePath, "go.mod") 80 | file, err := os.Open(goModPath) 81 | if err != nil { 82 | return nil, fmt.Errorf("error opening %s: %w", goModPath, err) 83 | } 84 | defer file.Close() 85 | 86 | moduleBytes, err := io.ReadAll(file) 87 | if err != nil { 88 | return nil, fmt.Errorf("error reading %s: %w", goModPath, err) 89 | } 90 | 91 | moduleFile, err := modfile.Parse(packagePath, moduleBytes, nil) 92 | if err != nil { 93 | return nil, fmt.Errorf("error parsing module file: %w", err) 94 | } 95 | 96 | if moduleFile.Module == nil { 97 | return nil, errors.New("parsed module should not be nil") 98 | } 99 | 100 | moduleFile.Module.Mod.Version = version 101 | return moduleFile, nil 102 | } 103 | 104 | func createZipArchive(packagePath string, moduleFile *modfile.File, outputDirectory string) error { 105 | zipFilePath := filepath.Join(outputDirectory, moduleFile.Module.Mod.Version+".zip") 106 | var zipContents bytes.Buffer 107 | 108 | if err := modzip.CreateFromDir(&zipContents, moduleFile.Module.Mod, packagePath); err != nil { 109 | return fmt.Errorf("create zip from dir: %w", err) 110 | } 111 | if err := os.WriteFile(zipFilePath, zipContents.Bytes(), 0o644); err != nil { 112 | return fmt.Errorf("writing zip file: %w", err) 113 | } 114 | return nil 115 | } 116 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | all: build 16 | .PHONY: all 17 | 18 | -include $(CONFIG) 19 | -include $(CONFIG)-token 20 | 21 | GIT_TAG ?= $(shell git describe --tags --always --dirty) 22 | 23 | # Image variables 24 | IMG_REGISTRY ?= gcr.io/k8s-staging-publishing-bot 25 | IMG_NAME = k8s-publishing-bot 26 | 27 | IMG_VERSION ?= v0.0.0-2 28 | 29 | # TODO(image): Consider renaming this variable 30 | DOCKER_REPO ?= $(IMG_REGISTRY)/$(IMG_NAME) 31 | NAMESPACE ?= 32 | TOKEN ?= 33 | KUBECTL ?= kubectl 34 | SCHEDULE ?= 0 5 * * * 35 | INTERVAL ?= 86400 36 | CPU_LIMITS ?= 2 37 | CPU_REQUESTS ?= 300m 38 | MEMORY_REQUESTS ?= 200Mi 39 | MEMORY_LIMITS ?= 1639Mi 40 | GOOS ?= linux 41 | 42 | build_cmd = mkdir -p _output && GOOS=$(GOOS) CGO_ENABLED=0 go build -o _output/$(1) ./cmd/$(1) 43 | prepare_spec = sed 's,DOCKER_IMAGE,$(DOCKER_REPO),g;s,CPU_LIMITS,$(CPU_LIMITS),g;s,CPU_REQUESTS,$(CPU_REQUESTS),g;s,MEMORY_REQUESTS,$(MEMORY_REQUESTS),g;s,MEMORY_LIMITS,$(MEMORY_LIMITS),g' 44 | 45 | SHELL := /bin/bash 46 | 47 | build: 48 | $(call build_cmd,collapsed-kube-commit-mapper) 49 | $(call build_cmd,publishing-bot) 50 | $(call build_cmd,sync-tags) 51 | $(call build_cmd,init-repo) 52 | $(call build_cmd,gomod-zip) 53 | $(call build_cmd,update-rules) 54 | .PHONY: build 55 | 56 | build-image: build 57 | docker build \ 58 | -t $(DOCKER_REPO):$(GIT_TAG) \ 59 | -t $(DOCKER_REPO):$(IMG_VERSION) \ 60 | -t $(DOCKER_REPO):latest \ 61 | . 62 | .PHONY: build-image 63 | 64 | push-image: 65 | docker push $(DOCKER_REPO):$(GIT_TAG) 66 | docker push $(DOCKER_REPO):$(IMG_VERSION) 67 | docker push $(DOCKER_REPO):latest 68 | .PHONY: push-image 69 | 70 | build-and-push-image: build-image push-image 71 | .PHONY: build-and-push-image 72 | 73 | clean: 74 | rm -rf _output 75 | .PHONY: clean 76 | 77 | update-deps: 78 | go mod tidy 79 | .PHONY: update-deps 80 | 81 | validate: 82 | if [ -f $(CONFIG)-rules-configmap.yaml ]; then \ 83 | go run ./cmd/validate-rules <(sed '1,/config: /d;s/^ //' $(CONFIG)-rules-configmap.yaml); \ 84 | else \ 85 | go run ./cmd/validate-rules $$(grep "rules-file: " $(CONFIG)-configmap.yaml | sed 's/.*rules-file: //'); \ 86 | fi 87 | .PHONY: validate 88 | 89 | init-deploy: validate 90 | $(KUBECTL) delete -n "$(NAMESPACE)" --ignore-not-found=true replicaset publisher 91 | $(KUBECTL) delete -n "$(NAMESPACE)" --ignore-not-found=true pod publisher 92 | while $(KUBECTL) get pod -n "$(NAMESPACE)" publisher -a &>/dev/null; do echo -n .; sleep 1; done 93 | $(KUBECTL) apply -n "$(NAMESPACE)" -f artifacts/manifests/storage-class.yaml || true 94 | $(KUBECTL) get StorageClass ssd 95 | $(KUBECTL) apply -n "$(NAMESPACE)" -f $(CONFIG)-configmap.yaml 96 | $(KUBECTL) apply -n "$(NAMESPACE)" -f $(CONFIG)-rules-configmap.yaml; \ 97 | $(KUBECTL) apply -n "$(NAMESPACE)" -f artifacts/manifests/pvc.yaml 98 | .PHONY: init-deploy 99 | 100 | run: init-deploy 101 | { cat artifacts/manifests/pod.yaml && sed 's/^/ /' artifacts/manifests/podspec.yaml; } | \ 102 | $(call prepare_spec) | $(KUBECTL) apply -n "$(NAMESPACE)" -f - 103 | .PHONY: run 104 | 105 | deploy: init-deploy 106 | $(KUBECTL) apply -n "$(NAMESPACE)" -f artifacts/manifests/service.yaml 107 | { cat artifacts/manifests/rs.yaml && sed 's/^/ /' artifacts/manifests/podspec.yaml; } | \ 108 | $(call prepare_spec) | sed 's/-interval=0/-interval=$(INTERVAL)/g' | \ 109 | $(KUBECTL) apply -n "$(NAMESPACE)" -f - 110 | .PHONY: deploy 111 | 112 | test: ## Run go tests 113 | go test -v -coverprofile=coverage.out ./... 114 | .PHONY: test 115 | -------------------------------------------------------------------------------- /pkg/git/mainline.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package git 18 | 19 | import ( 20 | "fmt" 21 | 22 | gogit "github.com/go-git/go-git/v5" 23 | "github.com/go-git/go-git/v5/plumbing" 24 | "github.com/go-git/go-git/v5/plumbing/object" 25 | "k8s.io/publishing-bot/pkg/cache" 26 | ) 27 | 28 | // FirstParent returns the first parent of a commit. For a merge commit this 29 | // is the parent which is usually depicted on the left. 30 | func FirstParent(r *gogit.Repository, c *object.Commit) (*object.Commit, error) { 31 | if c == nil { 32 | return nil, nil 33 | } 34 | if len(c.ParentHashes) == 0 { 35 | return nil, nil 36 | } 37 | p, err := cache.CommitObject(r, c.ParentHashes[0]) 38 | if err != nil { 39 | return nil, fmt.Errorf("failed to get %v: %w", c.ParentHashes[0], err) 40 | } 41 | return p, nil 42 | } 43 | 44 | // FirstParentList visits the ancestors of c using the FirstParent func. It returns the list 45 | // of visited commits. 46 | func FirstParentList(r *gogit.Repository, c *object.Commit) ([]*object.Commit, error) { 47 | l := []*object.Commit{} 48 | for c != nil { 49 | l = append(l, c) 50 | 51 | // continue with first parent if there is one 52 | next, err := FirstParent(r, c) 53 | if err != nil { 54 | return nil, fmt.Errorf("failed to get first parent of %s: %w", c.Hash, err) 55 | } 56 | c = next 57 | } 58 | return l, nil 59 | } 60 | 61 | // MergePoints creates a look-up table from feature branch commit hashes to their merge commits 62 | // onto the given mainline. 63 | func MergePoints(r *gogit.Repository, mainLine []*object.Commit) (map[plumbing.Hash]*object.Commit, error) { 64 | // create lookup table for the position in mainLine 65 | mainLinePos := map[plumbing.Hash]int{} 66 | for i, c := range mainLine { 67 | mainLinePos[c.Hash] = i 68 | } 69 | 70 | earliestMergePoints := map[plumbing.Hash]int{} // the earlist mainline commit index a given commit was merged into the mainline 71 | seen := map[plumbing.Hash]*object.Commit{} 72 | 73 | // pos is the position of the current mainline commit, h 74 | var visit func(pos int, h plumbing.Hash) error 75 | visit = func(pos int, h plumbing.Hash) error { 76 | // stop if we reached the mainline 77 | if _, isOnMainLine := mainLinePos[h]; isOnMainLine { 78 | return nil 79 | } 80 | 81 | // was h seen before as descendent of a mainline commit? It must have had 82 | // a better position as we saw it earlier. 83 | if _, seenBefore := earliestMergePoints[h]; seenBefore { 84 | return nil 85 | } 86 | 87 | earliestMergePoints[h] = pos 88 | 89 | // resolve hash 90 | c := seen[h] 91 | if c == nil { 92 | var err error 93 | c, err = cache.CommitObject(r, h) 94 | if err != nil { 95 | return fmt.Errorf("failed to get %s: %w", h.String(), err) 96 | } 97 | seen[h] = c 98 | } 99 | 100 | // recurse into parents 101 | for _, ph := range c.ParentHashes { 102 | err := visit(pos, ph) 103 | if err != nil { 104 | return err 105 | } 106 | } 107 | 108 | return nil 109 | } 110 | 111 | // recursively enumerate all reachable commits 112 | for pos := len(mainLine) - 1; pos >= 0; pos-- { 113 | c := mainLine[pos] 114 | earliestMergePoints[c.Hash] = pos 115 | seen[c.Hash] = c 116 | for _, ph := range c.ParentHashes { 117 | err := visit(pos, ph) 118 | if err != nil { 119 | return nil, err 120 | } 121 | } 122 | } 123 | 124 | // map commit hash to best merge point on mainline 125 | result := map[plumbing.Hash]*object.Commit{} 126 | for _, c := range seen { 127 | result[c.Hash] = mainLine[earliestMergePoints[c.Hash]] 128 | } 129 | 130 | return result, nil 131 | } 132 | -------------------------------------------------------------------------------- /cmd/publishing-bot/github.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "net/http" 23 | "strings" 24 | 25 | "github.com/golang/glog" 26 | "github.com/google/go-github/github" 27 | "golang.org/x/oauth2" 28 | ) 29 | 30 | func githubClient(token string) *github.Client { 31 | // create github client 32 | ctx := context.Background() 33 | ts := oauth2.StaticTokenSource( 34 | &oauth2.Token{AccessToken: token}, 35 | ) 36 | tc := oauth2.NewClient(ctx, ts) 37 | return github.NewClient(tc) 38 | } 39 | 40 | func ReportOnIssue(e error, logs, token, org, repo string, issue int) error { 41 | ctx := context.Background() 42 | client := githubClient(token) 43 | 44 | // filter out token, if it happens to be in the log (it shouldn't!) 45 | // TODO: Consider using log sanitizer from sigs.k8s.io/release-utils 46 | logs = strings.ReplaceAll(logs, token, "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") 47 | 48 | // who am I? 49 | myself, resp, err := client.Users.Get(ctx, "") 50 | if err != nil { 51 | return fmt.Errorf("failed to get own user: %w", err) 52 | } 53 | if resp.StatusCode != http.StatusOK { 54 | return fmt.Errorf("failed to get own user: HTTP code %d", resp.StatusCode) 55 | } 56 | 57 | // create new newComment 58 | body := transfromLogToGithubFormat(logs, 50, fmt.Sprintf("/reopen\n\nThe last publishing run failed: %v", e)) 59 | 60 | newComment, resp, err := client.Issues.CreateComment(ctx, org, repo, issue, &github.IssueComment{ 61 | Body: &body, 62 | }) 63 | if err != nil { 64 | return fmt.Errorf("failed to comment on issue #%d: %w", issue, err) 65 | } 66 | if resp.StatusCode >= 300 { 67 | return fmt.Errorf("failed to comment on issue #%d: HTTP code %d", issue, resp.StatusCode) 68 | } 69 | 70 | // delete all other comments from this user 71 | comments, resp, err := client.Issues.ListComments(ctx, org, repo, issue, &github.IssueListCommentsOptions{ 72 | ListOptions: github.ListOptions{PerPage: 100}, 73 | }) 74 | if err != nil { 75 | return fmt.Errorf("failed to get github comments of issue #%d: %w", issue, err) 76 | } 77 | if resp.StatusCode != http.StatusOK { 78 | return fmt.Errorf("failed to get github comments of issue #%d: HTTP code %d", issue, resp.StatusCode) 79 | } 80 | for _, c := range comments { 81 | if *c.User.ID != *myself.ID { 82 | glog.Infof("Skipping comment %d not by me, but %v", *c.ID, c.User.Name) 83 | continue 84 | } 85 | if *c.ID == *newComment.ID { 86 | continue 87 | } 88 | 89 | glog.Infof("Deleting comment %d", *c.ID) 90 | resp, err = client.Issues.DeleteComment(ctx, org, repo, *c.ID) 91 | if err != nil { 92 | return fmt.Errorf("failed to delete github comment %d of issue #%d: %w", *c.ID, issue, err) 93 | } 94 | if resp.StatusCode >= 300 { 95 | return fmt.Errorf("failed to delete github comment %d of issue #%d: HTTP code %d", *c.ID, issue, resp.StatusCode) 96 | } 97 | } 98 | 99 | return nil 100 | } 101 | 102 | func CloseIssue(token, org, repo string, issue int) error { 103 | ctx := context.Background() 104 | client := githubClient(token) 105 | 106 | _, resp, err := client.Issues.Edit(ctx, org, repo, issue, &github.IssueRequest{ 107 | State: github.String("closed"), 108 | }) 109 | if err != nil { 110 | return fmt.Errorf("failed to close issue #%d: %w", issue, err) 111 | } 112 | if resp.StatusCode >= 300 { 113 | return fmt.Errorf("failed to close issue #%d: HTTP code %d", issue, resp.StatusCode) 114 | } 115 | 116 | return nil 117 | } 118 | 119 | func transfromLogToGithubFormat(original string, maxLines int, headings ...string) string { 120 | logCount := 0 121 | transformed := newLogBuilderWithMaxBytes(65000, original). 122 | AddHeading(headings...). 123 | AddHeading("```"). 124 | Trim("\n"). 125 | Split("\n"). 126 | Reverse(). 127 | Filter(func(line string) bool { 128 | if logCount < maxLines { 129 | if strings.HasPrefix(line, "+") { 130 | logCount++ 131 | } 132 | return true 133 | } 134 | return false 135 | }). 136 | Reverse(). 137 | Join("\n"). 138 | AddTailing("```"). 139 | Log() 140 | return transformed 141 | } 142 | -------------------------------------------------------------------------------- /cmd/collapsed-kube-commit-mapper/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "os" 23 | "sort" 24 | "strings" 25 | 26 | gogit "github.com/go-git/go-git/v5" 27 | "github.com/go-git/go-git/v5/plumbing" 28 | "github.com/golang/glog" 29 | "k8s.io/publishing-bot/pkg/cache" 30 | "k8s.io/publishing-bot/pkg/git" 31 | ) 32 | 33 | func Usage() { 34 | fmt.Fprintf(os.Stderr, `Print a lookup table by printing each mainline k8s.io/kubernetes 35 | commit hash with its corresponding commit hash in the current branch 36 | (which is the result of a "git filter-branch --sub-directory"). It is 37 | expected that the commit messages on the current branch contain a 38 | "Kubernetes-commit: " line for the directly corresponding 39 | commit. Note, that a number of k8s.io/kubernetes mainline commits might 40 | be collapsed during filtering: 41 | 42 | HEAD 43 | | | 44 | H'<--------H 45 | z | 46 | y ,G 47 | F'<------*-F 48 | | ,-E 49 | x / ,D 50 | | / / | 51 | C'<----**--C 52 | j | 53 | i <----* | 54 | \--B 55 | '-A 56 | 57 | The sorted output looks like this: 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | ... 68 | 69 | Usage: %s --source-branch [-l] [--commit-message-tag ] 70 | `, os.Args[0]) 71 | flag.PrintDefaults() 72 | } 73 | 74 | func main() { 75 | commitMsgTag := flag.String("commit-message-tag", "Kubernetes-commit", "the git commit message tag used to point back to source commits") 76 | sourceBranch := flag.String("source-branch", "", "the source branch (fully qualified e.g. refs/remotes/origin/master) used as the filter-branch basis") 77 | showMessage := flag.Bool("l", false, "list the commit message after the two hashes") 78 | 79 | flag.Usage = Usage 80 | flag.Parse() 81 | 82 | if *sourceBranch == "" { 83 | glog.Fatalf("source-branch cannot be empty") 84 | } 85 | 86 | // open repo at "." 87 | r, err := gogit.PlainOpen(".") 88 | if err != nil { 89 | glog.Fatalf("Failed to open repo at .: %v", err) 90 | } 91 | 92 | // get HEAD 93 | dstRef, err := r.Head() 94 | if err != nil { 95 | glog.Fatalf("Failed to open HEAD: %v", err) 96 | } 97 | dstHead, err := cache.CommitObject(r, dstRef.Hash()) 98 | if err != nil { 99 | glog.Fatalf("Failed to resolve HEAD: %v", err) 100 | } 101 | 102 | // get first-parent commit list of upstream branch 103 | srcUpstreamBranch, err := r.ResolveRevision(plumbing.Revision(*sourceBranch)) 104 | if err != nil { 105 | glog.Fatalf("Failed to open upstream branch %s: %v", *sourceBranch, err) 106 | } 107 | srcHead, err := cache.CommitObject(r, *srcUpstreamBranch) 108 | if err != nil { 109 | glog.Fatalf("Failed to open upstream branch %s head: %v", *sourceBranch, err) 110 | } 111 | srcFirstParents, err := git.FirstParentList(r, srcHead) 112 | if err != nil { 113 | glog.Fatalf("Failed to get upstream branch %s first-parent list: %v", *sourceBranch, err) 114 | } 115 | 116 | // get first-parent commit list of HEAD 117 | dstFirstParents, err := git.FirstParentList(r, dstHead) 118 | if err != nil { 119 | glog.Fatalf("Failed to get first-parent commit list for %s: %v", dstHead.Hash, err) 120 | } 121 | 122 | sourceCommitToDstCommits, err := git.SourceCommitToDstCommits(r, *commitMsgTag, dstFirstParents, srcFirstParents) 123 | if err != nil { 124 | glog.Fatalf("Failed to map upstream branch %s to HEAD: %v", *sourceBranch, err) 125 | } 126 | 127 | // print out a look-up table 128 | // 129 | var lines []string 130 | for kh, dh := range sourceCommitToDstCommits { 131 | if *showMessage { 132 | c, err := cache.CommitObject(r, kh) 133 | if err != nil { 134 | // if this happen something above in the algorithm is broken 135 | glog.Fatalf("Failed to find k8s.io/kubernetes commit %s", kh) 136 | } 137 | lines = append(lines, fmt.Sprintf("%s %s %s", kh, dh, strings.SplitN(c.Message, "\n", 2)[0])) 138 | } else { 139 | lines = append(lines, fmt.Sprintf("%s %s", kh, dh)) 140 | } 141 | } 142 | sort.Strings(lines) // sort to allow binary search 143 | for _, l := range lines { 144 | fmt.Println(l) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /cmd/publishing-bot/publisher_logger.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Changing glog output directory via --log_dir doesn't work, because the flag 18 | // is parsed after the first invocation of glog, so the log file ends up in the 19 | // temporary directory. Hence, we manually duplicates glog ouptut. 20 | 21 | package main 22 | 23 | import ( 24 | "bytes" 25 | "fmt" 26 | "io" 27 | "os" 28 | "os/exec" 29 | "strings" 30 | "sync" 31 | "time" 32 | 33 | "github.com/golang/glog" 34 | "github.com/shurcooL/go/indentwriter" 35 | "gopkg.in/natefinch/lumberjack.v2" 36 | ) 37 | 38 | type plog struct { 39 | combinedBufAndFile io.Writer 40 | buf *bytes.Buffer 41 | } 42 | 43 | func newPublisherLog(buf *bytes.Buffer, logFileName string) (*plog, error) { 44 | logFile := &lumberjack.Logger{ 45 | Filename: logFileName, 46 | MaxAge: 7, 47 | } 48 | if err := logFile.Rotate(); err != nil { 49 | return nil, err 50 | } 51 | 52 | return &plog{newSyncWriter(muxWriter{buf, logFile}), buf}, nil 53 | } 54 | 55 | func (p *plog) write(s string) { 56 | //nolint:errcheck // TODO(lint): Should we be checking errors here? 57 | p.combinedBufAndFile.Write([]byte("[" + time.Now().Format(time.RFC822) + "]: ")) 58 | 59 | //nolint:errcheck // TODO(lint): Should we be checking errors here? 60 | p.combinedBufAndFile.Write([]byte(s)) 61 | 62 | //nolint:errcheck // TODO(lint): Should we be checking errors here? 63 | p.combinedBufAndFile.Write([]byte("\n")) 64 | } 65 | 66 | func (p *plog) Errorf(format string, args ...interface{}) { 67 | s := prefixFollowingLines(" ", fmt.Sprintf(format, args...)) 68 | glog.ErrorDepth(1, s) 69 | p.write(s) 70 | } 71 | 72 | func (p *plog) Infof(format string, args ...interface{}) { 73 | s := prefixFollowingLines(" ", fmt.Sprintf(format, args...)) 74 | glog.InfoDepth(1, s) 75 | p.write(s) 76 | } 77 | 78 | func (p *plog) Fatalf(format string, args ...interface{}) { 79 | s := prefixFollowingLines(" ", fmt.Sprintf(format, args...)) 80 | glog.FatalDepth(1, s) 81 | p.write(s) 82 | } 83 | 84 | func (p *plog) Run(c *exec.Cmd) error { 85 | p.Infof("%s", cmdStr(c)) 86 | 87 | errBuf := &bytes.Buffer{} 88 | 89 | stdoutLineWriter := newLineWriter(muxWriter{p.combinedBufAndFile, os.Stdout}) 90 | stderrLineWriter := newLineWriter(muxWriter{p.combinedBufAndFile, errBuf}) 91 | c.Stdout = indentwriter.New(stdoutLineWriter, 1) 92 | c.Stderr = indentwriter.New(stderrLineWriter, 1) 93 | 94 | err := c.Start() 95 | if err != nil { 96 | p.Errorf("failed to start %q: %v", c.Path, err) 97 | return err 98 | } 99 | err = c.Wait() 100 | if err != nil { 101 | p.Errorf("%s\n%s", err.Error(), errBuf.String()) 102 | } 103 | stdoutLineWriter.Flush() 104 | stderrLineWriter.Flush() 105 | return err 106 | } 107 | 108 | func (p *plog) Logs() string { 109 | return p.buf.String() 110 | } 111 | 112 | func (p *plog) Flush() { 113 | glog.Flush() 114 | } 115 | 116 | func prefixFollowingLines(p, s string) string { 117 | lines := strings.Split(s, "\n") 118 | for i := range lines { 119 | if i != 0 && lines[i] != "" { 120 | lines[i] = p + lines[i] 121 | } 122 | } 123 | return strings.Join(lines, "\n") 124 | } 125 | 126 | func cmdStr(cs *exec.Cmd) string { 127 | args := make([]string, len(cs.Args)) 128 | for i, s := range cs.Args { 129 | if strings.ContainsRune(s, ' ') { 130 | args[i] = fmt.Sprintf("%q", s) 131 | } else { 132 | args[i] = s 133 | } 134 | } 135 | return strings.Join(args, " ") 136 | } 137 | 138 | type muxWriter []io.Writer 139 | 140 | func (mw muxWriter) Write(b []byte) (int, error) { 141 | n := 0 142 | var err error 143 | for _, w := range mw { 144 | if n, err = w.Write(b); err != nil { 145 | return n, err 146 | } 147 | } 148 | return n, nil 149 | } 150 | 151 | func newLineWriter(writer io.Writer) lineWriter { 152 | return lineWriter{ 153 | buf: new(bytes.Buffer), 154 | writer: writer, 155 | } 156 | } 157 | 158 | type lineWriter struct { 159 | buf *bytes.Buffer 160 | writer io.Writer 161 | } 162 | 163 | func (lw lineWriter) Write(b []byte) (int, error) { 164 | n := 0 165 | for idx := range b { 166 | if err := lw.buf.WriteByte(b[idx]); err != nil { 167 | return n, err 168 | } 169 | if b[idx] == '\n' { 170 | if _, err := lw.Flush(); err != nil { 171 | return n, err 172 | } 173 | } 174 | n++ 175 | } 176 | return n, nil 177 | } 178 | 179 | //nolint:unparam // TODO(lint): result 0 (int) is never used 180 | func (lw lineWriter) Flush() (int, error) { 181 | written, err := lw.buf.WriteTo(lw.writer) 182 | lw.buf.Reset() 183 | return int(written), err 184 | } 185 | 186 | func newSyncWriter(writer io.Writer) syncWriter { 187 | return syncWriter{ 188 | writer: writer, 189 | lock: &sync.Mutex{}, 190 | } 191 | } 192 | 193 | type syncWriter struct { 194 | writer io.Writer 195 | lock *sync.Mutex 196 | } 197 | 198 | func (sw syncWriter) Write(b []byte) (int, error) { 199 | sw.lock.Lock() 200 | defer sw.lock.Unlock() 201 | return sw.writer.Write(b) 202 | } 203 | -------------------------------------------------------------------------------- /production.md: -------------------------------------------------------------------------------- 1 | # Kubernetes publishing-bot production instance notes 2 | 3 | ### What is this and what does it do? 4 | 5 | The publishing-bot for the Kubernetes project is running in the `publishing-bot` namespace on a CNCF sponsored GKE cluster `aaa` in the `kubernetes-public` project. 6 | 7 | ### How do i get access to this? 8 | 9 | If you need access to any of the following, please update [groups.yaml]. 10 | 11 | ### GKE instance 12 | 13 | publishing-bot is running in a GKE cluster named `aaa` in the `kubernetes-public` 14 | - [GKE project](https://console.cloud.google.com/kubernetes/list/overview?project=kubernetes-public) 15 | - [aaa cluster](https://console.cloud.google.com/kubernetes/clusters/details/us-central1/aaa/details?project=kubernetes-public) 16 | 17 | The cluster can be accessed by [k8s-infra-rbac-publishing-bot@kubernetes.io]. 18 | To access the cluster, please see these [instructions]. 19 | 20 | ### What images does it use? 21 | 22 | Publishing-bot [images] can be pushed by [k8s-infra-staging-publishing-bot@kubernetes.io]. 23 | 24 | ### What commands are in this repo and how/when do i use them? 25 | 26 | Make sure you are at the root of the publishing-bot repo before running these commands. 27 | 28 | #### Populating repos 29 | 30 | This script needs to be run whenever a new staging repo is added in kubernetes/kubernetes 31 | 32 | ```sh 33 | hack/fetch-all-latest-and-push.sh kubernetes 34 | ``` 35 | 36 | #### Deploying the bot 37 | 38 | ```sh 39 | make validate build-image push-image deploy CONFIG=configs/kubernetes 40 | ``` 41 | 42 | ### How to connect to the `aaa` cluster 43 | 44 | You can use the `Activate Cloud Shell` in the GCP console above and in that console, run the following command 45 | ``` 46 | gcloud container clusters get-credentials aaa --region us-central1 --project kubernetes-public 47 | ``` 48 | 49 | then run `kubectl` commands to ensure you can see what's running in the cluster. 50 | 51 | ### What is running there? 52 | 53 | The `publishing-bot` runs in a separate kubernetes namespace by the same name in the `aaa` cluster. 54 | The manifests [here](https://github.com/kubernetes/publishing-bot/tree/master/artifacts/manifests) have the definitions 55 | for these kubernetes resources. Example below: 56 | 57 | ````shell 58 | davanum@cloudshell:~ (kubernetes-public)$ kubectl get pv,pvc,replicaset,pod -n publishing-bot 59 | NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE 60 | persistentvolume/pvc-084a4d52-0a57-4f70-a76a-5d2d2667429d 100Gi RWO Delete Bound publishing-bot/publisher-gopath ssd 8h 61 | 62 | NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE 63 | persistentvolumeclaim/publisher-gopath Bound pvc-084a4d52-0a57-4f70-a76a-5d2d2667429d 100Gi RWO ssd 8h 64 | 65 | NAME DESIRED CURRENT READY AGE 66 | replicaset.apps/publisher 1 1 1 45d 67 | 68 | NAME READY STATUS RESTARTS AGE 69 | pod/publisher-cdvwj 1/1 Running 0 9h 70 | ```` 71 | 72 | ### How do i know if/when the bot fails? 73 | 74 | Follow this [Kubernetes issue #56876](https://github.com/kubernetes/kubernetes/issues/56876). When the bot fails it 75 | re-opens this issue with a fresh log. So if you are subscribed to this issue, you can see the bot open the issue 76 | when it fails. 77 | 78 | ### How do i see what the publishing bot is doing? 79 | 80 | you can stream the logs of the pod to see what the publishing-bot is doing 81 | ```shell 82 | kubectl -n publishing-bot logs pod/publisher-cdvwj -f 83 | ``` 84 | 85 | ### What is the persistent volume for? 86 | 87 | To do its work the publishing-bot has to download all the repositories and performs git surgery on them. So publishing-bot 88 | keeps the downloaded copy around and re-uses them. For example, if the pod gets killed the new pod can still work off 89 | of the downloaded git repositories on the persistent volume. Occasionally if we suspect the downloaded git repos are 90 | corrupted for some reason (say github flakiness), we may have to cleanup the pv/pvc. in other words, The volume is 91 | cache only. Wiping it is not harmful in general (other than for the time it takes to recreate all the data). 92 | 93 | ### How do i clean up the pvc? 94 | 95 | Step 1: Use the command to scale down the replicaset 96 | ```shell 97 | kubectl scale -n publishing-bot --replicas=0 replicaset publisher 98 | ``` 99 | 100 | Step 2: Delete the PVC 101 | ```shell 102 | kubectl delete -n publishing-bot persistentvolumeclaim/publisher-gopath 103 | ``` 104 | 105 | Step 3: Make sure the PVC is deleted and removed from the namespace 106 | ```shell 107 | kubectl get -n publishing-bot pvc 108 | ``` 109 | should not list any PVCs 110 | 111 | Step 4: Re-deploy the pvc again 112 | ```shell 113 | kubectl apply -n publishing-bot -f artifacts/manifests/pvc.yaml 114 | ``` 115 | 116 | Step 5: Scale up the replicaset 117 | ```shell 118 | kubectl scale -n publishing-bot --replicas=1 replicaset publisher 119 | ``` 120 | 121 | Step 6: Watch the pod start back up from `Pending` 122 | ```shell 123 | kubectl -n publishing-bot get pods 124 | ``` 125 | 126 | [k8s-infra-rbac-publishing-bot@kubernetes.io]: https://github.com/kubernetes/k8s.io/blob/7e72aa72f1548af9cf3dbe405f8c317fe637f361/groups/groups.yaml#L405-L418 127 | [k8s-infra-staging-publishing-bot@kubernetes.io]: https://github.com/kubernetes/k8s.io/blob/6a6b50f4d04124b02915bc2736b468def0de96e9/groups/groups.yaml#L992-L1001 128 | [images]: https://console.cloud.google.com/gcr/images/k8s-staging-publishing-bot/GLOBAL/k8s-publishing-bot 129 | [groups.yaml]: https://git.k8s.io/k8s.io/groups/groups.yaml 130 | [instructions]: https://git.k8s.io/k8s.io/running-in-community-clusters.md#access-the-cluster -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | run: 3 | concurrency: 6 4 | linters: 5 | default: none 6 | enable: 7 | - asasalint 8 | - asciicheck 9 | - bidichk 10 | - bodyclose 11 | - containedctx 12 | - contextcheck 13 | - decorder 14 | - dogsled 15 | - dupl 16 | - dupword 17 | - durationcheck 18 | - errcheck 19 | - errchkjson 20 | - errname 21 | - errorlint 22 | - exhaustive 23 | - forcetypeassert 24 | - ginkgolinter 25 | - gocheckcompilerdirectives 26 | - gochecknoinits 27 | - gochecksumtype 28 | - goconst 29 | - gocritic 30 | - gocyclo 31 | - godot 32 | - godox 33 | - goheader 34 | - gomoddirectives 35 | - gomodguard 36 | - goprintffuncname 37 | - gosmopolitan 38 | - govet 39 | - grouper 40 | - importas 41 | - inamedparam 42 | - ineffassign 43 | - interfacebloat 44 | - ireturn 45 | - loggercheck 46 | - makezero 47 | - mirror 48 | - misspell 49 | - musttag 50 | - nakedret 51 | - nilerr 52 | - noctx 53 | - nolintlint 54 | - nosprintfhostport 55 | - prealloc 56 | - predeclared 57 | - promlinter 58 | - protogetter 59 | - reassign 60 | - revive 61 | - rowserrcheck 62 | - sloglint 63 | - sqlclosecheck 64 | - staticcheck 65 | - tagalign 66 | - testableexamples 67 | - testifylint 68 | - thelper 69 | - tparallel 70 | - unconvert 71 | - unparam 72 | - unused 73 | - usestdlibvars 74 | - wastedassign 75 | - whitespace 76 | - zerologlint 77 | # - cyclop 78 | # - depguard 79 | # - exhaustruct 80 | # - forbidigo 81 | # - funlen 82 | # - gochecknoglobals 83 | # - gocognit 84 | # - gomnd 85 | # - gosec 86 | # - lll 87 | # - maintidx 88 | # - nestif 89 | # - nilnil 90 | # - nlreturn 91 | # - nonamedreturns 92 | # - paralleltest 93 | # - tagliatelle 94 | # - testpackage 95 | # - varnamelen 96 | # - wrapcheck 97 | # - wsl 98 | settings: 99 | errcheck: 100 | check-type-assertions: true 101 | check-blank: true 102 | gocritic: 103 | enabled-checks: 104 | - appendCombine 105 | - badLock 106 | - badRegexp 107 | - badSorting 108 | - boolExprSimplify 109 | - builtinShadow 110 | - builtinShadowDecl 111 | - commentedOutCode 112 | - commentedOutImport 113 | - deferInLoop 114 | - deferUnlambda 115 | - docStub 116 | - dupImport 117 | - dynamicFmtString 118 | - emptyDecl 119 | - emptyFallthrough 120 | - emptyStringTest 121 | - equalFold 122 | - evalOrder 123 | - exposedSyncMutex 124 | - externalErrorReassign 125 | - filepathJoin 126 | - hexLiteral 127 | - httpNoBody 128 | - hugeParam 129 | - importShadow 130 | - indexAlloc 131 | - initClause 132 | - methodExprCall 133 | - nestingReduce 134 | - nilValReturn 135 | - octalLiteral 136 | - paramTypeCombine 137 | - preferDecodeRune 138 | - preferFilepathJoin 139 | - preferFprint 140 | - preferStringWriter 141 | - preferWriteByte 142 | - ptrToRefParam 143 | - rangeExprCopy 144 | - rangeValCopy 145 | - redundantSprint 146 | - regexpPattern 147 | - regexpSimplify 148 | - returnAfterHttpError 149 | - ruleguard 150 | - sliceClear 151 | - sloppyReassign 152 | - sortSlice 153 | - sprintfQuotedString 154 | - sqlQuery 155 | - stringConcatSimplify 156 | - stringXbytes 157 | - stringsCompare 158 | - syncMapLoadAndDelete 159 | - timeExprSimplify 160 | - todoCommentWithoutDetail 161 | - tooManyResultsChecker 162 | - truncateCmp 163 | - typeAssertChain 164 | - typeDefFirst 165 | - typeUnparen 166 | - uncheckedInlineErr 167 | - unlabelStmt 168 | - unnamedResult 169 | - unnecessaryBlock 170 | - unnecessaryDefer 171 | - weakCond 172 | - whyNoLint 173 | - yodaStyleExpr 174 | # - appendAssign 175 | # - argOrder 176 | # - assignOp 177 | # - badCall 178 | # - badCond 179 | # - captLocal 180 | # - caseOrder 181 | # - codegenComment 182 | # - commentFormatting 183 | # - defaultCaseOrder 184 | # - deprecatedComment 185 | # - dupArg 186 | # - dupBranchBody 187 | # - dupCase 188 | # - dupSubExpr 189 | # - elseif 190 | # - exitAfterDefer 191 | # - flagDeref 192 | # - flagName 193 | # - ifElseChain 194 | # - mapKey 195 | # - newDeref 196 | # - offBy1 197 | # - regexpMust 198 | # - singleCaseSwitch 199 | # - sloppyLen 200 | # - sloppyTypeAssert 201 | # - switchTrue 202 | # - typeSwitchVar 203 | # - underef 204 | # - unlambda 205 | # - unslice 206 | # - valSwap 207 | # - wrapperFunc 208 | godox: 209 | keywords: 210 | - BUG 211 | - FIXME 212 | - HACK 213 | nolintlint: 214 | require-explanation: false 215 | require-specific: true 216 | allow-unused: false 217 | exclusions: 218 | generated: lax 219 | presets: 220 | - comments 221 | - common-false-positives 222 | - legacy 223 | - std-error-handling 224 | rules: 225 | - linters: 226 | - dupl 227 | - gocritic 228 | - golint 229 | # counterfeiter fakes are usually named 'fake_.go' 230 | path: fake_.*\.go 231 | - linters: 232 | - err113 233 | text: do not define dynamic errors 234 | paths: 235 | - third_party$ 236 | - builtin$ 237 | - examples$ 238 | issues: 239 | # Maximum issues count per one linter. 240 | # Set to 0 to disable. 241 | # Default: 50 242 | max-issues-per-linter: 0 243 | # Maximum count of issues with the same text. 244 | # Set to 0 to disable. 245 | # Default: 3 246 | max-same-issues: 0 247 | formatters: 248 | enable: 249 | - gci 250 | - gofmt 251 | - gofumpt 252 | - goimports 253 | exclusions: 254 | generated: lax 255 | paths: 256 | - third_party$ 257 | - builtin$ 258 | - examples$ 259 | -------------------------------------------------------------------------------- /cmd/update-rules/main_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | ) 23 | 24 | var ( 25 | testdataRules = "testdata/rules.yaml" 26 | testdataInvalidRules = "testdata/invalid_rules.yaml" 27 | remoteRules = "https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml" 28 | ) 29 | 30 | func TestLoad(t *testing.T) { 31 | tests := []struct { 32 | name string 33 | input string 34 | expectErr bool 35 | }{ 36 | { 37 | "local testdata valid rules file", 38 | testdataRules, 39 | false, 40 | }, 41 | { 42 | "local testdata invalid rules file with invalid go version", 43 | testdataInvalidRules, 44 | true, 45 | }, 46 | { 47 | "remote valid rules file", 48 | remoteRules, 49 | false, 50 | }, 51 | { 52 | "local invalid path to rules file", 53 | "/invalid/path.yaml", 54 | true, 55 | }, 56 | { 57 | "remote 404 rules files", 58 | "https://foo.bar/rules.yaml", 59 | true, 60 | }, 61 | } 62 | 63 | for _, tt := range tests { 64 | t.Run(tt.name, func(t *testing.T) { 65 | _, err := load(tt.input) 66 | if err != nil && !tt.expectErr { 67 | t.Errorf("error loading test rules file from %s , did not expect error", tt.input) 68 | } 69 | if err == nil && tt.expectErr { 70 | t.Errorf("expected error while loading rules from %s , but got none", tt.input) 71 | } 72 | }) 73 | } 74 | } 75 | 76 | func TestUpdateRules(t *testing.T) { 77 | tests := []struct { 78 | name string 79 | branch string 80 | goVersion string 81 | }{ 82 | { 83 | "new branch with go version", 84 | "release-1.XY", 85 | "1.17.1", 86 | }, 87 | { 88 | "new branch with go version", 89 | "release-1.XY", 90 | "1.17.1", 91 | }, 92 | { 93 | "new branch without go version", 94 | "release-1.XY", 95 | "", 96 | }, 97 | { 98 | "existing branch rule with go version update", 99 | "release-1.21", 100 | "1.16.1", 101 | }, 102 | { 103 | "master branch rule update for go version", 104 | "master", 105 | "1.16.4", 106 | }, 107 | } 108 | 109 | for _, tt := range tests { 110 | t.Run(tt.name, func(t *testing.T) { 111 | rules, err := load(testdataRules) 112 | if err != nil { 113 | t.Errorf("error loading test rules file %v", err) 114 | } 115 | UpdateRules(rules, tt.branch, tt.goVersion, false) 116 | 117 | for _, repoRule := range rules.Rules { 118 | var masterRulePresent, branchRulePresent bool 119 | var masterRuleIndex, branchRuleIndex int 120 | 121 | for i, branchRule := range repoRule.Branches { 122 | switch branchRule.Name { 123 | case "master": 124 | masterRulePresent = true 125 | masterRuleIndex = i 126 | case tt.branch: 127 | branchRulePresent = true 128 | branchRuleIndex = i 129 | } 130 | } 131 | switch masterRulePresent { 132 | case true: 133 | if !branchRulePresent && tt.branch != "master" { 134 | t.Errorf("error updating branch %s rule for repo %s", tt.branch, repoRule.DestinationRepository) 135 | } 136 | case false: 137 | if branchRulePresent { 138 | t.Errorf("incorrectly added branch %s rule for repo %s whose master branch rule does not exists", tt.branch, repoRule.DestinationRepository) 139 | } 140 | } 141 | 142 | if repoRule.Branches[branchRuleIndex].Source.Branch != tt.branch { 143 | t.Errorf("incorrect update to branch %s rule for source branch field for repo %s", tt.branch, repoRule.DestinationRepository) 144 | } 145 | 146 | if repoRule.Branches[masterRuleIndex].Source.Dir != repoRule.Branches[branchRuleIndex].Source.Dir { 147 | t.Errorf("incorrect update to branch %s rule for source dir field for repo %s", tt.branch, repoRule.DestinationRepository) 148 | } 149 | 150 | if repoRule.Branches[branchRuleIndex].GoVersion != tt.goVersion { 151 | t.Errorf("incorrect go version set for branch %s rule for repo %s", tt.branch, repoRule.DestinationRepository) 152 | } 153 | 154 | if len(repoRule.Branches[masterRuleIndex].Dependencies) != len(repoRule.Branches[branchRuleIndex].Dependencies) { 155 | t.Errorf("incorrect update to branch %s rule dependencies for repo %s", tt.branch, repoRule.DestinationRepository) 156 | } 157 | 158 | if len(repoRule.Branches[masterRuleIndex].RequiredPackages) != len(repoRule.Branches[branchRuleIndex].RequiredPackages) { 159 | t.Errorf("incorrect update to branch %s rule required packages for repo %s", tt.branch, repoRule.DestinationRepository) 160 | } 161 | 162 | if repoRule.Branches[masterRuleIndex].SmokeTest != repoRule.Branches[branchRuleIndex].SmokeTest { 163 | t.Errorf("incorrect update to branch %s rule smoke-test for repo %s", tt.branch, repoRule.DestinationRepository) 164 | } 165 | } 166 | }) 167 | } 168 | } 169 | 170 | func TestDeleteRules(t *testing.T) { 171 | tests := []struct { 172 | name string 173 | branch string 174 | goVersion string 175 | isBranchExist bool 176 | }{ 177 | { 178 | "deleting rule for non existing branch", 179 | "release-1.20", 180 | "1.17.1", 181 | true, 182 | }, 183 | { 184 | "deleting rule for non existing branch 1.25", 185 | "release-1.25", 186 | "1.17.1", 187 | false, 188 | }, 189 | } 190 | 191 | for _, tt := range tests { 192 | t.Run(tt.name, func(t *testing.T) { 193 | rules, err := load(testdataRules) 194 | if err != nil { 195 | t.Errorf("error loading test rules file %v", err) 196 | } 197 | UpdateRules(rules, tt.branch, tt.goVersion, true) 198 | if tt.isBranchExist { 199 | for _, repoRule := range rules.Rules { 200 | for _, branchRule := range repoRule.Branches { 201 | if branchRule.Name == tt.branch { 202 | t.Errorf("failed to delete %s branch rule from for repo %s", tt.name, repoRule.DestinationRepository) 203 | } 204 | } 205 | } 206 | } else { 207 | if loadedRules, err := load(testdataRules); err != nil { 208 | t.Errorf("error loading test rules file for comparison %v", err) 209 | } else if !reflect.DeepEqual(loadedRules, rules) { 210 | t.Errorf("rules changed after deleting a non existent branch %s", tt.branch) 211 | } 212 | } 213 | }) 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /cmd/init-repo/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "os" 23 | "os/exec" 24 | "path/filepath" 25 | "strings" 26 | 27 | "github.com/golang/glog" 28 | yaml "gopkg.in/yaml.v2" 29 | "k8s.io/publishing-bot/cmd/publishing-bot/config" 30 | ) 31 | 32 | var ( 33 | SystemGoPath = os.Getenv("GOPATH") 34 | BaseRepoPath = filepath.Join(SystemGoPath, "src", "k8s.io") 35 | ) 36 | 37 | func Usage() { 38 | fmt.Fprintf(os.Stderr, ` 39 | Usage: %s [-config ] [-source-repo ] [-source-org ] [-rules-file ] [-target-org ] 40 | 41 | Command line flags override config values. 42 | `, os.Args[0]) 43 | flag.PrintDefaults() 44 | } 45 | 46 | func main() { 47 | configFilePath := flag.String("config", "", "the config file in yaml format") 48 | githubHost := flag.String("github-host", "", "the address of github (defaults to github.com)") 49 | basePackage := flag.String("base-package", "", "the name of the package base (defaults to k8s.io when source repo is kubernetes, "+ 50 | "otherwise github-host/target-org)") 51 | repoName := flag.String("source-repo", "", "the name of the source repository (eg. kubernetes)") 52 | repoOrg := flag.String("source-org", "", "the name of the source repository organization, (eg. kubernetes)") 53 | rulesFile := flag.String("rules-file", "", "the file with repository rules") 54 | targetOrg := flag.String("target-org", "", `the target organization to publish into (e.g. "k8s-publishing-bot")`) 55 | 56 | flag.Usage = Usage 57 | flag.Parse() 58 | 59 | cfg := &config.Config{} 60 | if *configFilePath != "" { 61 | bs, err := os.ReadFile(*configFilePath) 62 | if err != nil { 63 | glog.Fatalf("Failed to load config file from %q: %v", *configFilePath, err) 64 | } 65 | if err := yaml.Unmarshal(bs, &cfg); err != nil { 66 | glog.Fatalf("Failed to parse config file at %q: %v", *configFilePath, err) 67 | } 68 | } 69 | 70 | if *targetOrg != "" { 71 | cfg.TargetOrg = *targetOrg 72 | } 73 | if *repoName != "" { 74 | cfg.SourceRepo = *repoName 75 | } 76 | if *repoOrg != "" { 77 | cfg.SourceOrg = *repoOrg 78 | } 79 | if *githubHost != "" { 80 | cfg.GithubHost = *githubHost 81 | } 82 | if *basePackage != "" { 83 | cfg.BasePackage = *basePackage 84 | } 85 | 86 | if cfg.GithubHost == "" { 87 | cfg.GithubHost = "github.com" 88 | } 89 | 90 | if cfg.GitDefaultBranch == "" { 91 | cfg.GitDefaultBranch = "master" 92 | } 93 | 94 | // defaulting when base package is not specified 95 | if cfg.BasePackage == "" { 96 | if cfg.SourceRepo == "kubernetes" { 97 | cfg.BasePackage = "k8s.io" 98 | } else { 99 | cfg.BasePackage = filepath.Join(cfg.GithubHost, cfg.TargetOrg) 100 | } 101 | } 102 | 103 | BaseRepoPath = filepath.Join(SystemGoPath, "src", cfg.BasePackage) 104 | 105 | if *rulesFile != "" { 106 | cfg.RulesFile = *rulesFile 107 | } 108 | 109 | if cfg.SourceRepo == "" || cfg.SourceOrg == "" { 110 | glog.Fatalf("source-org and source-repo cannot be empty") 111 | } 112 | 113 | if cfg.TargetOrg == "" { 114 | glog.Fatalf("Target organization cannot be empty") 115 | } 116 | 117 | // If RULE_FILE_PATH is detected, check if the source repository include rules files. 118 | if os.Getenv("RULE_FILE_PATH") != "" { 119 | cfg.RulesFile = filepath.Join(BaseRepoPath, cfg.SourceRepo, os.Getenv("RULE_FILE_PATH")) 120 | } 121 | 122 | if cfg.RulesFile == "" { 123 | glog.Fatalf("No rules file provided") 124 | } 125 | rules, err := config.LoadRules(cfg.RulesFile) 126 | if err != nil { 127 | glog.Fatalf("Failed to load rules: %v", err) 128 | } 129 | if err := config.Validate(rules); err != nil { 130 | glog.Fatalf("Invalid rules: %v", err) 131 | } 132 | 133 | if err := os.MkdirAll(BaseRepoPath, os.ModePerm); err != nil { 134 | glog.Fatalf("Failed to create source repo directory %s: %v", BaseRepoPath, err) 135 | } 136 | 137 | cloneSourceRepo(cfg) 138 | for _, rule := range rules.Rules { 139 | cloneForkRepo(cfg, rule.DestinationRepository) 140 | } 141 | } 142 | 143 | func cloneForkRepo(cfg *config.Config, repoName string) { 144 | forkRepoLocation := fmt.Sprintf("https://%s/%s/%s", cfg.GithubHost, cfg.TargetOrg, repoName) 145 | repoDir := filepath.Join(BaseRepoPath, repoName) 146 | 147 | if _, err := os.Stat(repoDir); err == nil { 148 | glog.Infof("Fork repository %q already cloned to %s, resetting remote URL ...", repoName, repoDir) 149 | setURLCmd := exec.Command("git", "remote", "set-url", "origin", forkRepoLocation) 150 | setURLCmd.Dir = repoDir 151 | run(setURLCmd) 152 | os.Remove(filepath.Join(repoDir, ".git", "index.lock")) 153 | } else { 154 | glog.Infof("Cloning fork repository %s ...", forkRepoLocation) 155 | run(exec.Command("git", "clone", forkRepoLocation)) 156 | } 157 | 158 | // set user in repo because old git version (compare https://github.com/git/git/commit/92bcbb9b338dd27f0fd4245525093c4bce867f3d) still look up user ids without 159 | setUsernameCmd := exec.Command("git", "config", "user.name", os.Getenv("GIT_COMMITTER_NAME")) 160 | setUsernameCmd.Dir = repoDir 161 | run(setUsernameCmd) 162 | setEmailCmd := exec.Command("git", "config", "user.email", os.Getenv("GIT_COMMITTER_EMAIL")) 163 | setEmailCmd.Dir = repoDir 164 | run(setEmailCmd) 165 | } 166 | 167 | // run wraps the cmd.Run() command and sets the standard output and common environment variables. 168 | // if the c.Dir is not set, the BaseRepoPath will be used as a base directory for the command. 169 | func run(c *exec.Cmd) { 170 | c.Stdout = os.Stdout 171 | c.Stderr = os.Stderr 172 | if c.Dir == "" { 173 | c.Dir = BaseRepoPath 174 | } 175 | if err := c.Run(); err != nil { 176 | glog.Fatalf("Command %q failed: %v", strings.Join(c.Args, " "), err) 177 | } 178 | } 179 | 180 | func cloneSourceRepo(cfg *config.Config) { 181 | repoLocation := fmt.Sprintf("https://%s/%s/%s", cfg.GithubHost, cfg.SourceOrg, cfg.SourceRepo) 182 | 183 | if _, err := os.Stat(filepath.Join(BaseRepoPath, cfg.SourceRepo)); err == nil { 184 | glog.Infof("Source repository %q already cloned, only setting remote", cfg.SourceRepo) 185 | remoteCmd := exec.Command("git", "remote", "set-url", "origin", repoLocation) 186 | remoteCmd.Dir = filepath.Join(BaseRepoPath, cfg.SourceRepo) 187 | run(remoteCmd) 188 | return 189 | } 190 | 191 | glog.Infof("Cloning source repository %s ...", repoLocation) 192 | cloneCmd := exec.Command("git", "clone", repoLocation) 193 | run(cloneCmd) 194 | } 195 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Publishing Bot 2 | 3 | [![sig-release-publishing-bot/build](https://testgrid.k8s.io/q/summary/sig-release-publishing-bot/build/tests_status?style=svg)](https://testgrid.k8s.io/sig-release-publishing-bot#build) 4 | [![](https://img.shields.io/uptimerobot/status/m779759348-04b1f4fd3bb5ce4a810670d2.svg?label=bot)](https://stats.uptimerobot.com/wm4Dyt8kY) 5 | [![](https://img.shields.io/uptimerobot/status/m779759340-0a6b2cb6fee352e75f58ba16.svg?label=last%20publishing%20run)](https://github.com/kubernetes/kubernetes/issues/56876) 6 | 7 | ## Overview 8 | 9 | The publishing bot publishes the code in `k8s.io/kubernetes/staging` to their own repositories. It guarantees that the master branches of the published repositories are compatible, i.e., if a user `go get` a published repository in a clean GOPATH, the repo is guaranteed to work. 10 | 11 | It pulls the latest k8s.io/kubernetes changes and runs `git filter-branch` to distill the commits that affect a staging repo. Then it cherry-picks merged PRs with their feature branch commits to the target repo. It records the SHA1 of the last cherrypicked commits in `Kubernetes-sha: ` lines in the commit messages. 12 | 13 | The robot is also responsible to update the `go-mod` and the `vendor/` directory for the target repos. 14 | 15 | ## Playbook 16 | 17 | ### Publishing a new repo or a new branch 18 | 19 | * Adapt the rules in [config/kubernetes-rules-configmap.yaml](configs/kubernetes-rules-configmap.yaml) 20 | * For Kubernetes, the configuration is located in the [kubernetes/kubernetes repository](https://github.com/kubernetes/kubernetes/blob/master/staging/publishing/rules.yaml) 21 | 22 | * For a new repo, add it to the repo list in [hack/repos.sh](hack/repos.sh) 23 | 24 | * [Test and deploy the changes](#testing-and-deploying-the-robot) 25 | 26 | ### Updating rules 27 | 28 | #### Adapting rules for a new branch 29 | 30 | If you're creating a new branch, you need to update the publishing-bot rules to reflect that. For Kubernetes, this means that you need to update the [`rules.yaml` file](https://github.com/kubernetes/kubernetes/blob/master/staging/publishing/rules.yaml) on the `master` branch. 31 | 32 | For each repository, add a new branch to the `branches` stanza. If the branch is using the same Go version as the [default Go version](https://github.com/kubernetes/kubernetes/blob/489fb9bee3f626b3eeb120a5af89ad8c2b2f1c20/staging/publishing/rules.yaml#L10), you don't need to specify the Go version for the branch (otherwise you need to do that). 33 | 34 | #### Adapting rules for a Go update 35 | 36 | If you're updating Go version for the master or release branches, you need to adapt the [`rules.yaml` file in kubernetes/kubernetes](https://github.com/kubernetes/kubernetes/blob/master/staging/publishing/rules.yaml) on the `master` branch. 37 | 38 | * If you're updating Go version for the master branch, you need to change the [default Go version](https://github.com/kubernetes/kubernetes/blob/489fb9bee3f626b3eeb120a5af89ad8c2b2f1c20/staging/publishing/rules.yaml#L10) to the new version. 39 | * If release branches that depend on the default Go version use a different (e.g. old) Go version, you need to explicitly set Go version for those branches (e.g. [like here](https://github.com/kubernetes/kubernetes/blob/489fb9bee3f626b3eeb120a5af89ad8c2b2f1c20/staging/publishing/rules.yaml#L37)) 40 | * If you're updating Go version for a previous release branch 41 | * if it's the same version as the default Go version, you don't need to specify the Go version for that branch 42 | * if it's **NOT** the same version as the default Go version, you need to explicitly specify the Go version for that branch (e.g. [like here](https://github.com/kubernetes/kubernetes/blob/489fb9bee3f626b3eeb120a5af89ad8c2b2f1c20/staging/publishing/rules.yaml#L37)) 43 | * Examples: https://github.com/kubernetes/kubernetes/pull/93998, https://github.com/kubernetes/kubernetes/pull/101232, https://github.com/kubernetes/kubernetes/pull/104226 44 | 45 | ### Testing and deploying the robot 46 | 47 | Currently we don't have tests for the bot. It relies on manual tests: 48 | 49 | * Fork the repos you are going the publish. 50 | * Run [hack/fetch-all-latest-and-push.sh](hack/fetch-all-latest-and-push.sh) from the bot root directory to update the branches of your repos. This will sync your forks with upstream. **CAUTION:** this might delete data in your forks. 51 | * Use [hack/create-repos.sh](hack/create-repos.sh) from the bot root directory to create any missing repos in the destination github org. 52 | 53 | * Create a config and a corresponding ConfigMap in [configs](configs), 54 | - by copying [configs/example](configs/example) and [configs/example-configmap.yaml](configs/example-configmap.yaml), 55 | - and by changing the Makefile constants in `configs/` 56 | - and the ConfigMap values in `configs/-configmap.yaml`. 57 | 58 | * Create a rule config and a corresponding ConfigMap in [configs](configs), 59 | - by copying [configs/example-rules-configmap.yaml](configs/example-rules-configmap.yaml), 60 | - and by changing the Makefile constants in `configs/` 61 | - and the ConfigMap values in `configs/-rules-configmap.yaml`. 62 | 63 | * Deploy the publishing bot by running make from the bot root directory, e.g. 64 | 65 | ```shell 66 | $ make build-image push-image CONFIG=configs/ 67 | $ make run CONFIG=configs/ TOKEN= 68 | ``` 69 | 70 | for a fire-and-forget pod. Or use 71 | 72 | ```shell 73 | $ make deploy CONFIG=configs/ TOKEN= 74 | ``` 75 | 76 | to run a ReplicaSet that publishes every 24h (you can change the `INTERVAL` config value for different intervals). 77 | 78 | This will not push to your org, but runs in dry-run mode. To run with a push, add `DRYRUN=false` to your `make` command line. 79 | 80 | ### Running in Production 81 | 82 | * Use one of the existing [configs](configs) and 83 | * launch `make deploy CONFIG=configs/kubernetes-nightly` 84 | 85 | **Caution:** Make sure that the bot github user CANNOT close arbitrary issues in the upstream repo. Otherwise, github will close, them triggered by `Fixes kubernetes/kubernetes#123` patterns in published commits. 86 | 87 | **Note:**: Details about running the publishing-bot for the Kubernetes project can be found in [production.md](production.md). 88 | 89 | 90 | ### Update rules 91 | 92 | To add new branch rules or update go version for configured destination repos, check [update-branch-rules](cmd/update-rules/README.md). 93 | 94 | ## Contributing 95 | 96 | Please see [CONTRIBUTING.md](CONTRIBUTING.md) for instructions on how to contribute. 97 | 98 | ## Known issues 99 | 100 | 1. Testing: currently we rely on manual testing. We should set up CI for it. 101 | 2. Automate release process (tracked at https://github.com/kubernetes/kubernetes/issues/49011): when kubernetes release, automatic update the configuration of the publishing robot. This probably means that the config must move into the Kubernetes repo, e.g. as a `.publishing.yaml` file. 102 | -------------------------------------------------------------------------------- /cmd/update-rules/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | 25 | "github.com/golang/glog" 26 | "gopkg.in/yaml.v2" 27 | "k8s.io/publishing-bot/cmd/publishing-bot/config" 28 | ) 29 | 30 | const GitDefaultBranch = "master" 31 | 32 | type options struct { 33 | branch string 34 | deleteRule bool 35 | rulesFile string 36 | goVersion string 37 | out string 38 | } 39 | 40 | func parseOptions() options { 41 | var o options 42 | flag.StringVar(&o.branch, "branch", "", "[required] Branch to update rules for, e.g. --branch release-x.yy") 43 | flag.StringVar(&o.rulesFile, "rules", "", "[required] URL or Path of the rules file to update rules for, e.g. --rules path/or/url/to/rules/file.yaml") 44 | flag.BoolVar(&o.deleteRule, "delete", false, "Remove old rules of deprecated branch") 45 | flag.StringVar(&o.goVersion, "go", "", "Golang version to pin for this branch, e.g. --go 1.16.1") 46 | flag.StringVar(&o.out, "o", "", "Path to export the updated rules to, e.g. -o /tmp/rules.yaml") 47 | 48 | examples := ` 49 | Examples: 50 | # Update rules for branch release-1.21 with go version 1.16.1 51 | update-rules -branch release-1.21 -go 1.16.4 -rules /go/src/k8s.io/kubernetes/staging/publishing/rules.yaml 52 | 53 | # Update rules using URL to input rules file 54 | update-rules -branch release-1.21 -go 1.16.4 -rules https://raw.githubusercontent.com/kubernetes/kubernetes/master/staging/publishing/rules.yaml 55 | 56 | # Update rules and export to /tmp/rules.yaml 57 | update-rules -branch release-1.22 -go 1.17.1 -o /tmp/rules.yaml -rules /go/src/k8s.io/kubernetes/staging/publishing/rules.yaml 58 | 59 | # Update rules to remove deprecated branch and export to /tmp/rules.yaml 60 | update-rules -branch release-1.22 -delete -o /tmp/rules.yaml -rules /go/src/k8s.io/kubernetes/staging/publishing/rules.yaml` 61 | 62 | flag.Usage = func() { 63 | fmt.Fprintf(os.Stdout, "\n Usage: update-rules --branch BRANCH --rules PATHorURL [--go VERSION | -o PATH | --delete ]") 64 | fmt.Fprintf(os.Stdout, "\n %s\n\n", examples) 65 | flag.PrintDefaults() 66 | } 67 | 68 | flag.Parse() 69 | 70 | if o.branch == "" { 71 | glog.Errorf("branch flag requires a non-empty value, e.g. --branch release-x.yy. Run `update-rules -h` for help!") 72 | os.Exit(2) 73 | } 74 | 75 | if o.rulesFile == "" { 76 | glog.Errorf("rules flag requires a non-empty value, e.g. --rules path/or/url/to/rules/file.yaml") 77 | os.Exit(2) 78 | } 79 | 80 | return o 81 | } 82 | 83 | func main() { 84 | o := parseOptions() 85 | 86 | // load and validate input rules file 87 | rules, err := load(o.rulesFile) 88 | if err != nil { 89 | glog.Fatal(err) 90 | } 91 | 92 | // update rules for all destination repos 93 | UpdateRules(rules, o.branch, o.goVersion, o.deleteRule) 94 | // validate rules after update 95 | if err := config.Validate(rules); err != nil { 96 | glog.Fatalf("update failed, found invalid rules after update: %v", err) 97 | } 98 | 99 | data, err := yaml.Marshal(rules) 100 | if err != nil { 101 | glog.Fatalf("error marshaling rules %v", err) 102 | } 103 | 104 | if o.out != "" { 105 | err = exportRules(o.out, data) 106 | if err != nil { 107 | glog.Fatalf("error exporting the rules %v", err) 108 | } 109 | } else { 110 | fmt.Fprintln(os.Stdout, string(data)) 111 | } 112 | } 113 | 114 | // load reads the input rules file and validates the rules. 115 | func load(rulesFile string) (*config.RepositoryRules, error) { 116 | rules, err := config.LoadRules(rulesFile) 117 | if err != nil { 118 | return nil, fmt.Errorf("error loading rules file %q: %w", rulesFile, err) 119 | } 120 | 121 | if err := config.Validate(rules); err != nil { 122 | return nil, fmt.Errorf("invalid rules file %q: %w", rulesFile, err) 123 | } 124 | return rules, nil 125 | } 126 | 127 | func UpdateRules(rules *config.RepositoryRules, branch, goVer string, deleteRule bool) { 128 | // run the update per destination repo in the rules 129 | for j, r := range rules.Rules { 130 | var deletedBranch bool 131 | // To Check and Remove the existing/deprecated branch 132 | if deleteRule { 133 | for i := range r.Branches { 134 | if rules.Rules[j].Branches[i].Name == branch { 135 | glog.Infof("remove rule %s for %s", branch, r.DestinationRepository) 136 | rules.Rules[j].Branches = append(rules.Rules[j].Branches[:i], rules.Rules[j].Branches[i+1:]...) 137 | deletedBranch = true 138 | break 139 | } 140 | } 141 | if !deletedBranch { 142 | glog.Infof("skipping delete of branch rule %s that doesn't exists for %s", branch, r.DestinationRepository) 143 | } 144 | continue 145 | } 146 | 147 | var mainBranchRuleFound bool 148 | var newBranchRule config.BranchRule 149 | // find the mainBranch rules 150 | for i := range r.Branches { 151 | br := r.Branches[i] 152 | if br.Name == GitDefaultBranch { 153 | cloneBranchRule(&br, &newBranchRule) 154 | mainBranchRuleFound = true 155 | break 156 | } 157 | } 158 | 159 | // if mainBranch rules not found for repo, it means it's removed from master tree, log warning and skip updating the rules 160 | if !mainBranchRuleFound { 161 | glog.Warningf("%s branch rules not found for repo %s, skipping to update branch %s rules", GitDefaultBranch, r.DestinationRepository, branch) 162 | continue 163 | } 164 | 165 | // update the rules for branch and its dependencies 166 | updateBranchRules(&newBranchRule, branch, goVer) 167 | 168 | var branchRuleExists bool 169 | // if the target branch rules already exists, update it 170 | for i := range r.Branches { 171 | br := r.Branches[i] 172 | if br.Name == branch { 173 | glog.Infof("found branch %s rules for destination repo %s, updating it", branch, r.DestinationRepository) 174 | r.Branches[i] = newBranchRule 175 | branchRuleExists = true 176 | break 177 | } 178 | } 179 | // new rules, append to destination's branches 180 | if !branchRuleExists { 181 | r.Branches = append(r.Branches, newBranchRule) 182 | } 183 | 184 | // update the rules for destination repo 185 | rules.Rules[j] = r 186 | } 187 | } 188 | 189 | func cloneBranchRule(in, out *config.BranchRule) { 190 | if in == nil { 191 | return 192 | } 193 | *out = *in 194 | if in.Dependencies != nil { 195 | out.Dependencies = make([]config.Dependency, len(in.Dependencies)) 196 | copy(out.Dependencies, in.Dependencies) 197 | } 198 | } 199 | 200 | func updateBranchRules(br *config.BranchRule, branch, goVersion string) { 201 | br.Name = branch 202 | br.Source.Branch = branch 203 | br.GoVersion = goVersion 204 | for k := range br.Dependencies { 205 | br.Dependencies[k].Branch = branch 206 | } 207 | } 208 | 209 | func exportRules(fPath string, data []byte) error { 210 | if err := os.MkdirAll(filepath.Dir(fPath), 0o755); err != nil { 211 | return err 212 | } 213 | return os.WriteFile(fPath, data, 0o644) 214 | } 215 | -------------------------------------------------------------------------------- /artifacts/scripts/construct.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2017 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script publishes the latest changes in the ${src_branch} of 18 | # k8s.io/kubernetes/staging/src/${repo} to the ${dst_branch} of 19 | # k8s.io/${repo}. 20 | # 21 | # dependent_k8s.io_repos are expected to be separated by ",", 22 | # e.g., "client-go,apimachinery". We will expand it to 23 | # "repo:commit,repo:commit..." in the future. 24 | # 25 | # ${kubernetes_remote} is the remote url of k8s.io/kubernetes that will be used 26 | # in .git/config in the local checkout of the ${repo}. 27 | # 28 | # is_library indicates is ${repo} is a library. 29 | # 30 | # The script assumes that the working directory is 31 | # $GOPATH/src/k8s.io/${repo}. 32 | # 33 | # The script is expected to be run by other publish scripts. 34 | 35 | set -o errexit 36 | set -o nounset 37 | set -o pipefail 38 | set -o xtrace 39 | 40 | if [ ! $# -eq 15 ]; then 41 | echo "usage: $0 repo src_branch dst_branch dependent_k8s.io_repos required_packages kubernetes_remote subdirectories source_repo_org source_repo_name base_package is_library recursive_delete_pattern skip_tags last_published_upstream_hash git_default_branch" 42 | exit 1 43 | fi 44 | 45 | # the target repo 46 | REPO="${1}" 47 | # src branch of k8s.io/kubernetes 48 | SRC_BRANCH="${2:-master}" 49 | # dst branch of k8s.io/${repo} 50 | DST_BRANCH="${3:-master}" 51 | # dependent k8s.io repos 52 | DEPS="${4}" 53 | # required packages that are manually copied completely into vendor/, e.g. k8s.io/code-generator or a sub-package. They must be dependencies as well, either via Go imports or via ${DEPS}. 54 | REQUIRED="${5}" 55 | # Remote url for Kubernetes. If empty, will fetch kubernetes 56 | # from https://github.com/kubernetes/kubernetes. 57 | SOURCE_REMOTE="${6}" 58 | # maps to staging/k8s.io/src/${REPO} or the subdirectories from which the repo needs to be published 59 | SUBDIRS="${7}" 60 | # source repository organization name (eg. kubernetes) 61 | SOURCE_REPO_ORG="${8}" 62 | # source repository name (eg. kubernetes) has to be set for the sync-tags 63 | SOURCE_REPO_NAME="${9}" 64 | 65 | shift 9 66 | 67 | # base package name (eg. k8s.io) 68 | BASE_PACKAGE="${1:-k8s.io}" 69 | # If ${REPO} is a library 70 | IS_LIBRARY="${2}" 71 | # A ls-files pattern like "*/BUILD *.ext pkg/foo.go Makefile" 72 | RECURSIVE_DELETE_PATTERN="${3}" 73 | # Skip syncing tags 74 | SKIP_TAGS="${4}" 75 | # last published upstream hash of this branch 76 | LAST_PUBLISHED_UPSTREAM_HASH="${5}" 77 | # name of the main branch. master for k8s.io/kubernetes 78 | GIT_DEFAULT_BRANCH="${6}" 79 | 80 | readonly REPO SRC_BRANCH DST_BRANCH DEPS REQUIRED SOURCE_REMOTE SOURCE_REPO_ORG SUBDIRS SOURCE_REPO_NAME BASE_PACKAGE IS_LIBRARY RECURSIVE_DELETE_PATTERN SKIP_TAGS LAST_PUBLISHED_UPSTREAM_HASH GIT_DEFAULT_BRANCH 81 | 82 | SCRIPT_DIR=$(dirname "${BASH_SOURCE}") 83 | source "${SCRIPT_DIR}"/util.sh 84 | 85 | if [ ! -f .git/info/attributes ]; then 86 | echo "Creating .git/info/attributes file to override .gitattributes files." 87 | mkdir -p .git/info 88 | echo "* -text" >> .git/info/attributes 89 | # switch over to new file endings 90 | rm -r * && git checkout . || true 91 | fi 92 | 93 | git config user.email "$GIT_COMMITTER_EMAIL" 94 | git config user.name "$GIT_COMMITTER_NAME" 95 | 96 | echo "Running garbage collection." 97 | git config gc.pruneExpire 3.days.ago 98 | git gc --auto 99 | 100 | # Remove corrupted lines in .git/packed-refs 101 | # Lines not starting with '^' caret are printed as-is. Lines 102 | # that do not start with a caret are printed ONLY if the previous 103 | # line did not start with a caret 104 | echo "Cleaning .git/packed-refs" 105 | cp .git/packed-refs .git/packed-refs.bak 106 | awk ' 107 | NR == 1 { 108 | prev_line_start_with_caret = 0 109 | } 110 | /^[^^]/ { 111 | prev_line_start_with_caret = 0 112 | print 113 | } 114 | /^\^/ { 115 | if (!prev_line_start_with_caret) { 116 | print 117 | } 118 | prev_line_start_with_caret = 1 119 | } 120 | ' .git/packed-refs.bak > .git/packed-refs 121 | rm .git/packed-refs.bak 122 | 123 | echo "Fetching from origin." 124 | git fetch origin --no-tags --prune 125 | echo "Cleaning up checkout." 126 | git rebase --abort >/dev/null || true 127 | rm -f .git/index.lock || true 128 | git reset -q --hard 129 | git clean -q -f -f -d 130 | git checkout -q $(git rev-parse HEAD) || true 131 | git branch -D "${DST_BRANCH}" >/dev/null || true 132 | git remote set-head origin -d >/dev/null # this let's filter-branch fail 133 | if git rev-parse origin/"${DST_BRANCH}" &>/dev/null; then 134 | echo "Switching to origin/${DST_BRANCH}." 135 | git branch -f "${DST_BRANCH}" origin/"${DST_BRANCH}" >/dev/null 136 | git checkout -q "${DST_BRANCH}" 137 | else 138 | # this is a new branch. Create an orphan branch without any commit. 139 | echo "Branch origin/${DST_BRANCH} not found. Creating orphan ${DST_BRANCH} branch." 140 | git checkout -q --orphan "${DST_BRANCH}" 141 | git rm -q --ignore-unmatch -rf . 142 | fi 143 | 144 | # fetch upstream kube and checkout $src_branch, name it filtered-branch 145 | echo "Fetching upstream changes." 146 | if git remote | grep -w -q upstream; then 147 | git remote set-url upstream "${SOURCE_REMOTE}" >/dev/null 148 | else 149 | git remote add upstream "${SOURCE_REMOTE}" >/dev/null 150 | fi 151 | git fetch -q upstream --no-tags --prune 152 | 153 | # sync if upstream changed 154 | UPSTREAM_HASH=$(git rev-parse upstream/${SRC_BRANCH}) 155 | if [ "${UPSTREAM_HASH}" != "${LAST_PUBLISHED_UPSTREAM_HASH}" ]; then 156 | echo "Upstream branch upstream/${SRC_BRANCH} moved from '${LAST_PUBLISHED_UPSTREAM_HASH}' to '${UPSTREAM_HASH}'. We have to sync." 157 | # sync_repo cherry-picks the commits that change 158 | # k8s.io/kubernetes/staging/src/k8s.io/${REPO} to the ${DST_BRANCH} 159 | sync_repo "${SOURCE_REPO_ORG}" "${SOURCE_REPO_NAME}" "${SUBDIRS}" "${SRC_BRANCH}" "${DST_BRANCH}" "${DEPS}" "${REQUIRED}" "${BASE_PACKAGE}" "${IS_LIBRARY}" "${RECURSIVE_DELETE_PATTERN}" "${GIT_DEFAULT_BRANCH}" 160 | else 161 | echo "Skipping sync because upstream/${SRC_BRANCH} at ${UPSTREAM_HASH} did not change since last sync." 162 | fi 163 | 164 | # add tags. 165 | LAST_BRANCH=$(git rev-parse --abbrev-ref HEAD) 166 | LAST_HEAD=$(git rev-parse HEAD) 167 | EXTRA_ARGS=() 168 | # the separator is used to handle branches with / in name 169 | PUSH_SCRIPT=../push-tags-${REPO}-${DST_BRANCH/\//_}.sh 170 | echo "#!/bin/bash" > ${PUSH_SCRIPT} 171 | chmod +x ${PUSH_SCRIPT} 172 | 173 | if [ -z "${SKIP_TAGS}" ]; then 174 | /sync-tags --prefix "$(echo ${SOURCE_REPO_NAME})-" \ 175 | --commit-message-tag $(echo ${SOURCE_REPO_NAME} | sed 's/^./\L\u&/')-commit \ 176 | --source-remote upstream --source-branch "${SRC_BRANCH}" \ 177 | --push-script ${PUSH_SCRIPT} \ 178 | --dependencies "${DEPS}" \ 179 | --mapping-output-file "../tag-${REPO}-{{.Tag}}-mapping" \ 180 | --publish-v0-semver \ 181 | -alsologtostderr \ 182 | "${EXTRA_ARGS[@]-}" 183 | if [ "${LAST_HEAD}" != "$(git rev-parse ${LAST_BRANCH})" ]; then 184 | echo "Unexpected: branch ${LAST_BRANCH} has diverted to $(git rev-parse HEAD) from ${LAST_HEAD} before tagging." 185 | exit 1 186 | fi 187 | fi 188 | 189 | git checkout ${LAST_BRANCH} 190 | -------------------------------------------------------------------------------- /cmd/publishing-bot/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "errors" 21 | "flag" 22 | "fmt" 23 | "os" 24 | "os/exec" 25 | "path/filepath" 26 | "strings" 27 | "time" 28 | 29 | "github.com/go-git/go-git/v5/storage" 30 | "github.com/golang/glog" 31 | "gopkg.in/yaml.v2" 32 | "k8s.io/publishing-bot/cmd/publishing-bot/config" 33 | ) 34 | 35 | func Usage() { 36 | fmt.Fprintf(os.Stderr, ` 37 | Usage: %s [-config ] [-dry-run] [-token-file ] [-interval ] 38 | [-source-repo ] [-target-org ] 39 | 40 | Command line flags override config values. 41 | `, os.Args[0]) 42 | flag.PrintDefaults() 43 | } 44 | 45 | //nolint:gocyclo // TODO(lint): cyclomatic complexity 38 of func `main` is high (> 30) 46 | func main() { 47 | configFilePath := flag.String("config", "", "the config file in yaml format") 48 | githubHost := flag.String("github-host", "", "the address of github (defaults to github.com)") 49 | basePackage := flag.String("base-package", "", "the name of the package base (defaults to k8s.io when source repo is kubernetes, "+ 50 | "otherwise github-host/target-org)") 51 | dryRun := flag.Bool("dry-run", false, "do not push anything to github") 52 | tokenFile := flag.String("token-file", "", "the file with the github token") 53 | rulesFile := flag.String("rules-file", "", "the file or URL with repository rules") 54 | // TODO: make absolute 55 | repoName := flag.String("source-repo", "", "the name of the source repository (eg. kubernetes)") 56 | repoOrg := flag.String("source-org", "", "the name of the source repository organization, (eg. kubernetes)") 57 | targetOrg := flag.String("target-org", "", `the target organization to publish into (e.g. "k8s-publishing-bot")`) 58 | basePublishScriptPath := flag.String("base-publish-script-path", "./publish_scripts", `the base path in source repo where bot will look for publishing scripts`) 59 | interval := flag.Uint("interval", 0, "loop with the given seconds of wait in between") 60 | serverPort := flag.Int("server-port", 0, "start a webserver on the given port listening on 0.0.0.0") 61 | 62 | flag.Usage = Usage 63 | flag.Parse() 64 | 65 | cfg := config.Config{} 66 | if *configFilePath != "" { 67 | bs, err := os.ReadFile(*configFilePath) 68 | if err != nil { 69 | glog.Fatalf("Failed to load config file from %q: %v", *configFilePath, err) 70 | } 71 | if err := yaml.Unmarshal(bs, &cfg); err != nil { 72 | glog.Fatalf("Failed to parse config file at %q: %v", *configFilePath, err) 73 | } 74 | } 75 | 76 | // override with flags 77 | if *dryRun { 78 | cfg.DryRun = true 79 | } 80 | if *targetOrg != "" { 81 | cfg.TargetOrg = *targetOrg 82 | } 83 | if *repoName != "" { 84 | cfg.SourceRepo = *repoName 85 | } 86 | if *repoOrg != "" { 87 | cfg.SourceOrg = *repoOrg 88 | } 89 | if *tokenFile != "" { 90 | cfg.TokenFile = *tokenFile 91 | } 92 | if *rulesFile != "" { 93 | cfg.RulesFile = *rulesFile 94 | } 95 | if *basePublishScriptPath != "" { 96 | cfg.BasePublishScriptPath = *basePublishScriptPath 97 | } 98 | if *githubHost != "" { 99 | cfg.GithubHost = *githubHost 100 | } 101 | if *basePackage != "" { 102 | cfg.BasePackage = *basePackage 103 | } 104 | 105 | // defaulting to github.com when it is not specified. 106 | if cfg.GithubHost == "" { 107 | cfg.GithubHost = "github.com" 108 | } 109 | 110 | if cfg.GitDefaultBranch == "" { 111 | cfg.GitDefaultBranch = "master" 112 | } 113 | 114 | var err error 115 | cfg.BasePublishScriptPath, err = filepath.Abs(cfg.BasePublishScriptPath) 116 | if err != nil { 117 | glog.Fatalf("Failed to get absolute path for base-publish-script-path %q: %v", cfg.BasePublishScriptPath, err) 118 | } 119 | 120 | if cfg.SourceRepo == "" || cfg.SourceOrg == "" { 121 | glog.Fatalf("source-org and source-repo cannot be empty") 122 | } 123 | 124 | if cfg.TargetOrg == "" { 125 | glog.Fatalf("Target organization cannot be empty") 126 | } 127 | 128 | // set the baseRepoPath 129 | gopath := os.Getenv("GOPATH") 130 | // defaulting when base package is not specified 131 | if cfg.BasePackage == "" { 132 | if cfg.SourceRepo == "kubernetes" { 133 | cfg.BasePackage = "k8s.io" 134 | } else { 135 | cfg.BasePackage = filepath.Join(cfg.GithubHost, cfg.TargetOrg) 136 | } 137 | } 138 | baseRepoPath := fmt.Sprintf("%s/%s/%s", gopath, "src", cfg.BasePackage) 139 | 140 | // If RULE_FILE_PATH is detected, check if the source repository include rules files. 141 | if os.Getenv("RULE_FILE_PATH") != "" { 142 | cfg.RulesFile = filepath.Join(baseRepoPath, cfg.SourceRepo, os.Getenv("RULE_FILE_PATH")) 143 | } 144 | 145 | if cfg.RulesFile == "" { 146 | glog.Fatalf("No rules file provided") 147 | } 148 | 149 | runChan := make(chan bool, 1) 150 | 151 | // start server 152 | server := Server{ 153 | Issue: cfg.GithubIssue, 154 | config: cfg, 155 | RunChan: runChan, 156 | } 157 | if *serverPort != 0 { 158 | server.Run(*serverPort) 159 | } 160 | 161 | githubIssueErrorf := glog.Fatalf 162 | if *interval != 0 { 163 | githubIssueErrorf = glog.Errorf 164 | } 165 | 166 | var publisherErr error 167 | 168 | for { 169 | waitfor := *interval 170 | last := time.Now() 171 | publisher := New(&cfg, baseRepoPath) 172 | 173 | if cfg.TokenFile != "" && cfg.GithubIssue != 0 && !cfg.DryRun { 174 | // load token 175 | bs, err := os.ReadFile(cfg.TokenFile) 176 | if err != nil { 177 | glog.Fatalf("Failed to load token file from %q: %v", cfg.TokenFile, err) 178 | } 179 | token := strings.Trim(string(bs), " \t\n") 180 | 181 | // run 182 | logs, hash, err := publisher.Run() 183 | server.SetHealth(err == nil, hash) 184 | if err != nil { 185 | glog.Infof("Failed to run publisher: %v", err) 186 | if err := ReportOnIssue(err, logs, token, cfg.TargetOrg, cfg.SourceRepo, cfg.GithubIssue); err != nil { 187 | githubIssueErrorf("Failed to report logs on github issue: %v", err) 188 | server.SetHealth(false, hash) 189 | } 190 | if strings.HasSuffix(err.Error(), storage.ErrReferenceHasChanged.Error()) { 191 | // TODO: If the issue is just "reference has changed concurrently", 192 | // then let us wait for 5 minutes and try again. We really need to dig 193 | // into the problem and fix the flakiness 194 | glog.Infof("Waiting for 5 minutes") 195 | waitfor = uint(5 * 60) 196 | } 197 | } else if err := CloseIssue(token, cfg.TargetOrg, cfg.SourceRepo, cfg.GithubIssue); err != nil { 198 | githubIssueErrorf("Failed to close issue: %v", err) 199 | server.SetHealth(false, hash) 200 | } 201 | } else { 202 | // run 203 | if _, _, publisherErr = publisher.Run(); publisherErr != nil { 204 | glog.Infof("Failed to run publisher: %v", publisherErr) 205 | } 206 | } 207 | 208 | if *interval == 0 { 209 | // This condition is specifically used by the CI to get the exit code 210 | // of the bot on an unsuccessful run. 211 | // 212 | // In production, the bot will not exit with a non-zero exit code, since 213 | // the interval will be always non-zero 214 | if publisherErr != nil { 215 | var exitErr *exec.ExitError 216 | if errors.As(publisherErr, &exitErr) { 217 | os.Exit(exitErr.ExitCode()) 218 | } 219 | 220 | os.Exit(1) 221 | } 222 | break 223 | } 224 | 225 | select { 226 | case <-runChan: 227 | case <-time.After(time.Duration(int(waitfor)-int(time.Since(last).Seconds())) * time.Second): 228 | } 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /cmd/publishing-bot/config/rules.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "regexp" 13 | "strconv" 14 | "strings" 15 | "time" 16 | 17 | "github.com/golang/glog" 18 | yaml "gopkg.in/yaml.v2" 19 | ) 20 | 21 | // Dependency of a piece of code. 22 | type Dependency struct { 23 | Repository string `yaml:"repository"` 24 | Branch string `yaml:"branch"` 25 | } 26 | 27 | func (c Dependency) String() string { 28 | repo := c.Repository 29 | if repo == "" { 30 | repo = "" 31 | } 32 | return fmt.Sprintf("[repository %s, branch %s]", repo, c.Branch) 33 | } 34 | 35 | // Source of a piece of code. 36 | type Source struct { 37 | Repository string `yaml:"repository,omitempty"` 38 | Branch string `yaml:"branch"` 39 | // Dir from repo root 40 | // It is preferred to use dirs instead of dir 41 | Dir string `yaml:"dir,omitempty"` 42 | // Directories from the repo root 43 | // If Dirs is present, it is given preference over Dir 44 | Dirs []string `yaml:"dirs,omitempty"` 45 | } 46 | 47 | func (c Source) String() string { 48 | repo := c.Repository 49 | if repo == "" { 50 | repo = "" 51 | } 52 | return fmt.Sprintf("[repository %s, branch %s, subdir {%s}]", repo, c.Branch, c.Dir) 53 | } 54 | 55 | type BranchRule struct { 56 | Name string `yaml:"name"` 57 | // a valid go version string like 1.10.2 or 1.10 58 | // 59 | // From go 1.21 onwards there is a change in the versioning format. 60 | // The version displayed by `go version` should be used here: 61 | // 1. 1.21.0 is valid and 1.21 is invalid 62 | // 2. 1.21rc1 and 1.21.0rc1 are valid 63 | GoVersion string `yaml:"go,omitempty"` 64 | // k8s.io/* repos the branch rule depends on 65 | Dependencies []Dependency `yaml:"dependencies,omitempty"` 66 | Source Source `yaml:"source"` 67 | RequiredPackages []string `yaml:"required-packages,omitempty"` 68 | // SmokeTest applies only to the specific branch 69 | SmokeTest string `yaml:"smoke-test,omitempty"` // a multiline bash script 70 | } 71 | 72 | // a collection of publishing rules for a single destination repo. 73 | type RepositoryRule struct { 74 | DestinationRepository string `yaml:"destination"` 75 | Branches []BranchRule `yaml:"branches"` 76 | // SmokeTest applies to all branches 77 | SmokeTest string `yaml:"smoke-test,omitempty"` // a multiline bash script 78 | Library bool `yaml:"library,omitempty"` 79 | // not updated when true 80 | Skip bool `yaml:"skipped,omitempty"` 81 | } 82 | 83 | type RepositoryRules struct { 84 | SkippedSourceBranches []string `yaml:"skip-source-branches,omitempty"` 85 | SkipGomod bool `yaml:"skip-gomod,omitempty"` 86 | SkipTags bool `yaml:"skip-tags,omitempty"` 87 | Rules []RepositoryRule `yaml:"rules"` 88 | 89 | // ls-files patterns like: */BUILD *.ext pkg/foo.go Makefile 90 | RecursiveDeletePatterns []string `yaml:"recursive-delete-patterns"` 91 | // a valid go version string like 1.10.2 or 1.10 92 | // if GoVersion is not specified in RepositoryRule, 93 | // DefaultGoVersion is used. 94 | DefaultGoVersion *string `yaml:"default-go-version,omitempty"` 95 | } 96 | 97 | // LoadRules loads the repository rules either from the remote HTTP location or 98 | // a local file path. 99 | func LoadRules(ruleFile string) (*RepositoryRules, error) { 100 | var content []byte 101 | 102 | if ruleURL, err := url.ParseRequestURI(ruleFile); err == nil && ruleURL.Host != "" { 103 | glog.Infof("loading rules file from url : %s", ruleURL) 104 | content, err = readFromURL(ruleURL) 105 | if err != nil { 106 | return nil, err 107 | } 108 | } else { 109 | glog.Infof("loading rules file : %s", ruleFile) 110 | content, err = os.ReadFile(ruleFile) 111 | if err != nil { 112 | return nil, err 113 | } 114 | } 115 | 116 | var rules RepositoryRules 117 | if err := yaml.Unmarshal(content, &rules); err != nil { 118 | return nil, err 119 | } 120 | 121 | return &rules, nil 122 | } 123 | 124 | // readFromURL reads the rule file from provided URL. 125 | func readFromURL(u *url.URL) ([]byte, error) { 126 | client := &http.Client{Transport: &http.Transport{ 127 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 128 | }} 129 | req, err := http.NewRequest(http.MethodGet, u.String(), http.NoBody) 130 | if err != nil { 131 | return nil, err 132 | } 133 | // timeout the request after 30 seconds 134 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 135 | defer cancel() 136 | resp, err := client.Do(req.WithContext(ctx)) 137 | if err != nil { 138 | return nil, err 139 | } 140 | defer resp.Body.Close() 141 | return io.ReadAll(resp.Body) 142 | } 143 | 144 | func validateRepoOrder(rules *RepositoryRules) (errs []error) { 145 | glog.Infof("validating repository order") 146 | indices := map[string]int{} 147 | for i, r := range rules.Rules { 148 | indices[r.DestinationRepository] = i 149 | } 150 | 151 | for i, r := range rules.Rules { 152 | for bri := range r.Branches { 153 | br := r.Branches[bri] 154 | for _, d := range br.Dependencies { 155 | if j, ok := indices[d.Repository]; !ok { 156 | errs = append(errs, fmt.Errorf("unknown dependency %q in repository rules of %q", d.Repository, r.DestinationRepository)) 157 | } else if j > i { 158 | errs = append(errs, fmt.Errorf("repository %q cannot depend on %q later in the rules file. Please define rules for %q above the rules for %q", r.DestinationRepository, d.Repository, d.Repository, r.DestinationRepository)) 159 | } 160 | } 161 | } 162 | } 163 | return errs 164 | } 165 | 166 | // validateGoVersions validates that all specified go versions are valid. 167 | func validateGoVersions(rules *RepositoryRules) (errs []error) { 168 | glog.Infof("validating go versions") 169 | if rules.DefaultGoVersion != nil { 170 | errs = append(errs, ensureValidGoVersion(*rules.DefaultGoVersion)) 171 | } 172 | 173 | for i := range rules.Rules { 174 | rule := rules.Rules[i] 175 | for j := range rule.Branches { 176 | branch := rule.Branches[j] 177 | if branch.GoVersion != "" { 178 | errs = append(errs, ensureValidGoVersion(branch.GoVersion)) 179 | } 180 | } 181 | } 182 | return errs 183 | } 184 | 185 | // goVerRegex is the regex for a valid go version. 186 | // go versions don't follow semver. Examples: 187 | // 1. 1.15.0 is invalid, 1.15 is valid 188 | // 2. 1.15.0-rc.1 is invalid, 1.15rc1 is valid 189 | // 190 | // From go 1.21 onwards there is a change in the versioning format 191 | // Ref: https://tip.golang.org/doc/toolchain#versions 192 | // 193 | // The version displayed by `go version` is what we care about and use in the config. 194 | // This is the version in the *name of the go tool chain* (of the form goV, V is what we 195 | // care about). For Go *language versions* >= 1.21, the following are the rules for versions 196 | // in the go tool chain name: 197 | // 1. 1.21 is invalid, and 1.21.0 is valid 198 | // 2. 1.21rc1 and 1.21.0rc1 are valid. 199 | var goVerRegex = regexp.MustCompile(`^(?P\d+)\.(?P\d+)(?:\.(?P\d+))?(?:(?P
alpha|beta|rc)\d+)?$`)
200 | 
201 | func ensureValidGoVersion(version string) error {
202 | 	match := goVerRegex.FindStringSubmatch(version)
203 | 	if len(match) == 0 {
204 | 		return fmt.Errorf("specified go version %s is invalid", version)
205 | 	}
206 | 
207 | 	var majorVersion, minorVersion, patchVersion int
208 | 	var preRelease string
209 | 	patchVersionExists := false
210 | 
211 | 	majorVersion, err := strconv.Atoi(match[1])
212 | 	if err != nil {
213 | 		return fmt.Errorf("error parsing major version '%s': %w", match[1], err)
214 | 	}
215 | 	minorVersion, err = strconv.Atoi(match[2])
216 | 	if err != nil {
217 | 		return fmt.Errorf("error parsing minor version '%s': %w", match[2], err)
218 | 	}
219 | 	if match[3] != "" {
220 | 		patchVersion, err = strconv.Atoi(match[3])
221 | 		if err != nil {
222 | 			return fmt.Errorf("error parsing patch version '%s': %w", match[3], err)
223 | 		}
224 | 		patchVersionExists = true
225 | 	}
226 | 	preRelease = match[4]
227 | 
228 | 	// for go versions <= 1.20, patch version .0 should not exist
229 | 	if majorVersion <= 1 && minorVersion <= 20 {
230 | 		if patchVersionExists && patchVersion == 0 {
231 | 			languageVersion := fmt.Sprintf("%d.%d", majorVersion, minorVersion)
232 | 			return fmt.Errorf("go language version %s below 1.21; should not have a 0th patch release, got %s", languageVersion, version)
233 | 		}
234 | 	}
235 | 
236 | 	// for go versions >= 1.21.0, patch versions should exist. If there is no patch version,
237 | 	// then it should be a prerelease
238 | 	if (majorVersion == 1 && minorVersion >= 21) || majorVersion >= 2 {
239 | 		if !patchVersionExists && preRelease == "" {
240 | 			return errors.New("patch version should always be present for go language version >= 1.21")
241 | 		}
242 | 	}
243 | 
244 | 	return nil
245 | }
246 | 
247 | // this makes sure that old dir field values are copied over to new dirs field.
248 | func fixDeprecatedFields(rules *RepositoryRules) {
249 | 	for i, rule := range rules.Rules {
250 | 		for j := range rule.Branches {
251 | 			branch := rule.Branches[j]
252 | 			if len(branch.Source.Dirs) == 0 && branch.Source.Dir != "" {
253 | 				rules.Rules[i].Branches[j].Source.Dirs = append(rules.Rules[i].Branches[j].Source.Dirs, branch.Source.Dir)
254 | 				// The Dir field is made empty so that it is not used later and only the Dirs
255 | 				// field is used.
256 | 				rules.Rules[i].Branches[j].Source.Dir = ""
257 | 			}
258 | 		}
259 | 	}
260 | }
261 | 
262 | func Validate(rules *RepositoryRules) error {
263 | 	errs := []error{}
264 | 
265 | 	errs = append(errs, validateRepoOrder(rules)...)
266 | 	errs = append(errs, validateGoVersions(rules)...)
267 | 
268 | 	fixDeprecatedFields(rules)
269 | 
270 | 	msgs := []string{}
271 | 	for _, err := range errs {
272 | 		if err != nil {
273 | 			msgs = append(msgs, err.Error())
274 | 		}
275 | 	}
276 | 	if len(msgs) > 0 {
277 | 		return fmt.Errorf("validation errors:\n- %s", strings.Join(msgs, "\n- "))
278 | 	}
279 | 	return nil
280 | }
281 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
  1 | dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
  2 | dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
  3 | github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
  4 | github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
  5 | github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
  6 | github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
  7 | github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
  8 | github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
  9 | github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 10 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 11 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 12 | github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 13 | github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 14 | github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
 15 | github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
 16 | github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
 17 | github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
 18 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 19 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 20 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 21 | github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
 22 | github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
 23 | github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 24 | github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 25 | github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 26 | github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 27 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 28 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 29 | github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM=
 30 | github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU=
 31 | github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
 32 | github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
 33 | github.com/go-git/go-git/v5 v5.16.3 h1:Z8BtvxZ09bYm/yYNgPKCzgWtaRqDTgIKRgIRHBfU6Z8=
 34 | github.com/go-git/go-git/v5 v5.16.3/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
 35 | github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I=
 36 | github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
 37 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
 38 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 39 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 40 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 41 | github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 42 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 43 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
 44 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 45 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 46 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 47 | github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 48 | github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 49 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 50 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 51 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 52 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 53 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 54 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 55 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 56 | github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
 57 | github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 58 | github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
 59 | github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
 60 | github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4=
 61 | github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A=
 62 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 63 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 64 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 65 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 66 | github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 67 | github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 68 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
 69 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 70 | github.com/shurcooL/go v0.0.0-20171108033853-004faa6b0118 h1:ygJUdybU9Z/Z7vMcMUL3wyXpYKlXvh8rGElC04RpJqw=
 71 | github.com/shurcooL/go v0.0.0-20171108033853-004faa6b0118/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 72 | github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 73 | github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
 74 | github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
 75 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 76 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 77 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 78 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 79 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 80 | github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 81 | github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 82 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 83 | golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
 84 | golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
 85 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
 86 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 87 | golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
 88 | golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
 89 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 90 | golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
 91 | golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 92 | golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
 93 | golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 94 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 95 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 96 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 97 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 98 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 99 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
100 | golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
101 | golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
102 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
103 | golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
104 | golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
105 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
106 | golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
107 | golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
108 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
109 | golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
110 | golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
111 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
112 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
113 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
114 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
115 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
116 | gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
117 | gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
118 | gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
119 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
120 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
121 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
122 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
123 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
124 | 


--------------------------------------------------------------------------------
/cmd/sync-tags/gomod.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2019 The Kubernetes Authors.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package main
 18 | 
 19 | import (
 20 | 	"encoding/json"
 21 | 	"errors"
 22 | 	"fmt"
 23 | 	"io"
 24 | 	"os"
 25 | 	"os/exec"
 26 | 	"path/filepath"
 27 | 	"strings"
 28 | 	"time"
 29 | 
 30 | 	gogit "github.com/go-git/go-git/v5"
 31 | 	"github.com/go-git/go-git/v5/plumbing"
 32 | )
 33 | 
 34 | // updateGomodWithTaggedDependencies gets the dependencies at the given tag and fills go.mod and go.sum.
 35 | // If anything is changed, it commits the changes. Returns true if go.mod changed.
 36 | func updateGomodWithTaggedDependencies(tag string, depsRepo []string, semverTag bool) (bool, error) {
 37 | 	found := map[string]bool{}
 38 | 	changed := false
 39 | 
 40 | 	depPackages, err := depsImportPaths(depsRepo)
 41 | 	if err != nil {
 42 | 		return changed, err
 43 | 	}
 44 | 
 45 | 	for _, dep := range depsRepo {
 46 | 		depPath := filepath.Join("..", dep)
 47 | 		dr, err := gogit.PlainOpen(depPath)
 48 | 		if err != nil {
 49 | 			return changed, fmt.Errorf("failed to open dependency repo at %q: %w", depPath, err)
 50 | 		}
 51 | 
 52 | 		depPkg, err := fullPackageName(depPath)
 53 | 		if err != nil {
 54 | 			return changed, fmt.Errorf("failed to get package at %s: %w", depPath, err)
 55 | 		}
 56 | 
 57 | 		commit, commitTime, err := localOrPublishedTaggedCommitHashAndTime(dr, tag)
 58 | 		if err != nil {
 59 | 			return changed, fmt.Errorf("failed to get tag %s for %q: %w", tag, depPkg, err)
 60 | 		}
 61 | 		rev := commit.String()
 62 | 		pseudoVersionOrTag := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12])
 63 | 
 64 | 		if semverTag {
 65 | 			pseudoVersionOrTag = tag
 66 | 		}
 67 | 
 68 | 		// check if we have the pseudoVersion/tag published already. if we don't, package it up
 69 | 		// and save to local mod download cache.
 70 | 		if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersionOrTag, commitTime); err != nil {
 71 | 			return changed, fmt.Errorf("failed to package %s dependency: %w", depPkg, err)
 72 | 		}
 73 | 
 74 | 		requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersionOrTag))
 75 | 		requireCommand.Env = append(os.Environ(), "GO111MODULE=on")
 76 | 		requireCommand.Stdout = os.Stdout
 77 | 		requireCommand.Stderr = os.Stderr
 78 | 		if err := requireCommand.Run(); err != nil {
 79 | 			return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %w", depPkg, pseudoVersionOrTag, err)
 80 | 		}
 81 | 
 82 | 		if semverTag {
 83 | 			dropReplaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-dropreplace", depPkg)
 84 | 			dropReplaceCommand.Env = append(os.Environ(), "GO111MODULE=on")
 85 | 			dropReplaceCommand.Stdout = os.Stdout
 86 | 			dropReplaceCommand.Stderr = os.Stderr
 87 | 			if err := dropReplaceCommand.Run(); err != nil {
 88 | 				return changed, fmt.Errorf("unable to drop %s in the replace section of go.mod: %w", depPkg, err)
 89 | 			}
 90 | 		} else {
 91 | 			replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersionOrTag))
 92 | 			replaceCommand.Env = append(os.Environ(), "GO111MODULE=on")
 93 | 			replaceCommand.Stdout = os.Stdout
 94 | 			replaceCommand.Stderr = os.Stderr
 95 | 			if err := replaceCommand.Run(); err != nil {
 96 | 				return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %w", depPkg, pseudoVersionOrTag, err)
 97 | 			}
 98 | 		}
 99 | 
100 | 		found[dep] = true
101 | 		fmt.Printf("Bumping %s in go.mod to %s.\n", depPkg, rev)
102 | 		changed = true
103 | 	}
104 | 
105 | 	for _, dep := range depsRepo {
106 | 		if !found[dep] {
107 | 			fmt.Printf("Warning: dependency %s not found in go.mod.\n", dep)
108 | 		}
109 | 	}
110 | 
111 | 	downloadCommand2 := exec.Command("go", "mod", "download")
112 | 	downloadCommand2.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
113 | 	downloadCommand2.Stdout = os.Stdout
114 | 	downloadCommand2.Stderr = os.Stderr
115 | 	if err := downloadCommand2.Run(); err != nil {
116 | 		return changed, fmt.Errorf("error running go mod download: %w", err)
117 | 	}
118 | 
119 | 	tidyCommand := exec.Command("go", "mod", "tidy")
120 | 	tidyCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPROXY=file://%s/pkg/mod/cache/download", os.Getenv("GOPATH")), fmt.Sprintf("GOPRIVATE=%s", depPackages))
121 | 	tidyCommand.Stdout = os.Stdout
122 | 	tidyCommand.Stderr = os.Stderr
123 | 	if err := tidyCommand.Run(); err != nil {
124 | 		return changed, fmt.Errorf("unable to run go mod tidy: %w", err)
125 | 	}
126 | 	fmt.Printf("Completed running go mod tidy for %s.\n", tag)
127 | 
128 | 	return changed, nil
129 | }
130 | 
131 | // depImportPaths returns a comma separated string with each dependencies' import path.
132 | // Eg. "k8s.io/api,k8s.io/apimachinery,k8s.io/client-go".
133 | func depsImportPaths(depsRepo []string) (string, error) {
134 | 	dir, err := os.Getwd()
135 | 	if err != nil {
136 | 		return "", fmt.Errorf("unable to get current working directory: %w", err)
137 | 	}
138 | 	d := strings.Split(dir, "/")
139 | 	basePackage := d[len(d)-2]
140 | 
141 | 	depImportPathList := []string{}
142 | 	for _, dep := range depsRepo {
143 | 		depImportPathList = append(depImportPathList, fmt.Sprintf("%s/%s", basePackage, dep))
144 | 	}
145 | 	return strings.Join(depImportPathList, ","), nil
146 | }
147 | 
148 | type ModuleInfo struct {
149 | 	Version string `json:"Version,omitempty"`
150 | 	Name    string `json:"Name,omitempty"`
151 | 	Short   string `json:"Short,omitempty"`
152 | 	Time    string `json:"Time,omitempty"`
153 | }
154 | 
155 | func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersionOrTag string, commitTime time.Time) error {
156 | 	cacheDir := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v", os.Getenv("GOPATH"), depPkg)
157 | 	goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersionOrTag)
158 | 
159 | 	if _, err := os.Stat(goModFile); err == nil {
160 | 		fmt.Printf("%s for %s is already packaged up.\n", pseudoVersionOrTag, depPkg)
161 | 		return nil
162 | 	} else if err != nil && !os.IsNotExist(err) {
163 | 		return fmt.Errorf("could not check if %s exists: %w", goModFile, err)
164 | 	}
165 | 
166 | 	fmt.Printf("Packaging up %s for %s into go mod cache.\n", pseudoVersionOrTag, depPkg)
167 | 
168 | 	// create the cache if it doesn't exist
169 | 	if err := os.MkdirAll(filepath.Dir(goModFile), os.FileMode(0o755)); err != nil {
170 | 		return fmt.Errorf("unable to create %s directory: %w", cacheDir, err)
171 | 	}
172 | 
173 | 	// checkout the dep repo to the commit at the tag
174 | 	checkoutCommand := exec.Command("git", "checkout", commit)
175 | 	checkoutCommand.Dir = fmt.Sprintf("%s/src/%s", os.Getenv("GOPATH"), depPkg)
176 | 	checkoutCommand.Stdout = os.Stdout
177 | 	checkoutCommand.Stderr = os.Stderr
178 | 	if err := checkoutCommand.Run(); err != nil {
179 | 		return fmt.Errorf("failed to checkout %s at %s: %w", depPkg, commit, err)
180 | 	}
181 | 
182 | 	// copy go.mod to the cache dir
183 | 	if err := copyFile(fmt.Sprintf("%s/go.mod", depPath), goModFile); err != nil {
184 | 		return fmt.Errorf("unable to copy %s file to %s to gomod cache for %s: %w", fmt.Sprintf("%s/go.mod", depPath), goModFile, depPkg, err)
185 | 	}
186 | 
187 | 	// create info file in the cache dir
188 | 	moduleInfo := ModuleInfo{
189 | 		Version: pseudoVersionOrTag,
190 | 		Name:    commit,
191 | 		Short:   commit[:12],
192 | 		Time:    commitTime.UTC().Format("2006-01-02T15:04:05Z"),
193 | 	}
194 | 
195 | 	moduleFile, err := json.Marshal(moduleInfo)
196 | 	if err != nil {
197 | 		return fmt.Errorf("error marshaling .info file for %s: %w", depPkg, err)
198 | 	}
199 | 	if err := os.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), moduleFile, 0o644); err != nil {
200 | 		return fmt.Errorf("failed to write %s file for %s: %w", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), depPkg, err)
201 | 	}
202 | 
203 | 	// create the zip file in the cache dir. This zip file has the same hash
204 | 	// as of the zip file that would have been created by go mod download.
205 | 	zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersionOrTag)
206 | 	zipCommand.Stdout = os.Stdout
207 | 	zipCommand.Stderr = os.Stderr
208 | 	if err := zipCommand.Run(); err != nil {
209 | 		return fmt.Errorf("failed to run gomod-zip for %s at %s: %w", depPkg, pseudoVersionOrTag, err)
210 | 	}
211 | 
212 | 	// append the pseudoVersion to the list file in the cache dir
213 | 	listFile, err := os.OpenFile(fmt.Sprintf("%s/list", cacheDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
214 | 	if err != nil {
215 | 		return fmt.Errorf("unable to open list file in %s: %w", cacheDir, err)
216 | 	}
217 | 	defer listFile.Close()
218 | 
219 | 	if _, err := fmt.Fprintf(listFile, "%s\n", pseudoVersionOrTag); err != nil {
220 | 		return fmt.Errorf("unable to write to list file in %s: %w", cacheDir, err)
221 | 	}
222 | 
223 | 	return nil
224 | }
225 | 
226 | func localOrPublishedTaggedCommitHashAndTime(r *gogit.Repository, tag string) (plumbing.Hash, time.Time, error) {
227 | 	if commit, commitTime, err := taggedCommitHashAndTime(r, tag); err == nil {
228 | 		return commit, commitTime, nil
229 | 	}
230 | 	return taggedCommitHashAndTime(r, "origin/"+tag)
231 | }
232 | 
233 | func taggedCommitHashAndTime(r *gogit.Repository, tag string) (plumbing.Hash, time.Time, error) {
234 | 	ref, err := r.Reference(plumbing.ReferenceName(fmt.Sprintf("refs/tags/%s", tag)), true)
235 | 	if err != nil {
236 | 		return plumbing.ZeroHash, time.Time{}, fmt.Errorf("failed to get refs/tags/%s: %w", tag, err)
237 | 	}
238 | 
239 | 	tagObject, err := r.TagObject(ref.Hash())
240 | 	if err != nil {
241 | 		if err != nil {
242 | 			return plumbing.ZeroHash, time.Time{}, fmt.Errorf("refs/tags/%s is invalid: %w", tag, err)
243 | 		}
244 | 	}
245 | 	commitAtTag, err := tagObject.Commit()
246 | 	if err != nil {
247 | 		return plumbing.ZeroHash, time.Time{}, fmt.Errorf("failed to get underlying commit for tag %s: %w", tag, err)
248 | 	}
249 | 	return commitAtTag.Hash, commitAtTag.Committer.When, nil
250 | }
251 | 
252 | func copyFile(src, dst string) error {
253 | 	in, err := os.Open(src)
254 | 	if err != nil {
255 | 		return fmt.Errorf("unable to open %s: %w", src, err)
256 | 	}
257 | 	defer in.Close()
258 | 
259 | 	out, err := os.Create(dst)
260 | 	if err != nil {
261 | 		return fmt.Errorf("unable to create %s: %w", dst, err)
262 | 	}
263 | 	defer out.Close()
264 | 
265 | 	_, err = io.Copy(out, in)
266 | 	if err != nil {
267 | 		return fmt.Errorf("unable to copy %s to %s: %w", src, dst, err)
268 | 	}
269 | 	return out.Close()
270 | }
271 | 
272 | // fullPackageName return the Golang full package name of dir inside the ${GOPATH}/src.
273 | func fullPackageName(dir string) (string, error) {
274 | 	gopath := os.Getenv("GOPATH")
275 | 	if gopath == "" {
276 | 		return "", errors.New("GOPATH is not set")
277 | 	}
278 | 
279 | 	absGopath, err := filepath.Abs(gopath)
280 | 	if err != nil {
281 | 		return "", fmt.Errorf("failed to make GOPATH %q absolute: %w", gopath, err)
282 | 	}
283 | 
284 | 	absDir, err := filepath.Abs(dir)
285 | 	if err != nil {
286 | 		return "", fmt.Errorf("failed to make %q absolute: %w", dir, err)
287 | 	}
288 | 
289 | 	if !strings.HasPrefix(filepath.ToSlash(absDir), filepath.ToSlash(absGopath)+"/src/") {
290 | 		return "", fmt.Errorf("path %q is no inside GOPATH %q", dir, gopath)
291 | 	}
292 | 
293 | 	return absDir[len(filepath.ToSlash(absGopath)+"/src/"):], nil
294 | }
295 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/cmd/publishing-bot/publisher.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2016 The Kubernetes Authors.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package main
 18 | 
 19 | import (
 20 | 	"bytes"
 21 | 	"errors"
 22 | 	"fmt"
 23 | 	"os"
 24 | 	"os/exec"
 25 | 	"path"
 26 | 	"path/filepath"
 27 | 	"strconv"
 28 | 	"strings"
 29 | 
 30 | 	gogit "github.com/go-git/go-git/v5"
 31 | 	"github.com/go-git/go-git/v5/plumbing"
 32 | 	"github.com/golang/glog"
 33 | 	"k8s.io/publishing-bot/cmd/publishing-bot/config"
 34 | 	"k8s.io/publishing-bot/pkg/golang"
 35 | )
 36 | 
 37 | // PublisherMunger publishes content from one repository to another one.
 38 | type PublisherMunger struct {
 39 | 	reposRules config.RepositoryRules
 40 | 	config     *config.Config
 41 | 	// plog duplicates the logs at glog and a file
 42 | 	plog *plog
 43 | 	// absolute path to the repos.
 44 | 	baseRepoPath string
 45 | }
 46 | 
 47 | // New will create a new munger.
 48 | func New(cfg *config.Config, baseRepoPath string) *PublisherMunger {
 49 | 	// create munger
 50 | 	return &PublisherMunger{
 51 | 		baseRepoPath: baseRepoPath,
 52 | 		config:       cfg,
 53 | 	}
 54 | }
 55 | 
 56 | // update the local checkout of the source repository. It returns the branch heads.
 57 | func (p *PublisherMunger) updateSourceRepo() (map[string]plumbing.Hash, error) {
 58 | 	repoDir := filepath.Join(p.baseRepoPath, p.config.SourceRepo)
 59 | 
 60 | 	// fetch origin
 61 | 	glog.Infof("Fetching origin at %s.", repoDir)
 62 | 	r, err := gogit.PlainOpen(repoDir)
 63 | 	if err != nil {
 64 | 		return nil, fmt.Errorf("failed to open repo at %s: %w", repoDir, err)
 65 | 	}
 66 | 	if err := r.Fetch(&gogit.FetchOptions{
 67 | 		Tags:     gogit.AllTags,
 68 | 		Progress: os.Stdout,
 69 | 	}); err != nil && !errors.Is(err, gogit.NoErrAlreadyUpToDate) {
 70 | 		return nil, fmt.Errorf("failed to fetch at %s: %w", repoDir, err)
 71 | 	}
 72 | 
 73 | 	// disable text conversion
 74 | 	// TODO: remove when go-git supports text conversion to be consistent with cli git
 75 | 	attrFile := filepath.Join(repoDir, ".git", "info", "attributes")
 76 | 	if _, err := os.Stat(attrFile); os.IsNotExist(err) {
 77 | 		glog.Infof("Disabling text conversion at %s.", repoDir)
 78 | 		err := os.MkdirAll(filepath.Join(repoDir, ".git", "info"), 0o755)
 79 | 		if err != nil {
 80 | 			return nil, fmt.Errorf("creating .git/info: %w", err)
 81 | 		}
 82 | 
 83 | 		if err := os.WriteFile(attrFile, []byte(`
 84 | * -text
 85 | `), 0o644); err != nil {
 86 | 			return nil, fmt.Errorf("failed to create .git/info/attributes: %w", err)
 87 | 		}
 88 | 
 89 | 		fis, err := os.ReadDir(repoDir)
 90 | 		if err != nil {
 91 | 			return nil, err
 92 | 		}
 93 | 		for _, fi := range fis {
 94 | 			if fi.Name() != ".git" {
 95 | 				if err := os.RemoveAll(filepath.Join(repoDir, fi.Name())); err != nil {
 96 | 					return nil, err
 97 | 				}
 98 | 			}
 99 | 		}
100 | 	}
101 | 
102 | 	// checkout head
103 | 	glog.Infof("Checking out HEAD at %s.", repoDir)
104 | 	w, err := r.Worktree()
105 | 	if err != nil {
106 | 		return nil, fmt.Errorf("failed to open worktree at %s: %w", repoDir, err)
107 | 	}
108 | 	head, err := r.Head()
109 | 	if err != nil {
110 | 		return nil, fmt.Errorf("failed to get head at %s: %w", repoDir, err)
111 | 	}
112 | 	if err := w.Checkout(&gogit.CheckoutOptions{Hash: head.Hash(), Force: true}); err != nil {
113 | 		return nil, fmt.Errorf("failed to checkout HEAD at %s: %w", repoDir, err)
114 | 	}
115 | 
116 | 	// create/update local branch for all origin branches. Those are fetches into the destination repos later (as upstream/).
117 | 	refs, err := r.Storer.IterReferences()
118 | 	if err != nil {
119 | 		return nil, fmt.Errorf("failed to get branches: %w", err)
120 | 	}
121 | 	glog.Infof("Updating local branches at %s.", repoDir)
122 | 	heads := map[string]plumbing.Hash{}
123 | 	if err = refs.ForEach(func(ref *plumbing.Reference) error {
124 | 		name := ref.Name().String()
125 | 
126 | 		originPrefix := "refs/remotes/origin/"
127 | 		if !strings.Contains(name, originPrefix) || ref.Type() != plumbing.HashReference {
128 | 			return nil
129 | 		}
130 | 
131 | 		shortName := strings.TrimPrefix(name, originPrefix)
132 | 		localBranch := plumbing.NewHashReference(plumbing.ReferenceName("refs/heads/"+shortName), ref.Hash())
133 | 		if err := r.Storer.SetReference(localBranch); err != nil {
134 | 			return fmt.Errorf("failed to create reference %s pointing to %s", localBranch.Name(), localBranch.Hash().String())
135 | 		}
136 | 
137 | 		heads[shortName] = localBranch.Hash()
138 | 
139 | 		return nil
140 | 	}); err != nil {
141 | 		return nil, fmt.Errorf("failed to process branches: %w", err)
142 | 	}
143 | 
144 | 	return heads, nil
145 | }
146 | 
147 | // update the active rules.
148 | func (p *PublisherMunger) updateRules() error {
149 | 	repoDir := filepath.Join(p.baseRepoPath, p.config.SourceRepo)
150 | 
151 | 	glog.Infof("Checking out %s at %s.", p.config.GitDefaultBranch, repoDir)
152 | 	cmd := exec.Command("git", "checkout", p.config.GitDefaultBranch)
153 | 	cmd.Dir = repoDir
154 | 	if _, err := cmd.CombinedOutput(); err != nil {
155 | 		return fmt.Errorf("failed to checkout %s: %w", p.config.GitDefaultBranch, err)
156 | 	}
157 | 
158 | 	rules, err := config.LoadRules(p.config.RulesFile)
159 | 	if err != nil {
160 | 		return err
161 | 	}
162 | 	if err := config.Validate(rules); err != nil {
163 | 		return err
164 | 	}
165 | 
166 | 	p.reposRules = *rules
167 | 	glog.Infof("Loaded %d repository rules from %s.", len(p.reposRules.Rules), p.config.RulesFile)
168 | 	return nil
169 | }
170 | 
171 | func (p *PublisherMunger) skippedBranch(b string) bool {
172 | 	for _, skipped := range p.reposRules.SkippedSourceBranches {
173 | 		if b == skipped {
174 | 			return true
175 | 		}
176 | 	}
177 | 	return false
178 | }
179 | 
180 | // git clone dstURL to dst if dst doesn't exist yet.
181 | func (p *PublisherMunger) ensureCloned(dst, dstURL string) error {
182 | 	if _, err := os.Stat(dst); err == nil {
183 | 		return nil
184 | 	}
185 | 
186 | 	cmd := exec.Command("mkdir", "-p", dst)
187 | 	if err := p.plog.Run(cmd); err != nil {
188 | 		return err
189 | 	}
190 | 	cmd = exec.Command("git", "clone", dstURL, dst)
191 | 	if err := p.plog.Run(cmd); err != nil {
192 | 		return err
193 | 	}
194 | 	cmd = exec.Command("/bin/bash", "-c", "git tag -l | xargs git tag -d")
195 | 	cmd.Dir = dst
196 | 	return p.plog.Run(cmd)
197 | }
198 | 
199 | func (p *PublisherMunger) runSmokeTests(smokeTest, oldHead, newHead string, branchEnv []string) error {
200 | 	if smokeTest != "" && oldHead != newHead {
201 | 		cmd := exec.Command("/bin/bash", "-xec", smokeTest)
202 | 		cmd.Env = append([]string(nil), branchEnv...) // make mutable
203 | 		cmd.Env = append(
204 | 			cmd.Env,
205 | 			"GO111MODULE=on",
206 | 			fmt.Sprintf("GOPROXY=file://%s/pkg/mod/cache/download", os.Getenv("GOPATH")),
207 | 		)
208 | 		if err := p.plog.Run(cmd); err != nil {
209 | 			// do not clean up to allow debugging with kubectl-exec.
210 | 			return err
211 | 		}
212 | 
213 | 		err := exec.Command("git", "reset", "--hard").Run()
214 | 		if err != nil {
215 | 			return err
216 | 		}
217 | 
218 | 		err = exec.Command("git", "clean", "-f", "-f", "-d").Run()
219 | 		if err != nil {
220 | 			return err
221 | 		}
222 | 	}
223 | 
224 | 	return nil
225 | }
226 | 
227 | // constructs all the repos, but does not push the changes to remotes.
228 | func (p *PublisherMunger) construct() error {
229 | 	sourceRemote := filepath.Join(p.baseRepoPath, p.config.SourceRepo, ".git")
230 | 
231 | 	if err := golang.InstallGoVersions(&p.reposRules); err != nil {
232 | 		return err
233 | 	}
234 | 
235 | 	for _, repoRule := range p.reposRules.Rules {
236 | 		if repoRule.Skip {
237 | 			continue
238 | 		}
239 | 
240 | 		// clone the destination repo
241 | 		dstDir := filepath.Join(p.baseRepoPath, repoRule.DestinationRepository, "")
242 | 		dstURL := fmt.Sprintf("https://%s/%s/%s.git", p.config.GithubHost, p.config.TargetOrg, repoRule.DestinationRepository)
243 | 		if err := p.ensureCloned(dstDir, dstURL); err != nil {
244 | 			p.plog.Errorf("%v", err)
245 | 			return err
246 | 		}
247 | 		p.plog.Infof("Successfully ensured %s exists", dstDir)
248 | 		if err := os.Chdir(dstDir); err != nil {
249 | 			return err
250 | 		}
251 | 
252 | 		// delete tags
253 | 		cmd := exec.Command("/bin/bash", "-c", "git tag | xargs git tag -d >/dev/null")
254 | 		if err := p.plog.Run(cmd); err != nil {
255 | 			return err
256 | 		}
257 | 
258 | 		formatDeps := func(deps []config.Dependency) string {
259 | 			var depStrings []string
260 | 			for _, dep := range deps {
261 | 				depStrings = append(depStrings, fmt.Sprintf("%s:%s", dep.Repository, dep.Branch))
262 | 			}
263 | 			return strings.Join(depStrings, ",")
264 | 		}
265 | 
266 | 		// construct branches
267 | 		for i := range repoRule.Branches {
268 | 			branchRule := repoRule.Branches[i]
269 | 			if p.skippedBranch(branchRule.Source.Branch) {
270 | 				continue
271 | 			}
272 | 			if len(branchRule.Source.Dirs) == 0 {
273 | 				branchRule.Source.Dirs = append(branchRule.Source.Dirs, ".")
274 | 				p.plog.Infof("%v: 'dir' cannot be empty, defaulting to '.'", branchRule)
275 | 			}
276 | 
277 | 			//nolint:errcheck // get old HEAD. Ignore errors as the branch might be non-existent
278 | 			oldHead, _ := exec.Command("git", "rev-parse", fmt.Sprintf("origin/%s", branchRule.Name)).Output()
279 | 
280 | 			goPath := os.Getenv("GOPATH")
281 | 			branchEnv := append([]string(nil), os.Environ()...) // make mutable
282 | 			if branchRule.GoVersion != "" {
283 | 				goRoot := filepath.Join(goPath, "go-"+branchRule.GoVersion)
284 | 				branchEnv = append(branchEnv, "GOROOT="+goRoot)
285 | 				goBin := filepath.Join(goRoot, "bin")
286 | 				branchEnv = updateEnv(branchEnv, "PATH", prependPath(goBin), goBin)
287 | 			}
288 | 
289 | 			skipTags := ""
290 | 			if p.reposRules.SkipTags {
291 | 				skipTags = "true"
292 | 				p.plog.Infof("synchronizing tags is disabled")
293 | 			}
294 | 
295 | 			// get old published hash to eventually skip cherry picking
296 | 			var lastPublishedUpstreamHash string
297 | 			bs, err := os.ReadFile(path.Join(p.baseRepoPath, publishedFileName(repoRule.DestinationRepository, branchRule.Name)))
298 | 			if err != nil && !os.IsNotExist(err) {
299 | 				return err
300 | 			}
301 | 			if err == nil {
302 | 				lastPublishedUpstreamHash = string(bs)
303 | 			}
304 | 
305 | 			// TODO: Refactor this to use environment variables instead
306 | 			repoPublishScriptPath := filepath.Join(p.config.BasePublishScriptPath, "construct.sh")
307 | 			cmd := exec.Command(repoPublishScriptPath,
308 | 				repoRule.DestinationRepository,
309 | 				branchRule.Source.Branch,
310 | 				branchRule.Name,
311 | 				formatDeps(branchRule.Dependencies),
312 | 				strings.Join(branchRule.RequiredPackages, ":"),
313 | 				sourceRemote,
314 | 				strings.Join(branchRule.Source.Dirs, ":"),
315 | 				p.config.SourceRepo,
316 | 				p.config.SourceRepo,
317 | 				p.config.BasePackage,
318 | 				strconv.FormatBool(repoRule.Library),
319 | 				strings.Join(p.reposRules.RecursiveDeletePatterns, " "),
320 | 				skipTags,
321 | 				lastPublishedUpstreamHash,
322 | 				p.config.GitDefaultBranch,
323 | 			)
324 | 			cmd.Env = append([]string(nil), branchEnv...) // make mutable
325 | 			if p.reposRules.SkipGomod {
326 | 				cmd.Env = append(cmd.Env, "PUBLISHER_BOT_SKIP_GOMOD=true")
327 | 			}
328 | 			if err := p.plog.Run(cmd); err != nil {
329 | 				return err
330 | 			}
331 | 
332 | 			//nolint:errcheck  // TODO(lint): Should we be checking errors here?
333 | 			newHead, _ := exec.Command("git", "rev-parse", "HEAD").Output()
334 | 
335 | 			p.plog.Infof("Running branch-specific smoke tests for branch %s", branchRule.Name)
336 | 			if err := p.runSmokeTests(branchRule.SmokeTest, string(oldHead), string(newHead), branchEnv); err != nil {
337 | 				return err
338 | 			}
339 | 
340 | 			p.plog.Infof("Running repo-specific smoke tests for branch %s", branchRule.Name)
341 | 			if err := p.runSmokeTests(repoRule.SmokeTest, string(oldHead), string(newHead), branchEnv); err != nil {
342 | 				return err
343 | 			}
344 | 
345 | 			p.plog.Infof("Successfully constructed %s", branchRule.Name)
346 | 		}
347 | 	}
348 | 	return nil
349 | }
350 | 
351 | func updateEnv(env []string, key string, change func(string) string, val string) []string {
352 | 	for i := range env {
353 | 		if strings.HasPrefix(env[i], key+"=") {
354 | 			ss := strings.SplitN(env[i], "=", 2)
355 | 			env[i] = fmt.Sprintf("%s=%s", key, change(ss[1]))
356 | 			return env
357 | 		}
358 | 	}
359 | 	return append(env, fmt.Sprintf("%s=%s", key, val))
360 | }
361 | 
362 | func prependPath(p string) func(string) string {
363 | 	return func(s string) string {
364 | 		if s == "" {
365 | 			return p
366 | 		}
367 | 		return p + ":" + s
368 | 	}
369 | }
370 | 
371 | // publish to remotes.
372 | func (p *PublisherMunger) publish(newUpstreamHeads map[string]plumbing.Hash) error {
373 | 	if p.config.DryRun {
374 | 		p.plog.Infof("Skipping push in dry-run mode")
375 | 		return nil
376 | 	}
377 | 
378 | 	if p.config.TokenFile == "" {
379 | 		return errors.New("token cannot be empty in non-dry-run mode")
380 | 	}
381 | 
382 | 	// NOTE: because some repos depend on each other, e.g., client-go depends on
383 | 	// apimachinery, they should be published atomically, but it's not supported
384 | 	// by github.
385 | 	for _, repoRules := range p.reposRules.Rules {
386 | 		if repoRules.Skip {
387 | 			continue
388 | 		}
389 | 
390 | 		dstDir := filepath.Join(p.baseRepoPath, repoRules.DestinationRepository, "")
391 | 		if err := os.Chdir(dstDir); err != nil {
392 | 			return err
393 | 		}
394 | 
395 | 		p.plog.Infof("Pushing branches for %s", repoRules.DestinationRepository)
396 | 		for i := range repoRules.Branches {
397 | 			branchRule := repoRules.Branches[i]
398 | 			if p.skippedBranch(branchRule.Source.Branch) {
399 | 				continue
400 | 			}
401 | 
402 | 			cmd := exec.Command(p.config.BasePublishScriptPath+"/push.sh", p.config.TokenFile, branchRule.Name)
403 | 			if err := p.plog.Run(cmd); err != nil {
404 | 				return err
405 | 			}
406 | 
407 | 			upstreamBranchHead, ok := newUpstreamHeads[branchRule.Source.Branch]
408 | 			if !ok {
409 | 				return fmt.Errorf("no upstream branch %q found", branchRule.Source.Branch)
410 | 			}
411 | 			if err := os.WriteFile(
412 | 				path.Join(
413 | 					path.Dir(dstDir),
414 | 					publishedFileName(repoRules.DestinationRepository, branchRule.Name),
415 | 				),
416 | 				[]byte(upstreamBranchHead.String()),
417 | 				0o644,
418 | 			); err != nil {
419 | 				return err
420 | 			}
421 | 		}
422 | 	}
423 | 	return nil
424 | }
425 | 
426 | func publishedFileName(repo, branch string) string {
427 | 	branch = strings.ReplaceAll(branch, "/", "_")
428 | 	return fmt.Sprintf("published-%s-%s", repo, branch)
429 | }
430 | 
431 | // Run constructs the repos and pushes them. It returns logs and the last master hash.
432 | func (p *PublisherMunger) Run() (logs, masterHead string, err error) {
433 | 	buf := bytes.NewBuffer(nil)
434 | 	if p.plog, err = newPublisherLog(buf, path.Join(p.baseRepoPath, "run.log")); err != nil {
435 | 		return "", "", err
436 | 	}
437 | 
438 | 	newUpstreamHeads, err := p.updateSourceRepo()
439 | 	if err != nil {
440 | 		p.plog.Errorf("%v", err)
441 | 		p.plog.Flush()
442 | 		return p.plog.Logs(), "", err
443 | 	}
444 | 
445 | 	if err := p.updateRules(); err != nil { // this comes after the source update because we might fetch the rules from there.
446 | 		p.plog.Errorf("%v", err)
447 | 		p.plog.Flush()
448 | 		return p.plog.Logs(), "", err
449 | 	}
450 | 
451 | 	if err := p.construct(); err != nil {
452 | 		p.plog.Errorf("%v", err)
453 | 		p.plog.Flush()
454 | 		return p.plog.Logs(), "", err
455 | 	}
456 | 
457 | 	if err := p.publish(newUpstreamHeads); err != nil {
458 | 		p.plog.Errorf("%v", err)
459 | 		p.plog.Flush()
460 | 		return p.plog.Logs(), "", err
461 | 	}
462 | 
463 | 	if h, ok := newUpstreamHeads["master"]; ok {
464 | 		masterHead = h.String()
465 | 	}
466 | 
467 | 	return p.plog.Logs(), masterHead, nil
468 | }
469 | 


--------------------------------------------------------------------------------