├── codecov.yaml ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── support_request.md │ ├── feature_request.md │ └── bug_report.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── presubmit_checks.yml ├── drghs ├── v1 │ ├── api_descriptor.pb │ ├── pagination.proto │ ├── admin_service.proto │ └── service_resources.proto ├── .gitignore ├── endpoints │ ├── models │ │ ├── priority.yaml │ │ ├── git_hub_user.yaml │ │ ├── request.yaml │ │ ├── compliance.yaml │ │ ├── type.yaml │ │ ├── compliance_response.yaml │ │ ├── get_issue_request.yaml │ │ ├── response.yaml │ │ ├── list_issues_request.yaml │ │ ├── git_hub_comment.yaml │ │ └── status.yaml │ ├── leif.yaml │ ├── samplr.yaml │ ├── maintnerd.yaml │ ├── devrelservices-admin.yaml │ └── cloudbuild.yaml ├── README.md ├── go.mod └── Makefile ├── .gitignore ├── drghs-worker ├── internal │ ├── api_descriptor.pb │ └── issue_service_internal.proto ├── .dockerignore ├── pkg │ ├── tokens │ │ ├── tokens.go │ │ ├── rotate_tokens.go │ │ └── tokens_test.go │ ├── googlers │ │ ├── googlers.go │ │ └── googlers_static.go │ └── utils │ │ ├── serialization_utils.go │ │ └── status_utils.go ├── utils │ ├── maint-bucket-migrate │ │ └── README.md │ ├── maint-bucket-reduce │ │ └── README.md │ └── serilization_utils_test.go ├── DEPLOY.md ├── maintner-sprvsr │ └── Dockerfile ├── maintner-swpr │ ├── Dockerfile │ ├── limit_transport.go │ ├── maint_test.go │ └── limit_transport_test.go ├── maintnerd │ ├── Dockerfile │ └── api │ │ ├── internalapi │ │ └── internal_api.go │ │ └── v1beta1 │ │ ├── page_utils.go │ │ ├── repos_paginator.go │ │ ├── issues_paginator.go │ │ └── comment_paginator.go ├── maintner-rtr │ └── Dockerfile └── README.md ├── samplr ├── git-go │ ├── go.mod │ ├── hash.go │ ├── go.sum │ ├── hash_test.go │ ├── object_type.go │ ├── file.go │ ├── commit.go │ ├── remote.go │ ├── reference.go │ └── remote_test.go ├── .gitignore ├── samplr-sprvsr │ └── Dockerfile ├── samplrd │ ├── Dockerfile │ └── samplrapi │ │ ├── page_utils.go │ │ ├── git_commit_paginator.go │ │ ├── snippet_paginator.go │ │ ├── tracked_repository_paginator.go │ │ └── snippet_version_paginator.go ├── samplr-rtr │ └── Dockerfile ├── samplrctl │ ├── testutil │ │ ├── commands.go │ │ └── testutil.go │ ├── cmd │ │ ├── commands.go │ │ ├── completion │ │ │ ├── completion.go │ │ │ └── completion_test.go │ │ ├── snippetversions │ │ │ └── snippet_versions.go │ │ └── snippets │ │ │ └── snippets.go │ ├── output │ │ ├── structured.go │ │ ├── output.go │ │ └── output_test.go │ ├── snippets │ │ └── snippets.go │ ├── snippetversions │ │ └── snippet_versions.go │ ├── main.go │ └── utils │ │ └── utils.go ├── watched_repository.go ├── go.mod ├── sample_metadata.go ├── git_repository_integration_test.go └── Makefile ├── sprvsr ├── README.md ├── sprvsr.go └── go.mod ├── .kokoro ├── go112.cfg ├── go113.cfg ├── common.cfg ├── trampoline.sh └── test.sh ├── go.vet.sh ├── go.get.sh ├── go.test.sh ├── go.integration.test.sh ├── devrelservices-admin ├── go.mod ├── Dockerfile ├── Makefile └── cloudbuild.yaml ├── repos ├── go.mod ├── repos.go └── repos_bucket.go ├── rtr └── go.mod ├── leif ├── README.md ├── go.mod ├── githubservices │ ├── mock_userservice.go │ ├── githubservices.go │ └── mock_reposervice.go ├── leifd │ └── leifapi │ │ ├── translation_utils.go │ │ └── pagination │ │ ├── page_utils.go │ │ ├── string_paginator.go │ │ └── slo_paginator.go ├── repository.go └── repos_disk.go ├── README.md └── CONTRIBUTING.md /codecov.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | codecov: 3 | ci: 4 | - source.cloud.google.com 5 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # By default all reviews go through orthros and jmdobry 2 | * @orthros, @jmdobry 3 | -------------------------------------------------------------------------------- /drghs/v1/api_descriptor.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/devrel-services/HEAD/drghs/v1/api_descriptor.pb -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | unit_test_coverage.txt 3 | coverage.txt 4 | coverage_integration.txt 5 | **/profile.out 6 | **/profile_integration.out 7 | -------------------------------------------------------------------------------- /drghs-worker/internal/api_descriptor.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/devrel-services/HEAD/drghs-worker/internal/api_descriptor.pb -------------------------------------------------------------------------------- /samplr/git-go/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/GoogleCloudPlatform/devrel-services/git-go 2 | 3 | require github.com/sirupsen/logrus v1.3.0 4 | 5 | go 1.13 6 | -------------------------------------------------------------------------------- /sprvsr/README.md: -------------------------------------------------------------------------------- 1 | # sprvsr 2 | 3 | Common utility function for maintner and samplr to interact with the Kubernetes 4 | API and create Services and Deployments for the various resources they manage. 5 | -------------------------------------------------------------------------------- /.kokoro/go112.cfg: -------------------------------------------------------------------------------- 1 | # Format: //devtools/kokoro/config/proto/build.proto 2 | 3 | # Configure the docker image for kokoro-trampoline. 4 | env_vars: { 5 | key: "TRAMPOLINE_IMAGE" 6 | value: "gcr.io/cloud-devrel-kokoro-resources/go112" 7 | } -------------------------------------------------------------------------------- /.kokoro/go113.cfg: -------------------------------------------------------------------------------- 1 | # Format: //devtools/kokoro/config/proto/build.proto 2 | 3 | # Configure the docker image for kokoro-trampoline. 4 | env_vars: { 5 | key: "TRAMPOLINE_IMAGE" 6 | value: "gcr.io/cloud-devrel-kokoro-resources/go113" 7 | } 8 | 9 | -------------------------------------------------------------------------------- /go.vet.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | dirs=( "devrelservices-admin" "drghs-worker" "leif" "repos" "rtr" "samplr" "sprvsr" ) 6 | 7 | for d in "${dirs[@]}"; do 8 | echo "Go vet-ing ./$d/..." 9 | ( 10 | cd "./$d" 11 | go vet ./... 12 | ) 13 | done 14 | 15 | -------------------------------------------------------------------------------- /go.get.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > coverage.txt 5 | 6 | dirs=( "devrelservices-admin" "drghs-worker" "leif" "repos" "rtr" "samplr" "sprvsr" ) 7 | 8 | for d in "${dirs[@]}"; do 9 | echo "Go getting ./$d/..." 10 | ( 11 | cd "./$d" 12 | go get ./... 13 | ) 14 | done 15 | 16 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/support_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Support request 3 | about: If you have a support contract with Google, please create an issue in the Google Cloud Support console. 4 | 5 | --- 6 | 7 | **PLEASE READ**: If you have a support contract with Google, please create an 8 | issue in the [support console](https://cloud.google.com/support/) instead of 9 | filing on GitHub. This will ensure a timely response. 10 | -------------------------------------------------------------------------------- /go.test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > unit_test_coverage.txt 5 | 6 | dirs=( "drghs-worker" "leif" "samplr" "sprvsr" ) 7 | 8 | for d in "${dirs[@]}"; do 9 | echo "Testing ./$d/..." 10 | ( 11 | cd "./$d" 12 | echo "$(pwd)" 13 | go test -v -race -coverprofile=../profile.out -covermode=atomic ./... 14 | if [ -f ../profile.out ]; then 15 | cat ../profile.out >> ../unit_test_coverage.txt 16 | rm ../profile.out 17 | fi 18 | ) 19 | done 20 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Thank you for opening a Pull Request! Before submitting your PR, there are a 2 | few things you can do to make sure it goes smoothly: 3 | 4 | - [ ] Make sure to open an issue as a [bug/issue](https://github.com/GoogleCloudPlatform/devrel-services/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea 5 | - [ ] Ensure the tests and linter pass 6 | - [ ] Code coverage does not decrease (if any source code was changed) 7 | - [ ] Appropriate docs were updated (if necessary) 8 | 9 | Fixes # 10 | -------------------------------------------------------------------------------- /go.integration.test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > integration_test_coverage.txt 5 | 6 | dirs=( "drghs-worker" "leif" "samplr" "sprvsr" ) 7 | 8 | for d in "${dirs[@]}"; do 9 | echo "Testing ./$d/..." 10 | ( 11 | cd "./$d" 12 | echo "$(pwd)" 13 | go test -v \ 14 | -tags integration \ 15 | -race \ 16 | -coverprofile=profile_integration.out \ 17 | -covermode=atomic \ 18 | ./... 19 | if [ -f profile_integration.out ]; then 20 | cat profile_integration.out >> integration_test_coverage.txt 21 | rm profile_integration.out 22 | fi 23 | ) 24 | done 25 | -------------------------------------------------------------------------------- /samplr/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | .vscode 16 | -------------------------------------------------------------------------------- /drghs-worker/.dockerignore: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | go.sum 16 | -------------------------------------------------------------------------------- /devrelservices-admin/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/GoogleCloudPlatform/devrel-services/devrelservices-admin 2 | 3 | go 1.13 4 | 5 | require ( 6 | cloud.google.com/go v0.49.0 7 | github.com/GoogleCloudPlatform/devrel-services/drghs v0.0.0-20191204181555-5cde750c6624 8 | github.com/sirupsen/logrus v1.4.2 9 | google.golang.org/grpc v1.27.1 10 | ) 11 | 12 | replace github.com/GoogleCloudPlatform/devrel-services/drghs => ../drghs 13 | 14 | replace github.com/GoogleCloudPlatform/devrel-services/rtr => ../rtr 15 | 16 | replace github.com/GoogleCloudPlatform/devrel-services/sprvsr => ../sprvsr 17 | 18 | replace github.com/GoogleCloudPlatform/devrel-services/repos => ../repos 19 | -------------------------------------------------------------------------------- /drghs/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Ignore the googleapis directory as we are vendoring it 16 | googleapis 17 | -------------------------------------------------------------------------------- /.kokoro/common.cfg: -------------------------------------------------------------------------------- 1 | # Format: //devtools/kokoro/config/proto/build.proto 2 | 3 | # Download trampoline resources. 4 | gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" 5 | 6 | # Use the trampoline script to run in docker. 7 | build_file: "devrel-services/.kokoro/trampoline.sh" 8 | 9 | # Configure the docker image for kokoro-trampoline. 10 | env_vars: { 11 | key: "TRAMPOLINE_IMAGE" 12 | value: "gcr.io/cloud-devrel-kokoro-resources/go113" 13 | } 14 | 15 | # Tell the trampoline which build file to use. 16 | env_vars: { 17 | key: "TRAMPOLINE_BUILD_FILE" 18 | value: "github/devrel-services/.kokoro/test.sh" 19 | } 20 | 21 | action { 22 | define_artifacts { 23 | regex: "**/*sponge_log.xml" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /drghs/endpoints/models/priority.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Priority: 16 | type: integer 17 | enum: 18 | - 0 19 | - 1 20 | - 2 21 | - 3 22 | - 4 23 | -------------------------------------------------------------------------------- /drghs/endpoints/models/git_hub_user.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | GitHubUser: 16 | type: object 17 | properties: 18 | ID: 19 | type: integer 20 | Login: 21 | type: string 22 | -------------------------------------------------------------------------------- /drghs/endpoints/models/request.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Request: 16 | type: object 17 | properties: 18 | Configs: 19 | type: array 20 | items: 21 | type: string 22 | -------------------------------------------------------------------------------- /drghs/endpoints/models/compliance.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Compliance: 16 | type: object 17 | properties: 18 | Compliant: 19 | type: boolean 20 | Duration: 21 | type: integer 22 | -------------------------------------------------------------------------------- /drghs/endpoints/models/type.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Type: 16 | type: string 17 | enum: 18 | - Bug 19 | - FR 20 | - Cleanup 21 | - Customer 22 | - Process 23 | - PR 24 | - 25 | -------------------------------------------------------------------------------- /drghs/endpoints/models/compliance_response.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ComplianceResponse: 16 | type: object 17 | properties: 18 | Compliant: 19 | type: boolean 20 | Duration: 21 | type: integer 22 | -------------------------------------------------------------------------------- /sprvsr/sprvsr.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sprvsr 16 | 17 | // Supervisor exists to supervise a set of deployments 18 | type Supervisor interface { 19 | Supervise(string, func(error)) error 20 | } 21 | -------------------------------------------------------------------------------- /drghs-worker/pkg/tokens/tokens.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tokens 16 | 17 | // TokenVendor describes an object that can provide API Tokens 18 | type TokenVendor interface { 19 | GetToken() (string, error) 20 | } 21 | -------------------------------------------------------------------------------- /repos/go.mod: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | module github.com/GoogleCloudPlatform/devrel-services/repos 16 | 17 | go 1.12 18 | 19 | require ( 20 | cloud.google.com/go v0.40.0 21 | google.golang.org/grpc v1.24.0 // indirect 22 | ) 23 | -------------------------------------------------------------------------------- /drghs/endpoints/models/get_issue_request.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | GetIssueRequest: 16 | type: object 17 | properties: 18 | Repo: 19 | type: string 20 | Issue: 21 | type: integer 22 | IncludeComments: 23 | type: boolean 24 | -------------------------------------------------------------------------------- /drghs-worker/utils/maint-bucket-migrate/README.md: -------------------------------------------------------------------------------- 1 | # maint-butcket-migrate 2 | 3 | This is a utility tool to migrate a set of mutation logs from one bucket to 4 | another. 5 | 6 | This works by reading a file describing the set of repositories to migrate, 7 | then calculating the "from" buckets and "to" buckets. 8 | 9 | This then uses the kubernetes api to stop the running "to" maintner instances, 10 | then it clears out the "to" bucket and copies the contents of the "from" bucket 11 | to the "to" bucket. 12 | 13 | Once all repositories have been migrated, the "supervisor" pod is restarted 14 | in order to restart all the deployments we deleted during the migration. 15 | 16 | > NOTE: The "from" and "to" buckets can be in different projects 17 | 18 | ## Usage 19 | 20 | `maint-bucket-migrate --file=migrate_repos.json --from-prefix="mtr-b-" 21 | --to-prefix="mtr-p-"` 22 | -------------------------------------------------------------------------------- /drghs-worker/pkg/googlers/googlers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package googlers 16 | 17 | // Resolver describes a struct that can be used to determine 18 | // if a given user is a Googler or not. 19 | type Resolver interface { 20 | IsGoogler(user string) bool 21 | Update() 22 | } 23 | -------------------------------------------------------------------------------- /drghs/endpoints/models/response.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Response: 16 | type: object 17 | properties: 18 | Error: 19 | type: string 20 | Issues: 21 | type: array 22 | items: 23 | $ref: "status.yaml#/Status" 24 | Issue: 25 | $ref: "status.yaml#/Status" 26 | -------------------------------------------------------------------------------- /rtr/go.mod: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | module github.com/GoogleCloudPlatform/devrel-services/rtr 16 | 17 | go 1.12 18 | 19 | require ( 20 | cloud.google.com/go v0.40.0 21 | github.com/gorilla/mux v1.7.2 22 | github.com/sirupsen/logrus v1.4.2 23 | github.com/urfave/negroni v1.0.0 24 | ) 25 | -------------------------------------------------------------------------------- /drghs/endpoints/models/list_issues_request.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ListIssuesRequest: 16 | type: object 17 | properties: 18 | Repo: 19 | type: string 20 | PullRequest: 21 | type: boolean 22 | Closed: 23 | type: boolean 24 | IncludeComments: 25 | type: boolean 26 | -------------------------------------------------------------------------------- /drghs/endpoints/leif.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | type: google.api.Service 16 | config_version: 3 17 | 18 | name: leif.endpoints.PROJECT_ID.cloud.goog 19 | title: leif gRPC API (TYPE) 20 | 21 | apis: 22 | - name: drghs.v1.SLOService 23 | 24 | endpoints: 25 | - name: leif.endpoints.PROJECT_ID.cloud.goog 26 | target: "IP_ADDR" 27 | -------------------------------------------------------------------------------- /leif/README.md: -------------------------------------------------------------------------------- 1 | # leif 2 | 3 | `leif` is a service designed to take a set of GitHub repositories to 4 | find and track their configuration files for service level objective (SLO) rules, 5 | periodically syncing to update the SLO rules, which are exposed over an API. 6 | 7 | ![leif](https://vignette.wikia.nocookie.net/animalcrossing/images/1/1c/Leif_NH.png/revision/latest/top-crop/width/360/height/360?cb=20200630055201) 8 | 9 | 10 | ### SLO rules configuration 11 | 12 | `leif` expects the configuration of the SLO rules to be defined in a JSON file named `issue_slo_rules.json` 13 | 14 | `leif` looks for the config file for the repository in the following places: 15 | 1. In the repository itself: `.github/issue_slo_rules.json` 16 | 2. If there is no config file in the respository, `leif` looks for the config file at the owner level: `/.github/issue_slo_rules.json` 17 | 18 | Note: an empty config file opts-out of SLO tracking 19 | -------------------------------------------------------------------------------- /drghs/endpoints/samplr.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | type: google.api.Service 16 | config_version: 3 17 | 18 | name: samplr.endpoints.PROJECT_ID.cloud.goog 19 | title: samplr gRPC API (TYPE) 20 | 21 | apis: 22 | - name: drghs.v1.SampleService 23 | 24 | endpoints: 25 | - name: samplr.endpoints.PROJECT_ID.cloud.goog 26 | target: "IP_ADDR" 27 | -------------------------------------------------------------------------------- /leif/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/GoogleCloudPlatform/devrel-services/leif 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/GoogleCloudPlatform/devrel-services/drghs v0.0.0-20200723024905-6c479f56d135 7 | github.com/GoogleCloudPlatform/devrel-services/repos v0.0.0-20200720163603-c134bef7ad58 8 | github.com/golang/protobuf v1.4.2 9 | github.com/google/cel-go v0.5.1 10 | github.com/google/go-cmp v0.5.0 11 | github.com/google/go-github v17.0.0+incompatible 12 | github.com/google/go-github/v32 v32.0.0 13 | github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 14 | github.com/mitchellh/mapstructure v1.3.2 15 | github.com/sirupsen/logrus v1.6.0 16 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 17 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a 18 | google.golang.org/grpc v1.27.1 19 | google.golang.org/protobuf v1.25.0 20 | ) 21 | 22 | replace github.com/GoogleCloudPlatform/devrel-services/drghs => ../drghs 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this library 4 | 5 | --- 6 | 7 | Thanks for stopping by to let us know something could be better! 8 | 9 | **PLEASE READ**: If you have a support contract with Google, please create an 10 | issue in the [support console](https://cloud.google.com/support/) instead of 11 | filing on GitHub. This will ensure a timely response. 12 | 13 | **Is your feature request related to a problem? Please describe.** 14 | A clear and concise description of what the problem is. 15 | Ex. I'm always frustrated when [...] 16 | **Describe the solution you'd like** 17 | A clear and concise description of what you want to happen. 18 | **Describe alternatives you've considered** 19 | A clear and concise description of any alternative solutions or 20 | features you've considered. 21 | **Additional context** 22 | Add any other context or screenshots about the feature request here. 23 | -------------------------------------------------------------------------------- /drghs/README.md: -------------------------------------------------------------------------------- 1 | Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | # Regenerating the Go proto files 15 | 16 | 1. Follow the setup instructions: 17 | 18 | https://grpc.io/docs/quickstart/go.html#before-you-begin 19 | 20 | 1. Generate the Go proto files: 21 | ``` 22 | protoc --go_out=plugins=grpc:v1/ --proto_path=v1/ --proto_path=../../googleapis --descriptor_set_out=v1/api_descriptor.pb v1/*.proto 23 | ``` 24 | -------------------------------------------------------------------------------- /drghs/endpoints/maintnerd.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2019 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | type: google.api.Service 17 | config_version: 3 18 | 19 | name: drghs.endpoints.PROJECT_ID.cloud.goog 20 | title: DevRel GitHub Services API (TYPE) 21 | 22 | apis: 23 | - name: drghs.v1.IssueService 24 | - name: drghs.v1.IssueServiceAdmin 25 | 26 | endpoints: 27 | - name: drghs.endpoints.PROJECT_ID.cloud.goog 28 | target: "IP_ADDR" 29 | -------------------------------------------------------------------------------- /drghs/endpoints/devrelservices-admin.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | type: google.api.Service 16 | config_version: 3 17 | 18 | name: devrelservices-admin.endpoints.PROJECT_ID.cloud.goog 19 | title: DevRel Services Admin gRPC API (TYPE) 20 | 21 | apis: 22 | - name: drghs.v1.DevRelServicesAdmin 23 | 24 | endpoints: 25 | - name: devrelservices-admin.endpoints.PROJECT_ID.cloud.goog 26 | target: "IP_ADDR" 27 | -------------------------------------------------------------------------------- /drghs/endpoints/models/git_hub_comment.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | GitHubComment: 16 | type: object 17 | properties: 18 | ID: 19 | type: integer 20 | User: 21 | $ref: "git_hub_user.yaml#/GitHubUser" 22 | Created: 23 | type: string 24 | format: date-time 25 | Updated: 26 | type: string 27 | format: date-time 28 | Body: 29 | type: string 30 | -------------------------------------------------------------------------------- /drghs/v1/pagination.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | import "google/protobuf/timestamp.proto"; 18 | 19 | package drghs.v1; 20 | 21 | message PageToken { 22 | // The offset where the next request should start. 23 | int32 offset = 1; 24 | 25 | // The time when the first page request was received. 26 | google.protobuf.Timestamp first_request_time_usec = 2; 27 | } 28 | -------------------------------------------------------------------------------- /.kokoro/trampoline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -eo pipefail 17 | 18 | # Always run the cleanup script, regardless of the success of bouncing into 19 | # the container. 20 | function cleanup() { 21 | chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh 22 | ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh 23 | echo "cleanup"; 24 | } 25 | trap cleanup EXIT 26 | 27 | python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" -------------------------------------------------------------------------------- /drghs-worker/DEPLOY.md: -------------------------------------------------------------------------------- 1 | Copyright 2019 Google LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | # Miscellaneous 16 | 17 | ## Creating secrets 18 | 19 | 1. Acquire service account JSON file. 20 | 2. Rename it to key.json 21 | 3. Point kubectl at the production cluster (see above). 22 | 4. Run: 23 | 24 | kubectl create secret generic service-account-maintnerd \ 25 | --from-file=key.json 26 | 27 | 28 | ## Update Go dependencies (they're pinned in the Dockerfile): 29 | 30 | make update-deps 31 | -------------------------------------------------------------------------------- /samplr/samplr-sprvsr/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.13 as build 16 | 17 | ENV GO111MODULE=on 18 | 19 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services 20 | COPY . . 21 | 22 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/samplr 23 | 24 | RUN go install github.com/GoogleCloudPlatform/devrel-services/samplr/samplr-sprvsr 25 | 26 | FROM gcr.io/distroless/base 27 | 28 | COPY --from=build /go/bin/samplr-sprvsr / 29 | 30 | CMD ["/samplr-sprvsr"] 31 | -------------------------------------------------------------------------------- /samplr/samplrd/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.13 as build 16 | 17 | 18 | ENV GO111MODULE=on 19 | 20 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/ 21 | COPY . . 22 | 23 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/samplr/samplrd 24 | 25 | RUN go install github.com/GoogleCloudPlatform/devrel-services/samplr/samplrd 26 | 27 | FROM golang:1.13-stretch 28 | 29 | WORKDIR / 30 | COPY --from=build /go/bin/samplrd . 31 | 32 | CMD ["/samplrd"] 33 | -------------------------------------------------------------------------------- /drghs-worker/internal/issue_service_internal.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package maintner.internal; 18 | 19 | // Internal Issue Service 20 | service InternalIssueService { 21 | rpc TombstoneIssues(TombstoneIssuesRequest) 22 | returns (TombstoneIssuesResponse) {} 23 | } 24 | 25 | message TombstoneIssuesRequest { 26 | string parent = 1; 27 | repeated int32 issue_numbers = 2; 28 | } 29 | 30 | message TombstoneIssuesResponse { int32 tombstoned_count = 1; } -------------------------------------------------------------------------------- /drghs-worker/maintner-sprvsr/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.13 as build 16 | 17 | ENV GO111MODULE=on 18 | 19 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services 20 | COPY . . 21 | 22 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/drghs-worker 23 | 24 | RUN go install github.com/GoogleCloudPlatform/devrel-services/drghs-worker/maintner-sprvsr 25 | 26 | FROM gcr.io/distroless/base 27 | COPY --from=build /go/bin/maintner-sprvsr / 28 | WORKDIR /src 29 | COPY . . 30 | CMD ["/maintner-sprvsr"] 31 | -------------------------------------------------------------------------------- /drghs-worker/maintner-swpr/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.13 as build 16 | 17 | ENV GO111MODULE=on 18 | 19 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services 20 | COPY . . 21 | 22 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/drghs-worker/maintner-swpr 23 | 24 | RUN go install github.com/GoogleCloudPlatform/devrel-services/drghs-worker/maintner-swpr 25 | 26 | FROM gcr.io/distroless/base 27 | COPY --from=build /go/bin/maintner-swpr / 28 | WORKDIR /src 29 | COPY . . 30 | ENTRYPOINT ["/maintner-swpr"] 31 | -------------------------------------------------------------------------------- /drghs-worker/pkg/utils/serialization_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "encoding/json" 19 | "strings" 20 | ) 21 | 22 | // UnmarshalBool takes a json.RawMessage and deserializes it 23 | // to a bool. If bytes is nil or, equal to "null" then 24 | // will return nil 25 | func UnmarshalBool(bytes json.RawMessage) *bool { 26 | if len(bytes) > 0 { 27 | s := string(bytes) 28 | s = strings.ToLower(s) 29 | if s != "null" { 30 | b := s == "true" 31 | return &b 32 | } 33 | } 34 | 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /drghs-worker/maintner-swpr/limit_transport.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "net/http" 19 | 20 | "golang.org/x/time/rate" 21 | ) 22 | 23 | type limitTransport struct { 24 | limiter *rate.Limiter 25 | base http.RoundTripper 26 | } 27 | 28 | func (t limitTransport) RoundTrip(r *http.Request) (*http.Response, error) { 29 | limiter := t.limiter 30 | if limiter != nil { 31 | log.Debug("in limitTransport. Round trip Waiting for limiter") 32 | if err := limiter.Wait(r.Context()); err != nil { 33 | return nil, err 34 | } 35 | } 36 | return t.base.RoundTrip(r) 37 | } 38 | -------------------------------------------------------------------------------- /drghs/v1/admin_service.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package drghs.v1; 18 | 19 | import "google/api/annotations.proto"; 20 | 21 | service DevRelServicesAdmin{ 22 | rpc UpdateTrackedRepos(UpdateTrackedReposRequest) returns (UpdateTrackedReposResponse) { 23 | option (google.api.http) = { 24 | post: "/api/v1/update" 25 | }; 26 | } 27 | } 28 | 29 | // Request message for [DevRelServicesAdmin.UpdateTrackedRepos]. 30 | message UpdateTrackedReposRequest{ 31 | } 32 | 33 | // Response message for [DevRelServicesAdmin.UpdateTrackedRepos]. 34 | message UpdateTrackedReposResponse{ 35 | } 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | Thanks for stopping by to let us know something could be better! 8 | 9 | **PLEASE READ**: If you have a support contract with Google, please create an 10 | issue in the [support console](https://cloud.google.com/support/) instead of 11 | filing on GitHub. This will ensure a timely response. 12 | 13 | Please run down the following list and make sure you've tried the usual 14 | "quick fixes": 15 | 16 | - Search the issues already opened: https://github.com/GoogleCloudPlatform/devrel-services/issues 17 | - Search the issues on our "catch-all" repository: https://github.com/googleapis/google-cloud-go 18 | - Search StackOverflow: http://stackoverflow.com/questions/tagged/google-cloud-platform+go 19 | 20 | If you are still having issues, please be sure to include as much information 21 | as possible: 22 | 23 | #### Environment details 24 | 25 | - OS: 26 | - Go version: 27 | - Kubernetes version: 28 | - Docker version: 29 | - `devrel-services` version: 30 | 31 | #### Steps to reproduce 32 | 33 | 1. ? 34 | 2. ? 35 | 36 | Making sure to follow these steps will guarantee the quickest 37 | resolution possible. 38 | 39 | Thanks! 40 | -------------------------------------------------------------------------------- /samplr/git-go/hash.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | import ( 18 | "encoding/hex" 19 | ) 20 | 21 | // Hash SHA1 hased content 22 | type Hash [20]byte 23 | 24 | // ZeroHash is Hash with value zero 25 | var ZeroHash Hash 26 | 27 | // NewHash return a new Hash from a hexadecimal hash representation 28 | func NewHash(s string) Hash { 29 | b, _ := hex.DecodeString(s) 30 | 31 | var h Hash 32 | copy(h[:], b) 33 | 34 | return h 35 | } 36 | 37 | // IsZero is a convience method to check if the Hash is the zero value 38 | func (h Hash) IsZero() bool { 39 | var empty Hash 40 | return h == empty 41 | } 42 | 43 | func (h Hash) String() string { 44 | return hex.EncodeToString(h[:]) 45 | } 46 | -------------------------------------------------------------------------------- /samplr/git-go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= 7 | github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 8 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 9 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 10 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 11 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= 12 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 13 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= 14 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 15 | -------------------------------------------------------------------------------- /samplr/git-go/hash_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | import "testing" 18 | 19 | func TestHashIsZero(t *testing.T) { 20 | hash := NewHash("f3f4e32b94c97ffd6b2e6f020d96931a9fe2c3f6") 21 | if hash.IsZero() { 22 | t.Errorf("Hash: %v was said to be the Zero Hash, when it is not", hash.String()) 23 | } 24 | if !ZeroHash.IsZero() { 25 | t.Errorf("The ZeroHas was said to not be the Zero Hash") 26 | } 27 | } 28 | 29 | func TestHashString(t *testing.T) { 30 | hash := NewHash("f3f4e32b94c97ffd6b2e6f020d96931a9fe2c3f6") 31 | hashStr := hash.String() 32 | if hashStr != "f3f4e32b94c97ffd6b2e6f020d96931a9fe2c3f6" { 33 | t.Errorf("Hash String. Expected %v, got %v", "f3f4e32b94c97ffd6b2e6f020d96931a9fe2c3f6", hashStr) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /drghs-worker/utils/maint-bucket-reduce/README.md: -------------------------------------------------------------------------------- 1 | # maint-bucket-reduce 2 | 3 | This script takes a set of mutation logs from disparate Google Cloud Storage 4 | buckets and moves them to a common bucket, but prefixed with the Owner and 5 | Repository they are tracking 6 | 7 | ## Usage 8 | 9 | ```bash 10 | ./maint-bucket-reduce --settings-bucket=devrel-maintner-settings \ 11 | --file=tracked_repos.json \ 12 | --from-prefix=mtr-d \ 13 | --to-bucket=maintner-bucket 14 | ``` 15 | 16 | 17 | ## Example 18 | 19 | Consider the following buckets 20 | 21 | | Bucket Name | Repository Tracked | 22 | | ------------ | ------------------ | 23 | | mtr-foo | org1/repo1 | 24 | | mtr-bar | org1/repo2 | 25 | | mtr-baz | org1/repo3 | 26 | | mtr-biz | org2/repo1 | 27 | 28 | And a new bucket `mtr-combined` 29 | 30 | When this script is run it will copy the mutation logs in each of those buckets 31 | to a new bucket (specified by the user). The new Bucket, `mtr-combined` will 32 | have the following structure: 33 | 34 | ``` 35 | mtr-combined 36 | +-- org1 37 | +-- repo1 38 | + --mutation log files 39 | +-- repo2 40 | +-- mutation log files 41 | +-- repo3 42 | +-- mutation log files 43 | +-- org2 44 | +-- repo1 45 | +-- mutation log files 46 | ``` 47 | 48 | 49 | -------------------------------------------------------------------------------- /drghs/go.mod: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Code generated by protoc-gen-go. DO NOT EDIT. 16 | // source: issue_service.proto 17 | 18 | module github.com/GoogleCloudPlatform/devrel-services/drghs 19 | 20 | require ( 21 | github.com/GoogleCloudPlatform/devrel-services/drghs v0.0.0-00010101000000-000000000000 // indirect 22 | github.com/golang/lint v0.0.0-20180702182130-06c8688daad7 // indirect 23 | github.com/golang/protobuf v1.4.2 24 | github.com/kisielk/gotool v1.0.0 // indirect 25 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a 26 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect 27 | golang.org/x/text v0.3.2 // indirect 28 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 29 | google.golang.org/grpc v1.27.1 30 | google.golang.org/protobuf v1.25.0 31 | ) 32 | 33 | go 1.13 34 | -------------------------------------------------------------------------------- /samplr/samplr-rtr/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.13 as build 16 | 17 | ENV GO111MODULE=on 18 | 19 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services 20 | COPY . . 21 | 22 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/samplr/samplr-rtr 23 | 24 | RUN go install github.com/GoogleCloudPlatform/devrel-services/samplr/samplr-rtr 25 | 26 | FROM alpine as health 27 | 28 | RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ 29 | wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ 30 | chmod +x /bin/grpc_health_probe 31 | 32 | FROM gcr.io/distroless/base 33 | 34 | COPY --from=build /go/bin/samplr-rtr / 35 | COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe 36 | 37 | CMD ["/samplr-rtr"] 38 | -------------------------------------------------------------------------------- /samplr/samplrctl/testutil/commands.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package testutil contains helpers for working with commands in tests. 16 | package testutil 17 | 18 | import ( 19 | "bytes" 20 | 21 | commands "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/cmd" 22 | 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | // CreateTestCommand create a Cobra command suitable for use in tests. 27 | func CreateTestCommand() *cobra.Command { 28 | cmd := &cobra.Command{} 29 | commands.AddFlags(cmd.PersistentFlags()) 30 | return cmd 31 | } 32 | 33 | // Execute will execute the provided command with args and return the output. 34 | func Execute(cmd *cobra.Command, args []string) (string, error) { 35 | output := new(bytes.Buffer) 36 | cmd.SetOutput(output) 37 | cmd.SetArgs(args) 38 | err := cmd.Execute() 39 | return output.String(), err 40 | } 41 | -------------------------------------------------------------------------------- /drghs-worker/maintnerd/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.13 as build 16 | 17 | ENV GO111MODULE=on 18 | 19 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services 20 | COPY . . 21 | 22 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/drghs-worker 23 | 24 | RUN go install github.com/GoogleCloudPlatform/devrel-services/drghs-worker/maintnerd 25 | 26 | FROM alpine as health 27 | 28 | RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ 29 | wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ 30 | chmod +x /bin/grpc_health_probe 31 | 32 | FROM golang:1.13 33 | 34 | COPY --from=build /go/bin/maintnerd / 35 | COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe 36 | 37 | WORKDIR /src 38 | COPY . . 39 | CMD ["/maintnerd"] 40 | -------------------------------------------------------------------------------- /devrelservices-admin/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.13 as build 16 | 17 | ENV GO111MODULE=on 18 | 19 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services 20 | COPY . . 21 | 22 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/devrelservices-admin 23 | 24 | RUN go install github.com/GoogleCloudPlatform/devrel-services/devrelservices-admin 25 | 26 | FROM alpine as health 27 | 28 | RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ 29 | wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ 30 | chmod +x /bin/grpc_health_probe 31 | 32 | FROM gcr.io/distroless/base 33 | 34 | COPY --from=build /go/bin/devrelservices-admin / 35 | COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe 36 | 37 | ENTRYPOINT ["/devrelservices-admin"] 38 | -------------------------------------------------------------------------------- /drghs-worker/maintner-rtr/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.13 as build 16 | 17 | ENV GO111MODULE=on 18 | 19 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services 20 | COPY . . 21 | 22 | WORKDIR /src/github.com/GoogleCloudPlatform/devrel-services/drghs-worker/maintner-rtr 23 | 24 | RUN go install github.com/GoogleCloudPlatform/devrel-services/drghs-worker/maintner-rtr 25 | 26 | FROM alpine as health 27 | 28 | RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ 29 | wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ 30 | chmod +x /bin/grpc_health_probe 31 | 32 | FROM gcr.io/distroless/base 33 | 34 | COPY --from=build /go/bin/maintner-rtr / 35 | COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe 36 | 37 | WORKDIR /src 38 | COPY . . 39 | CMD ["/maintner-rtr"] 40 | -------------------------------------------------------------------------------- /leif/githubservices/mock_userservice.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This package was modeled after the mocking strategy outlined at: 16 | // https://github.com/google/go-github/issues/113#issuecomment-46023864 17 | 18 | package githubservices 19 | 20 | import ( 21 | "context" 22 | "errors" 23 | 24 | "github.com/google/go-github/github" 25 | ) 26 | 27 | // MockGithubUserService is a struct that can replace github.UsersService for testing 28 | type MockGithubUserService struct { 29 | Response *github.Response 30 | Error error 31 | User string 32 | } 33 | 34 | // Get mocks the original github.UsersService.Get() 35 | // Checks whether the user is correct and returns the mocked response and error 36 | func (mgc *MockGithubUserService) Get(ctx context.Context, user string) (*github.User, *github.Response, error) { 37 | if user != mgc.User { 38 | return nil, nil, errors.New("user did not equal expected user: was: " + user) 39 | } 40 | return nil, mgc.Response, mgc.Error 41 | } 42 | -------------------------------------------------------------------------------- /.kokoro/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -eo pipefail 18 | set -x 19 | 20 | export GO111MODULE=on # Always use modules. 21 | TIMEOUT=45m 22 | 23 | go version 24 | date 25 | 26 | # Re-organize files 27 | export GOPATH=$PWD/gopath 28 | target=$GOPATH/src/github.com/GoogleCloudPlatform 29 | mkdir -p $target 30 | mv github/devrel-services $target 31 | cd $target/devrel-services 32 | 33 | testdirs=( repos rtr samplr sprvsr drghs-worker ) 34 | 35 | OUTFILE=$(pwd)/gotest.out 36 | for t in "${testdirs[@]}"; do 37 | # Subshell to avoid having to `cd ..` 38 | ( 39 | cd "$t" || exit 1 40 | echo "$t" 41 | go test -timeout $TIMEOUT -v ./... 2>&1 | tee -a $OUTFILE 42 | 43 | if [ $GOLANG_SAMPLES_GO_VET ]; then 44 | diff -u <(echo -n) <(gofmt -d .) 45 | # We are cd'd in the directory. Simply go vet ./... 46 | go vet ./... 47 | fi 48 | ) 49 | done 50 | 51 | # Do the easy stuff before running tests. Fail fast! 52 | 53 | date 54 | 55 | cat $OUTFILE > sponge_log.xml 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | UPDATE 2022-09-29: This repo is no longer being maintained or used. 3 |

4 | 5 | # devrel-github-services 6 | 7 | DevRel GitHub Services is a collection of tools and services aimed 8 | at helping DevRel **do** DevRel. These services in particular focus on the 9 | data aggregation portion of the DevRel process. 10 | Each service will have it's own README with a deeper description. 11 | 12 | ## drghs 13 | 14 | This is a common directory which defines specifications for 15 | [Cloud Endpoints](https://cloud.google.com/endpoints), as well as the 16 | [Protocol Buffer](https://developers.google.com/protocol-buffers/) and 17 | [gRPC](https://grpc.io) definitions. 18 | 19 | ## drghs-worker 20 | 21 | This folder contains an application that is a small fork of the 22 | [maintner](https://github.com/golang/build/tree/master/maintner) service 23 | written by the [Go](https://golang.org) team. The purpose of this service is 24 | to read a list of repositories, synchronize all the Issues and Pull Requests 25 | for each repository and expose an API to query these issues. 26 | It can be thought of as a giant in-memory cache. 27 | 28 | ## leif 29 | 30 | This service takes a set of GitHub repositories and scans them for service-level objective (SLO) rules and exposes an API to query them. It is designed to be deployed to a 31 | [Kubernetes](https://kubernetes.io) cluster. 32 | 33 | ## samplr 34 | 35 | This service takes a set of GitHub repositories and scans them for code 36 | snippets and exposes an API to query them. It is designed to be deployed to a 37 | [Kubernetes](https://kubernetes.io) cluster. 38 | -------------------------------------------------------------------------------- /samplr/samplrctl/cmd/commands.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package commands 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/spf13/cobra" 21 | "github.com/spf13/pflag" 22 | ) 23 | 24 | const ( 25 | verboseFlagName = "verbose" 26 | ) 27 | 28 | // AddFlags adds flags that are common across all commands. 29 | func AddFlags(s *pflag.FlagSet) { 30 | // s.BoolP(verboseFlagName, "v", false, "Sets Log level to \"Debug\"") 31 | } 32 | 33 | // CobraActionFunc represents a cobra command 34 | type CobraActionFunc func(cmd *cobra.Command, args []string) error 35 | 36 | // SamplrActionFunc provides a common type for Cobra functions that require a 37 | // stubby client connection and context. 38 | type SamplrActionFunc func(ctx context.Context, cmd *cobra.Command, args []string) error 39 | 40 | // CtxCommand allows the running of a command with a context 41 | func CtxCommand(ctx context.Context, f SamplrActionFunc) CobraActionFunc { 42 | return func(cmd *cobra.Command, args []string) error { 43 | return f(ctx, cmd, args) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /drghs-worker/pkg/tokens/rotate_tokens.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tokens 16 | 17 | import ( 18 | "fmt" 19 | "sync" 20 | ) 21 | 22 | // RotatingTokens provides thread safe access to a 23 | // rotating set of API keys. 24 | type RotatingTokens struct { 25 | keys []string 26 | mux sync.Mutex 27 | lastAccessed int 28 | } 29 | 30 | // NewRotatingVendor creates a new *RotatingTokens 31 | func NewRotatingVendor(tokens []string) *RotatingTokens { 32 | if tokens == nil { 33 | tokens = make([]string, 0) 34 | } 35 | return &RotatingTokens{keys: tokens} 36 | } 37 | 38 | // GetToken returns the next available in a thread-safe manner 39 | // or an error if there are no available tokens. 40 | func (r *RotatingTokens) GetToken() (string, error) { 41 | r.mux.Lock() 42 | defer r.mux.Unlock() 43 | if r.lastAccessed >= len(r.keys) { 44 | r.lastAccessed = 0 45 | } 46 | if len(r.keys) == 0 { 47 | return "", fmt.Errorf("no tokens") 48 | } 49 | key := r.keys[r.lastAccessed] 50 | r.lastAccessed++ 51 | return key, nil 52 | } 53 | -------------------------------------------------------------------------------- /samplr/watched_repository.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package samplr 16 | 17 | import "context" 18 | 19 | // WatchedRepository represents a repository being watched by the Corpus 20 | type WatchedRepository interface { 21 | // Unique Identifier of the Repository 22 | // TODO(colnnelson): Work out the details of this 23 | ID() string 24 | // Allows iterating over a WatchedRepository's snippets 25 | ForEachSnippet(func(snippet *Snippet) error) error 26 | // Allows iterating over a WatchedRepository's snippets that match the given filter 27 | ForEachSnippetF(func(snippet *Snippet) error, func(snippet *Snippet) bool) error 28 | // Allows iterating over a WatchedRepository's git commits 29 | ForEachGitCommit(func(commit *GitCommit) error) error 30 | // Allows iterating over a WatchedRepository's git commits that match the given filter 31 | ForEachGitCommitF(func(commit *GitCommit) error, func(commit *GitCommit) bool) error 32 | // The owner of the repository 33 | Owner() string 34 | // The name of the repository 35 | RepositoryName() string 36 | // Instructs the Repository to Update 37 | Update(ctx context.Context) error 38 | } 39 | -------------------------------------------------------------------------------- /leif/leifd/leifapi/translation_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package leifapi 16 | 17 | import ( 18 | drghs_v1 "github.com/GoogleCloudPlatform/devrel-services/drghs/v1" 19 | "github.com/GoogleCloudPlatform/devrel-services/leif" 20 | "github.com/golang/protobuf/ptypes" 21 | ) 22 | 23 | func makeOwnerPB(name string) (*drghs_v1.Owner, error) { 24 | return &drghs_v1.Owner{ 25 | Name: name, 26 | }, nil 27 | } 28 | 29 | func makeRepositoryPB(rname string) (*drghs_v1.Repository, error) { 30 | return &drghs_v1.Repository{ 31 | Name: rname, 32 | }, nil 33 | } 34 | 35 | func makeSloPB(slo *leif.SLORule) (*drghs_v1.SLO, error) { 36 | return &drghs_v1.SLO{ 37 | GithubLabels: slo.AppliesTo.GitHubLabels, 38 | ExcludedGithubLabels: slo.AppliesTo.ExcludedGitHubLabels, 39 | AppliesToIssues: slo.AppliesTo.Issues, 40 | AppliesToPrs: slo.AppliesTo.PRs, 41 | ResponseTime: ptypes.DurationProto(slo.ComplianceSettings.ResponseTime), 42 | ResolutionTime: ptypes.DurationProto(slo.ComplianceSettings.ResolutionTime), 43 | RequiresAssignee: slo.ComplianceSettings.RequiresAssignee, 44 | Responders: slo.ComplianceSettings.Responders, 45 | }, nil 46 | } 47 | -------------------------------------------------------------------------------- /drghs-worker/pkg/utils/status_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "golang.org/x/build/maintner" 19 | ) 20 | 21 | // IsApproved considers a GitHubIssue, loops through the 22 | // Review Events on the issue and, for each reivewer, 23 | // if the last review event was an "approved" event it returns true 24 | // if there are no reviews, this return false 25 | func IsApproved(issue *maintner.GitHubIssue) bool { 26 | if issue == nil { 27 | return false 28 | } 29 | 30 | reviewers := make(map[*maintner.GitHubUser]bool) 31 | 32 | // ForeachReview processes reviews in chronological 33 | // order. We can just call this serially and if a 34 | // reviewer ever requests changes after approving 35 | // this will still set the final review to 'false' 36 | issue.ForeachReview(func(review *maintner.GitHubReview) error { 37 | reviewers[review.Actor] = review.State == "APPROVED" 38 | return nil 39 | }) 40 | 41 | // If there are no reviewers, we shall state that it is not approved 42 | if len(reviewers) == 0 { 43 | return false 44 | } 45 | 46 | for _, approved := range reviewers { 47 | if !approved { 48 | return false 49 | } 50 | } 51 | 52 | return true 53 | } 54 | -------------------------------------------------------------------------------- /samplr/git-go/object_type.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | // ObjectType internal object type 18 | // Integer values from 0 to 7 map to those exposed by git. 19 | // AnyObject is used to represent any from 0 to 7. 20 | type ObjectType int8 21 | 22 | // Constants for Types of Objects 23 | const ( 24 | InvalidObject ObjectType = 0 25 | CommitObject ObjectType = 1 26 | TreeObject ObjectType = 2 27 | BlobObject ObjectType = 3 28 | TagObject ObjectType = 4 29 | // 5 reserved for future expansion 30 | OFSDeltaObject ObjectType = 6 31 | REFDeltaObject ObjectType = 7 32 | 33 | AnyObject ObjectType = -127 34 | ) 35 | 36 | // Bytes returns the ObjectType represented as bytes 37 | func (t ObjectType) Bytes() []byte { 38 | return []byte(t.String()) 39 | } 40 | func (t ObjectType) String() string { 41 | switch t { 42 | case CommitObject: 43 | return "commit" 44 | case TreeObject: 45 | return "tree" 46 | case BlobObject: 47 | return "blob" 48 | case TagObject: 49 | return "tag" 50 | case OFSDeltaObject: 51 | return "ofs-delta" 52 | case REFDeltaObject: 53 | return "ref-delta" 54 | case AnyObject: 55 | return "any" 56 | default: 57 | return "unknown" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /sprvsr/go.mod: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | module github.com/GoogleCloudPlatform/devrel-services/sprvsr 16 | 17 | go 1.12 18 | 19 | require ( 20 | cloud.google.com/go v0.40.0 21 | github.com/GoogleCloudPlatform/devrel-services/repos v0.0.0 22 | 23 | github.com/deckarep/golang-set v1.7.1 24 | github.com/google/go-cmp v0.3.0 25 | github.com/gorilla/mux v1.7.2 26 | github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17 // indirect 27 | github.com/matryer/is v1.2.0 28 | github.com/sirupsen/logrus v1.4.2 29 | github.com/urfave/negroni v1.0.0 30 | go4.org v0.0.0-20181109185143-00e24f1b2599 // indirect 31 | golang.org/x/build v0.0.0-20190201181641-63986c177d1f 32 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c 33 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 34 | gopkg.in/yaml.v2 v2.2.2 // indirect 35 | grpc.go4.org v0.0.0-20170609214715-11d0a25b4919 36 | k8s.io/api v0.0.0-20190528154508-67ef80593b24 37 | k8s.io/apimachinery v0.0.0-20190528154326-e59c2fb0a8e5 38 | k8s.io/client-go v0.0.0-20190528154735-79226fe1949a 39 | k8s.io/utils v0.0.0-20190529001817-6999998975a7 // indirect 40 | ) 41 | 42 | replace github.com/GoogleCloudPlatform/devrel-services/repos => ../repos 43 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Commits 26 | 27 | We use [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) 28 | to format our commits, and they are enforced by a GitHub check. If you 29 | accidentally make commits that do not conform to the spec, don't worry! 30 | You can edit your Pull Request title to be within the spec and the 31 | check will pass. 32 | 33 | ## Linting 34 | 35 | All Go code must be formatted with the 36 | [gofmt tool](https://golang.org/cmd/gofmt/). 37 | 38 | ## Testing 39 | 40 | Our Go code has test associated with them. If you write a function, or 41 | fix a bug, include a test to cover it. 42 | 43 | ## Community Guidelines 44 | 45 | This project follows 46 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 47 | -------------------------------------------------------------------------------- /drghs-worker/pkg/tokens/tokens_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tokens 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestGetstokens(t *testing.T) { 23 | tests := []struct { 24 | tokens []string 25 | err error 26 | }{ 27 | { 28 | tokens: []string{"One", "Two", "Three"}, 29 | err: nil, 30 | }, 31 | { 32 | tokens: []string{"One"}, 33 | err: nil, 34 | }, 35 | { 36 | tokens: []string{}, 37 | err: fmt.Errorf("no tokens"), 38 | }, 39 | } 40 | for _, test := range tests { 41 | store := NewRotatingVendor(test.tokens) 42 | 43 | for i := 0; i < len(test.tokens)*2; i++ { 44 | token, err := store.GetToken() 45 | if err != test.err { 46 | t.Errorf("Get token returned an unexpected error. Got %v, want %v", err, test.err) 47 | } 48 | if token != test.tokens[i%len(test.tokens)] { 49 | t.Errorf("GetKy returned an unexpected token. Got %v, want %v", token, test.tokens[i%len(test.tokens)]) 50 | } 51 | } 52 | 53 | if len(test.tokens) == 0 { 54 | _, err := store.GetToken() 55 | if err.Error() != test.err.Error() { 56 | t.Errorf("Get token returned an unexpected error. Got %v, want %v", err, test.err) 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /samplr/git-go/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | import "io" 18 | 19 | // FilesIter is a generic closable interface for iterating over files. 20 | type FilesIter interface { 21 | Next() (*File, error) 22 | ForEach(func(*File) error) error 23 | Close() 24 | } 25 | 26 | // File represents a git file 27 | type File struct { 28 | Name string 29 | Size int64 30 | contents string 31 | } 32 | 33 | // Contents returns the contents of a file 34 | func (f File) Contents() (string, error) { 35 | return f.contents, nil 36 | } 37 | 38 | type sliceFileIter struct { 39 | pos int 40 | series []File 41 | } 42 | 43 | func (iter *sliceFileIter) Next() (*File, error) { 44 | if iter.pos >= len(iter.series) { 45 | return nil, io.EOF 46 | } 47 | 48 | obj := iter.series[iter.pos] 49 | iter.pos++ 50 | return &obj, nil 51 | } 52 | 53 | func (iter *sliceFileIter) ForEach(fn func(*File) error) error { 54 | defer iter.Close() 55 | for _, r := range iter.series { 56 | if err := fn(&r); err != nil { 57 | if err == ErrStop { 58 | return nil 59 | } 60 | 61 | return err 62 | } 63 | } 64 | return nil 65 | } 66 | 67 | func (iter *sliceFileIter) Close() { 68 | iter.pos = len(iter.series) 69 | } 70 | -------------------------------------------------------------------------------- /repos/repos.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package repos 16 | 17 | import ( 18 | "context" 19 | "crypto/sha256" 20 | "fmt" 21 | ) 22 | 23 | // RepoList describes a struct that tracks repositories 24 | type RepoList interface { 25 | // UpdateTrackedRepos, updates the list of trakced repositories, returns 26 | // true if the list changed, false otherwise 27 | UpdateTrackedRepos(context.Context) (bool, error) 28 | GetTrackedRepos() []TrackedRepository 29 | } 30 | 31 | // TrackedRepository represents a repository tracked by Maintner or Samplr 32 | type TrackedRepository struct { 33 | Owner string `json:"owner"` 34 | Name string `json:"name"` 35 | DefaultBranch string `json:"defaultBranch"` 36 | IsTrackingIssues bool `json:"isTrackingIssues"` 37 | IsTrackingSamples bool `json:"isTrackingSamples"` 38 | } 39 | 40 | // RepoSha Creates a Sum224 of the TrackedRepository's name 41 | func (t TrackedRepository) RepoSha() string { 42 | sh := sha256.Sum224([]byte(fmt.Sprintf("%v/%v", t.Owner, t.Name))) 43 | return fmt.Sprintf("%x", sh) 44 | } 45 | 46 | // String returns the string representation of the TrackedRepository 47 | func (t TrackedRepository) String() string { 48 | return fmt.Sprintf("%v/%v", t.Owner, t.Name) 49 | } 50 | -------------------------------------------------------------------------------- /samplr/samplrctl/output/structured.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package output 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "io" 21 | "io/ioutil" 22 | "log" 23 | "strings" 24 | ) 25 | 26 | type structuredWriter struct { 27 | writer io.Writer 28 | } 29 | 30 | func (s *structuredWriter) Write(p []byte) (int, error) { 31 | return ioutil.Discard.Write(p) 32 | } 33 | 34 | func structured(w io.Writer) (io.Writer, bool) { 35 | if sw, ok := w.(*structuredWriter); ok { 36 | return sw.writer, true 37 | } 38 | return nil, false 39 | } 40 | 41 | func printStructured(w io.Writer, object interface{}) { 42 | j, err := json.MarshalIndent(object, "", " ") 43 | if err != nil { 44 | log.Fatalf("Failed to marshal output to JSON: %v", err) 45 | } 46 | fmt.Fprintln(w, string(j)) 47 | } 48 | 49 | func keysToCamelCase(m map[string]string) map[string]string { 50 | c := make(map[string]string, len(m)) 51 | for k, v := range m { 52 | c[toCamelCase(k)] = v 53 | } 54 | return c 55 | } 56 | 57 | func toCamelCase(s string) string { 58 | var b strings.Builder 59 | for _, w := range strings.Split(strings.ToLower(s), " ") { 60 | if len(w) == 0 { 61 | continue 62 | } 63 | if b.Len() == 0 { 64 | fmt.Fprint(&b, w) 65 | continue 66 | } 67 | fmt.Fprint(&b, strings.ToUpper(w[:1])+w[1:]) 68 | } 69 | return b.String() 70 | } 71 | -------------------------------------------------------------------------------- /samplr/samplrctl/snippets/snippets.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snippets 16 | 17 | import ( 18 | "io" 19 | "strconv" 20 | 21 | "github.com/GoogleCloudPlatform/devrel-services/samplr" 22 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/output" 23 | ) 24 | 25 | const ( 26 | nameName = "Name" 27 | languageName = "Language" 28 | versionsName = "Versions" 29 | isDeletedName = "Is Deleted" 30 | ) 31 | 32 | func snippetToMap(s *samplr.Snippet) map[string]string { 33 | return map[string]string{ 34 | nameName: s.Name, 35 | languageName: s.Language, 36 | versionsName: strconv.FormatInt(int64(len(s.Versions)), 10), 37 | isDeletedName: strconv.FormatBool(len(s.Primary.Content) == 0), 38 | } 39 | } 40 | 41 | // OutputSnippet writes a representation of the snippet to the given io.Writer 42 | func OutputSnippet(w io.Writer, s *samplr.Snippet) { 43 | output.PrintAllMap(w, snippetToMap(s)) 44 | } 45 | 46 | // OutputSnippets writes a representation of the given snippets to the io.Writer 47 | func OutputSnippets(w io.Writer, l []*samplr.Snippet) { 48 | mapList := make([]map[string]string, len(l)) 49 | for i, p := range l { 50 | mapList[i] = snippetToMap(p) 51 | } 52 | fields := []string{ 53 | nameName, 54 | languageName, 55 | versionsName, 56 | isDeletedName, 57 | } 58 | output.PrintList(w, fields, mapList) 59 | } 60 | -------------------------------------------------------------------------------- /samplr/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/GoogleCloudPlatform/devrel-services/samplr 2 | 3 | require ( 4 | cloud.google.com/go v0.61.0 5 | github.com/GoogleCloudPlatform/devrel-services/drghs v0.0.0 6 | github.com/GoogleCloudPlatform/devrel-services/git-go v0.0.0 7 | github.com/GoogleCloudPlatform/devrel-services/repos v0.0.0 8 | github.com/GoogleCloudPlatform/devrel-services/rtr v0.0.0 // indirect 9 | github.com/GoogleCloudPlatform/devrel-services/sprvsr v0.0.0 10 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.12.0 // indirect 11 | github.com/cespare/trie v0.0.0-20150610204604-3fe1a95cbba9 // indirect 12 | github.com/golang/protobuf v1.4.2 13 | github.com/google/cel-go v0.3.0 14 | github.com/google/go-cmp v0.5.2 15 | github.com/googleapis/gax-go v2.0.2+incompatible // indirect 16 | github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 // indirect 17 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 18 | github.com/sirupsen/logrus v1.4.2 19 | github.com/spf13/cobra v0.0.3 20 | github.com/spf13/pflag v1.0.3 21 | github.com/toqueteos/trie v0.0.0-20150530104557-56fed4a05683 // indirect 22 | go.opentelemetry.io/otel/sdk v0.13.0 // indirect 23 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 24 | google.golang.org/grpc v1.32.0 25 | gopkg.in/src-d/enry.v1 v1.6.7 26 | gopkg.in/toqueteos/substring.v1 v1.0.2 // indirect 27 | gopkg.in/yaml.v2 v2.2.2 28 | k8s.io/api v0.0.0-20190528154508-67ef80593b24 29 | k8s.io/apimachinery v0.0.0-20190528154326-e59c2fb0a8e5 30 | k8s.io/client-go v0.0.0-20190528154735-79226fe1949a 31 | ) 32 | 33 | replace github.com/GoogleCloudPlatform/devrel-services/drghs => ../drghs 34 | 35 | replace github.com/GoogleCloudPlatform/devrel-services/git-go => ./git-go 36 | 37 | replace github.com/GoogleCloudPlatform/devrel-services/repos => ../repos 38 | 39 | replace github.com/GoogleCloudPlatform/devrel-services/rtr => ../rtr 40 | 41 | replace github.com/GoogleCloudPlatform/devrel-services/sprvsr => ../sprvsr 42 | 43 | go 1.13 44 | -------------------------------------------------------------------------------- /drghs/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | .PHONY: all 16 | all: help 17 | 18 | .PHONY: build 19 | build: ## Generates the go protobuf files 20 | if [ -d "googleapis" ]; then \ 21 | rm -rf googleapis; \ 22 | fi; 23 | git clone --single-branch https://github.com/googleapis/googleapis googleapis 24 | protoc --go_out=plugins=grpc:v1/ --proto_path=v1/ --proto_path=googleapis/ --include_imports --descriptor_set_out=v1/api_descriptor.pb v1/*.proto; 25 | 26 | .PHONY: dev-deploy 27 | dev-deploy: check-project ## Deploys the endpoints configuration to dev 28 | gcloud builds submit \ 29 | --config=endpoints/cloudbuild.yaml \ 30 | --timeout=30m \ 31 | --substitutions _TYPE=DEV \ 32 | . 33 | 34 | .PHONY: prod-deploy 35 | prod-deploy: check-project check-deploy ## Deploys the endpoints configuration to prod. 36 | # do dangerous stuff 37 | gcloud builds submit \ 38 | --config=endpoints/cloudbuild.yaml \ 39 | --timeout=30m \ 40 | --substitutions _TYPE=PROD \ 41 | . 42 | 43 | .PHONY: check-deploy 44 | check-deploy: 45 | @echo -n "Are you sure? [yN] " && read ans && [ $$ans == y ] 46 | 47 | .PHONY: check-project 48 | check-project: 49 | @echo "Active project is: $$(gcloud config list --format 'value(core.project)')" 50 | @echo -n "Are you sure? [yN] " && read ans && [ $$ans == y ] 51 | 52 | .PHONY: help 53 | help: ## Prints help message 54 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 55 | -------------------------------------------------------------------------------- /drghs-worker/maintnerd/api/internalapi/internal_api.go: -------------------------------------------------------------------------------- 1 | package internalapi 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | maintner_internal "github.com/GoogleCloudPlatform/devrel-services/drghs-worker/internal" 9 | "golang.org/x/build/maintner" 10 | ) 11 | 12 | // TransferProxyServer implements maintner_internal.InternalIssueServiceServer 13 | type TransferProxyServer struct { 14 | c *maintner.Corpus 15 | } 16 | 17 | var _ maintner_internal.InternalIssueServiceServer = &TransferProxyServer{} 18 | 19 | // NewTransferProxyServer builds and returns a TransferProxyServer 20 | func NewTransferProxyServer(c *maintner.Corpus) *TransferProxyServer { 21 | return &TransferProxyServer{ 22 | c: c, 23 | } 24 | } 25 | 26 | // TombstoneIssues tombstones the requested issues that are in the corpus 27 | func (s *TransferProxyServer) TombstoneIssues(ctx context.Context, r *maintner_internal.TombstoneIssuesRequest) (*maintner_internal.TombstoneIssuesResponse, error) { 28 | var ntombstoned int32 29 | 30 | err := s.c.GitHub().ForeachRepo(func(repo *maintner.GitHubRepo) error { 31 | repoID := getRepoPath(repo) 32 | if !strings.HasPrefix(r.Parent, repoID) { 33 | // Not our repository... ignore 34 | fmt.Printf("Repo: %v not equal to parent: %v\n", repoID, r.Parent) 35 | return nil 36 | } 37 | 38 | for _, iss := range r.IssueNumbers { 39 | fmt.Printf("for repository %v requested to tombstone issue: %v", r.Parent, iss) 40 | // go through the corpus, find this repo, find the issue, tombstone it 41 | 42 | issue := repo.Issue(iss) 43 | if issue == nil { 44 | return fmt.Errorf("Issue: %v not found", iss) 45 | } 46 | err := repo.MarkTombstoned(issue) 47 | if err != nil { 48 | return err 49 | } 50 | ntombstoned++ 51 | } 52 | return nil 53 | }) 54 | 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | return &maintner_internal.TombstoneIssuesResponse{ 60 | TombstonedCount: ntombstoned, 61 | }, nil 62 | } 63 | 64 | func getRepoPath(ta *maintner.GitHubRepo) string { 65 | return fmt.Sprintf("%v/%v", ta.ID().Owner, ta.ID().Repo) 66 | } 67 | -------------------------------------------------------------------------------- /drghs-worker/pkg/googlers/googlers_static.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package googlers 16 | 17 | import "strings" 18 | 19 | // Static stores a list of Googlers in a hard-coded list 20 | type Static struct { 21 | // map from github login to ldap. 22 | googlers map[string]bool 23 | } 24 | 25 | // IsGoogler checks if the given username is a Googler or not. 26 | func (s *Static) IsGoogler(user string) bool { 27 | _, ok := s.googlers[user] 28 | return ok 29 | } 30 | 31 | // Update indicates the repository to update 32 | func (s *Static) Update() { 33 | return 34 | } 35 | 36 | // NewStatic instantiates and returns a new Static struct 37 | func NewStatic() *Static { 38 | googlers := make(map[string]bool) 39 | lines := strings.Split(users, "\n") 40 | for _, line := range lines { 41 | googlers[line] = true 42 | } 43 | return &Static{ 44 | googlers: googlers, 45 | } 46 | } 47 | 48 | const users = ` 49 | surferjeffatgoogle 50 | pfritzsche 51 | michaelawyu 52 | jadekler 53 | tmatsuo 54 | jba 55 | andrewsg 56 | jonparrott 57 | tbpg 58 | bshaffer 59 | dzlier-gcp 60 | ace-n 61 | jabubake 62 | kurtisvg 63 | jsimonweb 64 | ryanmats 65 | frankyn 66 | broady 67 | lesv 68 | afitz0 69 | sanche21 70 | simonz130 71 | djmailhot 72 | ahmetb 73 | jmdobry 74 | annie29 75 | sgreenberg 76 | davidcavazos 77 | nkashy1 78 | dizcology 79 | alixhami 80 | happyhuman 81 | sirtorry 82 | gguuss 83 | remi 84 | tswast 85 | elibixby 86 | nnegrey 87 | puneith 88 | oalami 89 | jeffmendoza 90 | rakyll 91 | jerjou 92 | p-buse 93 | waprin 94 | stephenplusplus 95 | ` 96 | -------------------------------------------------------------------------------- /samplr/samplrctl/snippetversions/snippet_versions.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snippetversions 16 | 17 | import ( 18 | "io" 19 | "strconv" 20 | "strings" 21 | 22 | "github.com/GoogleCloudPlatform/devrel-services/samplr" 23 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/output" 24 | ) 25 | 26 | const ( 27 | nameName = "Name" 28 | fileName = "File" 29 | linesName = "Lines" 30 | contentName = "Content" 31 | sizeName = "Size" 32 | shaName = "SHA" 33 | ) 34 | 35 | func snippetVersionToMap(s samplr.SnippetVersion) map[string]string { 36 | return map[string]string{ 37 | nameName: s.Name, 38 | fileName: s.File.FilePath, 39 | linesName: strings.Join(s.Lines, ","), 40 | contentName: s.Content, 41 | sizeName: strconv.FormatInt(int64(len(s.Content)), 10), 42 | shaName: s.File.GitCommit.Hash, 43 | } 44 | } 45 | 46 | // OutputSnippetVersion writes a representation of the SnippetVersion to the io.Writer 47 | func OutputSnippetVersion(w io.Writer, s samplr.SnippetVersion) { 48 | output.PrintAllMap(w, snippetVersionToMap(s)) 49 | } 50 | 51 | // OutputSnippetVersions writes a representation of the SnippetVersions to the io.Writer 52 | func OutputSnippetVersions(w io.Writer, l []samplr.SnippetVersion) { 53 | mapList := make([]map[string]string, len(l)) 54 | for i, p := range l { 55 | mapList[i] = snippetVersionToMap(p) 56 | } 57 | fields := []string{ 58 | nameName, 59 | fileName, 60 | linesName, 61 | sizeName, 62 | shaName, 63 | } 64 | output.PrintList(w, fields, mapList) 65 | } 66 | -------------------------------------------------------------------------------- /drghs-worker/utils/serilization_utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "encoding/json" 19 | "github.com/GoogleCloudPlatform/devrel-services/drghs-worker/pkg/utils" 20 | "testing" 21 | ) 22 | 23 | func TestUnmarshalBool(t *testing.T) { 24 | tests := []struct { 25 | message json.RawMessage 26 | expected bool 27 | isNil bool 28 | }{ 29 | { 30 | message: nil, 31 | expected: false, 32 | isNil: true, 33 | }, 34 | { 35 | message: []byte("null"), 36 | expected: false, 37 | isNil: true, 38 | }, 39 | { 40 | message: []byte("false"), 41 | expected: false, 42 | isNil: false, 43 | }, 44 | { 45 | message: []byte("TRUE"), 46 | expected: true, 47 | isNil: false, 48 | }, 49 | { 50 | message: []byte("true"), 51 | expected: true, 52 | isNil: false, 53 | }, 54 | { 55 | message: []byte("Random"), 56 | expected: false, 57 | isNil: false, 58 | }, 59 | } 60 | for _, test := range tests { 61 | val := utils.UnmarshalBool(test.message) 62 | 63 | if test.isNil && val != nil { 64 | t.Errorf("Unmarshal Bool for %v returned an unexpected value. Got %v, want nil", string(test.message), val) 65 | } 66 | 67 | if !test.isNil && val == nil { 68 | t.Errorf("Unmarshal Bool for %v returned an unexpected nil. Got nil, want %v", string(test.message), test.expected) 69 | } 70 | 71 | if val != nil && test.expected != *val { 72 | t.Errorf("Unmarshal Bool for %v returned an unexpected value. Got %v, want %v", string(test.message), *val, test.expected) 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /drghs-worker/maintnerd/api/v1beta1/page_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package v1beta1 16 | 17 | import ( 18 | b64 "encoding/base64" 19 | "errors" 20 | "time" 21 | 22 | drghs_v1 "github.com/GoogleCloudPlatform/devrel-services/drghs/v1" 23 | 24 | "github.com/golang/protobuf/proto" 25 | "github.com/golang/protobuf/ptypes" 26 | ) 27 | 28 | var ( 29 | // ErrNilPageToken is returned when a nil page token is passed 30 | ErrNilPageToken = errors.New("nil PageToken") 31 | ) 32 | 33 | func decodePageToken(req string) (*drghs_v1.PageToken, error) { 34 | pageToken := &drghs_v1.PageToken{} 35 | decstr, err := b64.StdEncoding.DecodeString(req) 36 | err = pageToken.XXX_Unmarshal(decstr) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return pageToken, nil 41 | } 42 | 43 | func makeFirstPageToken(t time.Time, idx int) (string, error) { 44 | tsp, err := ptypes.TimestampProto(t) 45 | if err != nil { 46 | return "", err 47 | } 48 | return makeNextPageToken(&drghs_v1.PageToken{ 49 | FirstRequestTimeUsec: tsp, 50 | Offset: int32(idx), 51 | }, idx) 52 | } 53 | 54 | func makeNextPageToken(prev *drghs_v1.PageToken, idx int) (string, error) { 55 | nextPageTokenStr := "" 56 | if prev == nil { 57 | return "", ErrNilPageToken 58 | } 59 | if idx > 0 { 60 | prev.Offset = int32(idx) 61 | pagetokenbytes, err := proto.Marshal(prev) 62 | if err != nil { 63 | return "", err 64 | } 65 | nextPageTokenStr = b64.StdEncoding.EncodeToString(pagetokenbytes) 66 | } 67 | return nextPageTokenStr, nil 68 | } 69 | 70 | func getPageSize(reqPageSize int) int { 71 | pagesize := 500 72 | if 0 < reqPageSize && reqPageSize < 500 { 73 | pagesize = reqPageSize 74 | } 75 | return pagesize 76 | } 77 | -------------------------------------------------------------------------------- /samplr/samplrctl/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "os" 20 | 21 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/cmd" 22 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/cmd/completion" 23 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/cmd/snippets" 24 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/cmd/snippetversions" 25 | 26 | "github.com/sirupsen/logrus" 27 | "github.com/spf13/cobra" 28 | ) 29 | 30 | var log *logrus.Logger 31 | 32 | func init() { 33 | log = logrus.New() 34 | log.Level = logrus.DebugLevel 35 | log.Out = os.Stdout 36 | } 37 | 38 | func main() { 39 | ctx := context.Background() 40 | 41 | cmd := Command(ctx, "samplrctl") 42 | if err := cmd.Execute(); err != nil { 43 | log.Fatal(err) 44 | } 45 | } 46 | 47 | // Command returns a *cobra.Command setup with the common set of commands 48 | // and configuration already done. 49 | func Command(ctx context.Context, name string) *cobra.Command { 50 | cmd := &cobra.Command{ 51 | Use: name, 52 | Short: "Command line interface for samplr", 53 | SilenceUsage: false, 54 | SilenceErrors: false, 55 | Args: cobra.MinimumNArgs(1), 56 | } 57 | cmd.SetOutput(os.Stdout) 58 | 59 | commands.AddFlags(cmd.PersistentFlags()) 60 | 61 | completion.AddCommand(ctx, cmd) 62 | snippets.AddCommand(ctx, cmd) 63 | snippetversions.AddCommand(ctx, cmd) 64 | 65 | updateHelpFlag(cmd) 66 | 67 | return cmd 68 | } 69 | 70 | func updateHelpFlag(cmd *cobra.Command) { 71 | cmd.Flags().BoolP("help", "h", false, "Help for "+cmd.Name()) 72 | for _, c := range cmd.Commands() { 73 | updateHelpFlag(c) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /samplr/samplrd/samplrapi/page_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package samplrapi 16 | 17 | import ( 18 | b64 "encoding/base64" 19 | "errors" 20 | drghs_v1 "github.com/GoogleCloudPlatform/devrel-services/drghs/v1" 21 | "time" 22 | 23 | "github.com/golang/protobuf/proto" 24 | "github.com/golang/protobuf/ptypes" 25 | ) 26 | 27 | var ( 28 | // ErrNilPageToken is returned when a PageToken is nil 29 | ErrNilPageToken = errors.New("nil PageToken") 30 | ) 31 | 32 | func decodePageToken(req string) (*drghs_v1.PageToken, error) { 33 | pageToken := &drghs_v1.PageToken{} 34 | decstr, err := b64.StdEncoding.DecodeString(req) 35 | err = pageToken.XXX_Unmarshal(decstr) 36 | if err != nil { 37 | return nil, err 38 | } 39 | return pageToken, nil 40 | } 41 | 42 | func makeFirstPageToken(t time.Time, idx int) (string, error) { 43 | tsp, err := ptypes.TimestampProto(t) 44 | if err != nil { 45 | log.Errorf("Could not make timestamp %v", err) 46 | return "", err 47 | } 48 | return makeNextPageToken(&drghs_v1.PageToken{ 49 | FirstRequestTimeUsec: tsp, 50 | Offset: int32(idx), 51 | }, idx) 52 | } 53 | 54 | func makeNextPageToken(prev *drghs_v1.PageToken, idx int) (string, error) { 55 | nextPageTokenStr := "" 56 | if prev == nil { 57 | return "", ErrNilPageToken 58 | } 59 | if idx > 0 { 60 | prev.Offset = int32(idx) 61 | pagetokenbytes, err := proto.Marshal(prev) 62 | if err != nil { 63 | return "", err 64 | } 65 | nextPageTokenStr = b64.StdEncoding.EncodeToString(pagetokenbytes) 66 | } 67 | return nextPageTokenStr, nil 68 | } 69 | 70 | func getPageSize(reqPageSize int) int { 71 | pagesize := 100 72 | if 0 < reqPageSize && reqPageSize < 100 { 73 | pagesize = reqPageSize 74 | } 75 | return pagesize 76 | } 77 | -------------------------------------------------------------------------------- /drghs/endpoints/models/status.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Status: 16 | type: object 17 | properties: 18 | RuleID: 19 | type: integer 20 | SLOID: 21 | type: integer 22 | Repo: 23 | type: string 24 | Priority: 25 | $ref: "priority.yaml#/Priority" 26 | Type: 27 | $ref: "type.yaml#/Type" 28 | PriorityUnknown: 29 | type: boolean 30 | Labels: 31 | type: array 32 | items: 33 | type: string 34 | LastGooglerUpdate: 35 | type: string 36 | format: date-time 37 | LastUserUpdate: 38 | type: string 39 | format: date-time 40 | Created: 41 | type: string 42 | format: date-time 43 | PullRequest: 44 | type: boolean 45 | Approved: 46 | type: boolean 47 | Closed: 48 | type: boolean 49 | ClosedBy: 50 | $ref: "git_hub_user.yaml#/GitHubUser" 51 | Blocked: 52 | type: boolean 53 | ReleaseBlocking: 54 | type: boolean 55 | Body: 56 | type: boolean 57 | Commit: 58 | type: string 59 | UpdateCompliance: 60 | $ref: "compliance_response.yaml#/ComplianceResponse" 61 | ResolutionCompliance: 62 | $ref: "compliance_response.yaml#/ComplianceResponse" 63 | ComplianceUpdates: 64 | $ref: "compliance.yaml#/Compliance" 65 | ComplianceResolution: 66 | $ref: "compliance.yaml#/Compliance" 67 | IssueID: 68 | type: integer 69 | URL: 70 | type: string 71 | Assignees: 72 | type: array 73 | items: 74 | $ref: "git_hub_user.yaml#/GitHubUser" 75 | Reporter: 76 | $ref: "git_hub_user.yaml#/GitHubUser" 77 | Title: 78 | type: string 79 | Comments: 80 | type: array 81 | items: 82 | $ref: "git_hub_comment.yaml#/GitHubComment" 83 | -------------------------------------------------------------------------------- /samplr/samplrctl/cmd/completion/completion.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package completion provides shell completion capabilities for the CLI. 16 | package completion 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "fmt" 22 | "io" 23 | 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | var ( 28 | errUnspecifiedShell = errors.New("Shell not specified") 29 | errTooManyArgs = errors.New("Too many arguments. Expected only the shell type") 30 | completionShells = map[string]func(w io.Writer, cmd *cobra.Command) error{ 31 | "bash": runCompletionBash, 32 | } 33 | ) 34 | 35 | type unsupportedShellErr struct { 36 | shellName string 37 | } 38 | 39 | func (e *unsupportedShellErr) Error() string { 40 | return fmt.Sprintf("Unsupported shell type %q", e.shellName) 41 | } 42 | 43 | // AddCommand adds the completion sub-command to the passed in root command. 44 | func AddCommand(ctx context.Context, root *cobra.Command) { 45 | shells := []string{} 46 | for s := range completionShells { 47 | shells = append(shells, s) 48 | } 49 | 50 | completion := &cobra.Command{ 51 | Use: "completion", 52 | Short: "Generate shell completion", 53 | Long: `Generate shell completion.`, 54 | ValidArgs: shells, 55 | RunE: runCompletion, 56 | } 57 | 58 | root.AddCommand(completion) 59 | } 60 | 61 | func runCompletion(c *cobra.Command, args []string) error { 62 | if len(args) == 0 { 63 | return errUnspecifiedShell 64 | } 65 | if len(args) > 1 { 66 | return errTooManyArgs 67 | } 68 | gen, found := completionShells[args[0]] 69 | if !found { 70 | return &unsupportedShellErr{shellName: args[0]} 71 | } 72 | return gen(c.OutOrStdout(), c) 73 | } 74 | 75 | func runCompletionBash(w io.Writer, cmd *cobra.Command) error { 76 | return cmd.Root().GenBashCompletion(w) 77 | } 78 | -------------------------------------------------------------------------------- /drghs-worker/maintnerd/api/v1beta1/repos_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package v1beta1 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | "time" 22 | 23 | drghs_v1 "github.com/GoogleCloudPlatform/devrel-services/drghs/v1" 24 | ) 25 | 26 | type repoPage struct { 27 | iss []*drghs_v1.Repository 28 | idx int 29 | } 30 | 31 | type repoPaginator struct { 32 | set map[time.Time]repoPage 33 | mu sync.Mutex 34 | } 35 | 36 | func (p *repoPaginator) PurgeOldRecords() { 37 | p.mu.Lock() 38 | defer p.mu.Unlock() 39 | now := time.Now() 40 | for t := range p.set { 41 | if now.Sub(t).Hours() > nHoursStale { 42 | delete(p.set, t) 43 | } 44 | } 45 | } 46 | 47 | func (p *repoPaginator) CreatePage(s []*drghs_v1.Repository) (time.Time, error) { 48 | p.mu.Lock() 49 | defer p.mu.Unlock() 50 | key := time.Now().UTC().Truncate(0) 51 | if _, ok := p.set[key]; ok { 52 | return time.Unix(0, 0), errors.New("Key already exists") 53 | } 54 | 55 | p.set[key] = repoPage{ 56 | iss: s, 57 | idx: 0, 58 | } 59 | return key, nil 60 | } 61 | 62 | func (p *repoPaginator) GetPage(key time.Time, n int) ([]*drghs_v1.Repository, int, error) { 63 | p.mu.Lock() 64 | defer p.mu.Unlock() 65 | 66 | key = key.UTC() 67 | if _, ok := p.set[key]; !ok { 68 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 69 | } 70 | val := p.set[key] 71 | 72 | nremain := len(val.iss) - val.idx 73 | 74 | if n > nremain { 75 | n = nremain 76 | } 77 | 78 | if n == 0 { 79 | return []*drghs_v1.Repository{}, -1, nil 80 | } 81 | 82 | retset := val.iss[val.idx:(val.idx + n)] 83 | val.idx = val.idx + n 84 | 85 | retidx := val.idx 86 | if val.idx == len(val.iss) { 87 | delete(p.set, key) 88 | retidx = -1 89 | } else { 90 | p.set[key] = val 91 | } 92 | 93 | return retset, retidx, nil 94 | } 95 | -------------------------------------------------------------------------------- /samplr/samplrctl/testutil/testutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package testutil provides common functionality for tests across the CLI. 16 | // 17 | // NOTE: this package is only for generic utilities that don't have dependencies on other packages 18 | // in the CLI. For utilities specific to another CLI package foo, create a foo/test package. 19 | package testutil 20 | 21 | import ( 22 | "regexp" 23 | "strings" 24 | ) 25 | 26 | // ContainsAll validates all provided substrs are contained in s. 27 | func ContainsAll(s string, substrs ...string) bool { 28 | for _, substr := range substrs { 29 | if !strings.Contains(s, substr) { 30 | return false 31 | } 32 | } 33 | return true 34 | } 35 | 36 | // ContainsAny validates any of the provided substrs is contained in s. 37 | func ContainsAny(s string, substrs ...string) bool { 38 | for _, substr := range substrs { 39 | if strings.Contains(s, substr) { 40 | return true 41 | } 42 | } 43 | return false 44 | } 45 | 46 | // ContainsRow validates that all provided fields appear, in order, in a tab-separated row in s. 47 | func ContainsRow(s string, fields ...string) (bool, error) { 48 | pattern := "(?m:^[\t ]*" 49 | for i, field := range fields { 50 | if i != 0 { 51 | pattern += `[\t ]+` 52 | } 53 | pattern += regexp.QuoteMeta(field) 54 | } 55 | pattern += "[\t ]*$)" 56 | r, err := regexp.Compile(pattern) 57 | if err != nil { 58 | return false, err 59 | } 60 | 61 | return r.FindString(s) != "", nil 62 | } 63 | 64 | // AnyLineContainsAll validates that some line of the output contains all the provided substrings. 65 | func AnyLineContainsAll(output string, substrs ...string) bool { 66 | for _, line := range strings.Split(output, "\n") { 67 | if ContainsAll(line, substrs...) { 68 | return true 69 | } 70 | } 71 | return false 72 | } 73 | -------------------------------------------------------------------------------- /drghs-worker/maintnerd/api/v1beta1/issues_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package v1beta1 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | "time" 22 | 23 | drghs_v1 "github.com/GoogleCloudPlatform/devrel-services/drghs/v1" 24 | ) 25 | 26 | type issuePage struct { 27 | iss []*drghs_v1.Issue 28 | idx int 29 | } 30 | 31 | type issuePaginator struct { 32 | set map[time.Time]issuePage 33 | mu sync.Mutex 34 | } 35 | 36 | const nHoursStale = 2 37 | 38 | func (p *issuePaginator) PurgeOldRecords() { 39 | p.mu.Lock() 40 | defer p.mu.Unlock() 41 | now := time.Now() 42 | for t := range p.set { 43 | if now.Sub(t).Hours() > nHoursStale { 44 | delete(p.set, t) 45 | } 46 | } 47 | } 48 | 49 | func (p *issuePaginator) CreatePage(s []*drghs_v1.Issue) (time.Time, error) { 50 | p.mu.Lock() 51 | defer p.mu.Unlock() 52 | key := time.Now().UTC().Truncate(0) 53 | if _, ok := p.set[key]; ok { 54 | return time.Unix(0, 0), errors.New("Key already exists") 55 | } 56 | 57 | p.set[key] = issuePage{ 58 | iss: s, 59 | idx: 0, 60 | } 61 | return key, nil 62 | } 63 | 64 | func (p *issuePaginator) GetPage(key time.Time, n int) ([]*drghs_v1.Issue, int, error) { 65 | p.mu.Lock() 66 | defer p.mu.Unlock() 67 | 68 | key = key.UTC() 69 | if _, ok := p.set[key]; !ok { 70 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 71 | } 72 | val := p.set[key] 73 | 74 | nremain := len(val.iss) - val.idx 75 | 76 | if n > nremain { 77 | n = nremain 78 | } 79 | 80 | if n == 0 { 81 | return []*drghs_v1.Issue{}, -1, nil 82 | } 83 | 84 | retset := val.iss[val.idx:(val.idx + n)] 85 | val.idx = val.idx + n 86 | 87 | retidx := val.idx 88 | if val.idx == len(val.iss) { 89 | delete(p.set, key) 90 | retidx = -1 91 | } else { 92 | p.set[key] = val 93 | } 94 | 95 | return retset, retidx, nil 96 | } 97 | -------------------------------------------------------------------------------- /drghs-worker/maintnerd/api/v1beta1/comment_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package v1beta1 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | "time" 22 | 23 | drghs_v1 "github.com/GoogleCloudPlatform/devrel-services/drghs/v1" 24 | ) 25 | 26 | type commentPage struct { 27 | iss []*drghs_v1.GitHubComment 28 | idx int 29 | } 30 | 31 | type commentPaginator struct { 32 | set map[time.Time]commentPage 33 | mu sync.Mutex 34 | } 35 | 36 | func (p *commentPaginator) PurgeOldRecords() { 37 | p.mu.Lock() 38 | defer p.mu.Unlock() 39 | now := time.Now() 40 | for t := range p.set { 41 | if now.Sub(t).Hours() > nHoursStale { 42 | delete(p.set, t) 43 | } 44 | } 45 | } 46 | 47 | func (p *commentPaginator) CreatePage(s []*drghs_v1.GitHubComment) (time.Time, error) { 48 | p.mu.Lock() 49 | defer p.mu.Unlock() 50 | key := time.Now().UTC().Truncate(0) 51 | if _, ok := p.set[key]; ok { 52 | return time.Unix(0, 0), errors.New("Key already exists") 53 | } 54 | 55 | p.set[key] = commentPage{ 56 | iss: s, 57 | idx: 0, 58 | } 59 | return key, nil 60 | } 61 | 62 | func (p *commentPaginator) GetPage(key time.Time, n int) ([]*drghs_v1.GitHubComment, int, error) { 63 | p.mu.Lock() 64 | defer p.mu.Unlock() 65 | 66 | key = key.UTC() 67 | if _, ok := p.set[key]; !ok { 68 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 69 | } 70 | val := p.set[key] 71 | 72 | nremain := len(val.iss) - val.idx 73 | 74 | if n > nremain { 75 | n = nremain 76 | } 77 | 78 | if n == 0 { 79 | return []*drghs_v1.GitHubComment{}, -1, nil 80 | } 81 | 82 | retset := val.iss[val.idx:(val.idx + n)] 83 | val.idx = val.idx + n 84 | 85 | retidx := val.idx 86 | if val.idx == len(val.iss) { 87 | delete(p.set, key) 88 | retidx = -1 89 | } else { 90 | p.set[key] = val 91 | } 92 | 93 | return retset, retidx, nil 94 | } 95 | -------------------------------------------------------------------------------- /samplr/samplrd/samplrapi/git_commit_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package samplrapi 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | "time" 22 | 23 | "github.com/GoogleCloudPlatform/devrel-services/samplr" 24 | ) 25 | 26 | type gitCommitPage struct { 27 | snps []*samplr.GitCommit 28 | idx int 29 | } 30 | 31 | type gitCommitPaginator struct { 32 | set map[time.Time]gitCommitPage 33 | mu sync.Mutex 34 | } 35 | 36 | func (p *gitCommitPaginator) PurgeOldRecords() { 37 | p.mu.Lock() 38 | defer p.mu.Unlock() 39 | now := time.Now() 40 | for t := range p.set { 41 | if now.Sub(t).Hours() > 2 { 42 | delete(p.set, t) 43 | } 44 | } 45 | } 46 | 47 | func (p *gitCommitPaginator) CreatePage(s []*samplr.GitCommit) (time.Time, error) { 48 | p.mu.Lock() 49 | defer p.mu.Unlock() 50 | key := time.Now().UTC().Truncate(0) 51 | if _, ok := p.set[key]; ok { 52 | return time.Unix(0, 0), errors.New("Key already exists") 53 | } 54 | 55 | p.set[key] = gitCommitPage{ 56 | snps: s, 57 | idx: 0, 58 | } 59 | return key, nil 60 | } 61 | 62 | func (p *gitCommitPaginator) GetPage(key time.Time, n int) ([]*samplr.GitCommit, int, error) { 63 | p.mu.Lock() 64 | defer p.mu.Unlock() 65 | 66 | key = key.UTC() 67 | if _, ok := p.set[key]; !ok { 68 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 69 | } 70 | val := p.set[key] 71 | 72 | nremain := len(val.snps) - val.idx 73 | log.Debugf("There are %v records remaining, requested %v", nremain, n) 74 | 75 | if n > nremain { 76 | n = nremain 77 | } 78 | 79 | if n == 0 { 80 | return nil, 0, errors.New("Get 0 from page") 81 | } 82 | 83 | retset := val.snps[val.idx:(val.idx + n)] 84 | val.idx = val.idx + n 85 | 86 | retidx := val.idx 87 | if val.idx == len(val.snps) { 88 | delete(p.set, key) 89 | retidx = -1 90 | } else { 91 | p.set[key] = val 92 | } 93 | 94 | return retset, retidx, nil 95 | } 96 | -------------------------------------------------------------------------------- /drghs-worker/maintner-swpr/maint_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "testing" 19 | 20 | drghs_v1 "github.com/GoogleCloudPlatform/devrel-services/drghs/v1" 21 | "github.com/GoogleCloudPlatform/devrel-services/repos" 22 | "github.com/google/go-cmp/cmp" 23 | "golang.org/x/time/rate" 24 | ) 25 | 26 | func TestRepoToTrackedRepo(t *testing.T) { 27 | cs := []struct { 28 | Repo *drghs_v1.Repository 29 | Want *repos.TrackedRepository 30 | Name string 31 | }{ 32 | { 33 | Name: "Expected", 34 | Repo: &drghs_v1.Repository{ 35 | Name: "GoogleCloudPlatform/devrel-services", 36 | }, 37 | Want: &repos.TrackedRepository{ 38 | Owner: "GoogleCloudPlatform", 39 | Name: "devrel-services", 40 | }, 41 | }, 42 | { 43 | Name: "Failure", 44 | Repo: &drghs_v1.Repository{ 45 | Name: "owners/GoogleCloudPlatform/repos/devrel-services", 46 | }, 47 | Want: nil, 48 | }, 49 | } 50 | for _, c := range cs { 51 | got := repoToTrackedRepo(c.Repo) 52 | if diff := cmp.Diff(c.Want, got); diff != "" { 53 | t.Errorf("Test: %v Repositories differ (-want +got)\n%s", c.Name, diff) 54 | } 55 | } 56 | } 57 | 58 | func TestBuildLimiter(t *testing.T) { 59 | cases := []struct { 60 | Name string 61 | NIssues int32 62 | WantLimit rate.Limit 63 | }{ 64 | { 65 | Name: "LowFrequency", 66 | NIssues: 864000, 67 | WantLimit: 0.1, 68 | }, 69 | { 70 | Name: "HighFrequency", 71 | NIssues: 86400000, 72 | WantLimit: 10, 73 | }, 74 | { 75 | Name: "EvenFrequency", 76 | NIssues: 8640000, 77 | WantLimit: 1, 78 | }, 79 | } 80 | for _, c := range cases { 81 | 82 | limiter := buildLimiter(c.NIssues) 83 | gotLimit := limiter.Limit() 84 | if gotLimit != c.WantLimit { 85 | t.Errorf("test: %v failed. limiter improperly set. got: %v want: %v", c.Name, gotLimit, c.WantLimit) 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /samplr/samplrd/samplrapi/snippet_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package samplrapi 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "github.com/GoogleCloudPlatform/devrel-services/samplr" 21 | "sync" 22 | "time" 23 | ) 24 | 25 | type snippetPage struct { 26 | snps []*samplr.Snippet 27 | idx int 28 | } 29 | 30 | type snippetPaginator struct { 31 | set map[time.Time]snippetPage 32 | mu sync.Mutex 33 | } 34 | 35 | func (p *snippetPaginator) PurgeOldRecords() { 36 | p.mu.Lock() 37 | defer p.mu.Unlock() 38 | now := time.Now() 39 | for t := range p.set { 40 | if now.Sub(t).Hours() > 2 { 41 | delete(p.set, t) 42 | } 43 | } 44 | } 45 | 46 | func (p *snippetPaginator) CreatePage(s []*samplr.Snippet) (time.Time, error) { 47 | p.mu.Lock() 48 | defer p.mu.Unlock() 49 | key := time.Now().UTC().Truncate(0) 50 | if _, ok := p.set[key]; ok { 51 | return time.Unix(0, 0), errors.New("Key already exists") 52 | } 53 | 54 | p.set[key] = snippetPage{ 55 | snps: s, 56 | idx: 0, 57 | } 58 | return key, nil 59 | } 60 | 61 | func (p *snippetPaginator) GetPage(key time.Time, n int) ([]*samplr.Snippet, int, error) { 62 | p.mu.Lock() 63 | defer p.mu.Unlock() 64 | 65 | key = key.UTC() 66 | if _, ok := p.set[key]; !ok { 67 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 68 | } 69 | val := p.set[key] 70 | 71 | nremain := len(val.snps) - val.idx 72 | log.Debugf("There are %v records remaining, requested %v", nremain, n) 73 | 74 | if n > nremain { 75 | n = nremain 76 | } 77 | 78 | if n == 0 { 79 | //return nil, 0, errors.New("Get 0 from page") 80 | return []*samplr.Snippet{}, -1, nil 81 | } 82 | 83 | retset := val.snps[val.idx:(val.idx + n)] 84 | val.idx = val.idx + n 85 | 86 | retidx := val.idx 87 | if val.idx == len(val.snps) { 88 | delete(p.set, key) 89 | retidx = -1 90 | } else { 91 | p.set[key] = val 92 | } 93 | 94 | return retset, retidx, nil 95 | } 96 | -------------------------------------------------------------------------------- /samplr/samplrd/samplrapi/tracked_repository_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package samplrapi 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | "time" 22 | ) 23 | 24 | type trackedRepositoryPage struct { 25 | trackedrepos []string 26 | idx int 27 | } 28 | 29 | type trackedRepositoryPaginator struct { 30 | set map[time.Time]trackedRepositoryPage 31 | mu sync.Mutex 32 | } 33 | 34 | func (p *trackedRepositoryPaginator) PurgeOldRecords() { 35 | p.mu.Lock() 36 | defer p.mu.Unlock() 37 | now := time.Now() 38 | for t := range p.set { 39 | if now.Sub(t).Hours() > 2 { 40 | delete(p.set, t) 41 | } 42 | } 43 | } 44 | 45 | func (p *trackedRepositoryPaginator) CreatePage(s []string) (time.Time, error) { 46 | p.mu.Lock() 47 | defer p.mu.Unlock() 48 | key := time.Now().UTC().Truncate(0) 49 | if _, ok := p.set[key]; ok { 50 | return time.Unix(0, 0), errors.New("Key already exists") 51 | } 52 | 53 | p.set[key] = trackedRepositoryPage{ 54 | trackedrepos: s, 55 | idx: 0, 56 | } 57 | return key, nil 58 | } 59 | 60 | func (p *trackedRepositoryPaginator) GetPage(key time.Time, n int) ([]string, int, error) { 61 | p.mu.Lock() 62 | defer p.mu.Unlock() 63 | 64 | key = key.UTC() 65 | if _, ok := p.set[key]; !ok { 66 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 67 | } 68 | val := p.set[key] 69 | 70 | nremain := len(val.trackedrepos) - val.idx 71 | log.Debugf("There are %v records remaining, requested %v", nremain, n) 72 | 73 | if n > nremain { 74 | n = nremain 75 | } 76 | 77 | if n == 0 { 78 | return nil, 0, errors.New("Get 0 from page") 79 | } 80 | 81 | retset := val.trackedrepos[val.idx:(val.idx + n)] 82 | val.idx = val.idx + n 83 | 84 | retidx := val.idx 85 | if val.idx == len(val.trackedrepos) { 86 | delete(p.set, key) 87 | retidx = -1 88 | } else { 89 | p.set[key] = val 90 | } 91 | 92 | return retset, retidx, nil 93 | } 94 | -------------------------------------------------------------------------------- /leif/githubservices/githubservices.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This package was modeled after the mocking strategy outlined at: 16 | // https://github.com/google/go-github/issues/113#issuecomment-46023864 17 | 18 | package githubservices 19 | 20 | import ( 21 | "context" 22 | "net/http" 23 | 24 | "github.com/google/go-github/github" 25 | ) 26 | 27 | // repoService is an interface defining the needed behaviour of the GitHub client 28 | // This way, the default client may be replaced for testing 29 | type repoService interface { 30 | Get(ctx context.Context, owner, repo string) (*github.Repository, *github.Response, error) 31 | GetContents(ctx context.Context, owner, repo, path string, opts *github.RepositoryContentGetOptions) (fileContent *github.RepositoryContent, directoryContent []*github.RepositoryContent, resp *github.Response, err error) 32 | ListByOrg(ctx context.Context, org string, opts *github.RepositoryListByOrgOptions) ([]*github.Repository, *github.Response, error) 33 | ListCollaborators(ctx context.Context, owner, repo string, opts *github.ListCollaboratorsOptions) ([]*github.User, *github.Response, error) 34 | } 35 | 36 | type userService interface { 37 | Get(ctx context.Context, user string) (*github.User, *github.Response, error) 38 | } 39 | 40 | // Client is a a wrapper around the GitHub client's RepositoriesService 41 | type Client struct { 42 | Repositories repoService 43 | Users userService 44 | } 45 | 46 | // NewClient creates a wrapper around the GitHub client's RepositoriesService 47 | // The RepositoriesService can be replaced for unit testing 48 | func NewClient(httpClient *http.Client, repoMock repoService, userMock userService) Client { 49 | if repoMock != nil || userMock != nil { 50 | return Client{ 51 | Repositories: repoMock, 52 | Users: userMock, 53 | } 54 | } 55 | client := github.NewClient(httpClient) 56 | 57 | return Client{ 58 | Repositories: client.Repositories, 59 | Users: client.Users, 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /samplr/samplrd/samplrapi/snippet_version_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package samplrapi 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | "time" 22 | 23 | "github.com/GoogleCloudPlatform/devrel-services/samplr" 24 | ) 25 | 26 | type snippetVersionPage struct { 27 | snps []samplr.SnippetVersion 28 | idx int 29 | } 30 | 31 | type snippetVersionPaginator struct { 32 | set map[time.Time]snippetVersionPage 33 | mu sync.Mutex 34 | } 35 | 36 | func (p *snippetVersionPaginator) PurgeOldRecords() { 37 | p.mu.Lock() 38 | defer p.mu.Unlock() 39 | now := time.Now() 40 | for t := range p.set { 41 | if now.Sub(t).Hours() > 2 { 42 | delete(p.set, t) 43 | } 44 | } 45 | } 46 | 47 | func (p *snippetVersionPaginator) CreatePage(s []samplr.SnippetVersion) (time.Time, error) { 48 | p.mu.Lock() 49 | defer p.mu.Unlock() 50 | key := time.Now().UTC().Truncate(0) 51 | if _, ok := p.set[key]; ok { 52 | return time.Unix(0, 0), errors.New("Key already exists") 53 | } 54 | 55 | p.set[key] = snippetVersionPage{ 56 | snps: s, 57 | idx: 0, 58 | } 59 | return key, nil 60 | } 61 | 62 | func (p *snippetVersionPaginator) GetPage(key time.Time, n int) ([]samplr.SnippetVersion, int, error) { 63 | p.mu.Lock() 64 | defer p.mu.Unlock() 65 | 66 | key = key.UTC() 67 | if _, ok := p.set[key]; !ok { 68 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 69 | } 70 | val := p.set[key] 71 | 72 | nremain := len(val.snps) - val.idx 73 | log.Debugf("There are %v records remaining, requested %v", nremain, n) 74 | 75 | if n > nremain { 76 | n = nremain 77 | } 78 | 79 | if n == 0 { 80 | //return nil, 0, errors.New("Get 0 from page") 81 | return []samplr.SnippetVersion{}, -1, nil 82 | } 83 | 84 | retset := val.snps[val.idx:(val.idx + n)] 85 | val.idx = val.idx + n 86 | 87 | retidx := val.idx 88 | if val.idx == len(val.snps) { 89 | delete(p.set, key) 90 | retidx = -1 91 | } else { 92 | p.set[key] = val 93 | } 94 | 95 | return retset, retidx, nil 96 | } 97 | -------------------------------------------------------------------------------- /samplr/samplrctl/output/output.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package output 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | "strings" 21 | "text/tabwriter" 22 | ) 23 | 24 | // Require a minimum of 2 spaces between columns. 25 | const padding = 2 26 | 27 | // PrintMap outputs a formatted object to the console. 28 | func PrintMap(w io.Writer, fields []string, m map[string]string) { 29 | if sw, ok := structured(w); ok { 30 | printStructured(sw, keysToCamelCase(m)) 31 | return 32 | } 33 | tw := tabwriter.NewWriter(w, 0, 0, padding, ' ', 0) 34 | for _, field := range fields { 35 | fmt.Fprintf(tw, "%v:\t%v\n", field, m[field]) 36 | } 37 | tw.Flush() 38 | } 39 | 40 | // PrintAllMap outputs a formatted object to the console. 41 | func PrintAllMap(w io.Writer, m map[string]string) { 42 | if sw, ok := structured(w); ok { 43 | printStructured(sw, keysToCamelCase(m)) 44 | return 45 | } 46 | tw := tabwriter.NewWriter(w, 0, 0, padding, ' ', 0) 47 | for field, val := range m { 48 | fmt.Fprintf(tw, "%v:\t%v\n", field, val) 49 | } 50 | tw.Flush() 51 | } 52 | 53 | // PrintList outputs a formatted list to the console. 54 | func PrintList(w io.Writer, fields []string, items []map[string]string) { 55 | if sw, ok := structured(w); ok { 56 | converted := make([]map[string]string, len(items)) 57 | for i, item := range items { 58 | converted[i] = keysToCamelCase(item) 59 | } 60 | printStructured(sw, converted) 61 | return 62 | } 63 | 64 | if len(items) == 0 { 65 | fmt.Fprintln(w, "no results") 66 | return 67 | } 68 | tw := tabwriter.NewWriter(w, 0, 0, padding, ' ', 0) 69 | 70 | for _, field := range fields { 71 | fmt.Fprintf(tw, "%v\t", field) 72 | } 73 | fmt.Fprintln(tw, "") 74 | 75 | for _, field := range fields { 76 | fmt.Fprintf(tw, "%v\t", strings.Repeat("=", len(field))) 77 | } 78 | fmt.Fprintln(tw, "") 79 | 80 | for _, m := range items { 81 | for _, f := range fields { 82 | fmt.Fprintf(tw, "%v\t", m[f]) 83 | } 84 | fmt.Fprintln(tw, "") 85 | } 86 | 87 | tw.Flush() 88 | } 89 | -------------------------------------------------------------------------------- /leif/leifd/leifapi/pagination/page_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package paginator 16 | 17 | import ( 18 | b64 "encoding/base64" 19 | "errors" 20 | "time" 21 | 22 | drghs_v1 "github.com/GoogleCloudPlatform/devrel-services/drghs/v1" 23 | 24 | "github.com/golang/protobuf/proto" 25 | "github.com/golang/protobuf/ptypes" 26 | ) 27 | 28 | var ( 29 | // ErrNilPageToken is returned when a PageToken is nil 30 | ErrNilPageToken = errors.New("nil PageToken") 31 | ) 32 | 33 | // DecodePageToken translates a token from a drghs_v1 api call to Go 34 | func DecodePageToken(req string) (*drghs_v1.PageToken, error) { 35 | pageToken := &drghs_v1.PageToken{} 36 | decstr, err := b64.StdEncoding.DecodeString(req) 37 | err = pageToken.XXX_Unmarshal(decstr) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return pageToken, nil 42 | } 43 | 44 | // MakeFirstPageToken creates a new string page token for the given key/time 45 | func MakeFirstPageToken(t time.Time, idx int) (string, error) { 46 | tsp, err := ptypes.TimestampProto(t) 47 | if err != nil { 48 | return "", err 49 | } 50 | return MakeNextPageToken(&drghs_v1.PageToken{ 51 | FirstRequestTimeUsec: tsp, 52 | Offset: int32(idx), 53 | }, idx) 54 | } 55 | 56 | // MakeNextPageToken creates a new string page token at the given index, based on prev 57 | func MakeNextPageToken(prev *drghs_v1.PageToken, idx int) (string, error) { 58 | nextPageTokenStr := "" 59 | if prev == nil { 60 | return "", ErrNilPageToken 61 | } 62 | if idx > 0 { 63 | prev.Offset = int32(idx) 64 | pagetokenbytes, err := proto.Marshal(prev) 65 | if err != nil { 66 | return "", err 67 | } 68 | nextPageTokenStr = b64.StdEncoding.EncodeToString(pagetokenbytes) 69 | } 70 | return nextPageTokenStr, nil 71 | } 72 | 73 | // GetPageSize returns the page size, setting a maximum page size of 100 74 | func GetPageSize(reqPageSize int) int { 75 | pagesize := 100 76 | if 0 < reqPageSize && reqPageSize < 100 { 77 | pagesize = reqPageSize 78 | } 79 | return pagesize 80 | } 81 | -------------------------------------------------------------------------------- /samplr/git-go/commit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | import ( 18 | "io" 19 | "time" 20 | ) 21 | 22 | // Commit stores data about a Git Commit 23 | type Commit struct { 24 | // Hash of the commit object. 25 | Hash Hash 26 | // Author is the original author of the commit. 27 | Author Signature 28 | // Committer is the one performing the commit, might be different from 29 | // Author. 30 | Committer Signature 31 | // Message is the commit message, contains arbitrary text. 32 | Message string 33 | files []File 34 | } 35 | 36 | // Files returns a FilesIter representing the Commit's files 37 | func (c Commit) Files() (FilesIter, error) { 38 | iter := &sliceFileIter{ 39 | pos: 0, 40 | series: c.files, 41 | } 42 | return iter, nil 43 | } 44 | 45 | // Signature is used to identify who and when created a commit or tag. 46 | type Signature struct { 47 | // Name represents a person name. It is an arbitrary string. 48 | Name string 49 | // Email is an email, but it cannot be assumed to be well-formed. 50 | Email string 51 | // When is the timestamp of the signature. 52 | When time.Time 53 | } 54 | 55 | // CommitIter is a generic closable interface for iterating over commits. 56 | type CommitIter interface { 57 | Next() (*Commit, error) 58 | ForEach(func(*Commit) error) error 59 | Close() 60 | } 61 | 62 | // sliceCommitIter iterates over an internaly held slice 63 | type sliceCommitIter struct { 64 | pos int 65 | series []*Commit 66 | } 67 | 68 | func (iter *sliceCommitIter) Next() (*Commit, error) { 69 | if iter.pos >= len(iter.series) { 70 | return nil, io.EOF 71 | } 72 | 73 | obj := iter.series[iter.pos] 74 | iter.pos++ 75 | return obj, nil 76 | } 77 | 78 | func (iter *sliceCommitIter) ForEach(fn func(*Commit) error) error { 79 | defer iter.Close() 80 | for _, r := range iter.series { 81 | if err := fn(r); err != nil { 82 | if err == ErrStop { 83 | return nil 84 | } 85 | 86 | return err 87 | } 88 | } 89 | return nil 90 | } 91 | 92 | func (iter *sliceCommitIter) Close() { 93 | iter.pos = len(iter.series) 94 | } 95 | -------------------------------------------------------------------------------- /samplr/samplrctl/cmd/snippetversions/snippet_versions.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snippetversions 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/GoogleCloudPlatform/devrel-services/samplr" 22 | commands "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/cmd" 23 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/snippetversions" 24 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/utils" 25 | 26 | "github.com/spf13/cobra" 27 | ) 28 | 29 | // AddCommand adds Snippet Version subcommands to the given command 30 | func AddCommand(ctx context.Context, cmd *cobra.Command) { 31 | 32 | snippetVersions := &cobra.Command{ 33 | Use: "versions", 34 | Short: "versions", 35 | Long: "versions", 36 | } 37 | 38 | snippetVersionsGet := &cobra.Command{ 39 | Use: "get", 40 | Short: "get", 41 | Long: "get", 42 | RunE: commands.CtxCommand(ctx, getSnippetVersion), 43 | Args: cobra.ExactArgs(3), 44 | } 45 | 46 | snippetVersionsList := &cobra.Command{ 47 | Use: "list", 48 | Short: "list", 49 | Long: "list", 50 | RunE: commands.CtxCommand(ctx, listSnippetVersions), 51 | Args: cobra.ExactArgs(2), 52 | } 53 | 54 | snippetVersions.AddCommand(snippetVersionsGet, snippetVersionsList) 55 | cmd.AddCommand(snippetVersions) 56 | } 57 | 58 | func getSnippetVersion(ctx context.Context, cmd *cobra.Command, args []string) error { 59 | return nil 60 | } 61 | 62 | func listSnippetVersions(ctx context.Context, cmd *cobra.Command, args []string) error { 63 | if len(args) != 2 { 64 | return fmt.Errorf("Need exactly 2 argumetns, got: %v", len(args)) 65 | } 66 | 67 | dir := args[0] 68 | sName := args[1] 69 | 70 | snps, err := utils.GetSnippets(ctx, dir) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | var snp *samplr.Snippet 76 | for _, s := range snps { 77 | if s.Name == sName { 78 | snp = s 79 | break 80 | } 81 | } 82 | 83 | if snp == nil { 84 | return fmt.Errorf("Could not find a snippet named: %v", sName) 85 | } 86 | 87 | snippetversions.OutputSnippetVersions(cmd.OutOrStderr(), snp.Versions) 88 | 89 | return nil 90 | } 91 | -------------------------------------------------------------------------------- /leif/repository.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This package was modeled after the mocking strategy outlined at: 16 | // https://github.com/google/go-github/issues/113#issuecomment-46023864 17 | 18 | package leif 19 | 20 | import ( 21 | "context" 22 | "time" 23 | 24 | "github.com/GoogleCloudPlatform/devrel-services/leif/githubservices" 25 | ) 26 | 27 | // Repository represents a GitHub repository and stores its SLO rules 28 | type Repository struct { 29 | name string 30 | ownerName string 31 | SLORules []*SLORule 32 | } 33 | 34 | // OwnerName returns the name of the repository's owner 35 | func (r *Repository) OwnerName() string { 36 | return r.ownerName 37 | } 38 | 39 | // RepoName returns the repository's name 40 | func (r *Repository) RepoName() string { 41 | return r.name 42 | } 43 | 44 | // Update reaches out to GitHub to update the SLO rules for the repository 45 | func (r *Repository) Update(ctx context.Context, owner Owner, ghClient *githubservices.Client) error { 46 | 47 | rules, err := findSLODoc(ctx, owner, r.name, ghClient) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | r.SLORules = rules 53 | return nil 54 | } 55 | 56 | // SLORule represents a service level objective (SLO) rule 57 | type SLORule struct { 58 | AppliesTo AppliesTo `json:"appliesTo"` 59 | ComplianceSettings ComplianceSettings `json:"complianceSettings"` 60 | } 61 | 62 | // AppliesTo stores structured data on which issues and/or pull requests a SLO applies to 63 | type AppliesTo struct { 64 | GitHubLabels []string `json:"gitHubLabels"` 65 | ExcludedGitHubLabels []string `json:"excludedGitHubLabels"` 66 | Issues bool `json:"issues"` 67 | PRs bool `json:"prs"` 68 | } 69 | 70 | // ComplianceSettings stores data on the requirements for an issue or pull request to be considered compliant with the SLO 71 | type ComplianceSettings struct { 72 | ResponseTime time.Duration `json:"responseTime"` 73 | ResolutionTime time.Duration `json:"resolutionTime"` 74 | RequiresAssignee bool `json:"requiresAssignee"` 75 | Responders []string `json:"responders"` 76 | } 77 | -------------------------------------------------------------------------------- /samplr/git-go/remote.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | ) 21 | 22 | const ( 23 | // RemoteOriginName is the name of the remote which is the "origin" of the repository 24 | RemoteOriginName = "origin" 25 | ) 26 | 27 | // RemoteConfig contains the configuration for a given remote repository. 28 | type RemoteConfig struct { 29 | // Name of the remote 30 | Name string 31 | // URLs the URLs of a remote repository. It must be non-empty. Fetch will 32 | // always use the first URL, while push will use all of them. 33 | URLs []string 34 | // Fetch the default set of "refspec" for fetch operation 35 | // Fetch []RefSpec 36 | } 37 | 38 | // Remote represents a connection to a remote repository. 39 | type Remote struct { 40 | c *RemoteConfig 41 | } 42 | 43 | // Config returns the RemoteConfig object used to instantiate this Remote. 44 | func (r *Remote) Config() *RemoteConfig { 45 | if r == nil { 46 | return nil 47 | } 48 | return r.c 49 | } 50 | 51 | func (r *Remote) String() string { 52 | var fetch, push string 53 | if len(r.c.URLs) > 0 { 54 | fetch = r.c.URLs[0] 55 | push = r.c.URLs[0] 56 | } 57 | 58 | return fmt.Sprintf("%s\t%s (fetch)\n%[1]s\t%[3]s (push)", r.c.Name, fetch, push) 59 | } 60 | 61 | // RemoteIter is a generic closable interface for iterating over Remotes. 62 | type RemoteIter interface { 63 | Next() (*Remote, error) 64 | ForEach(func(*Remote) error) error 65 | Close() 66 | } 67 | 68 | type sliceRemoteIter struct { 69 | pos int 70 | series []Remote 71 | } 72 | 73 | func (iter *sliceRemoteIter) Next() (*Remote, error) { 74 | if iter.pos >= len(iter.series) { 75 | return nil, io.EOF 76 | } 77 | 78 | obj := iter.series[iter.pos] 79 | iter.pos++ 80 | return &obj, nil 81 | } 82 | 83 | func (iter *sliceRemoteIter) Close() { 84 | iter.pos = len(iter.series) 85 | } 86 | 87 | func (iter *sliceRemoteIter) ForEach(fn func(*Remote) error) error { 88 | defer iter.Close() 89 | for _, r := range iter.series { 90 | if err := fn(&r); err != nil { 91 | if err == ErrStop { 92 | return nil 93 | } 94 | return err 95 | } 96 | } 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /samplr/samplrctl/utils/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "strings" 21 | 22 | git "github.com/GoogleCloudPlatform/devrel-services/git-go" 23 | "github.com/GoogleCloudPlatform/devrel-services/samplr" 24 | 25 | "golang.org/x/sync/errgroup" 26 | ) 27 | 28 | // GetSnippets retrieves the Snippets from the given directory path 29 | func GetSnippets(ctx context.Context, d string) ([]*samplr.Snippet, error) { 30 | r, err := git.PlainOpen(d) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | remotes, err := r.Remotes() 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | orgName := "" 41 | repoName := "" 42 | err = remotes.ForEach(func(re *git.Remote) error { 43 | if re.Config() == nil { 44 | return errors.New("Remote has a nil Config") 45 | } 46 | if re.Config().Name != "origin" { 47 | return nil 48 | } 49 | 50 | if len(re.Config().URLs) == 0 { 51 | return errors.New("No URLs associated with remote") 52 | } 53 | 54 | parts := strings.Split(re.Config().URLs[0], "/") 55 | if len(parts) < 3 { 56 | return errors.New("Remote has an invalid URL") 57 | } 58 | 59 | repoName = parts[len(parts)-1] 60 | orgName = parts[len(parts)-2] 61 | return nil 62 | }) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | refIter, err := r.Branches() 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | group, ctx := errgroup.WithContext(ctx) 73 | 74 | snps := make([]*samplr.Snippet, 0) 75 | 76 | refIter.ForEach(func(ref *git.Reference) error { 77 | if ref.Name() != git.Master { 78 | return nil 79 | } 80 | group.Go(func() error { 81 | cIter, err := r.Log(&git.LogOptions{From: ref.Hash()}) 82 | if err != nil { 83 | return err 84 | } 85 | snps, err = samplr.CalculateSnippets(orgName, repoName, cIter) 86 | if err != nil { 87 | return err 88 | } 89 | 90 | return nil 91 | }) 92 | return nil 93 | }) 94 | 95 | if err = group.Wait(); err != nil { 96 | return nil, err 97 | } 98 | return snps, nil 99 | } 100 | -------------------------------------------------------------------------------- /samplr/sample_metadata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package samplr 16 | 17 | import ( 18 | "bufio" 19 | "regexp" 20 | "strings" 21 | 22 | "gopkg.in/yaml.v2" 23 | ) 24 | 25 | var ( 26 | // Currently supported are "single line" comments that begin with 27 | // "#" or "//" 28 | smre = regexp.MustCompile(`^(#|//) sample-metadata:$`) 29 | ) 30 | 31 | // SampleMetadata is the root note of a SampleMeta 32 | type SampleMetadata struct { 33 | Meta SampleMeta `yaml:"sample-metadata"` 34 | } 35 | 36 | // SampleMeta stores structured metadata about a singular Sample. 37 | // It can have several Snippets associated with it. 38 | type SampleMeta struct { 39 | Title string `yaml:"title"` 40 | Description string `yaml:"description"` 41 | Usage string `yaml:"usage"` 42 | APIVersion string `yaml:"api_version"` 43 | Snippets []SnippetMetaRef `yaml:"snippets"` 44 | } 45 | 46 | // SnippetMetaRef stores strucutred data about a single Snippet 47 | type SnippetMetaRef struct { 48 | RegionTag string `yaml:"region_tag"` 49 | Description string `yaml:"description"` 50 | Usage string `yaml:"usage"` 51 | } 52 | 53 | func parseSampleMetadata(content string) (*SampleMetadata, error) { 54 | // Detect the metadata region 55 | scn := bufio.NewScanner(strings.NewReader(content)) 56 | comChr := "" 57 | ymlcom := strings.Builder{} 58 | for scn.Scan() { 59 | txt := scn.Text() 60 | if comChr == "" && smre.MatchString(txt) { 61 | m := smre.FindAllStringSubmatch(txt, -1) 62 | comChr = m[0][1] 63 | } 64 | 65 | if comChr != "" { 66 | if txt == "" || !strings.HasPrefix(txt, comChr) { 67 | break 68 | } 69 | // Replace JUST the first occurance of the comment character 70 | // in the string 71 | cln := strings.Replace(txt, comChr, "", 1) 72 | ymlcom.WriteString(cln) 73 | ymlcom.WriteString("\n") 74 | } 75 | } 76 | 77 | if comChr == "" { 78 | // We didn't find any metadata 79 | return nil, nil 80 | } 81 | 82 | // Unmarshal it into struct & return 83 | var sm SampleMetadata 84 | 85 | err := yaml.Unmarshal([]byte(ymlcom.String()), &sm) 86 | if err != nil { 87 | return nil, err 88 | } 89 | return &sm, nil 90 | } 91 | -------------------------------------------------------------------------------- /devrelservices-admin/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # __ __ _ _ _ 16 | # \ \ / /_ _ _ __(_) __ _| |__ | | ___ ___ 17 | # \ \ / / _` | '__| |/ _` | '_ \| |/ _ \ __| 18 | # \ V / (_| | | | | (_| | |_) | | __\__ \ 19 | # \_/ \__,_|_| |_|\__,_|_.__/|_|\___|___/ 20 | # 21 | 22 | # 23 | # All of the following can be overwritten with environemt variables 24 | # or passed through directly when invoking the relevent Make targets 25 | # 26 | 27 | # The (gcloud) test cluster that is being worked against 28 | GCP_CLUSTER_NAME ?= devrel-services 29 | GCP_CLUSTER_ZONE ?= us-central1-a 30 | # The service account to run as 31 | SERVICE_ACCOUNT_SECRET_NAME ?= service-account-maintnerd 32 | 33 | 34 | # _____ _ 35 | # |_ _|_ _ _ __ __ _ ___| |_ ___ 36 | # | |/ _` | '__/ _` |/ _ \ __/ __| 37 | # | | (_| | | | (_| | __/ |_\__ \ 38 | # |_|\__,_|_| \__, |\___|\__|___/ 39 | # |___/ 40 | # 41 | 42 | .PHONY: all 43 | all: help 44 | 45 | .PHONY: build 46 | build: ## Builds the docker image locally 47 | docker build -t devrelservices-admin:dev -f ./Dockerfile ../ 48 | 49 | .PHONY: test 50 | test: ## Tests the go module 51 | go test . 52 | 53 | .PHONY: deploy 54 | deploy: check-deploy check-project ## Builds and deploys to GKE using cloud build 55 | gcloud builds submit \ 56 | --timeout=1h \ 57 | --config=cloudbuild.yaml \ 58 | --substitutions _ZONE=$(GCP_CLUSTER_ZONE),_CLUSTER=$(GCP_CLUSTER_NAME),_SERVICE_ACCOUNT_SECRET=$(SERVICE_ACCOUNT_SECRET_NAME) \ 59 | ../ 60 | 61 | .PHONY: check-deploy 62 | check-deploy: printvars 63 | @echo -n "Are you sure? [yN] " && read ans && [ $$ans == y ] 64 | 65 | .PHONY: check-project 66 | check-project: 67 | @echo "Active project is: $$(gcloud config list --format 'value(core.project)')" 68 | @echo -n "Are you sure? [yN] " && read ans && [ $$ans == y ] 69 | 70 | .PHONY: printvars 71 | printvars: 72 | @$(foreach V,$(sort $(.VARIABLES)),\ 73 | $(if $(filter-out environment% default automatic,\ 74 | $(origin $V)),$(info $V=$($V) ($(value $V))))) 75 | 76 | 77 | .PHONY: help 78 | help: ## Prints help message 79 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 80 | 81 | -------------------------------------------------------------------------------- /samplr/git_repository_integration_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build integration 16 | 17 | package samplr 18 | 19 | import ( 20 | "context" 21 | "io/ioutil" 22 | "os" 23 | "testing" 24 | 25 | git "github.com/GoogleCloudPlatform/devrel-services/git-go" 26 | ) 27 | 28 | func TestRepositoryHasSnippet(t *testing.T) { 29 | cases := []struct { 30 | Name string 31 | URL string 32 | Branch string 33 | SnippetName string 34 | WantMinimumVersions int 35 | }{ 36 | { 37 | Name: "Handles Merges", 38 | URL: "https://github.com/GoogleCloudPlatform/dotnet-docs-samples", 39 | Branch: "master", 40 | SnippetName: "owners/GoogleCloudPlatform/repositories/dotnet-docs-samples/snippets/bigtable_hw_imports/languages/CSHARP", 41 | WantMinimumVersions: 2, 42 | }, 43 | } 44 | for _, c := range cases { 45 | dirname, err := ioutil.TempDir("", "samplr-") 46 | if err != nil { 47 | t.Errorf("Couldnt get a temp dir: %v", err) 48 | continue 49 | } 50 | defer os.RemoveAll(dirname) 51 | 52 | r, err := git.PlainClone(dirname, false, &git.CloneOptions{ 53 | URL: c.URL, 54 | }) 55 | if err != nil { 56 | t.Errorf("Error cloning %v: %v", c.URL, err) 57 | continue 58 | } 59 | cor := &Corpus{} 60 | wgh := watchedGitRepo{ 61 | repository: r, 62 | c: cor, 63 | id: c.URL, 64 | branchName: git.FullyQualifiedReferenceName(c.Branch), 65 | snippets: make(map[string][]*Snippet), 66 | commits: make(map[string][]*GitCommit), 67 | } 68 | 69 | ctx := context.Background() 70 | 71 | err = wgh.Update(ctx) 72 | if err != nil { 73 | t.Errorf("Error during update: %v", err) 74 | return 75 | } 76 | 77 | var found *Snippet 78 | // Find the snippet 79 | wgh.ForEachSnippet(func(snippet *Snippet) error { 80 | if snippet.Name == c.SnippetName { 81 | found = snippet 82 | } 83 | return nil 84 | }) 85 | 86 | if found == nil { 87 | t.Errorf("Could not find snippet named: %v", c.SnippetName) 88 | continue 89 | } 90 | 91 | if len(found.Versions) < c.WantMinimumVersions { 92 | t.Errorf("Snippet: %v has %v versions, want: %v", c.SnippetName, len(found.Versions), c.WantMinimumVersions) 93 | continue 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /devrelservices-admin/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | steps: 16 | # Build the admin image 17 | - name: 'gcr.io/cloud-builders/docker' 18 | args: ['build', '-t', 'gcr.io/$PROJECT_ID/devrelservices-admin:$BUILD_ID','-f', 'devrelservices-admin/Dockerfile', '.'] 19 | id: 'devrelservices-admin-build' 20 | waitFor: ['-'] 21 | 22 | # Push devrelservices-admin to Container Registry 23 | - name: 'gcr.io/cloud-builders/docker' 24 | args: ["push", "gcr.io/$PROJECT_ID/devrelservices-admin:$BUILD_ID"] 25 | waitFor: ['devrelservices-admin-build'] 26 | id: 'devrelservices-admin-push' 27 | 28 | # Prepare Kubernetes Manifest 29 | - name: 'gcr.io/cloud-builders/gcloud' 30 | entrypoint: 'bash' 31 | id: 'k8s-manifest-prep' 32 | waitFor: ['-'] # Start immediately 33 | args: 34 | - '-e' 35 | - '-c' 36 | - | 37 | # Dynamically create service name based on the current project ID 38 | SERVICE_NAME=devrelservices-admin.endpoints.$PROJECT_ID.cloud.goog 39 | 40 | # Deploy to Cloud Endpoints (note that some escaping is necessary 41 | # for bash variable references) 42 | #sed s/SERVICE_NAME/$$SERVICE_NAME/g api_config.yaml > target/api_config.yaml && \ 43 | # gcloud endpoints services deploy target/api_descriptor.pb target/api_config.yaml 44 | 45 | # Obtain the service config ID created by the deployment to Cloud Endpoints 46 | SERVICE_CONFIG_ID=`gcloud endpoints services describe $$SERVICE_NAME --format=value\(serviceConfig.id\)` 47 | # Substitute variables in Kubernetes manifest 48 | mkdir -p target 49 | cat devrelservices-admin/kubernetes/deployment.yaml | \ 50 | sed s/BUILD_ID/$BUILD_ID/g | \ 51 | sed s/PROJECT_ID/$PROJECT_ID/g | \ 52 | sed s/SERVICE_NAME/$$SERVICE_NAME/g | \ 53 | sed s/SERVICE_CONFIG_ID/$$SERVICE_CONFIG_ID/g | \ 54 | sed s/SERVICE_ACCOUNT_SECRET_NAME/$_SERVICE_ACCOUNT_SECRET/g \ 55 | > target/deployment.yaml 56 | 57 | # Perform Kubernetes Deployment 58 | - name: 'gcr.io/cloud-builders/kubectl' 59 | entrypoint: 'bash' 60 | id: 'k8s-push' 61 | waitFor: ['k8s-manifest-prep','devrelservices-admin-push'] 62 | args: 63 | - '-e' 64 | - '-c' 65 | - | 66 | gcloud container clusters get-credentials --project="$PROJECT_ID" --zone="$_ZONE" "$_CLUSTER" 67 | 68 | kubectl apply -f target/deployment.yaml 69 | 70 | images: 71 | - 'gcr.io/$PROJECT_ID/devrelservices-admin' 72 | -------------------------------------------------------------------------------- /samplr/samplrctl/cmd/completion/completion_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package completion 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "testing" 21 | 22 | "github.com/spf13/cobra" 23 | ) 24 | 25 | func TestGeneratesBash(t *testing.T) { 26 | ctx := context.Background() 27 | root := &cobra.Command{ 28 | Use: "root", 29 | } 30 | output := new(bytes.Buffer) 31 | root.SetOutput(output) 32 | root.SetArgs([]string{"completion", "bash"}) 33 | 34 | AddCommand(ctx, root) 35 | 36 | err := root.Execute() 37 | if err != nil { 38 | t.Errorf("Expected no error. Got: %v", err) 39 | } 40 | } 41 | 42 | func TestErrorsOnNoShell(t *testing.T) { 43 | ctx := context.Background() 44 | root := &cobra.Command{ 45 | Use: "root", 46 | SilenceErrors: false, 47 | SilenceUsage: true, 48 | } 49 | output := new(bytes.Buffer) 50 | root.SetOutput(output) 51 | root.SetArgs([]string{"completion"}) 52 | 53 | AddCommand(ctx, root) 54 | 55 | err := root.Execute() 56 | 57 | if err == nil { 58 | t.Error("Expected an error. Got: nil") 59 | } else if err != errUnspecifiedShell { 60 | t.Errorf("Expected Unspecified Shell error. Got %v", err) 61 | } 62 | } 63 | 64 | func TestErrorsOnUnsupportedShell(t *testing.T) { 65 | ctx := context.Background() 66 | root := &cobra.Command{ 67 | Use: "root", 68 | } 69 | output := new(bytes.Buffer) 70 | root.SetOutput(output) 71 | root.SetArgs([]string{"completion", "zsh"}) 72 | 73 | AddCommand(ctx, root) 74 | 75 | err := root.Execute() 76 | if err == nil { 77 | t.Error("Expected an error. Got: nil") 78 | } else if err, ok := err.(*unsupportedShellErr); !ok { 79 | t.Errorf("Expected an unsupportedShellErr. Got %v", err) 80 | } else if err.shellName != "zsh" { 81 | t.Errorf("Expected err.shellName to be zsh. Got %v", err.shellName) 82 | } 83 | } 84 | 85 | func TestErrorsOnTooManyArgs(t *testing.T) { 86 | ctx := context.Background() 87 | root := &cobra.Command{ 88 | Use: "root", 89 | } 90 | output := new(bytes.Buffer) 91 | root.SetOutput(output) 92 | root.SetArgs([]string{"completion", "bash", "zsh", "fish"}) 93 | 94 | AddCommand(ctx, root) 95 | 96 | err := root.Execute() 97 | if err == nil { 98 | t.Error("Expected an error. Got: nil") 99 | } else if err != errTooManyArgs { 100 | t.Errorf("Expected errTooManyArgs. Got %v", err) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /leif/repos_disk.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package leif 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | "io/ioutil" 22 | "reflect" 23 | "strings" 24 | "sync" 25 | 26 | "github.com/GoogleCloudPlatform/devrel-services/repos" 27 | ) 28 | 29 | // NewDiskRepo returns a RepoList based on a local file 30 | func NewDiskRepo(fileName string) repos.RepoList { 31 | return &diskRepoList{ 32 | fileName: fileName, 33 | reposList: make([]repos.TrackedRepository, 0), 34 | } 35 | } 36 | 37 | type diskRepoList struct { 38 | fileName string 39 | reposList []repos.TrackedRepository 40 | mu sync.RWMutex 41 | } 42 | 43 | func (r *diskRepoList) UpdateTrackedRepos(ctx context.Context) (bool, error) { 44 | newRepos, err := r.getRepos() 45 | if err != nil { 46 | return false, err 47 | } 48 | 49 | r.mu.Lock() 50 | defer r.mu.Unlock() 51 | if reflect.DeepEqual(newRepos, r.reposList) { 52 | return false, nil 53 | } 54 | 55 | r.reposList = newRepos 56 | return true, nil 57 | } 58 | 59 | func (r *diskRepoList) GetTrackedRepos() []repos.TrackedRepository { 60 | r.mu.RLock() 61 | defer r.mu.RUnlock() 62 | return r.reposList 63 | } 64 | 65 | type diskRepo struct { 66 | Repo string `json:"repo"` 67 | IsTrackingIssues bool `json:"is_tracking_issues"` 68 | IsTrackingSnippets bool `json:"is_tracking_snippets"` 69 | } 70 | 71 | func (r *diskRepoList) getRepos() ([]repos.TrackedRepository, error) { 72 | file, err := ioutil.ReadFile(r.fileName) 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | // Parse 78 | var data map[string][]diskRepo 79 | 80 | if err := json.Unmarshal(file, &data); err != nil { 81 | return nil, fmt.Errorf("Failed to unmarshal repos: %w", err) 82 | } 83 | 84 | fileRepos := data["repos"] 85 | 86 | convertedRepos := make([]repos.TrackedRepository, len(fileRepos)) 87 | for i, repo := range fileRepos { 88 | repoPath := strings.Split(repo.Repo, "/") 89 | if len(repoPath) != 2 { 90 | log.Printf("Bad format for repo %q", repo.Repo) 91 | continue 92 | } 93 | 94 | convertedRepos[i] = repos.TrackedRepository{ 95 | Owner: repoPath[0], 96 | Name: repoPath[1], 97 | IsTrackingIssues: repo.IsTrackingIssues, 98 | IsTrackingSnippets: repo.IsTrackingSnippets, 99 | } 100 | } 101 | 102 | return convertedRepos, nil 103 | } 104 | -------------------------------------------------------------------------------- /drghs/v1/service_resources.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package drghs.v1; 18 | 19 | import "resources.proto"; 20 | 21 | // Request message for [SampleService.ListRepositories][]. 22 | message ListRepositoriesRequest { 23 | // Required. The resource name of the owner associated with the 24 | // [Repositories][Repository], in the format `owners/*`. 25 | string parent = 1; 26 | // Optional. Limit the number of [Repositories][Repository] to include in the 27 | // response. Fewer repositories than requested might be returned. 28 | // 29 | // The maximum page size is `100`. If unspecified, the page size will be the 30 | // maximum. Further [Repositories][Repository] can subsequently be obtained 31 | // by including the [ListRepositoriesResponse.next_page_token][] in a 32 | // subsequent request. 33 | int32 page_size = 2; 34 | // Optional. To request the first page of results, `page_token` must be empty. 35 | // To request the next page of results, page_token must be the value of 36 | // [ListRepositoriesResponse.next_page_token][] returned by a previous call to 37 | // [Repositorieservice.ListRepositories][]. 38 | // 39 | // The page token is valid for only 2 hours. 40 | string page_token = 3; 41 | // Optional. Filter expression used to include in the response only those 42 | // resources that match the filter. Filter must be in following the format: 43 | // 44 | // field1=123 45 | // field2="Foo bar" 46 | // field3 IN ["value3", "value4"] 47 | // 48 | // Valid filter fields are: `repo` and `owner`. 49 | // 50 | string filter = 4; 51 | // Optional. Specify how the results should be sorted. The fields supported 52 | // for sorting are `name` and `size`. 53 | // The default ordering is by `name`. Prefix with `-` to specify 54 | // descending order, e.g. `-name`. 55 | string order_by = 5; 56 | } 57 | 58 | // Response message for [SampleService.ListRepositories][]. 59 | message ListRepositoriesResponse { 60 | // The list of [Repositories][Repository]. 61 | repeated drghs.v1.Repository repositories = 1; 62 | 63 | // A token to retrieve the next page of results, or empty if there are no 64 | // more results in the list. Pass this value in 65 | // [ListRepositoriesRequest.page_token][] to retrieve the next page of 66 | // results. 67 | string next_page_token = 2; 68 | 69 | // The total number of repositories that matched the query. 70 | int32 total = 3; 71 | } 72 | -------------------------------------------------------------------------------- /samplr/samplrctl/cmd/snippets/snippets.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snippets 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/GoogleCloudPlatform/devrel-services/samplr" 22 | commands "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/cmd" 23 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/snippets" 24 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/utils" 25 | 26 | "github.com/spf13/cobra" 27 | ) 28 | 29 | // AddCommand adds Snippets related commands to the given command 30 | func AddCommand(ctx context.Context, cmd *cobra.Command) { 31 | 32 | snippets := &cobra.Command{ 33 | Use: "snippets", 34 | Short: "interacts with snippets", 35 | Long: "interacts with snippets", 36 | } 37 | 38 | snippetsGet := &cobra.Command{ 39 | Use: "get", 40 | Short: "get", 41 | Long: "get", 42 | RunE: commands.CtxCommand(ctx, getSnippet), 43 | Args: cobra.ExactArgs(2), 44 | } 45 | 46 | snippetsList := &cobra.Command{ 47 | Use: "list", 48 | Short: "list", 49 | Long: "list", 50 | RunE: commands.CtxCommand(ctx, listSnippets), 51 | Args: cobra.ExactArgs(1), 52 | } 53 | 54 | snippets.AddCommand(snippetsGet) 55 | snippets.AddCommand(snippetsList) 56 | cmd.AddCommand(snippets) 57 | } 58 | 59 | func getSnippet(ctx context.Context, cmd *cobra.Command, args []string) error { 60 | if len(args) != 2 { 61 | return fmt.Errorf("Need exactly 2 arguments, got: %v", len(args)) 62 | } 63 | fmt.Println("list called") 64 | d := args[0] 65 | sName := args[1] 66 | 67 | snps, err := utils.GetSnippets(ctx, d) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | var snp *samplr.Snippet 73 | for _, s := range snps { 74 | if s.Name == sName { 75 | snp = s 76 | break 77 | } 78 | } 79 | 80 | if snp == nil { 81 | return fmt.Errorf("Could not find a snippet named: %v", sName) 82 | } 83 | 84 | snippets.OutputSnippet(cmd.OutOrStdout(), snp) 85 | 86 | return nil 87 | } 88 | 89 | func listSnippets(ctx context.Context, cmd *cobra.Command, args []string) error { 90 | if len(args) != 1 { 91 | return fmt.Errorf("Need exactly 1 argument, got: %v", len(args)) 92 | } 93 | 94 | d := args[0] 95 | 96 | snps, err := utils.GetSnippets(ctx, d) 97 | if err != nil { 98 | return err 99 | } 100 | 101 | snippets.OutputSnippets(cmd.OutOrStdout(), snps) 102 | 103 | return nil 104 | } 105 | -------------------------------------------------------------------------------- /drghs-worker/maintner-swpr/limit_transport_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "net/http" 19 | "testing" 20 | "time" 21 | 22 | "golang.org/x/time/rate" 23 | ) 24 | 25 | type mockRoundTripper struct { 26 | Response *http.Response 27 | Err error 28 | } 29 | 30 | func (t mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { 31 | return t.Response, t.Err 32 | } 33 | 34 | func TestLimitTransportLimits(t *testing.T) { 35 | 36 | cases := []struct { 37 | QPS float64 38 | Iterations int 39 | }{ 40 | { 41 | QPS: .5, 42 | Iterations: 4, 43 | }, 44 | { 45 | QPS: 1, 46 | Iterations: 3, 47 | }, 48 | { 49 | QPS: 2, 50 | Iterations: 5, 51 | }, 52 | { 53 | QPS: 1, 54 | Iterations: 1, 55 | }, 56 | { 57 | QPS: 2, 58 | Iterations: 1, 59 | }, 60 | } 61 | 62 | for _, c := range cases { 63 | dur := time.Duration(float64(time.Second) * (1.0 / c.QPS)) 64 | limit := rate.Every(dur) 65 | limiter := rate.NewLimiter(limit, 1) 66 | mrtr := mockRoundTripper{ 67 | Response: nil, 68 | Err: nil, 69 | } 70 | 71 | lt := limitTransport{ 72 | limiter: limiter, 73 | base: mrtr, 74 | } 75 | 76 | tst := time.Now() 77 | 78 | for i := 0; i < c.Iterations; i++ { 79 | r, err := lt.RoundTrip(&http.Request{}) 80 | if r != mrtr.Response { 81 | t.Errorf("limitTransport did not return expected response") 82 | } 83 | if err != mrtr.Err { 84 | t.Errorf("limitTransport did not return expected error. got: %v", err) 85 | } 86 | } 87 | 88 | got := time.Since(tst) 89 | want := time.Duration(float64(time.Second) * ((1.0 / c.QPS) * float64(c.Iterations-1))) 90 | if got < want { 91 | t.Errorf("limit transport did not limit our qps appropriately\nWant: %v Got: %v", want, got) 92 | } 93 | } 94 | } 95 | 96 | func TestLimitTransportHandlesNil(t *testing.T) { 97 | mrtr := mockRoundTripper{ 98 | Response: nil, 99 | Err: nil, 100 | } 101 | 102 | lt := limitTransport{ 103 | limiter: nil, 104 | base: mrtr, 105 | } 106 | 107 | max := 4 108 | for i := 0; i < max; i++ { 109 | r, err := lt.RoundTrip(&http.Request{}) 110 | if r != mrtr.Response { 111 | t.Errorf("limitTransport did not return expected response") 112 | } 113 | if err != mrtr.Err { 114 | t.Errorf("limitTransport did not return expected error") 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /drghs/endpoints/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | steps: 16 | 17 | # Create samplr endpoint 18 | - name: 'gcr.io/cloud-builders/gcloud' 19 | entrypoint: 'bash' 20 | id: 'samplr-endpoints-prep' 21 | waitFor: ['-'] # Start Immediately 22 | args: 23 | - '-e' 24 | - '-c' 25 | - | 26 | # Get the IP address to fill our spec with 27 | IPADDR=`gcloud compute addresses describe samplr-ip --global --format=value\(address\)` 28 | 29 | mkdir -p target 30 | cat endpoints/samplr.yaml | \ 31 | sed s/PROJECT_ID/$PROJECT_ID/g | \ 32 | sed s/IP_ADDR/$$IPADDR/g | \ 33 | sed s/TYPE/$_TYPE/g \ 34 | > target/samplr.yaml 35 | 36 | # Create maintner endpoint 37 | - name: 'gcr.io/cloud-builders/gcloud' 38 | entrypoint: 'bash' 39 | id: 'maintner-endpoints-prep' 40 | waitFor: ['-'] # Start Immediately 41 | args: 42 | - '-e' 43 | - '-c' 44 | - | 45 | # Get the IP address to fill our spec with 46 | IPADDR=`gcloud compute addresses describe maintnerd-ip --global --format=value\(address\)` 47 | 48 | mkdir -p target 49 | cat endpoints/maintnerd.yaml | \ 50 | sed s/PROJECT_ID/$PROJECT_ID/g | \ 51 | sed s/IP_ADDR/$$IPADDR/g | \ 52 | sed s/TYPE/$_TYPE/g \ 53 | > target/maintnerd.yaml 54 | 55 | # Create admin endpoint 56 | - name: 'gcr.io/cloud-builders/gcloud' 57 | entrypoint: 'bash' 58 | id: 'devrelservices-admin-endpoints-prep' 59 | waitFor: ['-'] # Start Immediately 60 | args: 61 | - '-e' 62 | - '-c' 63 | - | 64 | # Get the IP address to fill our spec with 65 | IPADDR=`gcloud compute addresses describe devrelservices-admin-ip --global --format=value\(address\)` 66 | 67 | mkdir -p target 68 | cat endpoints/devrelservices-admin.yaml | \ 69 | sed s/PROJECT_ID/$PROJECT_ID/g | \ 70 | sed s/IP_ADDR/$$IPADDR/g | \ 71 | sed s/TYPE/$_TYPE/g \ 72 | > target/devrelservices-admin.yaml 73 | 74 | # Deploy to endpoints 75 | - name: 'gcr.io/cloud-builders/gcloud' 76 | entrypoint: 'bash' 77 | id: 'endpoints-deploy' 78 | waitFor: ['samplr-endpoints-prep', 'maintner-endpoints-prep', 'devrelservices-admin-endpoints-prep'] 79 | args: 80 | - '-e' 81 | - '-c' 82 | - | 83 | # Deploy maintner 84 | gcloud endpoints services deploy v1/api_descriptor.pb target/maintnerd.yaml 85 | 86 | # Deploy samplr 87 | gcloud endpoints services deploy v1/api_descriptor.pb target/samplr.yaml 88 | 89 | # Deploy devrelservices-admin 90 | gcloud endpoints services deploy v1/api_descriptor.pb target/devrelservices-admin.yaml 91 | 92 | images: 93 | -------------------------------------------------------------------------------- /samplr/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # __ __ _ _ _ 16 | # \ \ / /_ _ _ __(_) __ _| |__ | | ___ ___ 17 | # \ \ / / _` | '__| |/ _` | '_ \| |/ _ \ __| 18 | # \ V / (_| | | | | (_| | |_) | | __\__ \ 19 | # \_/ \__,_|_| |_|\__,_|_.__/|_|\___|___/ 20 | # 21 | 22 | # 23 | # All of the following can be overwritten with environemt variables 24 | # or passed through directly when invoking the relevent Make targets 25 | # 26 | 27 | # The (gcloud) test cluster that is being worked against 28 | GCP_CLUSTER_NAME ?= devrel-dev-cluster 29 | GCP_CLUSTER_ZONE ?= us-central1-a 30 | # The service account to run as 31 | SERVICE_ACCOUNT_SECRET_NAME ?= service-account-maintnerd 32 | # Bucket settings for Repositories 33 | GCS_BUCKET_NAME ?= devrel-dev-settings 34 | REPOS_FILE_NAME ?= public_repos.json 35 | 36 | 37 | # _____ _ 38 | # |_ _|_ _ _ __ __ _ ___| |_ ___ 39 | # | |/ _` | '__/ _` |/ _ \ __/ __| 40 | # | | (_| | | | (_| | __/ |_\__ \ 41 | # |_|\__,_|_| \__, |\___|\__|___/ 42 | # |___/ 43 | # 44 | 45 | .PHONY: all 46 | all: help 47 | 48 | .PHONY: build 49 | build: ## Builds the docker image locally 50 | docker build -t samplrd:dev -f samplrd/Dockerfile ../ 51 | docker build -t samplr-rtr:dev -f samplr-rtr/Dockerfile ../ 52 | docker build -t samplr-sprvsr:dev -f samplr-sprvsr/Dockerfile ../ 53 | 54 | .PHONY: test 55 | test: ## Tests the go module 56 | go test . 57 | 58 | .PHONY: deploy 59 | deploy: check-deploy check-project ## Builds and deploys to GKE using cloud build 60 | gcloud builds submit \ 61 | --timeout=1h \ 62 | --config=cloudbuild.yaml \ 63 | --substitutions _ZONE=$(GCP_CLUSTER_ZONE),_CLUSTER=$(GCP_CLUSTER_NAME),_SERVICE_ACCOUNT_SECRET=$(SERVICE_ACCOUNT_SECRET_NAME),_BUCKET_NAME=$(GCS_BUCKET_NAME),_REPO_FILE_NAME=$(REPOS_FILE_NAME) \ 64 | ../ 65 | 66 | .PHONY: check-deploy 67 | check-deploy: printvars 68 | @echo -n "Are you sure? [yN] " && read ans && [ $$ans == y ] 69 | 70 | .PHONY: check-project 71 | check-project: 72 | @echo "Active project is: $$(gcloud config list --format 'value(core.project)')" 73 | @echo -n "Are you sure? [yN] " && read ans && [ $$ans == y ] 74 | 75 | .PHONY: printvars 76 | printvars: 77 | @$(foreach V,$(sort $(.VARIABLES)),\ 78 | $(if $(filter-out environment% default automatic,\ 79 | $(origin $V)),$(info $V=$($V) ($(value $V))))) 80 | 81 | 82 | .PHONY: help 83 | help: ## Prints help message 84 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 85 | 86 | -------------------------------------------------------------------------------- /samplr/samplrctl/output/output_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package output 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "testing" 21 | 22 | "github.com/GoogleCloudPlatform/devrel-services/samplr/samplrctl/testutil" 23 | ) 24 | 25 | const ( 26 | field1 = "Field 1" 27 | field2 = "Field 2" 28 | field3 = "Field 3" 29 | ) 30 | 31 | var ( 32 | object1 = map[string]string{ 33 | field1: "Value 1A", 34 | field2: "Value 2A", 35 | field3: "Value 3A", 36 | } 37 | 38 | object2 = map[string]string{ 39 | field1: "Value 1B", 40 | field2: "Value 2B", 41 | field3: "Value 3B", 42 | } 43 | ) 44 | 45 | func TestPrintMap(t *testing.T) { 46 | var b bytes.Buffer 47 | PrintMap(&b, []string{field1, field3}, object1) 48 | s := b.String() 49 | 50 | want := []string{field1, object1[field1], field3, object1[field3]} 51 | if !testutil.ContainsAll(s, want...) { 52 | t.Errorf("Printed: %q; missing: %v", s, want) 53 | } 54 | 55 | dontWant := []string{field2, object1[field2]} 56 | if testutil.ContainsAll(s, dontWant...) { 57 | t.Errorf("Printed: %q; don't want: %v:", s, dontWant) 58 | } 59 | } 60 | 61 | func TestPrintAllMap(t *testing.T) { 62 | var b bytes.Buffer 63 | PrintAllMap(&b, object1) 64 | s := b.String() 65 | 66 | want := []string{field1, object1[field1], field2, object1[field2], field3, object1[field3]} 67 | if !testutil.ContainsAll(s, want...) { 68 | t.Errorf("Printed: %q; missing: %v", s, want) 69 | } 70 | } 71 | 72 | func TestPrintList(t *testing.T) { 73 | var b bytes.Buffer 74 | PrintList(&b, []string{field1, field3}, []map[string]string{object1, object2}) 75 | s := b.String() 76 | 77 | wantRows := [][]string{ 78 | []string{field1, field3}, 79 | []string{object1[field1], object1[field3]}, 80 | []string{object2[field1], object2[field3]}, 81 | } 82 | 83 | for _, wantRow := range wantRows { 84 | if b, _ := testutil.ContainsRow(s, wantRow...); !b { 85 | t.Errorf("Printed: %q; missing: %v", s, wantRow) 86 | } 87 | } 88 | } 89 | 90 | func TestToCamelCase(t *testing.T) { 91 | tests := []struct { 92 | in string 93 | want string 94 | }{ 95 | {"string", "string"}, 96 | {"String", "string"}, 97 | {"With Spaces", "withSpaces"}, 98 | {"CAPS", "caps"}, 99 | {"Spaces AND caps", "spacesAndCaps"}, 100 | } 101 | 102 | for _, tt := range tests { 103 | t.Run(fmt.Sprintf("%s / %s", tt.in, tt.want), func(t *testing.T) { 104 | if got := toCamelCase(tt.in); got != tt.want { 105 | t.Errorf("Got %q; want %q", got, tt.want) 106 | } 107 | }) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /leif/githubservices/mock_reposervice.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This package was modeled after the mocking strategy outlined at: 16 | // https://github.com/google/go-github/issues/113#issuecomment-46023864 17 | 18 | package githubservices 19 | 20 | import ( 21 | "context" 22 | "errors" 23 | 24 | "github.com/google/go-github/github" 25 | ) 26 | 27 | // MockGithubRepositoryService is a struct that can replace github.RepositoriesService for testing 28 | type MockGithubRepositoryService struct { 29 | Content *github.RepositoryContent 30 | DirContent []*github.RepositoryContent 31 | Users []*github.User 32 | Response *github.Response 33 | Error error 34 | Owner string 35 | Repo string 36 | } 37 | 38 | // Get mocks the original github.RepositoriesService.Get() by returning the given mocked response and error 39 | func (mgc *MockGithubRepositoryService) Get(ctx context.Context, owner, repo string) (*github.Repository, *github.Response, error) { 40 | return nil, mgc.Response, mgc.Error 41 | } 42 | 43 | // GetContents mocks the original github.RepositoriesService.GetContents() 44 | // Checks whether the owner is correct and returns the mocked content, response and error 45 | func (mgc *MockGithubRepositoryService) GetContents(ctx context.Context, owner, repo, path string, opts *github.RepositoryContentGetOptions) (*github.RepositoryContent, []*github.RepositoryContent, *github.Response, error) { 46 | if owner != mgc.Owner { 47 | return nil, nil, nil, errors.New("owner did not equal expected owner: was: " + owner) 48 | } 49 | return mgc.Content, mgc.DirContent, mgc.Response, mgc.Error 50 | } 51 | 52 | // ListByOrg mocks the original github.RepositoriesService.ListByOrg() 53 | // Checks whether the owner is correct and returns the mocked error 54 | func (mgc *MockGithubRepositoryService) ListByOrg(ctx context.Context, org string, opts *github.RepositoryListByOrgOptions) ([]*github.Repository, *github.Response, error) { 55 | if org != mgc.Owner { 56 | return nil, nil, errors.New("org did not equal expected owner: was: " + org) 57 | } 58 | return nil, nil, mgc.Error 59 | } 60 | 61 | // ListCollaborators mocks the original github.RepositoriesService.ListCollaborators() 62 | // Checks whether the owner is correct and returns the mocked error 63 | func (mgc *MockGithubRepositoryService) ListCollaborators(ctx context.Context, owner, repo string, opts *github.ListCollaboratorsOptions) ([]*github.User, *github.Response, error) { 64 | if owner != mgc.Owner { 65 | return nil, nil, errors.New("org did not equal expected owner: was: " + owner) 66 | } 67 | return mgc.Users, nil, mgc.Error 68 | } 69 | -------------------------------------------------------------------------------- /drghs-worker/README.md: -------------------------------------------------------------------------------- 1 | # drghs-worker 2 | 3 | `drghs-worker` is a soft fork of the [maintner](https://github.com/golang/build/tree/master/maintner) service 4 | written by the [Go](https://golang.org) team. 5 | 6 | Whereas `maintner` is a single, monlithic service (and therfore mutation log) 7 | that records all Issues and Pull Requests for a set of repositories, 8 | `drghs-worker` has a single-tenancy theory. Each repository is given 9 | a single worker to read and check mutations. This allows the application to scale better and handle transient failures in a much 10 | more graceful manner. 11 | 12 | ## Main Components 13 | 14 | ### maintner-sprvsr 15 | 16 | This process is a "supervisor" to the rest of the cluster. It reads 17 | a list of repositories to track from a [Cloud Storage](https://cloud.google.com/storage/) bucket 18 | Then interacts with the Kubernetes API (within the cluster) to dynamically 19 | add and delete `Deployment`s and `Service`s for each repository listed in the file. 20 | 21 | > Note: because this service runs in the cluster the service account it runs as must have permissions to edit and delete Deployments and Services. 22 | 23 | ### maitntner-rtr 24 | 25 | This process is a reverse proxy that takes the incoming request, parses out 26 | the Owner and Repository from the request and proxies it to the `Service` in the cluster that is responsible for the repository 27 | 28 | ### maintnerd 29 | 30 | This is the "main" process that leverages the `corpus` from `maintner` and syncrhonizes the Issues and Pull Requests from GitHub and exposes the API to query them. 31 | 32 | ## Other tools 33 | 34 | ### maintmigrate 35 | 36 | This tool is used to take a mutation source in Cloud Storage, and create a subset 37 | of it by reading the source into memory and applying filters to it. 38 | 39 | > This was originally used to take our monolithic mutation source and split it to a single-tenancy model. 40 | 41 | ### maint-bucket-migrate 42 | 43 | This process is used to do a "one off" migration of a set of mutation logs from one set of buckets to another. 44 | 45 | ## Reseting data for a repo 46 | 47 | 1. Pause any job that may attempt to access the repo's data. 48 | 49 | 1. Find the deployment name for the repo (e.g. `googleapis/google-cloud-python`): 50 | 51 | kubectl get deployments -l owner=googleapis,repository=google-cloud-python 52 | 53 | Example output: 54 | 55 | NAME READY UP-TO-DATE AVAILABLE AGE 56 | mtr-d-abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234 1/1 1 1 31h 57 | 58 | 1. Scale down the deployment: 59 | 60 | kubectl scale deployment mtr-d-abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234 --replicas=0 61 | 62 | 1. Delete the mutation log: 63 | 64 | gsutil rm 'gs://[BUCKET_NAME]/googleapis/google-cloud-python/*' 65 | 66 | where `[BUCKET_NAME]` is the name of the Google Cloud Storage where the mutation logs are stored. 67 | 68 | 1. Scale up deployment: 69 | 70 | kubectl scale deployment mtr-d-abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234 --replicas=1 71 | 72 | 1. Wait for the repo's data to be re-fetched. 73 | 74 | 1. Resume any paused job. 75 | -------------------------------------------------------------------------------- /samplr/git-go/reference.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | ) 21 | 22 | // ReferenceType reference type's 23 | type ReferenceType int8 24 | 25 | const ( 26 | //InvalidReference is an Invalid Reference 27 | InvalidReference ReferenceType = 0 28 | // HashReference is a reference to a hash 29 | HashReference ReferenceType = 1 30 | // SymbolicReference is a symbolic reference 31 | SymbolicReference ReferenceType = 2 32 | ) 33 | 34 | const ( 35 | // HEAD is the name of the HEAD reference 36 | HEAD ReferenceName = "HEAD" 37 | // Master is the name of the Master reference 38 | Master ReferenceName = "refs/heads/master" 39 | // OriginMaster is the name of origin master 40 | OriginMaster ReferenceName = "refs/remotes/origin/master" 41 | ) 42 | 43 | // ReferenceName reference name's 44 | type ReferenceName string 45 | 46 | // FullyQualifiedReferenceName takes a simple branch name and returns 47 | // a ReferenceName appropriate for it. e.g. "main" => "refs/heads/main" 48 | func FullyQualifiedReferenceName(n string) ReferenceName { 49 | return ReferenceName(fmt.Sprintf("refs/heads/%v", n)) 50 | } 51 | 52 | // Reference represents a Git Reference 53 | type Reference struct { 54 | t ReferenceType 55 | n ReferenceName 56 | h Hash 57 | target ReferenceName 58 | } 59 | 60 | // ReferenceIter is a generic closable interface for iterating over References. 61 | type ReferenceIter interface { 62 | Next() (*Reference, error) 63 | ForEach(func(*Reference) error) error 64 | Close() 65 | } 66 | 67 | // Hash return the hash of a hash reference 68 | func (r *Reference) Hash() Hash { 69 | return r.h 70 | } 71 | 72 | // Name returns the Name of the Reference 73 | func (r *Reference) Name() ReferenceName { 74 | return r.n 75 | } 76 | 77 | func (r ReferenceName) String() string { 78 | return string(r) 79 | } 80 | 81 | type sliceRefIter struct { 82 | pos int 83 | series []Reference 84 | } 85 | 86 | func (iter *sliceRefIter) Next() (*Reference, error) { 87 | if iter.pos >= len(iter.series) { 88 | return nil, io.EOF 89 | } 90 | 91 | obj := iter.series[iter.pos] 92 | iter.pos++ 93 | return &obj, nil 94 | } 95 | 96 | func (iter *sliceRefIter) Close() { 97 | iter.pos = len(iter.series) 98 | } 99 | 100 | func (iter *sliceRefIter) ForEach(fn func(*Reference) error) error { 101 | defer iter.Close() 102 | for _, r := range iter.series { 103 | if err := fn(&r); err != nil { 104 | if err == ErrStop { 105 | return nil 106 | } 107 | return err 108 | } 109 | } 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /.github/workflows/presubmit_checks.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: "Presubmit Checks" 3 | jobs: 4 | lint: 5 | name: Lint 6 | strategy: 7 | matrix: 8 | go-version: [1.14.x] 9 | platform: [ubuntu-latest] 10 | runs-on: ${{ matrix.platform }} 11 | steps: 12 | - name: Install Go 13 | uses: actions/setup-go@v2 14 | with: 15 | go-version: ${{ matrix.go-version }} 16 | - name: Checkout code 17 | uses: actions/checkout@v2 18 | - name: Lint 19 | run: | 20 | go get -u golang.org/x/lint/golint 21 | golint -set_exit_status ./... 22 | vet: 23 | name: Vet 24 | needs: lint 25 | strategy: 26 | matrix: 27 | go-version: [1.14.x] 28 | platform: [ubuntu-latest] 29 | runs-on: ${{ matrix.platform }} 30 | steps: 31 | - name: Install Go 32 | uses: actions/setup-go@v2 33 | with: 34 | go-version: ${{ matrix.go-version }} 35 | - name: Checkout code 36 | uses: actions/checkout@v2 37 | # Vet 38 | - name: Vet 39 | run: ./go.vet.sh 40 | unit_tests: 41 | name: Unit Tests 42 | needs: lint 43 | strategy: 44 | matrix: 45 | go-version: [1.14.x] 46 | platform: [ubuntu-latest] 47 | runs-on: ${{ matrix.platform }} 48 | steps: 49 | - name: Install Go 50 | uses: actions/setup-go@v2 51 | with: 52 | go-version: ${{ matrix.go-version }} 53 | - name: Checkout code 54 | uses: actions/checkout@v2 55 | # Install Dependencies 56 | - uses: actions/cache@v1 57 | id: cache-deps 58 | with: 59 | path: ~/go/pkg/mod 60 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 61 | restore-keys: | 62 | ${{ runner.os }}-go- 63 | - name: Install dependencies 64 | run: ./go.get.sh 65 | # Unit Tests 66 | - name: Execute Unit Tests 67 | run: ./go.test.sh 68 | # Upload code coverage 69 | - name: Upload Unit Test Coverage 70 | uses: codecov/codecov-action@v1 71 | with: 72 | file: ./unit_test_coverage.txt 73 | token: ${{ secrets.CODECOV_TOKEN }} 74 | fail_ci_if_error: true 75 | integration_tests: 76 | name: Integration Tests 77 | needs: lint 78 | strategy: 79 | matrix: 80 | go-version: [1.14.x] 81 | platform: [ubuntu-latest] 82 | runs-on: ${{ matrix.platform }} 83 | steps: 84 | - name: Install Go 85 | uses: actions/setup-go@v2 86 | with: 87 | go-version: ${{ matrix.go-version }} 88 | - name: Checkout code 89 | uses: actions/checkout@v2 90 | # Install Dependencies 91 | - uses: actions/cache@v1 92 | id: cache-deps 93 | with: 94 | path: ~/go/pkg/mod 95 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 96 | restore-keys: | 97 | ${{ runner.os }}-go- 98 | - name: Install dependencies 99 | run: ./go.get.sh 100 | # Integration Tests 101 | - name: Execute Integration Tests 102 | run: ./go.integration.test.sh 103 | # Upload code coverage 104 | - name: Upload Integration Test Coverage 105 | uses: codecov/codecov-action@v1 106 | with: 107 | file: ./integration_test_coverage.txt 108 | token: ${{ secrets.CODECOV_TOKEN }} 109 | fail_ci_if_error: true 110 | -------------------------------------------------------------------------------- /repos/repos_bucket.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package repos 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | "io/ioutil" 22 | "log" 23 | "reflect" 24 | "strings" 25 | "sync" 26 | 27 | "cloud.google.com/go/storage" 28 | ) 29 | 30 | // NewBucketRepo returns a RepoList based on GCS Buckets 31 | func NewBucketRepo(bucketName string, fileName string) RepoList { 32 | return &bucketRepoList{ 33 | bucketName: bucketName, 34 | reposFileName: fileName, 35 | reposList: make([]TrackedRepository, 0), 36 | } 37 | } 38 | 39 | type bucketRepoList struct { 40 | bucketName string 41 | reposFileName string 42 | reposList []TrackedRepository 43 | mu sync.RWMutex 44 | } 45 | 46 | func (r *bucketRepoList) UpdateTrackedRepos(ctx context.Context) (bool, error) { 47 | newRepos, err := r.getRepos(ctx) 48 | if err != nil { 49 | return false, err 50 | } 51 | 52 | r.mu.Lock() 53 | defer r.mu.Unlock() 54 | if reflect.DeepEqual(newRepos, r.reposList) { 55 | return false, nil 56 | } 57 | r.reposList = newRepos 58 | return true, nil 59 | } 60 | 61 | func (r *bucketRepoList) GetTrackedRepos() []TrackedRepository { 62 | r.mu.RLock() 63 | defer r.mu.RUnlock() 64 | return r.reposList 65 | } 66 | 67 | type bucketRepo struct { 68 | Repo string `json:"repo"` 69 | DefaultBranch string `json:"default_branch"` 70 | IsTrackingIssues bool `json:"is_tracking_issues"` 71 | IsTrackingSamples bool `json:"is_tracking_samples"` 72 | } 73 | 74 | func (r *bucketRepoList) getRepos(ctx context.Context) ([]TrackedRepository, error) { 75 | // Creates a client. 76 | client, err := storage.NewClient(ctx) 77 | if err != nil { 78 | return nil, fmt.Errorf("Failed to create client: %v", err) 79 | } 80 | 81 | rc, err := client.Bucket(r.bucketName).Object(r.reposFileName).NewReader(ctx) 82 | if err != nil { 83 | return nil, fmt.Errorf("Failed to get bucket: %v", err) 84 | } 85 | defer rc.Close() 86 | 87 | data, err := ioutil.ReadAll(rc) 88 | if err != nil { 89 | return nil, fmt.Errorf("Failed to read repos: %v", err) 90 | } 91 | 92 | // Process data. 93 | 94 | var dat map[string][]bucketRepo 95 | 96 | if err := json.Unmarshal(data, &dat); err != nil { 97 | return nil, fmt.Errorf("Failed to unmarshal repos") 98 | } 99 | 100 | repos := dat["repos"] 101 | 102 | reps := make([]TrackedRepository, len(repos)) 103 | for i, re := range repos { 104 | parts := strings.Split(re.Repo, "/") 105 | if len(parts) != 2 { 106 | log.Printf("Bad format for repo %q", re.Repo) 107 | continue 108 | } 109 | 110 | tr := TrackedRepository{ 111 | Owner: parts[0], 112 | Name: parts[1], 113 | IsTrackingIssues: re.IsTrackingIssues, 114 | IsTrackingSamples: re.IsTrackingSamples, 115 | DefaultBranch: re.DefaultBranch, 116 | } 117 | reps[i] = tr 118 | } 119 | 120 | return reps, nil 121 | } 122 | -------------------------------------------------------------------------------- /samplr/git-go/remote_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | import ( 18 | "errors" 19 | "io" 20 | "testing" 21 | ) 22 | 23 | func TestSliceRemoteIterForEachReturnsInProperOrder(t *testing.T) { 24 | rems := []Remote{ 25 | Remote{}, 26 | Remote{}, 27 | Remote{}, 28 | } 29 | 30 | iter := &sliceRemoteIter{ 31 | pos: 0, 32 | series: rems, 33 | } 34 | 35 | pos := 0 36 | iter.ForEach(func(c *Remote) error { 37 | if rems[pos] != *c { 38 | t.Errorf("The order returned does not match for idex: %v", pos) 39 | } 40 | pos++ 41 | return nil 42 | }) 43 | } 44 | 45 | func TestSliceRemoteIterNextReturnsInProperOrder(t *testing.T) { 46 | rems := []Remote{ 47 | Remote{}, 48 | Remote{}, 49 | Remote{}, 50 | } 51 | 52 | iter := &sliceRemoteIter{ 53 | pos: 0, 54 | series: rems, 55 | } 56 | 57 | for idx := 0; idx < len(rems); idx++ { 58 | if c, err := iter.Next(); *c != rems[idx] || err != nil { 59 | t.Errorf("Expected %v, %v, got %v, %v", rems[idx], "nil", c, err) 60 | } 61 | } 62 | 63 | c, err := iter.Next() 64 | if c != nil || err != io.EOF { 65 | t.Errorf("Next at the end of the iter should return EOF. It returned %v, %v", c, err) 66 | } 67 | } 68 | 69 | func TestSliceRemoteIterForEachReturnsEarlyOnError(t *testing.T) { 70 | rems := []Remote{ 71 | Remote{}, 72 | Remote{}, 73 | Remote{}, 74 | } 75 | 76 | iter := &sliceRemoteIter{ 77 | pos: 0, 78 | series: rems, 79 | } 80 | 81 | expErr := errors.New("Short circuit") 82 | pos := 0 83 | err := iter.ForEach(func(c *Remote) error { 84 | if pos == 1 { 85 | return expErr 86 | } 87 | pos++ 88 | return nil 89 | }) 90 | 91 | if pos != 1 || err != expErr { 92 | t.Errorf("Expected to short circuit at position 1. Got: %v %v", pos, err) 93 | } 94 | } 95 | 96 | func TestSliceRemoteIterForEachReturnsEarlyOnErrorSignal(t *testing.T) { 97 | rems := []Remote{ 98 | Remote{}, 99 | Remote{}, 100 | Remote{}, 101 | } 102 | 103 | iter := &sliceRemoteIter{ 104 | pos: 0, 105 | series: rems, 106 | } 107 | 108 | pos := 0 109 | err := iter.ForEach(func(c *Remote) error { 110 | if pos == 1 { 111 | return ErrStop 112 | } 113 | pos++ 114 | return nil 115 | }) 116 | 117 | if pos != 1 || err != nil { 118 | t.Errorf("Expected to short circuit at position 1. Got: %v %v", pos, err) 119 | } 120 | } 121 | 122 | func TestSliceRemoteIterCloseEndsEarly(t *testing.T) { 123 | rems := []Remote{ 124 | Remote{}, 125 | Remote{}, 126 | Remote{}, 127 | } 128 | 129 | iter := &sliceRemoteIter{ 130 | pos: 0, 131 | series: rems, 132 | } 133 | 134 | c0, err := iter.Next() 135 | if *c0 != rems[0] || err != nil { 136 | t.Errorf("sliceRemoteIter returned unexpected values %v, %v", c0, err) 137 | } 138 | 139 | iter.Close() 140 | c1, err := iter.Next() 141 | if c1 != nil || err != io.EOF { 142 | t.Errorf("After closing, sliceRemoteIter returned unexpected values. Expected nil, io.EOF, got %v, %v", c1, err) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /leif/leifd/leifapi/pagination/string_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package paginator 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | "time" 22 | 23 | "github.com/sirupsen/logrus" 24 | ) 25 | 26 | type stringPage struct { 27 | items []string 28 | index int 29 | } 30 | 31 | // Strings is a paginator for strings 32 | type Strings struct { 33 | Log *logrus.Logger 34 | mu sync.Mutex 35 | set map[time.Time]stringPage 36 | didInit bool 37 | } 38 | 39 | // Init must be the first call to the paginator 40 | func (p *Strings) Init() error { 41 | p.mu.Lock() 42 | defer p.mu.Unlock() 43 | if p.didInit { 44 | return fmt.Errorf("Paginator already initialized") 45 | } 46 | p.set = make(map[time.Time]stringPage) 47 | p.didInit = true 48 | return nil 49 | } 50 | 51 | // PurgeOldRecords removes all pages that are more than 2 hours old 52 | func (p *Strings) PurgeOldRecords() { 53 | p.mu.Lock() 54 | defer p.mu.Unlock() 55 | now := time.Now() 56 | for t := range p.set { 57 | if now.Sub(t).Hours() > 2 { 58 | delete(p.set, t) 59 | } 60 | } 61 | } 62 | 63 | // CreatePage makes a new page in the paginator 64 | // It uses the current time as key and add the items to that key 65 | func (p *Strings) CreatePage(withItems []string) (time.Time, error) { 66 | p.mu.Lock() 67 | defer p.mu.Unlock() 68 | if !p.didInit { 69 | return time.Unix(0, 0), fmt.Errorf("Paginator not initialized") 70 | } 71 | key := time.Now().UTC().Truncate(0) 72 | if _, ok := p.set[key]; ok { 73 | return time.Unix(0, 0), errors.New("Key already exists") 74 | } 75 | 76 | p.set[key] = stringPage{ 77 | items: withItems, 78 | index: 0, 79 | } 80 | return key, nil 81 | } 82 | 83 | // GetPage gets the next numItems number of items from the given page/key 84 | // Key should be the key returned by a call to CreatePage 85 | // GetPage returns the items and the current index in the total items in the page 86 | func (p *Strings) GetPage(key time.Time, numItems int) ([]string, int, error) { 87 | p.mu.Lock() 88 | defer p.mu.Unlock() 89 | 90 | if !p.didInit { 91 | return nil, 0, fmt.Errorf("Paginator not initialized") 92 | } 93 | 94 | key = key.UTC() 95 | 96 | if _, ok := p.set[key]; !ok { 97 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 98 | } 99 | val := p.set[key] 100 | 101 | numItemsRemaining := len(val.items) - val.index 102 | if p.Log != nil { 103 | p.Log.Debugf("There are %v records remaining, requested %v", numItemsRemaining, numItems) 104 | } 105 | 106 | if numItems > numItemsRemaining { 107 | numItems = numItemsRemaining 108 | } 109 | 110 | if numItems == 0 { 111 | return nil, 0, errors.New("Get 0 from page") 112 | } 113 | 114 | retSet := val.items[val.index:(val.index + numItems)] 115 | val.index = val.index + numItems 116 | 117 | retIndex := val.index 118 | if val.index == len(val.items) { 119 | delete(p.set, key) 120 | retIndex = -1 121 | } else { 122 | p.set[key] = val 123 | } 124 | 125 | return retSet, retIndex, nil 126 | } 127 | -------------------------------------------------------------------------------- /leif/leifd/leifapi/pagination/slo_paginator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package paginator 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | "time" 22 | 23 | "github.com/GoogleCloudPlatform/devrel-services/leif" 24 | 25 | "github.com/sirupsen/logrus" 26 | ) 27 | 28 | type sloPage struct { 29 | items []*leif.SLORule 30 | index int 31 | } 32 | 33 | // Slo is a paginator for SLO rules 34 | type Slo struct { 35 | Log *logrus.Logger 36 | mu sync.Mutex 37 | set map[time.Time]sloPage 38 | didInit bool 39 | } 40 | 41 | // Init must be the first call to the paginator 42 | func (p *Slo) Init() error { 43 | p.mu.Lock() 44 | defer p.mu.Unlock() 45 | if p.didInit { 46 | return fmt.Errorf("Paginator already initialized") 47 | } 48 | p.set = make(map[time.Time]sloPage) 49 | p.didInit = true 50 | return nil 51 | } 52 | 53 | // PurgeOldRecords removes all pages that are more than 2 hours old 54 | func (p *Slo) PurgeOldRecords() { 55 | p.mu.Lock() 56 | defer p.mu.Unlock() 57 | now := time.Now() 58 | for t := range p.set { 59 | if now.Sub(t).Hours() > 2 { 60 | delete(p.set, t) 61 | } 62 | } 63 | } 64 | 65 | // CreatePage makes a new page in the paginator 66 | // It uses the current time as key and add the items to that key 67 | func (p *Slo) CreatePage(withItems []*leif.SLORule) (time.Time, error) { 68 | p.mu.Lock() 69 | defer p.mu.Unlock() 70 | 71 | if !p.didInit { 72 | return time.Unix(0, 0), fmt.Errorf("Paginator not initialized") 73 | } 74 | 75 | key := time.Now().UTC().Truncate(0) 76 | if _, ok := p.set[key]; ok { 77 | return time.Unix(0, 0), errors.New("Key already exists") 78 | } 79 | 80 | p.set[key] = sloPage{ 81 | items: withItems, 82 | index: 0, 83 | } 84 | return key, nil 85 | } 86 | 87 | // GetPage gets the next numItems number of items from the given page/key 88 | // Key should be the key returned by a call to CreatePage 89 | // GetPage returns the items and the current index in the total items in the page 90 | func (p *Slo) GetPage(key time.Time, numItems int) ([]*leif.SLORule, int, error) { 91 | p.mu.Lock() 92 | defer p.mu.Unlock() 93 | 94 | if !p.didInit { 95 | return nil, 0, fmt.Errorf("Paginator not initialized") 96 | } 97 | 98 | key = key.UTC() 99 | if _, ok := p.set[key]; !ok { 100 | return nil, 0, fmt.Errorf("Page key: %v not found", key) 101 | } 102 | val := p.set[key] 103 | 104 | numItemsRemaining := len(val.items) - val.index 105 | if p.Log != nil { 106 | p.Log.Debugf("There are %v records remaining, requested %v", numItemsRemaining, numItems) 107 | } 108 | 109 | if numItems > numItemsRemaining { 110 | numItems = numItemsRemaining 111 | } 112 | 113 | if numItems == 0 { 114 | return nil, 0, errors.New("Get 0 from page") 115 | } 116 | 117 | retSet := val.items[val.index:(val.index + numItems)] 118 | val.index = val.index + numItems 119 | 120 | retIndex := val.index 121 | if val.index == len(val.items) { 122 | delete(p.set, key) 123 | retIndex = -1 124 | } else { 125 | p.set[key] = val 126 | } 127 | 128 | return retSet, retIndex, nil 129 | } 130 | --------------------------------------------------------------------------------