├── contrib ├── grafana │ └── screenshot.png ├── prometheus │ └── rules.yaml └── kubernetes │ └── deployment.yaml ├── Dockerfile ├── .github └── workflows │ └── go.yml ├── pkg ├── github │ ├── milestone.go │ ├── issue.go │ ├── pullrequest.go │ └── repository.go ├── prow │ └── labels.go ├── client │ ├── client_labels.go │ ├── client.go │ ├── client_repository.go │ ├── client_issues.go │ ├── client_milestones.go │ ├── client_pullrequests.go │ ├── client_issues_gen.go │ ├── client_pullrequests_gen.go │ └── client_milestones_gen.go ├── fetcher │ ├── jobs.go │ ├── queue.go │ ├── jobs_issues.go │ ├── jobs_pullrequests.go │ ├── jobs_milestones.go │ └── fetcher.go └── metrics │ ├── metrics.go │ └── collector.go ├── util.go ├── hack ├── release.sh ├── client_issues_gen.go.tmpl ├── client_pullrequests_gen.go.tmpl ├── client_milestones_gen.go.tmpl ├── generate-client.go └── format-dashboards.sh ├── go.mod ├── LICENSE ├── go.sum ├── README.md └── main.go /contrib/grafana/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xrstf/github_exporter/HEAD/contrib/grafana/screenshot.png -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20.6-alpine as builder 2 | 3 | WORKDIR /app/ 4 | COPY . . 5 | RUN go build 6 | 7 | FROM alpine:3.17 8 | 9 | RUN apk --no-cache add ca-certificates 10 | COPY --from=builder /app/github_exporter . 11 | EXPOSE 9612 12 | ENTRYPOINT ["/github_exporter"] 13 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | name: Build 12 | runs-on: ubuntu-latest 13 | steps: 14 | - id: go 15 | name: Set up Go 1.x 16 | uses: actions/setup-go@v2 17 | with: 18 | go-version: ^1.20 19 | - name: Check out code into the Go module directory 20 | uses: actions/checkout@v2 21 | - name: Get dependencies 22 | run: go get -v -t -d ./... 23 | - name: Build 24 | run: go build -v . 25 | -------------------------------------------------------------------------------- /pkg/github/milestone.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package github 5 | 6 | import ( 7 | "time" 8 | 9 | "github.com/shurcooL/githubv4" 10 | ) 11 | 12 | type Milestone struct { 13 | Number int 14 | Title string 15 | State githubv4.MilestoneState 16 | CreatedAt time.Time 17 | UpdatedAt time.Time 18 | ClosedAt *time.Time 19 | DueOn *time.Time 20 | FetchedAt time.Time 21 | OpenIssues int 22 | ClosedIssues int 23 | OpenPullRequests int 24 | ClosedPullRequests int 25 | } 26 | -------------------------------------------------------------------------------- /pkg/github/issue.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package github 5 | 6 | import ( 7 | "strings" 8 | "time" 9 | 10 | "github.com/shurcooL/githubv4" 11 | ) 12 | 13 | type Issue struct { 14 | Number int 15 | Author string 16 | State githubv4.IssueState 17 | CreatedAt time.Time 18 | UpdatedAt time.Time 19 | FetchedAt time.Time 20 | Labels []string 21 | } 22 | 23 | func (i *Issue) HasLabel(label string) bool { 24 | label = strings.ToLower(label) 25 | 26 | for _, l := range i.Labels { 27 | if label == strings.ToLower(l) { 28 | return true 29 | } 30 | } 31 | 32 | return false 33 | } 34 | -------------------------------------------------------------------------------- /contrib/prometheus/rules.yaml: -------------------------------------------------------------------------------- 1 | # These are example rules to make the Grafana dashboard work. 2 | 3 | groups: 4 | - name: github-exporter 5 | rules: 6 | - record: ':github_exporter_repo_milestone_completion:' 7 | expr: | 8 | sum by (repo, number) (github_exporter_milestone_issues{state="closed"}) / 9 | (sum by (repo, number) (github_exporter_milestone_issues)) 10 | 11 | - record: ':github_exporter_repo_open_milestone_completion:' 12 | expr: | 13 | :github_exporter_repo_milestone_completion: * 14 | on (repo, number) (github_exporter_milestone_info{state="open"}) 15 | 16 | - record: 'sum:github_exporter_repo_language_size_bytes:repo' 17 | expr: | 18 | sum by (repo) (github_exporter_repo_language_size_bytes) 19 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package main 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "strings" 10 | ) 11 | 12 | type repository struct { 13 | owner string 14 | name string 15 | } 16 | 17 | func (r *repository) String() string { 18 | return fmt.Sprintf("%s/%s", r.owner, r.name) 19 | } 20 | 21 | type repositoryList []repository 22 | 23 | func (l *repositoryList) String() string { 24 | return fmt.Sprint(*l) 25 | } 26 | 27 | func (l *repositoryList) Set(value string) error { 28 | parts := strings.Split(value, "/") 29 | 30 | if len(parts) != 2 { 31 | return errors.New(`not a valid repository name, must be "owner/name"`) 32 | } 33 | 34 | *l = append(*l, repository{ 35 | owner: parts[0], 36 | name: parts[1], 37 | }) 38 | 39 | return nil 40 | } 41 | -------------------------------------------------------------------------------- /hack/release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # SPDX-FileCopyrightText: 2023 Christoph Mewes 4 | # SPDX-License-Identifier: MIT 5 | 6 | set -euo pipefail 7 | 8 | cd $(dirname $0)/.. 9 | 10 | version="${1:-}" 11 | version=${version#"v"} 12 | 13 | if [ -z "$version" ]; then 14 | echo "Usage: $0 VERSION" 15 | echo "Hint: Version prefix 'v' is automatically trimmed." 16 | exit 1 17 | fi 18 | 19 | if git tag | grep "v$version" >/dev/null; then 20 | echo "Version is already tagged." 21 | exit 1 22 | fi 23 | 24 | set_version() { 25 | yq --inplace ".spec.template.spec.containers[0].image=\"xrstf/github_exporter:$1\"" contrib/kubernetes/deployment.yaml 26 | } 27 | 28 | set_version "$version" 29 | git commit -am "version $version" 30 | git tag -m "version $version" "v$version" 31 | 32 | set_version "latest" 33 | git commit -am "back to dev" 34 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go.xrstf.de/github_exporter 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/prometheus/client_golang v1.16.0 7 | github.com/shurcooL/githubv4 v0.0.0-20230704064427-599ae7bbf278 8 | github.com/sirupsen/logrus v1.9.3 9 | golang.org/x/oauth2 v0.10.0 10 | ) 11 | 12 | require ( 13 | github.com/beorn7/perks v1.0.1 // indirect 14 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 15 | github.com/golang/protobuf v1.5.3 // indirect 16 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 17 | github.com/prometheus/client_model v0.3.0 // indirect 18 | github.com/prometheus/common v0.42.0 // indirect 19 | github.com/prometheus/procfs v0.10.1 // indirect 20 | github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 // indirect 21 | golang.org/x/net v0.12.0 // indirect 22 | golang.org/x/sys v0.10.0 // indirect 23 | google.golang.org/appengine v1.6.7 // indirect 24 | google.golang.org/protobuf v1.31.0 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /pkg/github/pullrequest.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package github 5 | 6 | import ( 7 | "strings" 8 | "time" 9 | 10 | "github.com/shurcooL/githubv4" 11 | ) 12 | 13 | type BuildContext struct { 14 | Name string 15 | State githubv4.StatusState 16 | } 17 | 18 | type PullRequest struct { 19 | Number int 20 | Author string 21 | State githubv4.PullRequestState 22 | CreatedAt time.Time 23 | UpdatedAt time.Time 24 | FetchedAt time.Time 25 | Labels []string 26 | Contexts []BuildContext 27 | } 28 | 29 | func (p *PullRequest) HasLabel(label string) bool { 30 | label = strings.ToLower(label) 31 | 32 | for _, l := range p.Labels { 33 | if label == strings.ToLower(l) { 34 | return true 35 | } 36 | } 37 | 38 | return false 39 | } 40 | 41 | func (p *PullRequest) Context(name string) *BuildContext { 42 | for i, ctx := range p.Contexts { 43 | if ctx.Name == name { 44 | return &p.Contexts[i] 45 | } 46 | } 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Christoph Mewes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /hack/client_issues_gen.go.tmpl: -------------------------------------------------------------------------------- 1 | // This file has been generated by hack/generate-client.sh 2 | // Do not edit manually! 3 | 4 | package client 5 | 6 | import ( 7 | "fmt" 8 | ) 9 | 10 | const ( 11 | MaxIssuesPerQuery = {{ .numFields }} 12 | ) 13 | 14 | type numberedIssueQuery struct { 15 | RateLimit rateLimit 16 | Repository struct { 17 | {{- range .fields }} 18 | Issue{{ . }} *graphqlIssue `graphql:"issue{{ . }}: issue(number: $number{{ . }}) @include(if: $has{{ . }})"` 19 | {{- end }} 20 | } `graphql:"repository(owner: $owner, name: $name)"` 21 | } 22 | 23 | func (r *numberedIssueQuery) GetAll() []graphqlIssue { 24 | result := []graphqlIssue{} 25 | 26 | for i := 0; i < MaxIssuesPerQuery; i++ { 27 | if issue := r.Get(i); issue != nil { 28 | result = append(result, *issue) 29 | } 30 | } 31 | 32 | return result 33 | } 34 | 35 | func (r *numberedIssueQuery) Get(index int) *graphqlIssue { 36 | switch index { 37 | {{- range .fields }} 38 | case {{ . }}: 39 | return r.Repository.Issue{{ . }} 40 | {{- end }} 41 | } 42 | 43 | panic(fmt.Sprintf("Index %d out of range [0,%d] when accessing issue request", index, MaxIssuesPerQuery-1)) 44 | } 45 | -------------------------------------------------------------------------------- /hack/client_pullrequests_gen.go.tmpl: -------------------------------------------------------------------------------- 1 | // This file has been generated by hack/generate-client.sh 2 | // Do not edit manually! 3 | 4 | package client 5 | 6 | import ( 7 | "fmt" 8 | ) 9 | 10 | const ( 11 | MaxPullRequestsPerQuery = {{ .numFields }} 12 | ) 13 | 14 | type numberedPullRequestQuery struct { 15 | RateLimit rateLimit 16 | Repository struct { 17 | {{- range .fields }} 18 | Pr{{ . }} *graphqlPullRequest `graphql:"pr{{ . }}: pullRequest(number: $number{{ . }}) @include(if: $has{{ . }})"` 19 | {{- end }} 20 | } `graphql:"repository(owner: $owner, name: $name)"` 21 | } 22 | 23 | func (r *numberedPullRequestQuery) GetAll() []graphqlPullRequest { 24 | result := []graphqlPullRequest{} 25 | 26 | for i := 0; i < MaxPullRequestsPerQuery; i++ { 27 | if pr := r.Get(i); pr != nil { 28 | result = append(result, *pr) 29 | } 30 | } 31 | 32 | return result 33 | } 34 | 35 | func (r *numberedPullRequestQuery) Get(index int) *graphqlPullRequest { 36 | switch index { 37 | {{- range .fields }} 38 | case {{ . }}: 39 | return r.Repository.Pr{{ . }} 40 | {{- end }} 41 | } 42 | 43 | panic(fmt.Sprintf("Index %d out of range [0,%d] when accessing PR request", index, MaxPullRequestsPerQuery-1)) 44 | } 45 | -------------------------------------------------------------------------------- /hack/client_milestones_gen.go.tmpl: -------------------------------------------------------------------------------- 1 | // This file has been generated by hack/generate-client.sh 2 | // Do not edit manually! 3 | 4 | package client 5 | 6 | import ( 7 | "fmt" 8 | ) 9 | 10 | const ( 11 | MaxMilestonesPerQuery = {{ .numFields }} 12 | ) 13 | 14 | type numberedMilestoneQuery struct { 15 | RateLimit rateLimit 16 | Repository struct { 17 | {{- range .fields }} 18 | Milestone{{ . }} *graphqlMilestone `graphql:"milestone{{ . }}: milestone(number: $number{{ . }}) @include(if: $has{{ . }})"` 19 | {{- end }} 20 | } `graphql:"repository(owner: $owner, name: $name)"` 21 | } 22 | 23 | func (r *numberedMilestoneQuery) GetAll() []graphqlMilestone { 24 | result := []graphqlMilestone{} 25 | 26 | for i := 0; i < MaxMilestonesPerQuery; i++ { 27 | if milestone := r.Get(i); milestone != nil { 28 | result = append(result, *milestone) 29 | } 30 | } 31 | 32 | return result 33 | } 34 | 35 | func (r *numberedMilestoneQuery) Get(index int) *graphqlMilestone { 36 | switch index { 37 | {{- range .fields }} 38 | case {{ . }}: 39 | return r.Repository.Milestone{{ . }} 40 | {{- end }} 41 | } 42 | 43 | panic(fmt.Sprintf("Index %d out of range [0,%d] when accessing milestone request", index, MaxMilestonesPerQuery-1)) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/prow/labels.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package prow 5 | 6 | import ( 7 | "fmt" 8 | "regexp" 9 | "strings" 10 | 11 | "go.xrstf.de/github_exporter/pkg/github" 12 | ) 13 | 14 | func PullRequestLabelNames() []string { 15 | return []string{"approved", "lgtm", "pending", "size", "kind", "priority", "team"} 16 | } 17 | 18 | func PullRequestLabels(pr *github.PullRequest) []string { 19 | return []string{ 20 | fmt.Sprintf("%v", pr.HasLabel("lgtm")), 21 | fmt.Sprintf("%v", pr.HasLabel("approved")), 22 | fmt.Sprintf("%v", prefixedLabel("do-not-merge", pr.Labels) != ""), 23 | prefixedLabel("size", pr.Labels), 24 | prefixedLabel("kind", pr.Labels), 25 | prefixedLabel("priority", pr.Labels), 26 | prefixedLabel("team", pr.Labels), 27 | } 28 | } 29 | 30 | func IssueLabelNames() []string { 31 | return []string{"kind", "priority", "team"} 32 | } 33 | 34 | func IssueLabels(issue *github.Issue) []string { 35 | return []string{ 36 | prefixedLabel("kind", issue.Labels), 37 | prefixedLabel("priority", issue.Labels), 38 | prefixedLabel("team", issue.Labels), 39 | } 40 | } 41 | 42 | func prefixedLabel(prefix string, labels []string) string { 43 | prefix = strings.ToLower(strings.TrimSuffix(prefix, "/")) 44 | regex := regexp.MustCompile(fmt.Sprintf(`^%s/(.+)$`, prefix)) 45 | 46 | for _, label := range labels { 47 | label := strings.ToLower(label) 48 | 49 | if match := regex.FindStringSubmatch(label); match != nil { 50 | return strings.ToLower(match[1]) 51 | } 52 | } 53 | 54 | return "" 55 | } 56 | -------------------------------------------------------------------------------- /hack/generate-client.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "go/format" 9 | "log" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "text/template" 14 | ) 15 | 16 | const ( 17 | fields = 100 18 | filename = "pkg/client/client_gen.go" 19 | ) 20 | 21 | func makeRange(min, max int) []int { 22 | a := make([]int, max-min+1) 23 | for i := range a { 24 | a[i] = min + i 25 | } 26 | return a 27 | } 28 | 29 | func main() { 30 | templates, err := filepath.Glob("hack/*.go.tmpl") 31 | if err != nil { 32 | log.Fatalf("Failed to find Go templates: %v", err) 33 | } 34 | 35 | data := map[string]interface{}{ 36 | "numFields": fields, 37 | "fields": makeRange(0, fields-1), 38 | } 39 | 40 | for _, templateFile := range templates { 41 | log.Printf("Rendering %s...", templateFile) 42 | 43 | content, err := os.ReadFile(templateFile) 44 | if err != nil { 45 | log.Fatalf("Failed to read client_gen.go.tmpl -- did you run this from the root directory?: %v", err) 46 | } 47 | 48 | tpl := template.Must(template.New("tpl").Parse(string(content))) 49 | 50 | var buf bytes.Buffer 51 | tpl.Execute(&buf, data) 52 | 53 | source, err := format.Source(buf.Bytes()) 54 | if err != nil { 55 | log.Fatalf("Failed to format generated code: %v", err) 56 | } 57 | 58 | filename := filepath.Join("pkg/client", strings.TrimSuffix(filepath.Base(templateFile), ".tmpl")) 59 | 60 | err = os.WriteFile(filename, source, 0644) 61 | if err != nil { 62 | log.Fatalf("Failed to write %s: %v", filename, err) 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/client/client_labels.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package client 5 | 6 | import ( 7 | "github.com/shurcooL/githubv4" 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type repositoryLabelsQuery struct { 12 | RateLimit rateLimit 13 | Repository struct { 14 | Labels struct { 15 | Nodes []struct { 16 | Name string 17 | } 18 | PageInfo struct { 19 | EndCursor githubv4.String 20 | HasNextPage bool 21 | } 22 | } `graphql:"labels(first: 100, after: $cursor)"` 23 | } `graphql:"repository(owner: $owner, name: $name)"` 24 | } 25 | 26 | func (c *Client) RepositoryLabels(owner string, name string) ([]string, error) { 27 | variables := map[string]interface{}{ 28 | "owner": githubv4.String(owner), 29 | "name": githubv4.String(name), 30 | "cursor": (*githubv4.String)(nil), 31 | } 32 | 33 | var q repositoryLabelsQuery 34 | 35 | labels := []string{} 36 | 37 | for { 38 | err := c.client.Query(c.ctx, &q, variables) 39 | c.countRequest(owner, name, q.RateLimit) 40 | 41 | c.log.WithFields(logrus.Fields{ 42 | "owner": owner, 43 | "name": name, 44 | "cursor": variables["cursor"], 45 | "cost": q.RateLimit.Cost, 46 | }).Debugf("RepositoryLabels()") 47 | 48 | if err != nil { 49 | return labels, err 50 | } 51 | 52 | for _, label := range q.Repository.Labels.Nodes { 53 | labels = append(labels, label.Name) 54 | } 55 | 56 | if !q.Repository.Labels.PageInfo.HasNextPage { 57 | break 58 | } 59 | 60 | variables["cursor"] = githubv4.NewString(q.Repository.Labels.PageInfo.EndCursor) 61 | } 62 | 63 | return labels, nil 64 | } 65 | -------------------------------------------------------------------------------- /hack/format-dashboards.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # SPDX-FileCopyrightText: 2023 Christoph Mewes 4 | # SPDX-License-Identifier: MIT 5 | 6 | set -euo pipefail 7 | 8 | cd $(dirname $0)/../ 9 | 10 | for filename in contrib/grafana/*.json; do 11 | tmpfile="$filename.tmp" 12 | 13 | cat "$filename" | \ 14 | jq '(.templating.list[] | select(.type=="query") | .options) = []' | \ 15 | jq '(.templating.list[] | select(.type=="query") | .refresh) = 2' | \ 16 | jq '(.templating.list[] | select(.type=="query") | .current) = {}' | \ 17 | jq '(.templating.list[] | select(.type=="datasource") | .current) = {}' | \ 18 | jq '(.templating.list[] | select(.type=="interval") | .current) = {}' | \ 19 | jq '(.panels[] | select(.scopedVars!=null) | .scopedVars) = {}' | \ 20 | jq '(.panels[].panels?[]? | select(.scopedVars!=null) | .scopedVars) = {}' | \ 21 | jq '(.templating.list[] | select(.type=="datasource") | .hide) = 2' | \ 22 | jq '(.annotations.list) = []' | \ 23 | jq '(.links) = []' | \ 24 | jq '(.refresh) = "1m"' | \ 25 | jq '(.time.from) = "now-1d"' | \ 26 | jq '(.editable) = true' | \ 27 | jq '(.panels[] | select(.type!="row") | .editable) = true' | \ 28 | jq '(.panels[] | select(.type!="row") | .transparent) = true' | \ 29 | jq '(.panels[] | select(.type!="row") | .timeRegions) = []' | \ 30 | jq '(.hideControls) = false' | \ 31 | jq '(.time.to) = "now"' | \ 32 | jq '(.timezone) = ""' | \ 33 | jq '(.graphTooltip) = 1' | \ 34 | jq '(.version) = 1' | \ 35 | jq 'del(.panels[] | select(.repeatPanelId!=null))' | \ 36 | jq 'del(.panels[].panels?[]? | select(.repeatPanelId!=null))' | \ 37 | jq 'del(.id)' | \ 38 | jq 'del(.iteration)' | \ 39 | jq --sort-keys '.' > "$tmpfile" 40 | 41 | mv "$tmpfile" "$filename" 42 | done 43 | -------------------------------------------------------------------------------- /pkg/fetcher/jobs.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package fetcher 5 | 6 | import ( 7 | "time" 8 | 9 | "go.xrstf.de/github_exporter/pkg/github" 10 | 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | const ( 15 | updateLabelsJobKey = "update-labels" 16 | updateRepoInfoJobKey = "update-repository-info" 17 | ) 18 | 19 | type jobQueue map[string]interface{} 20 | 21 | // processUpdateLabelsJob fetches the repository's labels and removes 22 | // the job afterwards. 23 | func (f *Fetcher) processUpdateLabelsJob(repo *github.Repository, log logrus.FieldLogger, job string) error { 24 | labels, err := f.client.RepositoryLabels(repo.Owner, repo.Name) 25 | 26 | log.Debugf("Fetched %d labels.", len(labels)) 27 | 28 | repo.SetLabels(labels) 29 | f.removeJob(repo, job) 30 | 31 | return err 32 | } 33 | 34 | // processUpdateRepoInfos fetches the repository's metadata. 35 | func (f *Fetcher) processUpdateRepoInfos(repo *github.Repository, log logrus.FieldLogger, job string) error { 36 | now := time.Now() 37 | 38 | info, err := f.client.RepositoryInfo(repo.Owner, repo.Name) 39 | 40 | if info != nil { 41 | _ = repo.Locked(func(r *github.Repository) error { 42 | r.FetchedAt = &now 43 | r.DiskUsageBytes = info.DiskUsage * 1024 // convert kbytes to bytes 44 | r.Forks = info.Forks 45 | r.Stargazers = info.Stargazers 46 | r.Watchers = info.Watchers 47 | r.IsPrivate = info.IsPrivate 48 | r.IsArchived = info.IsArchived 49 | r.IsDisabled = info.IsDisabled 50 | r.IsFork = info.IsFork 51 | r.IsLocked = info.IsLocked 52 | r.IsMirror = info.IsMirror 53 | r.IsTemplate = info.IsTemplate 54 | r.Languages = info.Languages 55 | 56 | return nil 57 | }) 58 | } 59 | 60 | f.removeJob(repo, job) 61 | 62 | return err 63 | } 64 | -------------------------------------------------------------------------------- /contrib/kubernetes/deployment.yaml: -------------------------------------------------------------------------------- 1 | # Create a secret containing your GitHub token first: 2 | # kubectl create secret generic github-token --from-literal=token=YOUR_TOKEN_HERE 3 | 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: github-exporter 8 | labels: 9 | app.kubernetes.io/name: github-exporter 10 | app.kubernetes.io/version: '0.3.3' 11 | spec: 12 | replicas: 1 13 | strategy: 14 | type: Recreate 15 | selector: 16 | matchLabels: 17 | app.kubernetes.io/name: github-exporter 18 | template: 19 | metadata: 20 | labels: 21 | app.kubernetes.io/name: github-exporter 22 | annotations: 23 | prometheus.io/scrape: 'true' 24 | prometheus.io/port: '9612' 25 | spec: 26 | containers: 27 | - name: github-exporter 28 | image: 'xrstf/github_exporter:latest' 29 | args: 30 | - -listen=0.0.0.0:9162 31 | - -repo=xrstf/github_exporter 32 | env: 33 | - name: GITHUB_TOKEN 34 | valueFrom: 35 | secretKeyRef: 36 | name: github-token 37 | key: token 38 | ports: 39 | - name: metrics 40 | containerPort: 9162 41 | protocol: TCP 42 | livenessProbe: 43 | httpGet: 44 | path: /metrics 45 | port: metrics 46 | readinessProbe: 47 | httpGet: 48 | path: /metrics 49 | port: metrics 50 | resources: 51 | requests: 52 | cpu: 50m 53 | memory: 64Mi 54 | limits: 55 | cpu: 1 56 | memory: 128Mi 57 | securityContext: 58 | runAsNonRoot: true 59 | runAsUser: 65534 60 | -------------------------------------------------------------------------------- /pkg/fetcher/queue.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package fetcher 5 | 6 | type integerQueue map[int]struct{} 7 | 8 | func (q integerQueue) fillSliceUpTo(list []int, max int) []int { 9 | for number := range q { 10 | list = append(list, number) 11 | if len(list) >= max { 12 | break 13 | } 14 | } 15 | 16 | return list 17 | } 18 | 19 | type prioritizedIntegerQueue struct { 20 | priority integerQueue 21 | regular integerQueue 22 | } 23 | 24 | func newPrioritizedIntegerQueue() prioritizedIntegerQueue { 25 | return prioritizedIntegerQueue{ 26 | priority: integerQueue{}, 27 | regular: integerQueue{}, 28 | } 29 | } 30 | 31 | func (q *prioritizedIntegerQueue) priorityEnqueue(numbers []int) { 32 | q.enqueue(numbers, q.priority) 33 | } 34 | 35 | func (q *prioritizedIntegerQueue) regularEnqueue(numbers []int) { 36 | q.enqueue(numbers, q.regular) 37 | } 38 | 39 | func (q *prioritizedIntegerQueue) enqueue(numbers []int, queue integerQueue) { 40 | for _, number := range numbers { 41 | queue[number] = struct{}{} 42 | } 43 | } 44 | 45 | func (q *prioritizedIntegerQueue) prioritySize() int { 46 | return len(q.priority) 47 | } 48 | 49 | func (q *prioritizedIntegerQueue) regularSize() int { 50 | return len(q.regular) 51 | } 52 | 53 | func (q *prioritizedIntegerQueue) getBatch(minBatchSize int, maxBatchSize int) []int { 54 | // get the first N random priority items 55 | items := q.priority.fillSliceUpTo([]int{}, maxBatchSize) 56 | 57 | // if the batch is already full, stop and return 58 | if len(items) >= maxBatchSize { 59 | return items 60 | } 61 | 62 | // otherwise, continue to add random regular items 63 | items = q.regular.fillSliceUpTo(items, maxBatchSize) 64 | 65 | // if we have reached the min batch size, it's good enough 66 | // and we can return 67 | if len(items) >= minBatchSize { 68 | return items 69 | } 70 | 71 | // not enough items at all 72 | return nil 73 | } 74 | 75 | func (q *prioritizedIntegerQueue) dequeue(numbers []int) { 76 | for _, number := range numbers { 77 | delete(q.priority, number) 78 | delete(q.regular, number) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /pkg/client/client.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package client 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | 11 | "github.com/shurcooL/githubv4" 12 | "github.com/sirupsen/logrus" 13 | "golang.org/x/oauth2" 14 | ) 15 | 16 | type rateLimit struct { 17 | Cost int 18 | Remaining int 19 | } 20 | 21 | var stopFetching = errors.New("stop fetching data pls") 22 | 23 | type Client struct { 24 | ctx context.Context 25 | client *githubv4.Client 26 | log logrus.FieldLogger 27 | realnames bool 28 | requests map[string]int 29 | remainingPoints int 30 | totalCosts map[string]int 31 | } 32 | 33 | func NewClient(ctx context.Context, log logrus.FieldLogger, token string, realnames bool) (*Client, error) { 34 | if token == "" { 35 | return nil, errors.New("token cannot be empty") 36 | } 37 | 38 | src := oauth2.StaticTokenSource( 39 | &oauth2.Token{ 40 | AccessToken: token, 41 | }, 42 | ) 43 | httpClient := oauth2.NewClient(ctx, src) 44 | client := githubv4.NewClient(httpClient) 45 | 46 | return &Client{ 47 | ctx: ctx, 48 | client: client, 49 | log: log, 50 | realnames: realnames, 51 | requests: map[string]int{}, 52 | remainingPoints: 0, 53 | totalCosts: map[string]int{}, 54 | }, nil 55 | } 56 | 57 | func (c *Client) GetRemainingPoints() int { 58 | return c.remainingPoints 59 | } 60 | 61 | func (c *Client) GetRequestCounts() map[string]int { 62 | return c.requests 63 | } 64 | 65 | func (c *Client) GetTotalCosts() map[string]int { 66 | return c.totalCosts 67 | } 68 | 69 | func (c *Client) countRequest(owner string, name string, rateLimit rateLimit) { 70 | key := fmt.Sprintf("%s/%s", owner, name) 71 | 72 | val := c.requests[key] 73 | c.requests[key] = val + 1 74 | 75 | val = c.totalCosts[key] 76 | c.totalCosts[key] = val + rateLimit.Cost 77 | 78 | c.remainingPoints = rateLimit.Remaining 79 | } 80 | 81 | func getNumberedQueryVariables(numbers []int, max int) map[string]interface{} { 82 | if len(numbers) > max { 83 | panic(fmt.Sprintf("List contains more (%d) than possible (%d) PR numbers.", len(numbers), max)) 84 | } 85 | 86 | variables := map[string]interface{}{} 87 | 88 | for i := 0; i < max; i++ { 89 | number := 0 90 | has := false 91 | 92 | if i < len(numbers) { 93 | number = numbers[i] 94 | has = true 95 | } 96 | 97 | variables[fmt.Sprintf("number%d", i)] = githubv4.Int(number) 98 | variables[fmt.Sprintf("has%d", i)] = githubv4.Boolean(has) 99 | } 100 | 101 | return variables 102 | } 103 | -------------------------------------------------------------------------------- /pkg/client/client_repository.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package client 5 | 6 | import ( 7 | "github.com/shurcooL/githubv4" 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type repositoryInfoQuery struct { 12 | RateLimit rateLimit 13 | Repository struct { 14 | DiskUsage int 15 | ForkCount int 16 | Stargazers struct { 17 | TotalCount int 18 | } 19 | Watchers struct { 20 | TotalCount int 21 | } 22 | IsPrivate bool 23 | IsArchived bool 24 | IsDisabled bool 25 | IsFork bool 26 | IsLocked bool 27 | IsMirror bool 28 | IsTemplate bool 29 | Languages struct { 30 | Edges []struct { 31 | Size int 32 | Node struct { 33 | Name string 34 | } 35 | } 36 | } `graphql:"languages(first: 100)"` 37 | } `graphql:"repository(owner: $owner, name: $name)"` 38 | } 39 | 40 | type RepositoryInfo struct { 41 | // DiskUsage is returned in KBytes 42 | DiskUsage int 43 | Forks int 44 | Stargazers int 45 | Watchers int 46 | IsPrivate bool 47 | IsArchived bool 48 | IsDisabled bool 49 | IsFork bool 50 | IsLocked bool 51 | IsMirror bool 52 | IsTemplate bool 53 | Languages map[string]int 54 | } 55 | 56 | func (c *Client) RepositoryInfo(owner string, name string) (*RepositoryInfo, error) { 57 | variables := map[string]interface{}{ 58 | "owner": githubv4.String(owner), 59 | "name": githubv4.String(name), 60 | } 61 | 62 | var q repositoryInfoQuery 63 | 64 | err := c.client.Query(c.ctx, &q, variables) 65 | c.countRequest(owner, name, q.RateLimit) 66 | 67 | c.log.WithFields(logrus.Fields{ 68 | "owner": owner, 69 | "name": name, 70 | "cost": q.RateLimit.Cost, 71 | }).Debugf("RepositoryInfo()") 72 | 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | info := &RepositoryInfo{ 78 | DiskUsage: q.Repository.DiskUsage, 79 | Forks: q.Repository.ForkCount, 80 | Stargazers: q.Repository.Stargazers.TotalCount, 81 | Watchers: q.Repository.Watchers.TotalCount, 82 | IsPrivate: q.Repository.IsPrivate, 83 | IsArchived: q.Repository.IsArchived, 84 | IsDisabled: q.Repository.IsDisabled, 85 | IsFork: q.Repository.IsFork, 86 | IsLocked: q.Repository.IsLocked, 87 | IsMirror: q.Repository.IsMirror, 88 | IsTemplate: q.Repository.IsTemplate, 89 | Languages: map[string]int{}, 90 | } 91 | 92 | for _, lang := range q.Repository.Languages.Edges { 93 | info.Languages[lang.Node.Name] = lang.Size 94 | } 95 | 96 | return info, nil 97 | } 98 | 99 | type repositoriesNamesQuery struct { 100 | RateLimit rateLimit 101 | RepositoryOwner struct { 102 | Repositories struct { 103 | Nodes []struct { 104 | Name string 105 | } 106 | } `graphql:"repositories(last: 100, isFork: false, isLocked: false, affiliations: OWNER)"` 107 | } `graphql:"repositoryOwner(login: $login)"` 108 | } 109 | 110 | func (c *Client) RepositoriesNames(login string) ([]string, error) { 111 | variables := map[string]interface{}{ 112 | "login": githubv4.String(login), 113 | } 114 | 115 | var q repositoriesNamesQuery 116 | 117 | err := c.client.Query(c.ctx, &q, variables) 118 | 119 | if err != nil { 120 | c.log.Error(err) 121 | return nil, err 122 | } 123 | 124 | repos := []string{} 125 | for _, node := range q.RepositoryOwner.Repositories.Nodes { 126 | repos = append(repos, node.Name) 127 | } 128 | 129 | return repos, nil 130 | } 131 | -------------------------------------------------------------------------------- /pkg/client/client_issues.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package client 5 | 6 | import ( 7 | "strings" 8 | "time" 9 | 10 | "go.xrstf.de/github_exporter/pkg/github" 11 | 12 | "github.com/shurcooL/githubv4" 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | type graphqlIssue struct { 17 | Number int 18 | State githubv4.IssueState 19 | CreatedAt time.Time 20 | UpdatedAt time.Time 21 | 22 | Author struct { 23 | Login string 24 | User struct { 25 | ID string 26 | } `graphql:"... on User"` 27 | } 28 | 29 | Labels struct { 30 | Nodes []struct { 31 | Name string 32 | } 33 | } `graphql:"labels(first: 50)"` 34 | } 35 | 36 | func (c *Client) convertIssue(api graphqlIssue, fetchedAt time.Time) github.Issue { 37 | issue := github.Issue{ 38 | Number: api.Number, 39 | Author: api.Author.User.ID, 40 | State: api.State, 41 | CreatedAt: api.CreatedAt, 42 | UpdatedAt: api.UpdatedAt, 43 | FetchedAt: fetchedAt, 44 | Labels: []string{}, 45 | } 46 | 47 | if c.realnames { 48 | issue.Author = api.Author.Login 49 | } 50 | 51 | for _, label := range api.Labels.Nodes { 52 | issue.Labels = append(issue.Labels, label.Name) 53 | } 54 | 55 | return issue 56 | } 57 | 58 | func (c *Client) GetRepositoryIssues(owner string, name string, numbers []int) ([]github.Issue, error) { 59 | variables := getNumberedQueryVariables(numbers, MaxIssuesPerQuery) 60 | variables["owner"] = githubv4.String(owner) 61 | variables["name"] = githubv4.String(name) 62 | 63 | var q numberedIssueQuery 64 | 65 | err := c.client.Query(c.ctx, &q, variables) 66 | c.countRequest(owner, name, q.RateLimit) 67 | 68 | c.log.WithFields(logrus.Fields{ 69 | "owner": owner, 70 | "name": name, 71 | "issues": len(numbers), 72 | "cost": q.RateLimit.Cost, 73 | }).Debugf("GetRepositoryIssues()") 74 | 75 | if err != nil && !strings.Contains(err.Error(), "Could not resolve to an Issue") { 76 | return nil, err 77 | } 78 | 79 | now := time.Now() 80 | issues := []github.Issue{} 81 | for _, issue := range q.GetAll() { 82 | issues = append(issues, c.convertIssue(issue, now)) 83 | } 84 | 85 | return issues, nil 86 | } 87 | 88 | type listIssuesQuery struct { 89 | RateLimit rateLimit 90 | Repository struct { 91 | Issues struct { 92 | Nodes []graphqlIssue 93 | PageInfo struct { 94 | EndCursor githubv4.String 95 | HasNextPage bool 96 | } 97 | } `graphql:"issues(states: $states, first: 100, orderBy: {field: UPDATED_AT, direction: DESC}, after: $cursor)"` 98 | } `graphql:"repository(owner: $owner, name: $name)"` 99 | } 100 | 101 | func (c *Client) ListIssues(owner string, name string, states []githubv4.IssueState, cursor string) ([]github.Issue, string, error) { 102 | if states == nil { 103 | states = []githubv4.IssueState{ 104 | githubv4.IssueStateOpen, 105 | githubv4.IssueStateClosed, 106 | } 107 | } 108 | 109 | variables := map[string]interface{}{ 110 | "owner": githubv4.String(owner), 111 | "name": githubv4.String(name), 112 | "states": states, 113 | } 114 | 115 | if cursor == "" { 116 | variables["cursor"] = (*githubv4.String)(nil) 117 | } else { 118 | variables["cursor"] = githubv4.String(cursor) 119 | } 120 | 121 | var q listIssuesQuery 122 | 123 | err := c.client.Query(c.ctx, &q, variables) 124 | c.countRequest(owner, name, q.RateLimit) 125 | 126 | c.log.WithFields(logrus.Fields{ 127 | "owner": owner, 128 | "name": name, 129 | "cursor": cursor, 130 | "cost": q.RateLimit.Cost, 131 | }).Debugf("ListIssues()") 132 | 133 | if err != nil { 134 | return nil, "", err 135 | } 136 | 137 | now := time.Now() 138 | issues := []github.Issue{} 139 | for _, node := range q.Repository.Issues.Nodes { 140 | issues = append(issues, c.convertIssue(node, now)) 141 | } 142 | 143 | cursor = "" 144 | if q.Repository.Issues.PageInfo.HasNextPage { 145 | cursor = string(q.Repository.Issues.PageInfo.EndCursor) 146 | } 147 | 148 | return issues, cursor, nil 149 | } 150 | -------------------------------------------------------------------------------- /pkg/client/client_milestones.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package client 5 | 6 | import ( 7 | "strings" 8 | "time" 9 | 10 | "go.xrstf.de/github_exporter/pkg/github" 11 | 12 | "github.com/shurcooL/githubv4" 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | type graphqlMilestone struct { 17 | Number int 18 | Title string 19 | State githubv4.MilestoneState 20 | CreatedAt time.Time 21 | UpdatedAt time.Time 22 | ClosedAt *time.Time 23 | DueOn *time.Time 24 | 25 | OpenIssues struct { 26 | TotalCount int 27 | } `graphql:"openIssues: issues(states: OPEN)"` 28 | 29 | ClosedIssues struct { 30 | TotalCount int 31 | } `graphql:"closedIssues: issues(states: CLOSED)"` 32 | 33 | OpenPullRequests struct { 34 | TotalCount int 35 | } `graphql:"openPullRequests: pullRequests(states: OPEN)"` 36 | 37 | ClosedPullRequests struct { 38 | TotalCount int 39 | } `graphql:"closedPullRequests: pullRequests(states: [MERGED, CLOSED])"` 40 | } 41 | 42 | func (c *Client) convertMilestone(api graphqlMilestone, fetchedAt time.Time) github.Milestone { 43 | return github.Milestone{ 44 | Number: api.Number, 45 | Title: api.Title, 46 | State: api.State, 47 | CreatedAt: api.CreatedAt, 48 | UpdatedAt: api.UpdatedAt, 49 | ClosedAt: api.ClosedAt, 50 | DueOn: api.DueOn, 51 | FetchedAt: fetchedAt, 52 | OpenIssues: api.OpenIssues.TotalCount, 53 | ClosedIssues: api.ClosedIssues.TotalCount, 54 | OpenPullRequests: api.OpenPullRequests.TotalCount, 55 | ClosedPullRequests: api.ClosedPullRequests.TotalCount, 56 | } 57 | } 58 | 59 | func (c *Client) GetRepositoryMilestones(owner string, name string, numbers []int) ([]github.Milestone, error) { 60 | variables := getNumberedQueryVariables(numbers, MaxMilestonesPerQuery) 61 | variables["owner"] = githubv4.String(owner) 62 | variables["name"] = githubv4.String(name) 63 | 64 | var q numberedMilestoneQuery 65 | 66 | err := c.client.Query(c.ctx, &q, variables) 67 | c.countRequest(owner, name, q.RateLimit) 68 | 69 | c.log.WithFields(logrus.Fields{ 70 | "owner": owner, 71 | "name": name, 72 | "milestones": len(numbers), 73 | "cost": q.RateLimit.Cost, 74 | }).Debugf("GetRepositoryMilestones()") 75 | 76 | // As of 2020-06-12, the GitHub API does not return an error, but instead just sets 77 | // the milestone field to null if it was not found. For safety we keep the check anyway. 78 | if err != nil && !strings.Contains(err.Error(), "Could not resolve to a Milestone") { 79 | return nil, err 80 | } 81 | 82 | now := time.Now() 83 | milestones := []github.Milestone{} 84 | for _, milestone := range q.GetAll() { 85 | milestones = append(milestones, c.convertMilestone(milestone, now)) 86 | } 87 | 88 | return milestones, nil 89 | } 90 | 91 | type listMilestonesQuery struct { 92 | RateLimit rateLimit 93 | Repository struct { 94 | Milestones struct { 95 | Nodes []graphqlMilestone 96 | PageInfo struct { 97 | EndCursor githubv4.String 98 | HasNextPage bool 99 | } 100 | } `graphql:"milestones(states: $states, first: 100, orderBy: {field: UPDATED_AT, direction: DESC}, after: $cursor)"` 101 | } `graphql:"repository(owner: $owner, name: $name)"` 102 | } 103 | 104 | func (c *Client) ListMilestones(owner string, name string, states []githubv4.MilestoneState, cursor string) ([]github.Milestone, string, error) { 105 | if states == nil { 106 | states = []githubv4.MilestoneState{ 107 | githubv4.MilestoneStateOpen, 108 | githubv4.MilestoneStateClosed, 109 | } 110 | } 111 | 112 | variables := map[string]interface{}{ 113 | "owner": githubv4.String(owner), 114 | "name": githubv4.String(name), 115 | "states": states, 116 | } 117 | 118 | if cursor == "" { 119 | variables["cursor"] = (*githubv4.String)(nil) 120 | } else { 121 | variables["cursor"] = githubv4.String(cursor) 122 | } 123 | 124 | var q listMilestonesQuery 125 | 126 | err := c.client.Query(c.ctx, &q, variables) 127 | c.countRequest(owner, name, q.RateLimit) 128 | 129 | c.log.WithFields(logrus.Fields{ 130 | "owner": owner, 131 | "name": name, 132 | "cursor": cursor, 133 | "cost": q.RateLimit.Cost, 134 | }).Debugf("ListMilestones()") 135 | 136 | if err != nil { 137 | return nil, "", err 138 | } 139 | 140 | now := time.Now() 141 | milestones := []github.Milestone{} 142 | for _, node := range q.Repository.Milestones.Nodes { 143 | milestones = append(milestones, c.convertMilestone(node, now)) 144 | } 145 | 146 | cursor = "" 147 | if q.Repository.Milestones.PageInfo.HasNextPage { 148 | cursor = string(q.Repository.Milestones.PageInfo.EndCursor) 149 | } 150 | 151 | return milestones, cursor, nil 152 | } 153 | -------------------------------------------------------------------------------- /pkg/client/client_pullrequests.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package client 5 | 6 | import ( 7 | "strings" 8 | "time" 9 | 10 | "go.xrstf.de/github_exporter/pkg/github" 11 | 12 | "github.com/shurcooL/githubv4" 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | type graphqlPullRequest struct { 17 | Number int 18 | State githubv4.PullRequestState 19 | CreatedAt time.Time 20 | UpdatedAt time.Time 21 | 22 | Author struct { 23 | Login string 24 | User struct { 25 | ID string 26 | } `graphql:"... on User"` 27 | } 28 | 29 | Labels struct { 30 | Nodes []struct { 31 | Name string 32 | } 33 | } `graphql:"labels(first: 50)"` 34 | 35 | Commits struct { 36 | Nodes []struct { 37 | Commit struct { 38 | Status struct { 39 | Contexts []struct { 40 | Context string 41 | State githubv4.StatusState 42 | } 43 | } 44 | } 45 | } 46 | } `graphql:"commits(last: 1)"` 47 | } 48 | 49 | func (c *Client) convertPullRequest(api graphqlPullRequest, fetchedAt time.Time) github.PullRequest { 50 | pr := github.PullRequest{ 51 | Number: api.Number, 52 | Author: api.Author.User.ID, 53 | State: api.State, 54 | CreatedAt: api.CreatedAt, 55 | UpdatedAt: api.UpdatedAt, 56 | FetchedAt: fetchedAt, 57 | Labels: []string{}, 58 | Contexts: []github.BuildContext{}, 59 | } 60 | 61 | if c.realnames { 62 | pr.Author = api.Author.Login 63 | } 64 | 65 | for _, label := range api.Labels.Nodes { 66 | pr.Labels = append(pr.Labels, label.Name) 67 | } 68 | 69 | if len(api.Commits.Nodes) > 0 { 70 | for _, context := range api.Commits.Nodes[0].Commit.Status.Contexts { 71 | pr.Contexts = append(pr.Contexts, github.BuildContext{ 72 | Name: context.Context, 73 | State: context.State, 74 | }) 75 | } 76 | } 77 | 78 | return pr 79 | } 80 | 81 | func (c *Client) GetRepositoryPullRequests(owner string, name string, numbers []int) ([]github.PullRequest, error) { 82 | variables := getNumberedQueryVariables(numbers, MaxPullRequestsPerQuery) 83 | variables["owner"] = githubv4.String(owner) 84 | variables["name"] = githubv4.String(name) 85 | 86 | var q numberedPullRequestQuery 87 | 88 | err := c.client.Query(c.ctx, &q, variables) 89 | c.countRequest(owner, name, q.RateLimit) 90 | 91 | c.log.WithFields(logrus.Fields{ 92 | "owner": owner, 93 | "name": name, 94 | "prs": len(numbers), 95 | "cost": q.RateLimit.Cost, 96 | }).Debugf("GetRepositoryPullRequests()") 97 | 98 | if err != nil && !strings.Contains(err.Error(), "Could not resolve to a PullRequest") { 99 | return nil, err 100 | } 101 | 102 | now := time.Now() 103 | prs := []github.PullRequest{} 104 | for _, pr := range q.GetAll() { 105 | prs = append(prs, c.convertPullRequest(pr, now)) 106 | } 107 | 108 | return prs, nil 109 | } 110 | 111 | type listPullRequestsQuery struct { 112 | RateLimit rateLimit 113 | Repository struct { 114 | PullRequests struct { 115 | Nodes []graphqlPullRequest 116 | PageInfo struct { 117 | EndCursor githubv4.String 118 | HasNextPage bool 119 | } 120 | } `graphql:"pullRequests(states: $states, first: 100, orderBy: {field: UPDATED_AT, direction: DESC}, after: $cursor)"` 121 | } `graphql:"repository(owner: $owner, name: $name)"` 122 | } 123 | 124 | func (c *Client) ListPullRequests(owner string, name string, states []githubv4.PullRequestState, cursor string) ([]github.PullRequest, string, error) { 125 | if states == nil { 126 | states = []githubv4.PullRequestState{ 127 | githubv4.PullRequestStateClosed, 128 | githubv4.PullRequestStateMerged, 129 | githubv4.PullRequestStateOpen, 130 | } 131 | } 132 | 133 | variables := map[string]interface{}{ 134 | "owner": githubv4.String(owner), 135 | "name": githubv4.String(name), 136 | "states": states, 137 | } 138 | 139 | if cursor == "" { 140 | variables["cursor"] = (*githubv4.String)(nil) 141 | } else { 142 | variables["cursor"] = githubv4.String(cursor) 143 | } 144 | 145 | var q listPullRequestsQuery 146 | 147 | err := c.client.Query(c.ctx, &q, variables) 148 | c.countRequest(owner, name, q.RateLimit) 149 | 150 | c.log.WithFields(logrus.Fields{ 151 | "owner": owner, 152 | "name": name, 153 | "cursor": cursor, 154 | "cost": q.RateLimit.Cost, 155 | }).Debugf("ListPullRequests()") 156 | 157 | if err != nil { 158 | return nil, "", err 159 | } 160 | 161 | now := time.Now() 162 | prs := []github.PullRequest{} 163 | for _, node := range q.Repository.PullRequests.Nodes { 164 | prs = append(prs, c.convertPullRequest(node, now)) 165 | } 166 | 167 | cursor = "" 168 | if q.Repository.PullRequests.PageInfo.HasNextPage { 169 | cursor = string(q.Repository.PullRequests.PageInfo.EndCursor) 170 | } 171 | 172 | return prs, cursor, nil 173 | } 174 | -------------------------------------------------------------------------------- /pkg/github/repository.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package github 5 | 6 | import ( 7 | "fmt" 8 | "sync" 9 | "time" 10 | 11 | "github.com/shurcooL/githubv4" 12 | ) 13 | 14 | type Repository struct { 15 | Owner string 16 | Name string 17 | 18 | PullRequests map[int]PullRequest 19 | Issues map[int]Issue 20 | Milestones map[int]Milestone 21 | Labels []string 22 | DiskUsageBytes int 23 | Forks int 24 | Stargazers int 25 | Watchers int 26 | IsPrivate bool 27 | IsArchived bool 28 | IsDisabled bool 29 | IsFork bool 30 | IsLocked bool 31 | IsMirror bool 32 | IsTemplate bool 33 | Languages map[string]int 34 | FetchedAt *time.Time 35 | 36 | lock sync.RWMutex 37 | } 38 | 39 | func NewRepository(owner string, name string) *Repository { 40 | return &Repository{ 41 | Owner: owner, 42 | Name: name, 43 | PullRequests: map[int]PullRequest{}, 44 | Issues: map[int]Issue{}, 45 | Milestones: map[int]Milestone{}, 46 | Labels: []string{}, 47 | Languages: map[string]int{}, 48 | lock: sync.RWMutex{}, 49 | } 50 | } 51 | 52 | func (d *Repository) FullName() string { 53 | return fmt.Sprintf("%s/%s", d.Owner, d.Name) 54 | } 55 | 56 | func (d *Repository) SetLabels(Labels []string) { 57 | d.lock.Lock() 58 | defer d.lock.Unlock() 59 | 60 | d.Labels = Labels 61 | } 62 | 63 | func (d *Repository) AddPullRequests(prs []PullRequest) { 64 | d.lock.Lock() 65 | defer d.lock.Unlock() 66 | 67 | for _, pr := range prs { 68 | d.PullRequests[pr.Number] = pr 69 | } 70 | } 71 | 72 | func (d *Repository) DeletePullRequests(numbers []int) { 73 | d.lock.Lock() 74 | defer d.lock.Unlock() 75 | 76 | for _, number := range numbers { 77 | delete(d.PullRequests, number) 78 | } 79 | } 80 | 81 | func (d *Repository) GetPullRequests(states ...githubv4.PullRequestState) []PullRequest { 82 | d.lock.RLock() 83 | defer d.lock.RUnlock() 84 | 85 | numbers := []PullRequest{} 86 | for _, pr := range d.PullRequests { 87 | include := false 88 | 89 | if len(states) == 0 { 90 | include = true 91 | } else { 92 | for _, state := range states { 93 | if pr.State == state { 94 | include = true 95 | break 96 | } 97 | } 98 | } 99 | 100 | if include { 101 | numbers = append(numbers, pr) 102 | } 103 | } 104 | 105 | return numbers 106 | } 107 | 108 | func (d *Repository) AddIssues(issues []Issue) { 109 | d.lock.Lock() 110 | defer d.lock.Unlock() 111 | 112 | for _, issue := range issues { 113 | d.Issues[issue.Number] = issue 114 | } 115 | } 116 | 117 | func (d *Repository) DeleteIssues(numbers []int) { 118 | d.lock.Lock() 119 | defer d.lock.Unlock() 120 | 121 | for _, number := range numbers { 122 | delete(d.Issues, number) 123 | } 124 | } 125 | 126 | func (d *Repository) GetIssues(states ...githubv4.IssueState) []Issue { 127 | d.lock.RLock() 128 | defer d.lock.RUnlock() 129 | 130 | numbers := []Issue{} 131 | for _, issue := range d.Issues { 132 | include := false 133 | 134 | if len(states) == 0 { 135 | include = true 136 | } else { 137 | for _, state := range states { 138 | if issue.State == state { 139 | include = true 140 | break 141 | } 142 | } 143 | } 144 | 145 | if include { 146 | numbers = append(numbers, issue) 147 | } 148 | } 149 | 150 | return numbers 151 | } 152 | 153 | func (d *Repository) AddMilestones(milestones []Milestone) { 154 | d.lock.Lock() 155 | defer d.lock.Unlock() 156 | 157 | for _, milestone := range milestones { 158 | d.Milestones[milestone.Number] = milestone 159 | } 160 | } 161 | 162 | func (d *Repository) DeleteMilestones(numbers []int) { 163 | d.lock.Lock() 164 | defer d.lock.Unlock() 165 | 166 | for _, number := range numbers { 167 | delete(d.Milestones, number) 168 | } 169 | } 170 | 171 | func (d *Repository) GetMilestones(states ...githubv4.MilestoneState) []Milestone { 172 | d.lock.RLock() 173 | defer d.lock.RUnlock() 174 | 175 | numbers := []Milestone{} 176 | for _, milestone := range d.Milestones { 177 | include := false 178 | 179 | if len(states) == 0 { 180 | include = true 181 | } else { 182 | for _, state := range states { 183 | if milestone.State == state { 184 | include = true 185 | break 186 | } 187 | } 188 | } 189 | 190 | if include { 191 | numbers = append(numbers, milestone) 192 | } 193 | } 194 | 195 | return numbers 196 | } 197 | 198 | func (d *Repository) Locked(callback func(*Repository) error) error { 199 | d.lock.Lock() 200 | defer d.lock.Unlock() 201 | 202 | return callback(d) 203 | } 204 | 205 | func (d *Repository) RLocked(callback func(*Repository) error) error { 206 | d.lock.RLock() 207 | defer d.lock.RUnlock() 208 | 209 | return callback(d) 210 | } 211 | -------------------------------------------------------------------------------- /pkg/fetcher/jobs_issues.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package fetcher 5 | 6 | import ( 7 | "time" 8 | 9 | "go.xrstf.de/github_exporter/pkg/github" 10 | 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | const ( 15 | scanIssuesJobKey = "scan-issues" 16 | updateIssuesJobKey = "update-issues" 17 | findUpdatedIssuesJobKey = "find-updated-issues" 18 | ) 19 | 20 | type updateIssuesJobMeta struct { 21 | numbers []int 22 | } 23 | 24 | // processUpdateIssuesJob updates a list of already fetched 25 | // issues to ensure they stay up to date. This is done for all open 26 | // issues. 27 | func (f *Fetcher) processUpdateIssuesJob(repo *github.Repository, log logrus.FieldLogger, job string, data interface{}) error { 28 | meta := data.(updateIssuesJobMeta) 29 | 30 | issues, err := f.client.GetRepositoryIssues(repo.Owner, repo.Name, meta.numbers) 31 | 32 | fetchedNumbers := []int{} 33 | fetchedNumbersMap := map[int]struct{}{} 34 | for _, issue := range issues { 35 | fetchedNumbers = append(fetchedNumbers, issue.Number) 36 | fetchedNumbersMap[issue.Number] = struct{}{} 37 | } 38 | 39 | log.Debugf("Fetched %d out of %d issues.", len(fetchedNumbers), len(meta.numbers)) 40 | 41 | deleted := []int{} 42 | for _, number := range meta.numbers { 43 | if _, ok := fetchedNumbersMap[number]; !ok { 44 | deleted = append(deleted, number) 45 | } 46 | } 47 | 48 | if len(issues) > 0 { 49 | repo.AddIssues(issues) 50 | } 51 | 52 | // only delete not found issues from our local cache if the request was a success, otherwise 53 | // we would remove all issues if e.g. GitHub is unavailable 54 | if err == nil && len(deleted) > 0 { 55 | repo.DeleteIssues(deleted) 56 | } 57 | 58 | f.removeJob(repo, job) 59 | f.dequeueIssues(repo, meta.numbers) 60 | 61 | return err 62 | } 63 | 64 | // processFindUpdatedIssuesJob fetches the 100 most recently updated 65 | // issues in the given repository and updates repo. The job will be removed 66 | // from the job queue afterwards and all fetched issues will be removed from 67 | // the priority/regular issue queues. 68 | func (f *Fetcher) processFindUpdatedIssuesJob(repo *github.Repository, log logrus.FieldLogger, job string) error { 69 | fetchedNumbers := []int{} 70 | 71 | issues, _, err := f.client.ListIssues(repo.Owner, repo.Name, nil, "") 72 | for _, issue := range issues { 73 | fetchedNumbers = append(fetchedNumbers, issue.Number) 74 | } 75 | 76 | log.Debugf("Fetched %d recently updated issues.", len(fetchedNumbers)) 77 | 78 | repo.AddIssues(issues) 79 | 80 | f.removeJob(repo, job) 81 | f.dequeueIssues(repo, fetchedNumbers) 82 | 83 | return err 84 | } 85 | 86 | type scanIssuesJobMeta struct { 87 | max int 88 | fetched int 89 | cursor string 90 | } 91 | 92 | // processScanIssuesJob is the initial job for every repository. 93 | // It lists all existing issues and adds them to repo. 94 | // 95 | // Because the initial scan is vital for proper functioning of every 96 | // other job, this job must succeed before anything else can happen 97 | // with a repository. For this reason a failed scan job is re-queued 98 | // a few seconds later. 99 | func (f *Fetcher) processScanIssuesJob(repo *github.Repository, log logrus.FieldLogger, job string, data interface{}) error { 100 | meta := data.(scanIssuesJobMeta) 101 | fullName := repo.FullName() 102 | fetchedNumbers := []int{} 103 | 104 | issues, cursor, err := f.client.ListIssues(repo.Owner, repo.Name, nil, meta.cursor) 105 | 106 | // if a max limit was set, enforce it (using ">=" here makes 107 | // it so that we stop cleanly when the list of issues is exactly 108 | // the right amount that was left to fetch) 109 | if meta.max > 0 && len(issues)+meta.fetched >= meta.max { 110 | issues = issues[:meta.max-meta.fetched] 111 | cursor = "" 112 | } 113 | 114 | for _, issue := range issues { 115 | fetchedNumbers = append(fetchedNumbers, issue.Number) 116 | } 117 | 118 | repo.AddIssues(issues) 119 | f.dequeueIssues(repo, fetchedNumbers) 120 | 121 | // always delete the job, no matter the outcome 122 | f.lock.Lock() 123 | delete(f.jobQueues[fullName], job) 124 | f.lock.Unlock() 125 | 126 | // batch query was successful 127 | if err == nil { 128 | log.WithField("new-cursor", cursor).Debugf("Fetched %d issues.", len(issues)) 129 | 130 | // queue the query for the next page 131 | if cursor != "" { 132 | f.enqueueJob(repo, job, scanIssuesJobMeta{ 133 | max: meta.max, 134 | fetched: meta.fetched + len(issues), 135 | cursor: cursor, 136 | }) 137 | } 138 | 139 | return nil 140 | } 141 | 142 | retryAfter := 30 * time.Second 143 | log.Errorf("Failed to list issues, will retry in %s: %v", retryAfter.String(), err) 144 | 145 | // query failed, re-try later 146 | go func() { 147 | time.Sleep(retryAfter) 148 | f.enqueueJob(repo, job, data) 149 | }() 150 | 151 | return err 152 | } 153 | -------------------------------------------------------------------------------- /pkg/fetcher/jobs_pullrequests.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package fetcher 5 | 6 | import ( 7 | "time" 8 | 9 | "go.xrstf.de/github_exporter/pkg/github" 10 | 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | const ( 15 | scanPullRequestsJobKey = "scan-pull-requests" 16 | updatePullRequestsJobKey = "update-pull-requests" 17 | findUpdatedPullRequestsJobKey = "find-updated-pull-requests" 18 | ) 19 | 20 | type updatePullRequestsJobMeta struct { 21 | numbers []int 22 | } 23 | 24 | // processUpdatePullRequestsJob updates a list of already fetched 25 | // PRs to ensure they stay up to date. This is done for all open 26 | // PRs. 27 | func (f *Fetcher) processUpdatePullRequestsJob(repo *github.Repository, log logrus.FieldLogger, job string, data interface{}) error { 28 | meta := data.(updatePullRequestsJobMeta) 29 | 30 | prs, err := f.client.GetRepositoryPullRequests(repo.Owner, repo.Name, meta.numbers) 31 | 32 | fetchedNumbers := []int{} 33 | fetchedNumbersMap := map[int]struct{}{} 34 | for _, pr := range prs { 35 | fetchedNumbers = append(fetchedNumbers, pr.Number) 36 | fetchedNumbersMap[pr.Number] = struct{}{} 37 | } 38 | 39 | log.Debugf("Fetched %d out of %d PRs.", len(fetchedNumbers), len(meta.numbers)) 40 | 41 | deleted := []int{} 42 | for _, number := range meta.numbers { 43 | if _, ok := fetchedNumbersMap[number]; !ok { 44 | deleted = append(deleted, number) 45 | } 46 | } 47 | 48 | if len(prs) > 0 { 49 | repo.AddPullRequests(prs) 50 | } 51 | 52 | // only delete not found PRs from our local cache if the request was a success, otherwise 53 | // we would remove all PRs if e.g. GitHub is unavailable 54 | if err == nil && len(deleted) > 0 { 55 | repo.DeletePullRequests(deleted) 56 | } 57 | 58 | f.removeJob(repo, job) 59 | f.dequeuePullRequests(repo, meta.numbers) 60 | 61 | return err 62 | } 63 | 64 | // processFindUpdatedPullRequestsJob fetches the 100 most recently updated 65 | // PRs in the given repository and updates repo. The job will be removed 66 | // from the job queue afterwards and all fetched PRs will be removed from 67 | // the priority/regular PR queues. 68 | func (f *Fetcher) processFindUpdatedPullRequestsJob(repo *github.Repository, log logrus.FieldLogger, job string) error { 69 | fetchedNumbers := []int{} 70 | 71 | prs, _, err := f.client.ListPullRequests(repo.Owner, repo.Name, nil, "") 72 | for _, pr := range prs { 73 | fetchedNumbers = append(fetchedNumbers, pr.Number) 74 | } 75 | 76 | log.Debugf("Fetched %d recently updated PRs.", len(fetchedNumbers)) 77 | 78 | repo.AddPullRequests(prs) 79 | 80 | f.removeJob(repo, job) 81 | f.dequeuePullRequests(repo, fetchedNumbers) 82 | 83 | return err 84 | } 85 | 86 | type scanPullRequestsJobMeta struct { 87 | max int 88 | fetched int 89 | cursor string 90 | } 91 | 92 | // processScanPullRequestsJob is the initial job for every repository. 93 | // It lists all existing pull requests and adds them to repo. 94 | // 95 | // Because the initial scan is vital for proper functioning of every 96 | // other job, this job must succeed before anything else can happen 97 | // with a repository. For this reason a failed scan job is re-queued 98 | // a few seconds later. 99 | func (f *Fetcher) processScanPullRequestsJob(repo *github.Repository, log logrus.FieldLogger, job string, data interface{}) error { 100 | meta := data.(scanPullRequestsJobMeta) 101 | fullName := repo.FullName() 102 | fetchedNumbers := []int{} 103 | 104 | prs, cursor, err := f.client.ListPullRequests(repo.Owner, repo.Name, nil, meta.cursor) 105 | 106 | // if a max limit was set, enforce it (using ">=" here makes 107 | // it so that we stop cleanly when the list of PRs is exactly 108 | // the right amount that was left to fetch) 109 | if meta.max > 0 && len(prs)+meta.fetched >= meta.max { 110 | prs = prs[:meta.max-meta.fetched] 111 | cursor = "" 112 | } 113 | 114 | for _, pr := range prs { 115 | fetchedNumbers = append(fetchedNumbers, pr.Number) 116 | } 117 | 118 | repo.AddPullRequests(prs) 119 | f.dequeuePullRequests(repo, fetchedNumbers) 120 | 121 | // always delete the job, no matter the outcome 122 | f.lock.Lock() 123 | delete(f.jobQueues[fullName], job) 124 | f.lock.Unlock() 125 | 126 | // batch query was successful 127 | if err == nil { 128 | log.WithField("new-cursor", cursor).Debugf("Fetched %d PRs.", len(prs)) 129 | 130 | // queue the query for the next page 131 | if cursor != "" { 132 | f.enqueueJob(repo, job, scanPullRequestsJobMeta{ 133 | max: meta.max, 134 | fetched: meta.fetched + len(prs), 135 | cursor: cursor, 136 | }) 137 | } 138 | 139 | return nil 140 | } 141 | 142 | retryAfter := 30 * time.Second 143 | log.Errorf("Failed to list PRs, will retry in %s: %v", retryAfter.String(), err) 144 | 145 | // query failed, re-try later 146 | go func() { 147 | time.Sleep(retryAfter) 148 | f.enqueueJob(repo, job, data) 149 | }() 150 | 151 | return err 152 | } 153 | -------------------------------------------------------------------------------- /pkg/fetcher/jobs_milestones.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package fetcher 5 | 6 | import ( 7 | "time" 8 | 9 | "go.xrstf.de/github_exporter/pkg/github" 10 | 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | const ( 15 | scanMilestonesJobKey = "scan-milestones" 16 | updateMilestonesJobKey = "update-milestones" 17 | findUpdatedMilestonesJobKey = "find-updated-milestones" 18 | ) 19 | 20 | type updateMilestonesJobMeta struct { 21 | numbers []int 22 | } 23 | 24 | // processUpdateMilestonesJob updates a list of already fetched 25 | // milestones to ensure they stay up to date. This is done for all open 26 | // milestones. 27 | func (f *Fetcher) processUpdateMilestonesJob(repo *github.Repository, log logrus.FieldLogger, job string, data interface{}) error { 28 | meta := data.(updateMilestonesJobMeta) 29 | 30 | milestones, err := f.client.GetRepositoryMilestones(repo.Owner, repo.Name, meta.numbers) 31 | 32 | fetchedNumbers := []int{} 33 | fetchedNumbersMap := map[int]struct{}{} 34 | for _, milestone := range milestones { 35 | fetchedNumbers = append(fetchedNumbers, milestone.Number) 36 | fetchedNumbersMap[milestone.Number] = struct{}{} 37 | } 38 | 39 | log.Debugf("Fetched %d out of %d milestones.", len(fetchedNumbers), len(meta.numbers)) 40 | 41 | deleted := []int{} 42 | for _, number := range meta.numbers { 43 | if _, ok := fetchedNumbersMap[number]; !ok { 44 | deleted = append(deleted, number) 45 | } 46 | } 47 | 48 | if len(milestones) > 0 { 49 | repo.AddMilestones(milestones) 50 | } 51 | 52 | // only delete not found milestones from our local cache if the request was a success, otherwise 53 | // we would remove all milestones if e.g. GitHub is unavailable 54 | if err == nil && len(deleted) > 0 { 55 | repo.DeleteMilestones(deleted) 56 | } 57 | 58 | f.removeJob(repo, job) 59 | f.dequeueMilestones(repo, meta.numbers) 60 | 61 | return err 62 | } 63 | 64 | // processFindUpdatedMilestonesJob fetches the 100 most recently updated 65 | // milestones in the given repository and updates repo. The job will be removed 66 | // from the job queue afterwards and all fetched milestones will be removed from 67 | // the priority/regular milestone queues. 68 | func (f *Fetcher) processFindUpdatedMilestonesJob(repo *github.Repository, log logrus.FieldLogger, job string) error { 69 | fetchedNumbers := []int{} 70 | 71 | milestones, _, err := f.client.ListMilestones(repo.Owner, repo.Name, nil, "") 72 | for _, milestone := range milestones { 73 | fetchedNumbers = append(fetchedNumbers, milestone.Number) 74 | } 75 | 76 | log.Debugf("Fetched %d recently updated milestones.", len(fetchedNumbers)) 77 | 78 | repo.AddMilestones(milestones) 79 | 80 | f.removeJob(repo, job) 81 | f.dequeueMilestones(repo, fetchedNumbers) 82 | 83 | return err 84 | } 85 | 86 | type scanMilestonesJobMeta struct { 87 | max int 88 | fetched int 89 | cursor string 90 | } 91 | 92 | // processScanMilestonesJob is the initial job for every repository. 93 | // It lists all existing milestones and adds them to repo. 94 | // 95 | // Because the initial scan is vital for proper functioning of every 96 | // other job, this job must succeed before anything else can happen 97 | // with a repository. For this reason a failed scan job is re-queued 98 | // a few seconds later. 99 | func (f *Fetcher) processScanMilestonesJob(repo *github.Repository, log logrus.FieldLogger, job string, data interface{}) error { 100 | meta := data.(scanMilestonesJobMeta) 101 | fullName := repo.FullName() 102 | fetchedNumbers := []int{} 103 | 104 | milestones, cursor, err := f.client.ListMilestones(repo.Owner, repo.Name, nil, meta.cursor) 105 | 106 | // if a max limit was set, enforce it (using ">=" here makes 107 | // it so that we stop cleanly when the list of milestones is exactly 108 | // the right amount that was left to fetch) 109 | if meta.max > 0 && len(milestones)+meta.fetched >= meta.max { 110 | milestones = milestones[:meta.max-meta.fetched] 111 | cursor = "" 112 | } 113 | 114 | for _, milestone := range milestones { 115 | fetchedNumbers = append(fetchedNumbers, milestone.Number) 116 | } 117 | 118 | repo.AddMilestones(milestones) 119 | f.dequeueMilestones(repo, fetchedNumbers) 120 | 121 | // always delete the job, no matter the outcome 122 | f.lock.Lock() 123 | delete(f.jobQueues[fullName], job) 124 | f.lock.Unlock() 125 | 126 | // batch query was successful 127 | if err == nil { 128 | log.WithField("new-cursor", cursor).Debugf("Fetched %d milestones.", len(milestones)) 129 | 130 | // queue the query for the next page 131 | if cursor != "" { 132 | f.enqueueJob(repo, job, scanMilestonesJobMeta{ 133 | max: meta.max, 134 | fetched: meta.fetched + len(milestones), 135 | cursor: cursor, 136 | }) 137 | } 138 | 139 | return nil 140 | } 141 | 142 | retryAfter := 30 * time.Second 143 | log.Errorf("Failed to list milestones, will retry in %s: %v", retryAfter.String(), err) 144 | 145 | // query failed, re-try later 146 | go func() { 147 | time.Sleep(retryAfter) 148 | f.enqueueJob(repo, job, data) 149 | }() 150 | 151 | return err 152 | } 153 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 2 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 3 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 4 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 9 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 10 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 11 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 12 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 13 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 14 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 15 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 16 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 17 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 18 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 19 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 20 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 21 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 22 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 23 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 24 | github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= 25 | github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= 26 | github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= 27 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 28 | github.com/shurcooL/githubv4 v0.0.0-20230704064427-599ae7bbf278 h1:kdEGVAV4sO46DPtb8k793jiecUEhaX9ixoIBt41HEGU= 29 | github.com/shurcooL/githubv4 v0.0.0-20230704064427-599ae7bbf278/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= 30 | github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 h1:B1PEwpArrNp4dkQrfxh/abbBAOZBVp0ds+fBEOUOqOc= 31 | github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= 32 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 33 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 34 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 35 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 36 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 37 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 38 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 39 | golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= 40 | golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= 41 | golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= 42 | golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= 43 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 44 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 45 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 46 | golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= 47 | golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 48 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 49 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 50 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 51 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 52 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= 53 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 54 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 55 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 56 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 57 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 58 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 59 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 60 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 61 | -------------------------------------------------------------------------------- /pkg/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package metrics 5 | 6 | import ( 7 | "go.xrstf.de/github_exporter/pkg/prow" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | ) 11 | 12 | var ( 13 | ////////////////////////////////////////////// 14 | // repository 15 | 16 | repositoryDiskUsage = prometheus.NewDesc( 17 | "github_exporter_repo_disk_usage_bytes", 18 | "Repository size in bytes", 19 | []string{"repo"}, 20 | nil, 21 | ) 22 | 23 | repositoryForks = prometheus.NewDesc( 24 | "github_exporter_repo_forks", 25 | "Number of forks of this repository", 26 | []string{"repo"}, 27 | nil, 28 | ) 29 | 30 | repositoryStargazers = prometheus.NewDesc( 31 | "github_exporter_repo_stargazers", 32 | "Number of stargazers for this repository", 33 | []string{"repo"}, 34 | nil, 35 | ) 36 | 37 | repositoryWatchers = prometheus.NewDesc( 38 | "github_exporter_repo_watchers", 39 | "Number of watchers for this repository", 40 | []string{"repo"}, 41 | nil, 42 | ) 43 | 44 | repositoryPrivate = prometheus.NewDesc( 45 | "github_exporter_repo_is_private", 46 | "1 if the repository is private, 0 otherwise", 47 | []string{"repo"}, 48 | nil, 49 | ) 50 | 51 | repositoryArchived = prometheus.NewDesc( 52 | "github_exporter_repo_is_archived", 53 | "1 if the repository is archived, 0 otherwise", 54 | []string{"repo"}, 55 | nil, 56 | ) 57 | 58 | repositoryDisabled = prometheus.NewDesc( 59 | "github_exporter_repo_is_disabled", 60 | "1 if the repository is disabled, 0 otherwise", 61 | []string{"repo"}, 62 | nil, 63 | ) 64 | 65 | repositoryFork = prometheus.NewDesc( 66 | "github_exporter_repo_is_fork", 67 | "1 if the repository is a fork, 0 otherwise", 68 | []string{"repo"}, 69 | nil, 70 | ) 71 | 72 | repositoryLocked = prometheus.NewDesc( 73 | "github_exporter_repo_is_locked", 74 | "1 if the repository is locked, 0 otherwise", 75 | []string{"repo"}, 76 | nil, 77 | ) 78 | 79 | repositoryMirror = prometheus.NewDesc( 80 | "github_exporter_repo_is_mirror", 81 | "1 if the repository is a mirror, 0 otherwise", 82 | []string{"repo"}, 83 | nil, 84 | ) 85 | 86 | repositoryTemplate = prometheus.NewDesc( 87 | "github_exporter_repo_is_template", 88 | "1 if the repository is a template, 0 otherwise", 89 | []string{"repo"}, 90 | nil, 91 | ) 92 | 93 | repositoryLanguageSize = prometheus.NewDesc( 94 | "github_exporter_repo_language_size_bytes", 95 | "Number of bytes in the repository detected as using a given language", 96 | []string{"repo", "language"}, 97 | nil, 98 | ) 99 | 100 | ////////////////////////////////////////////// 101 | // pull requests 102 | 103 | pullRequestInfo *prometheus.Desc 104 | 105 | pullRequestLabelCount = prometheus.NewDesc( 106 | "github_exporter_pr_label_count", 107 | "Total count of Pull Requests using a given label", 108 | []string{"repo", "label", "state"}, 109 | nil, 110 | ) 111 | 112 | pullRequestCreatedAt = prometheus.NewDesc( 113 | "github_exporter_pr_created_at", 114 | "UNIX timestamp of a Pull Request's creation time", 115 | []string{"repo", "number"}, 116 | nil, 117 | ) 118 | 119 | pullRequestUpdatedAt = prometheus.NewDesc( 120 | "github_exporter_pr_updated_at", 121 | "UNIX timestamp of a Pull Request's last update time", 122 | []string{"repo", "number"}, 123 | nil, 124 | ) 125 | 126 | pullRequestFetchedAt = prometheus.NewDesc( 127 | "github_exporter_pr_fetched_at", 128 | "UNIX timestamp of a Pull Request's last fetch time (when it was retrieved from the API)", 129 | []string{"repo", "number"}, 130 | nil, 131 | ) 132 | 133 | pullRequestQueueSize = prometheus.NewDesc( 134 | "github_exporter_pr_queue_size", 135 | "Number of pull requests currently queued for an update", 136 | []string{"repo", "queue"}, 137 | nil, 138 | ) 139 | 140 | ////////////////////////////////////////////// 141 | // issues 142 | 143 | issueInfo *prometheus.Desc 144 | 145 | issueLabelCount = prometheus.NewDesc( 146 | "github_exporter_issue_label_count", 147 | "Total count of Pull Requests using a given label", 148 | []string{"repo", "label", "state"}, 149 | nil, 150 | ) 151 | 152 | issueCreatedAt = prometheus.NewDesc( 153 | "github_exporter_issue_created_at", 154 | "UNIX timestamp of an Issue's creation time", 155 | []string{"repo", "number"}, 156 | nil, 157 | ) 158 | 159 | issueUpdatedAt = prometheus.NewDesc( 160 | "github_exporter_issue_updated_at", 161 | "UNIX timestamp of an Issue's last update time", 162 | []string{"repo", "number"}, 163 | nil, 164 | ) 165 | 166 | issueFetchedAt = prometheus.NewDesc( 167 | "github_exporter_issue_fetched_at", 168 | "UNIX timestamp of an Issue's last fetch time (when it was retrieved from the API)", 169 | []string{"repo", "number"}, 170 | nil, 171 | ) 172 | 173 | issueQueueSize = prometheus.NewDesc( 174 | "github_exporter_issue_queue_size", 175 | "Number of issues currently queued for an update", 176 | []string{"repo", "queue"}, 177 | nil, 178 | ) 179 | 180 | ////////////////////////////////////////////// 181 | // milestones 182 | 183 | milestoneInfo = prometheus.NewDesc( 184 | "github_exporter_milestone_info", 185 | "Various milestone related meta information with the static value 1", 186 | []string{"repo", "number", "state", "title"}, 187 | nil, 188 | ) 189 | 190 | milestoneIssues = prometheus.NewDesc( 191 | "github_exporter_milestone_issues", 192 | "Total number issues (includes pull requests) belonging to a milestone, grouped by kind and state", 193 | []string{"repo", "number", "kind", "state"}, 194 | nil, 195 | ) 196 | 197 | milestoneCreatedAt = prometheus.NewDesc( 198 | "github_exporter_milestone_created_at", 199 | "UNIX timestamp of a Milestone's creation time", 200 | []string{"repo", "number"}, 201 | nil, 202 | ) 203 | 204 | milestoneUpdatedAt = prometheus.NewDesc( 205 | "github_exporter_milestone_updated_at", 206 | "UNIX timestamp of a Milestone's last update time", 207 | []string{"repo", "number"}, 208 | nil, 209 | ) 210 | 211 | milestoneFetchedAt = prometheus.NewDesc( 212 | "github_exporter_milestone_fetched_at", 213 | "UNIX timestamp of a Milestone's last fetch time (when it was retrieved from the API)", 214 | []string{"repo", "number"}, 215 | nil, 216 | ) 217 | 218 | milestoneClosedAt = prometheus.NewDesc( 219 | "github_exporter_milestone_closed_at", 220 | "UNIX timestamp of a Milestone's close time (0 if the milestone is open)", 221 | []string{"repo", "number"}, 222 | nil, 223 | ) 224 | 225 | milestoneDueOn = prometheus.NewDesc( 226 | "github_exporter_milestone_due_on", 227 | "UNIX timestamp of a Milestone's due date (0 if there is no due date set)", 228 | []string{"repo", "number"}, 229 | nil, 230 | ) 231 | 232 | milestoneQueueSize = prometheus.NewDesc( 233 | "github_exporter_milestone_queue_size", 234 | "Number of milestones currently queued for an update", 235 | []string{"repo", "queue"}, 236 | nil, 237 | ) 238 | 239 | ////////////////////////////////////////////// 240 | // exporter-related 241 | 242 | githubPointsRemaining = prometheus.NewDesc( 243 | "github_exporter_api_points_remaining", 244 | "Number of currently remaining GitHub API points", 245 | nil, 246 | nil, 247 | ) 248 | 249 | githubRequestsTotal = prometheus.NewDesc( 250 | "github_exporter_api_requests_total", 251 | "Total number of requests against the GitHub API", 252 | []string{"repo"}, 253 | nil, 254 | ) 255 | 256 | githubCostsTotal = prometheus.NewDesc( 257 | "github_exporter_api_costs_total", 258 | "Total sum of API credits spent for all performed API requests", 259 | []string{"repo"}, 260 | nil, 261 | ) 262 | ) 263 | 264 | func init() { 265 | prLabels := []string{"repo", "number", "author", "state"} 266 | prLabels = append(prLabels, prow.PullRequestLabelNames()...) 267 | 268 | pullRequestInfo = prometheus.NewDesc( 269 | "github_exporter_pr_info", 270 | "Various Pull Request related meta information with the static value 1", 271 | prLabels, 272 | nil, 273 | ) 274 | 275 | issueLabels := []string{"repo", "number", "author", "state"} 276 | issueLabels = append(issueLabels, prow.IssueLabelNames()...) 277 | 278 | issueInfo = prometheus.NewDesc( 279 | "github_exporter_issue_info", 280 | "Various issue related meta information with the static value 1", 281 | issueLabels, 282 | nil, 283 | ) 284 | } 285 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # xrstf's GitHub Exporter for Prometheus 2 | 3 | This exporter exposes Prometheus metrics for a list of pre-configured GitHub repositories. 4 | The focus is on providing more insights about issues, pull requests and milestones. 5 | 6 | ![Grafana Screenshot](https://github.com/xrstf/github_exporter/blob/main/contrib/grafana/screenshot.png?raw=true) 7 | 8 | It uses GitHub's API v4 and tries its best to not exceed the request quotas, but for large 9 | repositories (5k+ PRs) it's recommended to tweak the settings a bit. 10 | 11 | ## Operation 12 | 13 | The goal of this particular exporter is to provide metrics for **all** pull requests, issues 14 | and milestones (collectively called "items" from here on) within a given set of repositories. 15 | At the same time, **open** items should be refreshed much more often and quickly than older 16 | data. 17 | 18 | To achieve this, the exporter upon startup scans all repositories for all items. After 19 | this is complete, it will 20 | 21 | * fetch the most recently updated 100 items (to detect new elements and elements 22 | whose status has changed), 23 | * re-fetch all open items frequently (every 5 minutes by default) and 24 | * re-fetch **all** items every 12 hours by default. 25 | 26 | While the scheduling for the re-fetches happens concurrently in multiple go routines, 27 | the fetching itself is done sequentially to avoid triggering GitHub's anti-abuse system. 28 | 29 | Fetching open items has higher priority, so that even large amounts of old items 30 | cannot interfere with the freshness of open items. 31 | 32 | It is possible to limit the initial scan (using `-pr-depth`, `-issue-depth` and `-milestone-depth`), 33 | so that for very large repositories not all items are fetched. But this only limits the 34 | initial scan, over time the exporter will learn about new items and not forget the old ones 35 | (and since it always keeps all items up-to-date, the number of items fetched will slooooowly 36 | over time grow). 37 | 38 | Jobs are always removed from the queue, even if they failed. The exporter relies on the 39 | goroutines to re-schedule them later anyway, and this prevents flooding GitHub when the 40 | API has issues or misconfiguration occurs. Job queues can only contain one job per kind, 41 | so even if the API is down for an hour, the queue will not fill up with the re-fetch job. 42 | 43 | ## Installation 44 | 45 | You need Go 1.14 installed on your machine. 46 | 47 | ``` 48 | go get go.xrstf.de/github_exporter 49 | ``` 50 | 51 | A Docker image is available as [`xrstf/github_exporter`](https://hub.docker.com/r/xrstf/github_exporter). 52 | 53 | ## Usage 54 | 55 | You need an OAuth2 token to authenticate against the API. Make it available 56 | as the `GITHUB_TOKEN` environment variable. 57 | 58 | By default, the exporter listens on `0.0.0.0:9612`. 59 | 60 | All configuration happens via commandline arguments. At the bare minimum, you need to 61 | specify a single repository to scrape: 62 | 63 | ``` 64 | ./github_exporter -repo myself/my-repository 65 | ``` 66 | 67 | You can configure multiple `-repo` (which is also recommended over running the exporter 68 | multiple times in parallel, so a single exporter can serialize all API requests) and 69 | tweak the exporter further using the available flags: 70 | 71 | ``` 72 | Usage of ./github_exporter: 73 | -debug 74 | enable more verbose logging 75 | -issue-depth int 76 | max number of issues to fetch per repository upon startup (-1 disables the limit, 0 disables issue fetching entirely) (default -1) 77 | -issue-refresh-interval duration 78 | time in between issue refreshes (default 5m0s) 79 | -issue-resync-interval duration 80 | time in between full issue re-syncs (default 12h0m0s) 81 | -listen string 82 | address and port to listen on (default ":9612") 83 | -milestone-depth int 84 | max number of milestones to fetch per repository upon startup (-1 disables the limit, 0 disables milestone fetching entirely) (default -1) 85 | -milestone-refresh-interval duration 86 | time in between milestone refreshes (default 5m0s) 87 | -milestone-resync-interval duration 88 | time in between full milestone re-syncs (default 12h0m0s) 89 | -pr-depth int 90 | max number of pull requests to fetch per repository upon startup (-1 disables the limit, 0 disables PR fetching entirely) (default -1) 91 | -pr-refresh-interval duration 92 | time in between PR refreshes (default 5m0s) 93 | -pr-resync-interval duration 94 | time in between full PR re-syncs (default 12h0m0s) 95 | -realnames 96 | use usernames instead of internal IDs for author labels (this will make metrics contain personally identifiable information) 97 | -repo value 98 | repository (owner/name format) to include, can be given multiple times 99 | -owner string 100 | github login (username or organization) of the owner of the repositories that will be included. Excludes forked and locked repo, includes 100 first private & public repos 101 | ``` 102 | 103 | ## Metrics 104 | 105 | **All** metrics are labelled with `repo=(full repo name)`, for example 106 | `repo="xrstf/github_exporter"`. 107 | 108 | For each repository, the following metrics are available: 109 | 110 | * `github_exporter_repo_disk_usage_bytes` 111 | * `github_exporter_repo_forks` 112 | * `github_exporter_repo_stargazers` 113 | * `github_exporter_repo_watchers` 114 | * `github_exporter_repo_is_private` 115 | * `github_exporter_repo_is_archived` 116 | * `github_exporter_repo_is_disabled` 117 | * `github_exporter_repo_is_fork` 118 | * `github_exporter_repo_is_locked` 119 | * `github_exporter_repo_is_mirror` 120 | * `github_exporter_repo_is_template` 121 | * `github_exporter_repo_language_size_bytes` is additionally labelled with `language`. 122 | 123 | For pull requests, these metrics are available: 124 | 125 | * `github_exporter_pr_info` contains lots of metadata labels and always has a constant 126 | value of `1`. Labels are: 127 | 128 | * `number` is the PR's number. 129 | * `state` is one of `open`, `closed` or `merged`. 130 | * `author` is the author ID (or username if `-realnames` is configured). 131 | 132 | In addition, the exporter recognizes a few common label conventions, namely: 133 | 134 | * `size/*` is reflected as a `size` label (e.g. the `size/xs` label on GitHub becomes 135 | a `size="xs"` label on the Prometheus metric). 136 | * `team/*` is reflected as a `team` label. 137 | * `kind/*` is reflected as a `kind` label. 138 | * `priority/*` is reflected as a `priority` label. 139 | * `approved` is reflected as a boolean `approved` label. 140 | * `lgtm` is reflected as a boolean `lgtm` label. 141 | * `do-no-merge/*` is reflected as a boolean `pending` label. 142 | 143 | * `github_exporter_pr_label_count` is the number of PRs that have a given label 144 | and state. This counts all labels individually, not just those recognized for 145 | the `_info` metric. 146 | 147 | * `github_exporter_pr_created_at` is the UNIX timestamp of when the PR was 148 | created on GitHub. This metric only has `repo` and `number` labels. 149 | 150 | * `github_exporter_pr_updated_at` is the UNIX timestamp of when the PR was 151 | last updated on GitHub. This metric only has `repo` and `number` labels. 152 | 153 | * `github_exporter_pr_fetched_at` is the UNIX timestamp of when the PR was 154 | last fetched from the GitHub API. This metric only has `repo` and `number` labels. 155 | 156 | The PR metrics are mirrored for issues: 157 | 158 | * `github_exporter_issue_info` 159 | * `github_exporter_issue_label_count` 160 | * `github_exporter_issue_created_at` 161 | * `github_exporter_issue_updated_at` 162 | * `github_exporter_issue_fetched_at` 163 | 164 | The metrics for milestones are similar: 165 | 166 | * `github_exporter_milestone_info` has `repo`, `number`, `title` and `state` labels. 167 | * `github_exporter_milestone_issues` counts the number of open/closed issues/PRs 168 | for a given milestone, so it has `repo`, `number`, `kind` (issue or pullrequest) 169 | and `state` labels. 170 | * `github_exporter_milestone_created_at` 171 | * `github_exporter_milestone_updated_at` 172 | * `github_exporter_milestone_fetched_at` 173 | * `github_exporter_milestone_closed_at` is optional and 0 if the milestone is open. 174 | * `github_exporter_milestone_due_on` is optional and 0 if no due date is set. 175 | 176 | And a few more metrics for monitoring the exporter itself are available as well: 177 | 178 | * `github_exporter_pr_queue_size` is the number of PRs currently queued for 179 | being fetched from the API. This is split via the `queue` label into `priority` 180 | (open PRs) and `regular` (older PRs). 181 | * `github_exporter_issue_queue_size` is the same as for the PR queue. 182 | * `github_exporter_milestone_queue_size` is the same as for the PR queue. 183 | * `github_exporter_api_requests_total` counts the number of API requests per 184 | repository. 185 | * `github_exporter_api_costs_total` is the sum of costs (in API points) that have 186 | been used, grouped by `repo`. 187 | * `github_exporter_api_points_remaining` is a gauge representing the remaining 188 | API points. 5k points can be consumed per hour, with resets after 1 hour. 189 | 190 | ## Long-term storage 191 | 192 | If you plan on performing long-term analysis over repositories, make sure to put proper 193 | recording rules into place so that queries can be performed quickly. The exporter 194 | intentionally does not pre-aggregate most things, as to not spam Prometheus or restrict 195 | the available information. 196 | 197 | A few example rules can be found in `contrib/prometheus/rules.yaml`. 198 | 199 | ## License 200 | 201 | MIT 202 | -------------------------------------------------------------------------------- /pkg/metrics/collector.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package metrics 5 | 6 | import ( 7 | "strconv" 8 | "strings" 9 | 10 | "go.xrstf.de/github_exporter/pkg/client" 11 | "go.xrstf.de/github_exporter/pkg/fetcher" 12 | "go.xrstf.de/github_exporter/pkg/github" 13 | "go.xrstf.de/github_exporter/pkg/prow" 14 | 15 | "github.com/prometheus/client_golang/prometheus" 16 | "github.com/shurcooL/githubv4" 17 | ) 18 | 19 | var ( 20 | AllPullRequestStates = []string{ 21 | string(githubv4.PullRequestStateOpen), 22 | string(githubv4.PullRequestStateClosed), 23 | string(githubv4.PullRequestStateMerged), 24 | } 25 | 26 | AllIssueStates = []string{ 27 | string(githubv4.IssueStateOpen), 28 | string(githubv4.IssueStateClosed), 29 | } 30 | 31 | AllMilestoneStates = []string{ 32 | string(githubv4.MilestoneStateOpen), 33 | string(githubv4.MilestoneStateClosed), 34 | } 35 | ) 36 | 37 | type Collector struct { 38 | repos map[string]*github.Repository 39 | fetcher *fetcher.Fetcher 40 | client *client.Client 41 | } 42 | 43 | func NewCollector(repos map[string]*github.Repository, fetcher *fetcher.Fetcher, client *client.Client) *Collector { 44 | return &Collector{ 45 | repos: repos, 46 | fetcher: fetcher, 47 | client: client, 48 | } 49 | } 50 | 51 | func (mc *Collector) Describe(ch chan<- *prometheus.Desc) { 52 | prometheus.DescribeByCollect(mc, ch) 53 | } 54 | 55 | func (mc *Collector) Collect(ch chan<- prometheus.Metric) { 56 | requestCounts := mc.client.GetRequestCounts() 57 | costs := mc.client.GetTotalCosts() 58 | 59 | for _, repo := range mc.repos { 60 | // do not publish metrics for repos for which we have not even fetched 61 | // the bare minimum of information 62 | if repo.FetchedAt == nil { 63 | continue 64 | } 65 | 66 | fullName := repo.FullName() 67 | 68 | _ = repo.RLocked(func(r *github.Repository) error { 69 | return mc.collectRepository(ch, r) 70 | }) 71 | 72 | ch <- constMetric(githubRequestsTotal, prometheus.CounterValue, float64(requestCounts[fullName]), fullName) 73 | ch <- constMetric(githubCostsTotal, prometheus.CounterValue, float64(costs[fullName]), fullName) 74 | } 75 | 76 | ch <- constMetric(githubPointsRemaining, prometheus.GaugeValue, float64(mc.client.GetRemainingPoints())) 77 | } 78 | 79 | func (mc *Collector) collectRepository(ch chan<- prometheus.Metric, repo *github.Repository) error { 80 | if err := mc.collectRepoInfo(ch, repo); err != nil { 81 | return err 82 | } 83 | 84 | if err := mc.collectRepoPullRequests(ch, repo); err != nil { 85 | return err 86 | } 87 | 88 | if err := mc.collectRepoIssues(ch, repo); err != nil { 89 | return err 90 | } 91 | 92 | if err := mc.collectRepoMilestones(ch, repo); err != nil { 93 | return err 94 | } 95 | 96 | return nil 97 | } 98 | 99 | func boolVal(b bool) float64 { 100 | if b { 101 | return 1 102 | } 103 | 104 | return 0 105 | } 106 | 107 | func (mc *Collector) collectRepoInfo(ch chan<- prometheus.Metric, repo *github.Repository) error { 108 | repoName := repo.FullName() 109 | 110 | ch <- constMetric(repositoryDiskUsage, prometheus.GaugeValue, float64(repo.DiskUsageBytes), repoName) 111 | ch <- constMetric(repositoryForks, prometheus.GaugeValue, float64(repo.Forks), repoName) 112 | ch <- constMetric(repositoryStargazers, prometheus.GaugeValue, float64(repo.Stargazers), repoName) 113 | ch <- constMetric(repositoryWatchers, prometheus.GaugeValue, float64(repo.Watchers), repoName) 114 | ch <- constMetric(repositoryPrivate, prometheus.GaugeValue, boolVal(repo.IsPrivate), repoName) 115 | ch <- constMetric(repositoryArchived, prometheus.GaugeValue, boolVal(repo.IsArchived), repoName) 116 | ch <- constMetric(repositoryDisabled, prometheus.GaugeValue, boolVal(repo.IsDisabled), repoName) 117 | ch <- constMetric(repositoryFork, prometheus.GaugeValue, boolVal(repo.IsFork), repoName) 118 | ch <- constMetric(repositoryLocked, prometheus.GaugeValue, boolVal(repo.IsLocked), repoName) 119 | ch <- constMetric(repositoryMirror, prometheus.GaugeValue, boolVal(repo.IsMirror), repoName) 120 | ch <- constMetric(repositoryTemplate, prometheus.GaugeValue, boolVal(repo.IsTemplate), repoName) 121 | 122 | for language, size := range repo.Languages { 123 | ch <- constMetric(repositoryLanguageSize, prometheus.GaugeValue, float64(size), repoName, language) 124 | } 125 | 126 | return nil 127 | } 128 | 129 | func (mc *Collector) collectRepoPullRequests(ch chan<- prometheus.Metric, repo *github.Repository) error { 130 | totals := newStateLabelMap(repo, AllPullRequestStates) 131 | repoName := repo.FullName() 132 | 133 | for number, pr := range repo.PullRequests { 134 | num := strconv.Itoa(number) 135 | 136 | for _, label := range pr.Labels { 137 | totals[string(pr.State)][label]++ 138 | } 139 | 140 | infoLabels := []string{ 141 | repoName, 142 | num, 143 | pr.Author, 144 | strings.ToLower(string(pr.State)), 145 | } 146 | infoLabels = append(infoLabels, prow.PullRequestLabels(&pr)...) 147 | 148 | ch <- constMetric(pullRequestInfo, prometheus.GaugeValue, 1, infoLabels...) 149 | ch <- constMetric(pullRequestCreatedAt, prometheus.GaugeValue, float64(pr.CreatedAt.Unix()), repoName, num) 150 | ch <- constMetric(pullRequestUpdatedAt, prometheus.GaugeValue, float64(pr.UpdatedAt.Unix()), repoName, num) 151 | ch <- constMetric(pullRequestFetchedAt, prometheus.GaugeValue, float64(pr.FetchedAt.Unix()), repoName, num) 152 | } 153 | 154 | totals.ToMetrics(ch, repo, pullRequestLabelCount) 155 | 156 | ch <- constMetric(pullRequestQueueSize, prometheus.GaugeValue, float64(mc.fetcher.PriorityPullRequestQueueSize(repo)), repoName, "priority") 157 | ch <- constMetric(pullRequestQueueSize, prometheus.GaugeValue, float64(mc.fetcher.RegularPullRequestQueueSize(repo)), repoName, "regular") 158 | 159 | return nil 160 | } 161 | 162 | func (mc *Collector) collectRepoIssues(ch chan<- prometheus.Metric, repo *github.Repository) error { 163 | totals := newStateLabelMap(repo, AllIssueStates) 164 | repoName := repo.FullName() 165 | 166 | for number, issue := range repo.Issues { 167 | num := strconv.Itoa(number) 168 | 169 | for _, label := range issue.Labels { 170 | totals[string(issue.State)][label]++ 171 | } 172 | 173 | infoLabels := []string{ 174 | repoName, 175 | num, 176 | issue.Author, 177 | strings.ToLower(string(issue.State)), 178 | } 179 | infoLabels = append(infoLabels, prow.IssueLabels(&issue)...) 180 | 181 | ch <- constMetric(issueInfo, prometheus.GaugeValue, 1, infoLabels...) 182 | ch <- constMetric(issueCreatedAt, prometheus.GaugeValue, float64(issue.CreatedAt.Unix()), repoName, num) 183 | ch <- constMetric(issueUpdatedAt, prometheus.GaugeValue, float64(issue.UpdatedAt.Unix()), repoName, num) 184 | ch <- constMetric(issueFetchedAt, prometheus.GaugeValue, float64(issue.FetchedAt.Unix()), repoName, num) 185 | } 186 | 187 | totals.ToMetrics(ch, repo, issueLabelCount) 188 | 189 | ch <- constMetric(issueQueueSize, prometheus.GaugeValue, float64(mc.fetcher.PriorityIssueQueueSize(repo)), repoName, "priority") 190 | ch <- constMetric(issueQueueSize, prometheus.GaugeValue, float64(mc.fetcher.RegularIssueQueueSize(repo)), repoName, "regular") 191 | 192 | return nil 193 | } 194 | 195 | func (mc *Collector) collectRepoMilestones(ch chan<- prometheus.Metric, repo *github.Repository) error { 196 | repoName := repo.FullName() 197 | openState := strings.ToLower(string(githubv4.MilestoneStateOpen)) 198 | closedState := strings.ToLower(string(githubv4.MilestoneStateClosed)) 199 | 200 | for number, milestone := range repo.Milestones { 201 | num := strconv.Itoa(number) 202 | 203 | var closedAt int64 204 | if milestone.ClosedAt != nil { 205 | closedAt = milestone.ClosedAt.Unix() 206 | } 207 | 208 | var dueOn int64 209 | if milestone.DueOn != nil { 210 | dueOn = milestone.DueOn.Unix() 211 | } 212 | 213 | ch <- constMetric(milestoneInfo, prometheus.GaugeValue, 1, repoName, num, strings.ToLower(string(milestone.State)), milestone.Title) 214 | ch <- constMetric(milestoneCreatedAt, prometheus.GaugeValue, float64(milestone.CreatedAt.Unix()), repoName, num) 215 | ch <- constMetric(milestoneUpdatedAt, prometheus.GaugeValue, float64(milestone.UpdatedAt.Unix()), repoName, num) 216 | ch <- constMetric(milestoneClosedAt, prometheus.GaugeValue, float64(closedAt), repoName, num) 217 | ch <- constMetric(milestoneDueOn, prometheus.GaugeValue, float64(dueOn), repoName, num) 218 | ch <- constMetric(milestoneFetchedAt, prometheus.GaugeValue, float64(milestone.FetchedAt.Unix()), repoName, num) 219 | ch <- constMetric(milestoneIssues, prometheus.GaugeValue, float64(milestone.OpenIssues), repoName, num, "issue", openState) 220 | ch <- constMetric(milestoneIssues, prometheus.GaugeValue, float64(milestone.ClosedIssues), repoName, num, "issue", closedState) 221 | ch <- constMetric(milestoneIssues, prometheus.GaugeValue, float64(milestone.OpenPullRequests), repoName, num, "pullrequest", openState) 222 | ch <- constMetric(milestoneIssues, prometheus.GaugeValue, float64(milestone.ClosedPullRequests), repoName, num, "pullrequest", closedState) 223 | } 224 | 225 | ch <- constMetric(milestoneQueueSize, prometheus.GaugeValue, float64(mc.fetcher.PriorityMilestoneQueueSize(repo)), repoName, "priority") 226 | ch <- constMetric(milestoneQueueSize, prometheus.GaugeValue, float64(mc.fetcher.RegularMilestoneQueueSize(repo)), repoName, "regular") 227 | 228 | return nil 229 | } 230 | 231 | // constMetric just helps reducing code noise 232 | func constMetric(desc *prometheus.Desc, valueType prometheus.ValueType, value float64, labelValues ...string) prometheus.Metric { 233 | return prometheus.MustNewConstMetric(desc, valueType, value, labelValues...) 234 | } 235 | 236 | type stateLabelMap map[string]map[string]int 237 | 238 | func newStateLabelMap(repo *github.Repository, states []string) stateLabelMap { 239 | result := stateLabelMap{} 240 | 241 | for _, state := range states { 242 | result[state] = map[string]int{} 243 | 244 | for _, label := range repo.Labels { 245 | result[state][label] = 0 246 | } 247 | } 248 | 249 | return result 250 | } 251 | 252 | func (m stateLabelMap) ToMetrics(ch chan<- prometheus.Metric, repo *github.Repository, metric *prometheus.Desc) { 253 | repoName := repo.FullName() 254 | 255 | for state, counts := range m { 256 | for label, count := range counts { 257 | ch <- prometheus.MustNewConstMetric(metric, prometheus.GaugeValue, float64(count), repoName, label, strings.ToLower(state)) 258 | } 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "flag" 9 | "fmt" 10 | "net/http" 11 | "os" 12 | "time" 13 | 14 | "go.xrstf.de/github_exporter/pkg/client" 15 | "go.xrstf.de/github_exporter/pkg/fetcher" 16 | "go.xrstf.de/github_exporter/pkg/github" 17 | "go.xrstf.de/github_exporter/pkg/metrics" 18 | 19 | "github.com/prometheus/client_golang/prometheus" 20 | "github.com/prometheus/client_golang/prometheus/promhttp" 21 | "github.com/shurcooL/githubv4" 22 | "github.com/sirupsen/logrus" 23 | ) 24 | 25 | type options struct { 26 | repositories repositoryList 27 | owner string 28 | realnames bool 29 | repoRefreshInterval time.Duration 30 | prRefreshInterval time.Duration 31 | prResyncInterval time.Duration 32 | prDepth int 33 | issueRefreshInterval time.Duration 34 | issueResyncInterval time.Duration 35 | issueDepth int 36 | milestoneRefreshInterval time.Duration 37 | milestoneResyncInterval time.Duration 38 | milestoneDepth int 39 | listenAddr string 40 | debugLog bool 41 | } 42 | 43 | type AppContext struct { 44 | ctx context.Context 45 | client *client.Client 46 | fetcher *fetcher.Fetcher 47 | options *options 48 | } 49 | 50 | func main() { 51 | opt := options{ 52 | repoRefreshInterval: 5 * time.Minute, 53 | prRefreshInterval: 5 * time.Minute, 54 | prResyncInterval: 12 * time.Hour, 55 | prDepth: -1, 56 | issueRefreshInterval: 5 * time.Minute, 57 | issueResyncInterval: 12 * time.Hour, 58 | issueDepth: -1, 59 | milestoneRefreshInterval: 5 * time.Minute, 60 | milestoneResyncInterval: 12 * time.Hour, 61 | milestoneDepth: -1, 62 | listenAddr: ":9612", 63 | } 64 | 65 | flag.Var(&opt.repositories, "repo", "repository (owner/name format) to include, can be given multiple times") 66 | flag.StringVar(&opt.owner, "owner", opt.owner, "github login (username or organization) of the owner of the repositories that will be included. Excludes forked and locked repo, includes 100 first private & public repos") 67 | flag.BoolVar(&opt.realnames, "realnames", opt.realnames, "use usernames instead of internal IDs for author labels (this will make metrics contain personally identifiable information)") 68 | flag.DurationVar(&opt.repoRefreshInterval, "repo-refresh-interval", opt.repoRefreshInterval, "time in between repository metadata refreshes") 69 | flag.IntVar(&opt.prDepth, "pr-depth", opt.prDepth, "max number of pull requests to fetch per repository upon startup (-1 disables the limit, 0 disables PR fetching entirely)") 70 | flag.DurationVar(&opt.prRefreshInterval, "pr-refresh-interval", opt.prRefreshInterval, "time in between PR refreshes") 71 | flag.DurationVar(&opt.prResyncInterval, "pr-resync-interval", opt.prResyncInterval, "time in between full PR re-syncs") 72 | flag.IntVar(&opt.issueDepth, "issue-depth", opt.issueDepth, "max number of issues to fetch per repository upon startup (-1 disables the limit, 0 disables issue fetching entirely)") 73 | flag.DurationVar(&opt.issueRefreshInterval, "issue-refresh-interval", opt.issueRefreshInterval, "time in between issue refreshes") 74 | flag.DurationVar(&opt.issueResyncInterval, "issue-resync-interval", opt.issueResyncInterval, "time in between full issue re-syncs") 75 | flag.IntVar(&opt.milestoneDepth, "milestone-depth", opt.milestoneDepth, "max number of milestones to fetch per repository upon startup (-1 disables the limit, 0 disables milestone fetching entirely)") 76 | flag.DurationVar(&opt.milestoneRefreshInterval, "milestone-refresh-interval", opt.milestoneRefreshInterval, "time in between milestone refreshes") 77 | flag.DurationVar(&opt.milestoneResyncInterval, "milestone-resync-interval", opt.milestoneResyncInterval, "time in between full milestone re-syncs") 78 | flag.StringVar(&opt.listenAddr, "listen", opt.listenAddr, "address and port to listen on") 79 | flag.BoolVar(&opt.debugLog, "debug", opt.debugLog, "enable more verbose logging") 80 | flag.Parse() 81 | 82 | // setup logging 83 | var log = logrus.New() 84 | log.SetFormatter(&logrus.TextFormatter{ 85 | FullTimestamp: true, 86 | TimestampFormat: time.RFC1123, 87 | }) 88 | 89 | if opt.debugLog { 90 | log.SetLevel(logrus.DebugLevel) 91 | } 92 | 93 | // validate CLI flags 94 | if opt.owner == "" && len(opt.repositories) == 0 { 95 | log.Fatal("No -repo nor -owner defined.") 96 | } 97 | 98 | if opt.prRefreshInterval >= opt.prResyncInterval { 99 | log.Fatal("-pr-refresh-interval must be < than -pr-resync-interval.") 100 | } 101 | 102 | if opt.issueRefreshInterval >= opt.issueResyncInterval { 103 | log.Fatal("-issue-refresh-interval must be < than -issue-resync-interval.") 104 | } 105 | 106 | if opt.milestoneRefreshInterval >= opt.milestoneResyncInterval { 107 | log.Fatal("-milestone-refresh-interval must be < than -milestone-resync-interval.") 108 | } 109 | 110 | token := os.Getenv("GITHUB_TOKEN") 111 | if len(token) == 0 { 112 | log.Fatal("No GITHUB_TOKEN environment variable defined.") 113 | } 114 | 115 | // setup API client 116 | ctx := context.Background() 117 | 118 | client, err := client.NewClient(ctx, log.WithField("component", "client"), token, opt.realnames) 119 | if err != nil { 120 | log.Fatalf("Failed to create API client: %v", err) 121 | } 122 | 123 | appCtx := AppContext{ 124 | ctx: ctx, 125 | client: client, 126 | options: &opt, 127 | } 128 | 129 | // start fetching data in the background, but start metrics 130 | // server as soon as possible 131 | go setup(appCtx, log) 132 | 133 | log.Printf("Starting server on %s…", opt.listenAddr) 134 | 135 | http.Handle("/metrics", promhttp.Handler()) 136 | log.Fatal(http.ListenAndServe(opt.listenAddr, nil)) 137 | } 138 | 139 | func setup(ctx AppContext, log logrus.FieldLogger) { 140 | repositories := map[string]*github.Repository{} 141 | 142 | if ctx.options.owner != "" { 143 | log.Infof("Fetching all repositories for %s", ctx.options.owner) 144 | repoNames, err := ctx.client.RepositoriesNames(ctx.options.owner) 145 | if err != nil { 146 | log.Fatalf("Failed to recover repositories: %v", err) 147 | } 148 | for _, repoName := range repoNames { 149 | repositories[fmt.Sprintf("%s/%s", ctx.options.owner, repoName)] = github.NewRepository(ctx.options.owner, repoName) 150 | } 151 | } 152 | 153 | // create a PR database for each repo 154 | for _, repo := range ctx.options.repositories { 155 | repositories[repo.String()] = github.NewRepository(repo.owner, repo.name) 156 | } 157 | 158 | // setup the single-threaded fetcher 159 | ctx.fetcher = fetcher.NewFetcher(ctx.client, repositories, log.WithField("component", "fetcher")) 160 | go ctx.fetcher.Worker() 161 | 162 | prometheus.MustRegister(metrics.NewCollector(repositories, ctx.fetcher, ctx.client)) 163 | 164 | // perform the initial scan sequentially across all repositories, otherwise 165 | // it's likely that we trigger GitHub's anti abuse system 166 | log.Info("Initializing repositories…") 167 | 168 | hasLabelledMetrics := ctx.options.prDepth != 0 || ctx.options.issueDepth != 0 || ctx.options.milestoneDepth != 0 169 | 170 | for identifier, repo := range repositories { 171 | repoLog := log.WithField("repo", identifier) 172 | 173 | repoLog.Info("Scheduling initial scans…") 174 | ctx.fetcher.EnqueueRepoUpdate(repo) 175 | 176 | // keep repository metadata up-to-date 177 | go refreshRepositoryInfoWorker(ctx, repoLog, repo) 178 | 179 | if hasLabelledMetrics { 180 | ctx.fetcher.EnqueueLabelUpdate(repo) 181 | } 182 | 183 | if ctx.options.prDepth != 0 { 184 | ctx.fetcher.EnqueuePullRequestScan(repo, ctx.options.prDepth) 185 | 186 | // keep refreshing open PRs 187 | go refreshPullRequestsWorker(ctx, repoLog, repo) 188 | 189 | // in a much larger interval, crawl all existing PRs to detect deletions and changes 190 | // after a PR has been merged 191 | go resyncPullRequestsWorker(ctx, repoLog, repo) 192 | } 193 | 194 | if ctx.options.issueDepth != 0 { 195 | ctx.fetcher.EnqueueIssueScan(repo, ctx.options.issueDepth) 196 | 197 | // keep refreshing open issues 198 | go refreshIssuesWorker(ctx, repoLog, repo) 199 | 200 | // in a much larger interval, crawl all existing issues to detect status changes 201 | go resyncIssuesWorker(ctx, repoLog, repo) 202 | } 203 | 204 | if ctx.options.milestoneDepth != 0 { 205 | ctx.fetcher.EnqueueMilestoneScan(repo, ctx.options.milestoneDepth) 206 | 207 | // keep refreshing open milestones 208 | go refreshMilestonesWorker(ctx, repoLog, repo) 209 | 210 | // in a much larger interval, crawl all existing milestones to detect status changes 211 | go resyncMilestonesWorker(ctx, repoLog, repo) 212 | } 213 | } 214 | } 215 | 216 | func refreshRepositoryInfoWorker(ctx AppContext, log logrus.FieldLogger, repo *github.Repository) { 217 | for range time.NewTicker(ctx.options.repoRefreshInterval).C { 218 | log.Debug("Refreshing repository metadata…") 219 | ctx.fetcher.EnqueueRepoUpdate(repo) 220 | } 221 | } 222 | 223 | // refreshRepositoriesWorker refreshes all OPEN pull requests, because changes 224 | // to the build contexts do not change the updatedAt timestamp on GitHub and we 225 | // want to closely track the mergability. It also fetches the last 50 updated 226 | // PRs to find cases where a PR was merged and is not open anymore. 227 | func refreshPullRequestsWorker(ctx AppContext, log logrus.FieldLogger, repo *github.Repository) { 228 | for range time.NewTicker(ctx.options.prRefreshInterval).C { 229 | log.Debug("Refreshing open pull requests…") 230 | 231 | numbers := []int{} 232 | for _, pr := range repo.GetPullRequests(githubv4.PullRequestStateOpen) { 233 | numbers = append(numbers, pr.Number) 234 | } 235 | 236 | ctx.fetcher.EnqueuePriorityPullRequests(repo, numbers) 237 | ctx.fetcher.EnqueueUpdatedPullRequests(repo) 238 | } 239 | } 240 | 241 | func resyncPullRequestsWorker(ctx AppContext, log logrus.FieldLogger, repo *github.Repository) { 242 | for range time.NewTicker(ctx.options.prResyncInterval).C { 243 | log.Info("Synchronizing repository pull requests…") 244 | 245 | numbers := []int{} 246 | for _, pr := range repo.GetPullRequests(githubv4.PullRequestStateClosed, githubv4.PullRequestStateMerged) { 247 | numbers = append(numbers, pr.Number) 248 | } 249 | 250 | ctx.fetcher.EnqueueRegularPullRequests(repo, numbers) 251 | ctx.fetcher.EnqueueLabelUpdate(repo) 252 | } 253 | } 254 | 255 | func refreshIssuesWorker(ctx AppContext, log logrus.FieldLogger, repo *github.Repository) { 256 | for range time.NewTicker(ctx.options.issueRefreshInterval).C { 257 | log.Debug("Refreshing open pull issues…") 258 | 259 | numbers := []int{} 260 | for _, issue := range repo.GetIssues(githubv4.IssueStateOpen) { 261 | numbers = append(numbers, issue.Number) 262 | } 263 | 264 | ctx.fetcher.EnqueuePriorityIssues(repo, numbers) 265 | ctx.fetcher.EnqueueUpdatedIssues(repo) 266 | } 267 | } 268 | 269 | func resyncIssuesWorker(ctx AppContext, log logrus.FieldLogger, repo *github.Repository) { 270 | for range time.NewTicker(ctx.options.issueResyncInterval).C { 271 | log.Info("Synchronizing repository issues…") 272 | 273 | numbers := []int{} 274 | for _, issue := range repo.GetIssues(githubv4.IssueStateClosed) { 275 | numbers = append(numbers, issue.Number) 276 | } 277 | 278 | ctx.fetcher.EnqueueRegularIssues(repo, numbers) 279 | ctx.fetcher.EnqueueLabelUpdate(repo) 280 | } 281 | } 282 | 283 | func refreshMilestonesWorker(ctx AppContext, log logrus.FieldLogger, repo *github.Repository) { 284 | for range time.NewTicker(ctx.options.milestoneRefreshInterval).C { 285 | log.Debug("Refreshing open pull milestones…") 286 | 287 | numbers := []int{} 288 | for _, milestone := range repo.GetMilestones(githubv4.MilestoneStateOpen) { 289 | numbers = append(numbers, milestone.Number) 290 | } 291 | 292 | ctx.fetcher.EnqueuePriorityMilestones(repo, numbers) 293 | ctx.fetcher.EnqueueUpdatedMilestones(repo) 294 | } 295 | } 296 | 297 | func resyncMilestonesWorker(ctx AppContext, log logrus.FieldLogger, repo *github.Repository) { 298 | for range time.NewTicker(ctx.options.milestoneResyncInterval).C { 299 | log.Info("Synchronizing repository milestones…") 300 | 301 | numbers := []int{} 302 | for _, milestone := range repo.GetMilestones(githubv4.MilestoneStateClosed) { 303 | numbers = append(numbers, milestone.Number) 304 | } 305 | 306 | ctx.fetcher.EnqueueRegularMilestones(repo, numbers) 307 | ctx.fetcher.EnqueueLabelUpdate(repo) 308 | } 309 | } 310 | -------------------------------------------------------------------------------- /pkg/fetcher/fetcher.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2023 Christoph Mewes 2 | // SPDX-License-Identifier: MIT 3 | 4 | package fetcher 5 | 6 | import ( 7 | "sync" 8 | "time" 9 | 10 | "go.xrstf.de/github_exporter/pkg/client" 11 | "go.xrstf.de/github_exporter/pkg/github" 12 | 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | type Fetcher struct { 17 | client *client.Client 18 | log logrus.FieldLogger 19 | repositories map[string]*github.Repository 20 | jobQueues map[string]jobQueue 21 | pullRequestQueues map[string]prioritizedIntegerQueue 22 | issueQueues map[string]prioritizedIntegerQueue 23 | milestoneQueues map[string]prioritizedIntegerQueue 24 | lock sync.RWMutex 25 | } 26 | 27 | func NewFetcher(client *client.Client, repos map[string]*github.Repository, log logrus.FieldLogger) *Fetcher { 28 | return &Fetcher{ 29 | client: client, 30 | log: log, 31 | repositories: repos, 32 | jobQueues: makeJobQueues(repos), 33 | pullRequestQueues: makePrioritizedIntegerQueues(repos), 34 | issueQueues: makePrioritizedIntegerQueues(repos), 35 | milestoneQueues: makePrioritizedIntegerQueues(repos), 36 | lock: sync.RWMutex{}, 37 | } 38 | } 39 | 40 | func makePrioritizedIntegerQueues(repos map[string]*github.Repository) map[string]prioritizedIntegerQueue { 41 | queues := map[string]prioritizedIntegerQueue{} 42 | 43 | for fullName := range repos { 44 | queues[fullName] = newPrioritizedIntegerQueue() 45 | } 46 | 47 | return queues 48 | } 49 | 50 | func makeJobQueues(repos map[string]*github.Repository) map[string]jobQueue { 51 | queues := map[string]jobQueue{} 52 | 53 | for fullName := range repos { 54 | queues[fullName] = jobQueue{} 55 | } 56 | 57 | return queues 58 | } 59 | 60 | func (f *Fetcher) EnqueueRepoUpdate(r *github.Repository) { 61 | f.enqueueJob(r, updateRepoInfoJobKey, nil) 62 | } 63 | 64 | func (f *Fetcher) EnqueueLabelUpdate(r *github.Repository) { 65 | f.enqueueJob(r, updateLabelsJobKey, nil) 66 | } 67 | 68 | func (f *Fetcher) EnqueueUpdatedPullRequests(r *github.Repository) { 69 | f.enqueueJob(r, findUpdatedPullRequestsJobKey, nil) 70 | } 71 | 72 | func (f *Fetcher) EnqueuePullRequestScan(r *github.Repository, max int) { 73 | f.enqueueJob(r, scanPullRequestsJobKey, scanPullRequestsJobMeta{ 74 | max: max, 75 | }) 76 | } 77 | 78 | func (f *Fetcher) enqueueUpdatedPullRequests(r *github.Repository, numbers []int) { 79 | f.enqueueJob(r, updatePullRequestsJobKey, updatePullRequestsJobMeta{ 80 | numbers: numbers, 81 | }) 82 | } 83 | 84 | func (f *Fetcher) EnqueueUpdatedIssues(r *github.Repository) { 85 | f.enqueueJob(r, findUpdatedIssuesJobKey, nil) 86 | } 87 | 88 | func (f *Fetcher) EnqueueIssueScan(r *github.Repository, max int) { 89 | f.enqueueJob(r, scanIssuesJobKey, scanIssuesJobMeta{ 90 | max: max, 91 | }) 92 | } 93 | 94 | func (f *Fetcher) enqueueUpdatedIssues(r *github.Repository, numbers []int) { 95 | f.enqueueJob(r, updateIssuesJobKey, updateIssuesJobMeta{ 96 | numbers: numbers, 97 | }) 98 | } 99 | 100 | func (f *Fetcher) EnqueueUpdatedMilestones(r *github.Repository) { 101 | f.enqueueJob(r, findUpdatedMilestonesJobKey, nil) 102 | } 103 | 104 | func (f *Fetcher) EnqueueMilestoneScan(r *github.Repository, max int) { 105 | f.enqueueJob(r, scanMilestonesJobKey, scanMilestonesJobMeta{ 106 | max: max, 107 | }) 108 | } 109 | 110 | func (f *Fetcher) enqueueUpdatedMilestones(r *github.Repository, numbers []int) { 111 | f.enqueueJob(r, updateMilestonesJobKey, updateMilestonesJobMeta{ 112 | numbers: numbers, 113 | }) 114 | } 115 | 116 | func (f *Fetcher) enqueueJob(r *github.Repository, key string, data interface{}) { 117 | f.lock.Lock() 118 | defer f.lock.Unlock() 119 | 120 | f.log.WithField("repo", r.FullName()).WithField("job", key).Debug("Enqueueing job.") 121 | 122 | f.jobQueues[r.FullName()][key] = data 123 | } 124 | 125 | func (f *Fetcher) EnqueuePriorityPullRequests(r *github.Repository, numbers []int) { 126 | f.enqueue(r, numbers, f.pullRequestQueues, true) 127 | } 128 | 129 | func (f *Fetcher) EnqueueRegularPullRequests(r *github.Repository, numbers []int) { 130 | f.enqueue(r, numbers, f.pullRequestQueues, false) 131 | } 132 | 133 | func (f *Fetcher) EnqueuePriorityIssues(r *github.Repository, numbers []int) { 134 | f.enqueue(r, numbers, f.issueQueues, true) 135 | } 136 | 137 | func (f *Fetcher) EnqueueRegularIssues(r *github.Repository, numbers []int) { 138 | f.enqueue(r, numbers, f.issueQueues, false) 139 | } 140 | 141 | func (f *Fetcher) EnqueuePriorityMilestones(r *github.Repository, numbers []int) { 142 | f.enqueue(r, numbers, f.milestoneQueues, true) 143 | } 144 | 145 | func (f *Fetcher) EnqueueRegularMilestones(r *github.Repository, numbers []int) { 146 | f.enqueue(r, numbers, f.milestoneQueues, false) 147 | } 148 | 149 | func (f *Fetcher) enqueue(r *github.Repository, numbers []int, queues map[string]prioritizedIntegerQueue, priority bool) { 150 | queue, ok := queues[r.FullName()] 151 | if !ok { 152 | f.log.Fatalf("No queue defined for repository %s", r.FullName()) 153 | } 154 | 155 | f.lock.Lock() 156 | defer f.lock.Unlock() 157 | 158 | f.log.WithField("repo", r.FullName()).Debugf("Enqueueing %d items for updating.", len(numbers)) 159 | 160 | if priority { 161 | queue.priorityEnqueue(numbers) 162 | } else { 163 | queue.regularEnqueue(numbers) 164 | } 165 | } 166 | 167 | func (f *Fetcher) PriorityPullRequestQueueSize(r *github.Repository) int { 168 | return f.queueSize(r, f.pullRequestQueues, true) 169 | } 170 | 171 | func (f *Fetcher) RegularPullRequestQueueSize(r *github.Repository) int { 172 | return f.queueSize(r, f.pullRequestQueues, false) 173 | } 174 | 175 | func (f *Fetcher) PriorityIssueQueueSize(r *github.Repository) int { 176 | return f.queueSize(r, f.issueQueues, true) 177 | } 178 | 179 | func (f *Fetcher) RegularIssueQueueSize(r *github.Repository) int { 180 | return f.queueSize(r, f.issueQueues, false) 181 | } 182 | 183 | func (f *Fetcher) PriorityMilestoneQueueSize(r *github.Repository) int { 184 | return f.queueSize(r, f.milestoneQueues, true) 185 | } 186 | 187 | func (f *Fetcher) RegularMilestoneQueueSize(r *github.Repository) int { 188 | return f.queueSize(r, f.milestoneQueues, false) 189 | } 190 | 191 | func (f *Fetcher) queueSize(r *github.Repository, queues map[string]prioritizedIntegerQueue, priority bool) int { 192 | queue, ok := queues[r.FullName()] 193 | if !ok { 194 | f.log.Fatalf("No queue defined for repository %s", r.FullName()) 195 | } 196 | 197 | f.lock.RLock() 198 | defer f.lock.RUnlock() 199 | 200 | if priority { 201 | return queue.prioritySize() 202 | } else { 203 | return queue.regularSize() 204 | } 205 | } 206 | 207 | func (f *Fetcher) Worker() { 208 | lastForceFlush := time.Now() 209 | maxWaitTime := 1 * time.Minute 210 | 211 | for { 212 | // the job queue has priority over crawling numbered PRs from the other queues 213 | repo, job, data := f.getNextJob() 214 | 215 | // there is a job ready to be processed 216 | if repo != nil { 217 | err := f.processJob(repo, job, data) 218 | if err != nil { 219 | f.log.Errorf("Failed to process job: %v", err) 220 | } 221 | 222 | continue 223 | } 224 | 225 | // if there was no job, try to create a job to update the existing 226 | // numbered PRs 227 | repo, candidates := f.getPullRequestBatch(10, client.MaxPullRequestsPerQuery) 228 | 229 | // a repository has amassed enough PRs to warrant a new job 230 | if repo != nil { 231 | f.enqueueUpdatedPullRequests(repo, candidates) 232 | continue 233 | } 234 | 235 | // try batching up issues next 236 | repo, candidates = f.getIssueBatch(10, client.MaxIssuesPerQuery) 237 | if repo != nil { 238 | f.enqueueUpdatedIssues(repo, candidates) 239 | continue 240 | } 241 | 242 | // try batching up milestones next 243 | repo, candidates = f.getMilestoneBatch(10, client.MaxMilestonesPerQuery) 244 | if repo != nil { 245 | f.enqueueUpdatedMilestones(repo, candidates) 246 | continue 247 | } 248 | 249 | // no repo has enough items for a good batch; in order to not burn 250 | // CPU cycles, we will wait a bit and check again. But we don't wait 251 | // forever, otherwise repositories with very few PRs might never get 252 | // updated. 253 | if time.Since(lastForceFlush) < maxWaitTime { 254 | time.Sleep(1 * time.Second) 255 | continue 256 | } 257 | 258 | // we waited long enough, give up and accept 1-element batches 259 | repo, candidates = f.getPullRequestBatch(1, client.MaxPullRequestsPerQuery) 260 | 261 | // got a mini batch 262 | if repo != nil { 263 | f.enqueueUpdatedPullRequests(repo, candidates) 264 | continue 265 | } 266 | 267 | repo, candidates = f.getIssueBatch(1, client.MaxIssuesPerQuery) 268 | if repo != nil { 269 | f.enqueueUpdatedIssues(repo, candidates) 270 | continue 271 | } 272 | 273 | repo, candidates = f.getMilestoneBatch(1, client.MaxMilestonesPerQuery) 274 | if repo != nil { 275 | f.enqueueUpdatedMilestones(repo, candidates) 276 | continue 277 | } 278 | 279 | // all repository queues are entirely empty, we finished the 280 | // force flush and can remember the time; this means on the next 281 | // iteration we will begin to sleep again. 282 | lastForceFlush = time.Now() 283 | 284 | f.log.Debug("All queues emptied, force flush completed.") 285 | } 286 | } 287 | 288 | var priorityJobs = []string{ 289 | // prioritizing this makes useful metrics available earlier 290 | updateRepoInfoJobKey, 291 | 292 | // the scan jobs have priority over everything else, as other jobs are based on it 293 | scanIssuesJobKey, 294 | scanPullRequestsJobKey, 295 | scanMilestonesJobKey, 296 | } 297 | 298 | func (f *Fetcher) getNextJob() (*github.Repository, string, interface{}) { 299 | f.lock.RLock() 300 | defer f.lock.RUnlock() 301 | 302 | for fullName, queue := range f.jobQueues { 303 | for _, job := range priorityJobs { 304 | if data, ok := queue[job]; ok { 305 | return f.repositories[fullName], job, data 306 | } 307 | } 308 | 309 | for job, data := range queue { 310 | return f.repositories[fullName], job, data 311 | } 312 | } 313 | 314 | return nil, "", nil 315 | } 316 | 317 | func (f *Fetcher) getPullRequestBatch(minBatchSize int, maxBatchSize int) (*github.Repository, []int) { 318 | return f.getBatch(f.pullRequestQueues, minBatchSize, maxBatchSize) 319 | } 320 | 321 | func (f *Fetcher) getIssueBatch(minBatchSize int, maxBatchSize int) (*github.Repository, []int) { 322 | return f.getBatch(f.issueQueues, minBatchSize, maxBatchSize) 323 | } 324 | 325 | func (f *Fetcher) getMilestoneBatch(minBatchSize int, maxBatchSize int) (*github.Repository, []int) { 326 | return f.getBatch(f.milestoneQueues, minBatchSize, maxBatchSize) 327 | } 328 | 329 | func (f *Fetcher) getBatch(queues map[string]prioritizedIntegerQueue, minBatchSize int, maxBatchSize int) (*github.Repository, []int) { 330 | f.lock.RLock() 331 | defer f.lock.RUnlock() 332 | 333 | for fullName := range f.repositories { 334 | queue := queues[fullName] 335 | 336 | batch := queue.getBatch(minBatchSize, maxBatchSize) 337 | if batch != nil { 338 | return f.repositories[fullName], batch 339 | } 340 | } 341 | 342 | // no repository has (combined) enough items to satisfy minBatchSize 343 | return nil, nil 344 | } 345 | 346 | func (f *Fetcher) processJob(repo *github.Repository, job string, data interface{}) error { 347 | var err error 348 | 349 | log := f.log.WithField("repo", repo.FullName()).WithField("job", job) 350 | log.Debug("Processing job…") 351 | 352 | switch job { 353 | case updateLabelsJobKey: 354 | err = f.processUpdateLabelsJob(repo, log, job) 355 | case updateRepoInfoJobKey: 356 | err = f.processUpdateRepoInfos(repo, log, job) 357 | case updatePullRequestsJobKey: 358 | err = f.processUpdatePullRequestsJob(repo, log, job, data) 359 | case findUpdatedPullRequestsJobKey: 360 | err = f.processFindUpdatedPullRequestsJob(repo, log, job) 361 | case scanPullRequestsJobKey: 362 | err = f.processScanPullRequestsJob(repo, log, job, data) 363 | case updateIssuesJobKey: 364 | err = f.processUpdateIssuesJob(repo, log, job, data) 365 | case findUpdatedIssuesJobKey: 366 | err = f.processFindUpdatedIssuesJob(repo, log, job) 367 | case scanIssuesJobKey: 368 | err = f.processScanIssuesJob(repo, log, job, data) 369 | case updateMilestonesJobKey: 370 | err = f.processUpdateMilestonesJob(repo, log, job, data) 371 | case findUpdatedMilestonesJobKey: 372 | err = f.processFindUpdatedMilestonesJob(repo, log, job) 373 | case scanMilestonesJobKey: 374 | err = f.processScanMilestonesJob(repo, log, job, data) 375 | default: 376 | f.log.Fatalf("Encountered unknown job type %q for repo %q", job, repo.FullName()) 377 | } 378 | 379 | return err 380 | } 381 | 382 | func (f *Fetcher) removeJob(repo *github.Repository, job string) { 383 | f.log.WithField("job", job).Debugf("Removing job.") 384 | 385 | fullName := repo.FullName() 386 | 387 | f.lock.Lock() 388 | defer f.lock.Unlock() 389 | 390 | delete(f.jobQueues[fullName], job) 391 | } 392 | 393 | func (f *Fetcher) dequeuePullRequests(repo *github.Repository, numbers []int) { 394 | f.log.Debugf("Removing %d fetched PRs.", len(numbers)) 395 | f.dequeue(repo, f.pullRequestQueues, numbers) 396 | } 397 | 398 | func (f *Fetcher) dequeueIssues(repo *github.Repository, numbers []int) { 399 | f.log.Debugf("Removing %d fetched issues.", len(numbers)) 400 | f.dequeue(repo, f.issueQueues, numbers) 401 | } 402 | 403 | func (f *Fetcher) dequeueMilestones(repo *github.Repository, numbers []int) { 404 | f.log.Debugf("Removing %d fetched milestones.", len(numbers)) 405 | f.dequeue(repo, f.milestoneQueues, numbers) 406 | } 407 | 408 | func (f *Fetcher) dequeue(repo *github.Repository, queues map[string]prioritizedIntegerQueue, numbers []int) { 409 | fullName := repo.FullName() 410 | 411 | f.lock.Lock() 412 | defer f.lock.Unlock() 413 | 414 | queue := queues[fullName] 415 | queue.dequeue(numbers) 416 | } 417 | -------------------------------------------------------------------------------- /pkg/client/client_issues_gen.go: -------------------------------------------------------------------------------- 1 | // This file has been generated by hack/generate-client.sh 2 | // Do not edit manually! 3 | 4 | package client 5 | 6 | import ( 7 | "fmt" 8 | ) 9 | 10 | const ( 11 | MaxIssuesPerQuery = 100 12 | ) 13 | 14 | type numberedIssueQuery struct { 15 | RateLimit rateLimit 16 | Repository struct { 17 | Issue0 *graphqlIssue `graphql:"issue0: issue(number: $number0) @include(if: $has0)"` 18 | Issue1 *graphqlIssue `graphql:"issue1: issue(number: $number1) @include(if: $has1)"` 19 | Issue2 *graphqlIssue `graphql:"issue2: issue(number: $number2) @include(if: $has2)"` 20 | Issue3 *graphqlIssue `graphql:"issue3: issue(number: $number3) @include(if: $has3)"` 21 | Issue4 *graphqlIssue `graphql:"issue4: issue(number: $number4) @include(if: $has4)"` 22 | Issue5 *graphqlIssue `graphql:"issue5: issue(number: $number5) @include(if: $has5)"` 23 | Issue6 *graphqlIssue `graphql:"issue6: issue(number: $number6) @include(if: $has6)"` 24 | Issue7 *graphqlIssue `graphql:"issue7: issue(number: $number7) @include(if: $has7)"` 25 | Issue8 *graphqlIssue `graphql:"issue8: issue(number: $number8) @include(if: $has8)"` 26 | Issue9 *graphqlIssue `graphql:"issue9: issue(number: $number9) @include(if: $has9)"` 27 | Issue10 *graphqlIssue `graphql:"issue10: issue(number: $number10) @include(if: $has10)"` 28 | Issue11 *graphqlIssue `graphql:"issue11: issue(number: $number11) @include(if: $has11)"` 29 | Issue12 *graphqlIssue `graphql:"issue12: issue(number: $number12) @include(if: $has12)"` 30 | Issue13 *graphqlIssue `graphql:"issue13: issue(number: $number13) @include(if: $has13)"` 31 | Issue14 *graphqlIssue `graphql:"issue14: issue(number: $number14) @include(if: $has14)"` 32 | Issue15 *graphqlIssue `graphql:"issue15: issue(number: $number15) @include(if: $has15)"` 33 | Issue16 *graphqlIssue `graphql:"issue16: issue(number: $number16) @include(if: $has16)"` 34 | Issue17 *graphqlIssue `graphql:"issue17: issue(number: $number17) @include(if: $has17)"` 35 | Issue18 *graphqlIssue `graphql:"issue18: issue(number: $number18) @include(if: $has18)"` 36 | Issue19 *graphqlIssue `graphql:"issue19: issue(number: $number19) @include(if: $has19)"` 37 | Issue20 *graphqlIssue `graphql:"issue20: issue(number: $number20) @include(if: $has20)"` 38 | Issue21 *graphqlIssue `graphql:"issue21: issue(number: $number21) @include(if: $has21)"` 39 | Issue22 *graphqlIssue `graphql:"issue22: issue(number: $number22) @include(if: $has22)"` 40 | Issue23 *graphqlIssue `graphql:"issue23: issue(number: $number23) @include(if: $has23)"` 41 | Issue24 *graphqlIssue `graphql:"issue24: issue(number: $number24) @include(if: $has24)"` 42 | Issue25 *graphqlIssue `graphql:"issue25: issue(number: $number25) @include(if: $has25)"` 43 | Issue26 *graphqlIssue `graphql:"issue26: issue(number: $number26) @include(if: $has26)"` 44 | Issue27 *graphqlIssue `graphql:"issue27: issue(number: $number27) @include(if: $has27)"` 45 | Issue28 *graphqlIssue `graphql:"issue28: issue(number: $number28) @include(if: $has28)"` 46 | Issue29 *graphqlIssue `graphql:"issue29: issue(number: $number29) @include(if: $has29)"` 47 | Issue30 *graphqlIssue `graphql:"issue30: issue(number: $number30) @include(if: $has30)"` 48 | Issue31 *graphqlIssue `graphql:"issue31: issue(number: $number31) @include(if: $has31)"` 49 | Issue32 *graphqlIssue `graphql:"issue32: issue(number: $number32) @include(if: $has32)"` 50 | Issue33 *graphqlIssue `graphql:"issue33: issue(number: $number33) @include(if: $has33)"` 51 | Issue34 *graphqlIssue `graphql:"issue34: issue(number: $number34) @include(if: $has34)"` 52 | Issue35 *graphqlIssue `graphql:"issue35: issue(number: $number35) @include(if: $has35)"` 53 | Issue36 *graphqlIssue `graphql:"issue36: issue(number: $number36) @include(if: $has36)"` 54 | Issue37 *graphqlIssue `graphql:"issue37: issue(number: $number37) @include(if: $has37)"` 55 | Issue38 *graphqlIssue `graphql:"issue38: issue(number: $number38) @include(if: $has38)"` 56 | Issue39 *graphqlIssue `graphql:"issue39: issue(number: $number39) @include(if: $has39)"` 57 | Issue40 *graphqlIssue `graphql:"issue40: issue(number: $number40) @include(if: $has40)"` 58 | Issue41 *graphqlIssue `graphql:"issue41: issue(number: $number41) @include(if: $has41)"` 59 | Issue42 *graphqlIssue `graphql:"issue42: issue(number: $number42) @include(if: $has42)"` 60 | Issue43 *graphqlIssue `graphql:"issue43: issue(number: $number43) @include(if: $has43)"` 61 | Issue44 *graphqlIssue `graphql:"issue44: issue(number: $number44) @include(if: $has44)"` 62 | Issue45 *graphqlIssue `graphql:"issue45: issue(number: $number45) @include(if: $has45)"` 63 | Issue46 *graphqlIssue `graphql:"issue46: issue(number: $number46) @include(if: $has46)"` 64 | Issue47 *graphqlIssue `graphql:"issue47: issue(number: $number47) @include(if: $has47)"` 65 | Issue48 *graphqlIssue `graphql:"issue48: issue(number: $number48) @include(if: $has48)"` 66 | Issue49 *graphqlIssue `graphql:"issue49: issue(number: $number49) @include(if: $has49)"` 67 | Issue50 *graphqlIssue `graphql:"issue50: issue(number: $number50) @include(if: $has50)"` 68 | Issue51 *graphqlIssue `graphql:"issue51: issue(number: $number51) @include(if: $has51)"` 69 | Issue52 *graphqlIssue `graphql:"issue52: issue(number: $number52) @include(if: $has52)"` 70 | Issue53 *graphqlIssue `graphql:"issue53: issue(number: $number53) @include(if: $has53)"` 71 | Issue54 *graphqlIssue `graphql:"issue54: issue(number: $number54) @include(if: $has54)"` 72 | Issue55 *graphqlIssue `graphql:"issue55: issue(number: $number55) @include(if: $has55)"` 73 | Issue56 *graphqlIssue `graphql:"issue56: issue(number: $number56) @include(if: $has56)"` 74 | Issue57 *graphqlIssue `graphql:"issue57: issue(number: $number57) @include(if: $has57)"` 75 | Issue58 *graphqlIssue `graphql:"issue58: issue(number: $number58) @include(if: $has58)"` 76 | Issue59 *graphqlIssue `graphql:"issue59: issue(number: $number59) @include(if: $has59)"` 77 | Issue60 *graphqlIssue `graphql:"issue60: issue(number: $number60) @include(if: $has60)"` 78 | Issue61 *graphqlIssue `graphql:"issue61: issue(number: $number61) @include(if: $has61)"` 79 | Issue62 *graphqlIssue `graphql:"issue62: issue(number: $number62) @include(if: $has62)"` 80 | Issue63 *graphqlIssue `graphql:"issue63: issue(number: $number63) @include(if: $has63)"` 81 | Issue64 *graphqlIssue `graphql:"issue64: issue(number: $number64) @include(if: $has64)"` 82 | Issue65 *graphqlIssue `graphql:"issue65: issue(number: $number65) @include(if: $has65)"` 83 | Issue66 *graphqlIssue `graphql:"issue66: issue(number: $number66) @include(if: $has66)"` 84 | Issue67 *graphqlIssue `graphql:"issue67: issue(number: $number67) @include(if: $has67)"` 85 | Issue68 *graphqlIssue `graphql:"issue68: issue(number: $number68) @include(if: $has68)"` 86 | Issue69 *graphqlIssue `graphql:"issue69: issue(number: $number69) @include(if: $has69)"` 87 | Issue70 *graphqlIssue `graphql:"issue70: issue(number: $number70) @include(if: $has70)"` 88 | Issue71 *graphqlIssue `graphql:"issue71: issue(number: $number71) @include(if: $has71)"` 89 | Issue72 *graphqlIssue `graphql:"issue72: issue(number: $number72) @include(if: $has72)"` 90 | Issue73 *graphqlIssue `graphql:"issue73: issue(number: $number73) @include(if: $has73)"` 91 | Issue74 *graphqlIssue `graphql:"issue74: issue(number: $number74) @include(if: $has74)"` 92 | Issue75 *graphqlIssue `graphql:"issue75: issue(number: $number75) @include(if: $has75)"` 93 | Issue76 *graphqlIssue `graphql:"issue76: issue(number: $number76) @include(if: $has76)"` 94 | Issue77 *graphqlIssue `graphql:"issue77: issue(number: $number77) @include(if: $has77)"` 95 | Issue78 *graphqlIssue `graphql:"issue78: issue(number: $number78) @include(if: $has78)"` 96 | Issue79 *graphqlIssue `graphql:"issue79: issue(number: $number79) @include(if: $has79)"` 97 | Issue80 *graphqlIssue `graphql:"issue80: issue(number: $number80) @include(if: $has80)"` 98 | Issue81 *graphqlIssue `graphql:"issue81: issue(number: $number81) @include(if: $has81)"` 99 | Issue82 *graphqlIssue `graphql:"issue82: issue(number: $number82) @include(if: $has82)"` 100 | Issue83 *graphqlIssue `graphql:"issue83: issue(number: $number83) @include(if: $has83)"` 101 | Issue84 *graphqlIssue `graphql:"issue84: issue(number: $number84) @include(if: $has84)"` 102 | Issue85 *graphqlIssue `graphql:"issue85: issue(number: $number85) @include(if: $has85)"` 103 | Issue86 *graphqlIssue `graphql:"issue86: issue(number: $number86) @include(if: $has86)"` 104 | Issue87 *graphqlIssue `graphql:"issue87: issue(number: $number87) @include(if: $has87)"` 105 | Issue88 *graphqlIssue `graphql:"issue88: issue(number: $number88) @include(if: $has88)"` 106 | Issue89 *graphqlIssue `graphql:"issue89: issue(number: $number89) @include(if: $has89)"` 107 | Issue90 *graphqlIssue `graphql:"issue90: issue(number: $number90) @include(if: $has90)"` 108 | Issue91 *graphqlIssue `graphql:"issue91: issue(number: $number91) @include(if: $has91)"` 109 | Issue92 *graphqlIssue `graphql:"issue92: issue(number: $number92) @include(if: $has92)"` 110 | Issue93 *graphqlIssue `graphql:"issue93: issue(number: $number93) @include(if: $has93)"` 111 | Issue94 *graphqlIssue `graphql:"issue94: issue(number: $number94) @include(if: $has94)"` 112 | Issue95 *graphqlIssue `graphql:"issue95: issue(number: $number95) @include(if: $has95)"` 113 | Issue96 *graphqlIssue `graphql:"issue96: issue(number: $number96) @include(if: $has96)"` 114 | Issue97 *graphqlIssue `graphql:"issue97: issue(number: $number97) @include(if: $has97)"` 115 | Issue98 *graphqlIssue `graphql:"issue98: issue(number: $number98) @include(if: $has98)"` 116 | Issue99 *graphqlIssue `graphql:"issue99: issue(number: $number99) @include(if: $has99)"` 117 | } `graphql:"repository(owner: $owner, name: $name)"` 118 | } 119 | 120 | func (r *numberedIssueQuery) GetAll() []graphqlIssue { 121 | result := []graphqlIssue{} 122 | 123 | for i := 0; i < MaxIssuesPerQuery; i++ { 124 | if issue := r.Get(i); issue != nil { 125 | result = append(result, *issue) 126 | } 127 | } 128 | 129 | return result 130 | } 131 | 132 | func (r *numberedIssueQuery) Get(index int) *graphqlIssue { 133 | switch index { 134 | case 0: 135 | return r.Repository.Issue0 136 | case 1: 137 | return r.Repository.Issue1 138 | case 2: 139 | return r.Repository.Issue2 140 | case 3: 141 | return r.Repository.Issue3 142 | case 4: 143 | return r.Repository.Issue4 144 | case 5: 145 | return r.Repository.Issue5 146 | case 6: 147 | return r.Repository.Issue6 148 | case 7: 149 | return r.Repository.Issue7 150 | case 8: 151 | return r.Repository.Issue8 152 | case 9: 153 | return r.Repository.Issue9 154 | case 10: 155 | return r.Repository.Issue10 156 | case 11: 157 | return r.Repository.Issue11 158 | case 12: 159 | return r.Repository.Issue12 160 | case 13: 161 | return r.Repository.Issue13 162 | case 14: 163 | return r.Repository.Issue14 164 | case 15: 165 | return r.Repository.Issue15 166 | case 16: 167 | return r.Repository.Issue16 168 | case 17: 169 | return r.Repository.Issue17 170 | case 18: 171 | return r.Repository.Issue18 172 | case 19: 173 | return r.Repository.Issue19 174 | case 20: 175 | return r.Repository.Issue20 176 | case 21: 177 | return r.Repository.Issue21 178 | case 22: 179 | return r.Repository.Issue22 180 | case 23: 181 | return r.Repository.Issue23 182 | case 24: 183 | return r.Repository.Issue24 184 | case 25: 185 | return r.Repository.Issue25 186 | case 26: 187 | return r.Repository.Issue26 188 | case 27: 189 | return r.Repository.Issue27 190 | case 28: 191 | return r.Repository.Issue28 192 | case 29: 193 | return r.Repository.Issue29 194 | case 30: 195 | return r.Repository.Issue30 196 | case 31: 197 | return r.Repository.Issue31 198 | case 32: 199 | return r.Repository.Issue32 200 | case 33: 201 | return r.Repository.Issue33 202 | case 34: 203 | return r.Repository.Issue34 204 | case 35: 205 | return r.Repository.Issue35 206 | case 36: 207 | return r.Repository.Issue36 208 | case 37: 209 | return r.Repository.Issue37 210 | case 38: 211 | return r.Repository.Issue38 212 | case 39: 213 | return r.Repository.Issue39 214 | case 40: 215 | return r.Repository.Issue40 216 | case 41: 217 | return r.Repository.Issue41 218 | case 42: 219 | return r.Repository.Issue42 220 | case 43: 221 | return r.Repository.Issue43 222 | case 44: 223 | return r.Repository.Issue44 224 | case 45: 225 | return r.Repository.Issue45 226 | case 46: 227 | return r.Repository.Issue46 228 | case 47: 229 | return r.Repository.Issue47 230 | case 48: 231 | return r.Repository.Issue48 232 | case 49: 233 | return r.Repository.Issue49 234 | case 50: 235 | return r.Repository.Issue50 236 | case 51: 237 | return r.Repository.Issue51 238 | case 52: 239 | return r.Repository.Issue52 240 | case 53: 241 | return r.Repository.Issue53 242 | case 54: 243 | return r.Repository.Issue54 244 | case 55: 245 | return r.Repository.Issue55 246 | case 56: 247 | return r.Repository.Issue56 248 | case 57: 249 | return r.Repository.Issue57 250 | case 58: 251 | return r.Repository.Issue58 252 | case 59: 253 | return r.Repository.Issue59 254 | case 60: 255 | return r.Repository.Issue60 256 | case 61: 257 | return r.Repository.Issue61 258 | case 62: 259 | return r.Repository.Issue62 260 | case 63: 261 | return r.Repository.Issue63 262 | case 64: 263 | return r.Repository.Issue64 264 | case 65: 265 | return r.Repository.Issue65 266 | case 66: 267 | return r.Repository.Issue66 268 | case 67: 269 | return r.Repository.Issue67 270 | case 68: 271 | return r.Repository.Issue68 272 | case 69: 273 | return r.Repository.Issue69 274 | case 70: 275 | return r.Repository.Issue70 276 | case 71: 277 | return r.Repository.Issue71 278 | case 72: 279 | return r.Repository.Issue72 280 | case 73: 281 | return r.Repository.Issue73 282 | case 74: 283 | return r.Repository.Issue74 284 | case 75: 285 | return r.Repository.Issue75 286 | case 76: 287 | return r.Repository.Issue76 288 | case 77: 289 | return r.Repository.Issue77 290 | case 78: 291 | return r.Repository.Issue78 292 | case 79: 293 | return r.Repository.Issue79 294 | case 80: 295 | return r.Repository.Issue80 296 | case 81: 297 | return r.Repository.Issue81 298 | case 82: 299 | return r.Repository.Issue82 300 | case 83: 301 | return r.Repository.Issue83 302 | case 84: 303 | return r.Repository.Issue84 304 | case 85: 305 | return r.Repository.Issue85 306 | case 86: 307 | return r.Repository.Issue86 308 | case 87: 309 | return r.Repository.Issue87 310 | case 88: 311 | return r.Repository.Issue88 312 | case 89: 313 | return r.Repository.Issue89 314 | case 90: 315 | return r.Repository.Issue90 316 | case 91: 317 | return r.Repository.Issue91 318 | case 92: 319 | return r.Repository.Issue92 320 | case 93: 321 | return r.Repository.Issue93 322 | case 94: 323 | return r.Repository.Issue94 324 | case 95: 325 | return r.Repository.Issue95 326 | case 96: 327 | return r.Repository.Issue96 328 | case 97: 329 | return r.Repository.Issue97 330 | case 98: 331 | return r.Repository.Issue98 332 | case 99: 333 | return r.Repository.Issue99 334 | } 335 | 336 | panic(fmt.Sprintf("Index %d out of range [0,%d] when accessing issue request", index, MaxIssuesPerQuery-1)) 337 | } 338 | -------------------------------------------------------------------------------- /pkg/client/client_pullrequests_gen.go: -------------------------------------------------------------------------------- 1 | // This file has been generated by hack/generate-client.sh 2 | // Do not edit manually! 3 | 4 | package client 5 | 6 | import ( 7 | "fmt" 8 | ) 9 | 10 | const ( 11 | MaxPullRequestsPerQuery = 100 12 | ) 13 | 14 | type numberedPullRequestQuery struct { 15 | RateLimit rateLimit 16 | Repository struct { 17 | Pr0 *graphqlPullRequest `graphql:"pr0: pullRequest(number: $number0) @include(if: $has0)"` 18 | Pr1 *graphqlPullRequest `graphql:"pr1: pullRequest(number: $number1) @include(if: $has1)"` 19 | Pr2 *graphqlPullRequest `graphql:"pr2: pullRequest(number: $number2) @include(if: $has2)"` 20 | Pr3 *graphqlPullRequest `graphql:"pr3: pullRequest(number: $number3) @include(if: $has3)"` 21 | Pr4 *graphqlPullRequest `graphql:"pr4: pullRequest(number: $number4) @include(if: $has4)"` 22 | Pr5 *graphqlPullRequest `graphql:"pr5: pullRequest(number: $number5) @include(if: $has5)"` 23 | Pr6 *graphqlPullRequest `graphql:"pr6: pullRequest(number: $number6) @include(if: $has6)"` 24 | Pr7 *graphqlPullRequest `graphql:"pr7: pullRequest(number: $number7) @include(if: $has7)"` 25 | Pr8 *graphqlPullRequest `graphql:"pr8: pullRequest(number: $number8) @include(if: $has8)"` 26 | Pr9 *graphqlPullRequest `graphql:"pr9: pullRequest(number: $number9) @include(if: $has9)"` 27 | Pr10 *graphqlPullRequest `graphql:"pr10: pullRequest(number: $number10) @include(if: $has10)"` 28 | Pr11 *graphqlPullRequest `graphql:"pr11: pullRequest(number: $number11) @include(if: $has11)"` 29 | Pr12 *graphqlPullRequest `graphql:"pr12: pullRequest(number: $number12) @include(if: $has12)"` 30 | Pr13 *graphqlPullRequest `graphql:"pr13: pullRequest(number: $number13) @include(if: $has13)"` 31 | Pr14 *graphqlPullRequest `graphql:"pr14: pullRequest(number: $number14) @include(if: $has14)"` 32 | Pr15 *graphqlPullRequest `graphql:"pr15: pullRequest(number: $number15) @include(if: $has15)"` 33 | Pr16 *graphqlPullRequest `graphql:"pr16: pullRequest(number: $number16) @include(if: $has16)"` 34 | Pr17 *graphqlPullRequest `graphql:"pr17: pullRequest(number: $number17) @include(if: $has17)"` 35 | Pr18 *graphqlPullRequest `graphql:"pr18: pullRequest(number: $number18) @include(if: $has18)"` 36 | Pr19 *graphqlPullRequest `graphql:"pr19: pullRequest(number: $number19) @include(if: $has19)"` 37 | Pr20 *graphqlPullRequest `graphql:"pr20: pullRequest(number: $number20) @include(if: $has20)"` 38 | Pr21 *graphqlPullRequest `graphql:"pr21: pullRequest(number: $number21) @include(if: $has21)"` 39 | Pr22 *graphqlPullRequest `graphql:"pr22: pullRequest(number: $number22) @include(if: $has22)"` 40 | Pr23 *graphqlPullRequest `graphql:"pr23: pullRequest(number: $number23) @include(if: $has23)"` 41 | Pr24 *graphqlPullRequest `graphql:"pr24: pullRequest(number: $number24) @include(if: $has24)"` 42 | Pr25 *graphqlPullRequest `graphql:"pr25: pullRequest(number: $number25) @include(if: $has25)"` 43 | Pr26 *graphqlPullRequest `graphql:"pr26: pullRequest(number: $number26) @include(if: $has26)"` 44 | Pr27 *graphqlPullRequest `graphql:"pr27: pullRequest(number: $number27) @include(if: $has27)"` 45 | Pr28 *graphqlPullRequest `graphql:"pr28: pullRequest(number: $number28) @include(if: $has28)"` 46 | Pr29 *graphqlPullRequest `graphql:"pr29: pullRequest(number: $number29) @include(if: $has29)"` 47 | Pr30 *graphqlPullRequest `graphql:"pr30: pullRequest(number: $number30) @include(if: $has30)"` 48 | Pr31 *graphqlPullRequest `graphql:"pr31: pullRequest(number: $number31) @include(if: $has31)"` 49 | Pr32 *graphqlPullRequest `graphql:"pr32: pullRequest(number: $number32) @include(if: $has32)"` 50 | Pr33 *graphqlPullRequest `graphql:"pr33: pullRequest(number: $number33) @include(if: $has33)"` 51 | Pr34 *graphqlPullRequest `graphql:"pr34: pullRequest(number: $number34) @include(if: $has34)"` 52 | Pr35 *graphqlPullRequest `graphql:"pr35: pullRequest(number: $number35) @include(if: $has35)"` 53 | Pr36 *graphqlPullRequest `graphql:"pr36: pullRequest(number: $number36) @include(if: $has36)"` 54 | Pr37 *graphqlPullRequest `graphql:"pr37: pullRequest(number: $number37) @include(if: $has37)"` 55 | Pr38 *graphqlPullRequest `graphql:"pr38: pullRequest(number: $number38) @include(if: $has38)"` 56 | Pr39 *graphqlPullRequest `graphql:"pr39: pullRequest(number: $number39) @include(if: $has39)"` 57 | Pr40 *graphqlPullRequest `graphql:"pr40: pullRequest(number: $number40) @include(if: $has40)"` 58 | Pr41 *graphqlPullRequest `graphql:"pr41: pullRequest(number: $number41) @include(if: $has41)"` 59 | Pr42 *graphqlPullRequest `graphql:"pr42: pullRequest(number: $number42) @include(if: $has42)"` 60 | Pr43 *graphqlPullRequest `graphql:"pr43: pullRequest(number: $number43) @include(if: $has43)"` 61 | Pr44 *graphqlPullRequest `graphql:"pr44: pullRequest(number: $number44) @include(if: $has44)"` 62 | Pr45 *graphqlPullRequest `graphql:"pr45: pullRequest(number: $number45) @include(if: $has45)"` 63 | Pr46 *graphqlPullRequest `graphql:"pr46: pullRequest(number: $number46) @include(if: $has46)"` 64 | Pr47 *graphqlPullRequest `graphql:"pr47: pullRequest(number: $number47) @include(if: $has47)"` 65 | Pr48 *graphqlPullRequest `graphql:"pr48: pullRequest(number: $number48) @include(if: $has48)"` 66 | Pr49 *graphqlPullRequest `graphql:"pr49: pullRequest(number: $number49) @include(if: $has49)"` 67 | Pr50 *graphqlPullRequest `graphql:"pr50: pullRequest(number: $number50) @include(if: $has50)"` 68 | Pr51 *graphqlPullRequest `graphql:"pr51: pullRequest(number: $number51) @include(if: $has51)"` 69 | Pr52 *graphqlPullRequest `graphql:"pr52: pullRequest(number: $number52) @include(if: $has52)"` 70 | Pr53 *graphqlPullRequest `graphql:"pr53: pullRequest(number: $number53) @include(if: $has53)"` 71 | Pr54 *graphqlPullRequest `graphql:"pr54: pullRequest(number: $number54) @include(if: $has54)"` 72 | Pr55 *graphqlPullRequest `graphql:"pr55: pullRequest(number: $number55) @include(if: $has55)"` 73 | Pr56 *graphqlPullRequest `graphql:"pr56: pullRequest(number: $number56) @include(if: $has56)"` 74 | Pr57 *graphqlPullRequest `graphql:"pr57: pullRequest(number: $number57) @include(if: $has57)"` 75 | Pr58 *graphqlPullRequest `graphql:"pr58: pullRequest(number: $number58) @include(if: $has58)"` 76 | Pr59 *graphqlPullRequest `graphql:"pr59: pullRequest(number: $number59) @include(if: $has59)"` 77 | Pr60 *graphqlPullRequest `graphql:"pr60: pullRequest(number: $number60) @include(if: $has60)"` 78 | Pr61 *graphqlPullRequest `graphql:"pr61: pullRequest(number: $number61) @include(if: $has61)"` 79 | Pr62 *graphqlPullRequest `graphql:"pr62: pullRequest(number: $number62) @include(if: $has62)"` 80 | Pr63 *graphqlPullRequest `graphql:"pr63: pullRequest(number: $number63) @include(if: $has63)"` 81 | Pr64 *graphqlPullRequest `graphql:"pr64: pullRequest(number: $number64) @include(if: $has64)"` 82 | Pr65 *graphqlPullRequest `graphql:"pr65: pullRequest(number: $number65) @include(if: $has65)"` 83 | Pr66 *graphqlPullRequest `graphql:"pr66: pullRequest(number: $number66) @include(if: $has66)"` 84 | Pr67 *graphqlPullRequest `graphql:"pr67: pullRequest(number: $number67) @include(if: $has67)"` 85 | Pr68 *graphqlPullRequest `graphql:"pr68: pullRequest(number: $number68) @include(if: $has68)"` 86 | Pr69 *graphqlPullRequest `graphql:"pr69: pullRequest(number: $number69) @include(if: $has69)"` 87 | Pr70 *graphqlPullRequest `graphql:"pr70: pullRequest(number: $number70) @include(if: $has70)"` 88 | Pr71 *graphqlPullRequest `graphql:"pr71: pullRequest(number: $number71) @include(if: $has71)"` 89 | Pr72 *graphqlPullRequest `graphql:"pr72: pullRequest(number: $number72) @include(if: $has72)"` 90 | Pr73 *graphqlPullRequest `graphql:"pr73: pullRequest(number: $number73) @include(if: $has73)"` 91 | Pr74 *graphqlPullRequest `graphql:"pr74: pullRequest(number: $number74) @include(if: $has74)"` 92 | Pr75 *graphqlPullRequest `graphql:"pr75: pullRequest(number: $number75) @include(if: $has75)"` 93 | Pr76 *graphqlPullRequest `graphql:"pr76: pullRequest(number: $number76) @include(if: $has76)"` 94 | Pr77 *graphqlPullRequest `graphql:"pr77: pullRequest(number: $number77) @include(if: $has77)"` 95 | Pr78 *graphqlPullRequest `graphql:"pr78: pullRequest(number: $number78) @include(if: $has78)"` 96 | Pr79 *graphqlPullRequest `graphql:"pr79: pullRequest(number: $number79) @include(if: $has79)"` 97 | Pr80 *graphqlPullRequest `graphql:"pr80: pullRequest(number: $number80) @include(if: $has80)"` 98 | Pr81 *graphqlPullRequest `graphql:"pr81: pullRequest(number: $number81) @include(if: $has81)"` 99 | Pr82 *graphqlPullRequest `graphql:"pr82: pullRequest(number: $number82) @include(if: $has82)"` 100 | Pr83 *graphqlPullRequest `graphql:"pr83: pullRequest(number: $number83) @include(if: $has83)"` 101 | Pr84 *graphqlPullRequest `graphql:"pr84: pullRequest(number: $number84) @include(if: $has84)"` 102 | Pr85 *graphqlPullRequest `graphql:"pr85: pullRequest(number: $number85) @include(if: $has85)"` 103 | Pr86 *graphqlPullRequest `graphql:"pr86: pullRequest(number: $number86) @include(if: $has86)"` 104 | Pr87 *graphqlPullRequest `graphql:"pr87: pullRequest(number: $number87) @include(if: $has87)"` 105 | Pr88 *graphqlPullRequest `graphql:"pr88: pullRequest(number: $number88) @include(if: $has88)"` 106 | Pr89 *graphqlPullRequest `graphql:"pr89: pullRequest(number: $number89) @include(if: $has89)"` 107 | Pr90 *graphqlPullRequest `graphql:"pr90: pullRequest(number: $number90) @include(if: $has90)"` 108 | Pr91 *graphqlPullRequest `graphql:"pr91: pullRequest(number: $number91) @include(if: $has91)"` 109 | Pr92 *graphqlPullRequest `graphql:"pr92: pullRequest(number: $number92) @include(if: $has92)"` 110 | Pr93 *graphqlPullRequest `graphql:"pr93: pullRequest(number: $number93) @include(if: $has93)"` 111 | Pr94 *graphqlPullRequest `graphql:"pr94: pullRequest(number: $number94) @include(if: $has94)"` 112 | Pr95 *graphqlPullRequest `graphql:"pr95: pullRequest(number: $number95) @include(if: $has95)"` 113 | Pr96 *graphqlPullRequest `graphql:"pr96: pullRequest(number: $number96) @include(if: $has96)"` 114 | Pr97 *graphqlPullRequest `graphql:"pr97: pullRequest(number: $number97) @include(if: $has97)"` 115 | Pr98 *graphqlPullRequest `graphql:"pr98: pullRequest(number: $number98) @include(if: $has98)"` 116 | Pr99 *graphqlPullRequest `graphql:"pr99: pullRequest(number: $number99) @include(if: $has99)"` 117 | } `graphql:"repository(owner: $owner, name: $name)"` 118 | } 119 | 120 | func (r *numberedPullRequestQuery) GetAll() []graphqlPullRequest { 121 | result := []graphqlPullRequest{} 122 | 123 | for i := 0; i < MaxPullRequestsPerQuery; i++ { 124 | if pr := r.Get(i); pr != nil { 125 | result = append(result, *pr) 126 | } 127 | } 128 | 129 | return result 130 | } 131 | 132 | func (r *numberedPullRequestQuery) Get(index int) *graphqlPullRequest { 133 | switch index { 134 | case 0: 135 | return r.Repository.Pr0 136 | case 1: 137 | return r.Repository.Pr1 138 | case 2: 139 | return r.Repository.Pr2 140 | case 3: 141 | return r.Repository.Pr3 142 | case 4: 143 | return r.Repository.Pr4 144 | case 5: 145 | return r.Repository.Pr5 146 | case 6: 147 | return r.Repository.Pr6 148 | case 7: 149 | return r.Repository.Pr7 150 | case 8: 151 | return r.Repository.Pr8 152 | case 9: 153 | return r.Repository.Pr9 154 | case 10: 155 | return r.Repository.Pr10 156 | case 11: 157 | return r.Repository.Pr11 158 | case 12: 159 | return r.Repository.Pr12 160 | case 13: 161 | return r.Repository.Pr13 162 | case 14: 163 | return r.Repository.Pr14 164 | case 15: 165 | return r.Repository.Pr15 166 | case 16: 167 | return r.Repository.Pr16 168 | case 17: 169 | return r.Repository.Pr17 170 | case 18: 171 | return r.Repository.Pr18 172 | case 19: 173 | return r.Repository.Pr19 174 | case 20: 175 | return r.Repository.Pr20 176 | case 21: 177 | return r.Repository.Pr21 178 | case 22: 179 | return r.Repository.Pr22 180 | case 23: 181 | return r.Repository.Pr23 182 | case 24: 183 | return r.Repository.Pr24 184 | case 25: 185 | return r.Repository.Pr25 186 | case 26: 187 | return r.Repository.Pr26 188 | case 27: 189 | return r.Repository.Pr27 190 | case 28: 191 | return r.Repository.Pr28 192 | case 29: 193 | return r.Repository.Pr29 194 | case 30: 195 | return r.Repository.Pr30 196 | case 31: 197 | return r.Repository.Pr31 198 | case 32: 199 | return r.Repository.Pr32 200 | case 33: 201 | return r.Repository.Pr33 202 | case 34: 203 | return r.Repository.Pr34 204 | case 35: 205 | return r.Repository.Pr35 206 | case 36: 207 | return r.Repository.Pr36 208 | case 37: 209 | return r.Repository.Pr37 210 | case 38: 211 | return r.Repository.Pr38 212 | case 39: 213 | return r.Repository.Pr39 214 | case 40: 215 | return r.Repository.Pr40 216 | case 41: 217 | return r.Repository.Pr41 218 | case 42: 219 | return r.Repository.Pr42 220 | case 43: 221 | return r.Repository.Pr43 222 | case 44: 223 | return r.Repository.Pr44 224 | case 45: 225 | return r.Repository.Pr45 226 | case 46: 227 | return r.Repository.Pr46 228 | case 47: 229 | return r.Repository.Pr47 230 | case 48: 231 | return r.Repository.Pr48 232 | case 49: 233 | return r.Repository.Pr49 234 | case 50: 235 | return r.Repository.Pr50 236 | case 51: 237 | return r.Repository.Pr51 238 | case 52: 239 | return r.Repository.Pr52 240 | case 53: 241 | return r.Repository.Pr53 242 | case 54: 243 | return r.Repository.Pr54 244 | case 55: 245 | return r.Repository.Pr55 246 | case 56: 247 | return r.Repository.Pr56 248 | case 57: 249 | return r.Repository.Pr57 250 | case 58: 251 | return r.Repository.Pr58 252 | case 59: 253 | return r.Repository.Pr59 254 | case 60: 255 | return r.Repository.Pr60 256 | case 61: 257 | return r.Repository.Pr61 258 | case 62: 259 | return r.Repository.Pr62 260 | case 63: 261 | return r.Repository.Pr63 262 | case 64: 263 | return r.Repository.Pr64 264 | case 65: 265 | return r.Repository.Pr65 266 | case 66: 267 | return r.Repository.Pr66 268 | case 67: 269 | return r.Repository.Pr67 270 | case 68: 271 | return r.Repository.Pr68 272 | case 69: 273 | return r.Repository.Pr69 274 | case 70: 275 | return r.Repository.Pr70 276 | case 71: 277 | return r.Repository.Pr71 278 | case 72: 279 | return r.Repository.Pr72 280 | case 73: 281 | return r.Repository.Pr73 282 | case 74: 283 | return r.Repository.Pr74 284 | case 75: 285 | return r.Repository.Pr75 286 | case 76: 287 | return r.Repository.Pr76 288 | case 77: 289 | return r.Repository.Pr77 290 | case 78: 291 | return r.Repository.Pr78 292 | case 79: 293 | return r.Repository.Pr79 294 | case 80: 295 | return r.Repository.Pr80 296 | case 81: 297 | return r.Repository.Pr81 298 | case 82: 299 | return r.Repository.Pr82 300 | case 83: 301 | return r.Repository.Pr83 302 | case 84: 303 | return r.Repository.Pr84 304 | case 85: 305 | return r.Repository.Pr85 306 | case 86: 307 | return r.Repository.Pr86 308 | case 87: 309 | return r.Repository.Pr87 310 | case 88: 311 | return r.Repository.Pr88 312 | case 89: 313 | return r.Repository.Pr89 314 | case 90: 315 | return r.Repository.Pr90 316 | case 91: 317 | return r.Repository.Pr91 318 | case 92: 319 | return r.Repository.Pr92 320 | case 93: 321 | return r.Repository.Pr93 322 | case 94: 323 | return r.Repository.Pr94 324 | case 95: 325 | return r.Repository.Pr95 326 | case 96: 327 | return r.Repository.Pr96 328 | case 97: 329 | return r.Repository.Pr97 330 | case 98: 331 | return r.Repository.Pr98 332 | case 99: 333 | return r.Repository.Pr99 334 | } 335 | 336 | panic(fmt.Sprintf("Index %d out of range [0,%d] when accessing PR request", index, MaxPullRequestsPerQuery-1)) 337 | } 338 | -------------------------------------------------------------------------------- /pkg/client/client_milestones_gen.go: -------------------------------------------------------------------------------- 1 | // This file has been generated by hack/generate-client.sh 2 | // Do not edit manually! 3 | 4 | package client 5 | 6 | import ( 7 | "fmt" 8 | ) 9 | 10 | const ( 11 | MaxMilestonesPerQuery = 100 12 | ) 13 | 14 | type numberedMilestoneQuery struct { 15 | RateLimit rateLimit 16 | Repository struct { 17 | Milestone0 *graphqlMilestone `graphql:"milestone0: milestone(number: $number0) @include(if: $has0)"` 18 | Milestone1 *graphqlMilestone `graphql:"milestone1: milestone(number: $number1) @include(if: $has1)"` 19 | Milestone2 *graphqlMilestone `graphql:"milestone2: milestone(number: $number2) @include(if: $has2)"` 20 | Milestone3 *graphqlMilestone `graphql:"milestone3: milestone(number: $number3) @include(if: $has3)"` 21 | Milestone4 *graphqlMilestone `graphql:"milestone4: milestone(number: $number4) @include(if: $has4)"` 22 | Milestone5 *graphqlMilestone `graphql:"milestone5: milestone(number: $number5) @include(if: $has5)"` 23 | Milestone6 *graphqlMilestone `graphql:"milestone6: milestone(number: $number6) @include(if: $has6)"` 24 | Milestone7 *graphqlMilestone `graphql:"milestone7: milestone(number: $number7) @include(if: $has7)"` 25 | Milestone8 *graphqlMilestone `graphql:"milestone8: milestone(number: $number8) @include(if: $has8)"` 26 | Milestone9 *graphqlMilestone `graphql:"milestone9: milestone(number: $number9) @include(if: $has9)"` 27 | Milestone10 *graphqlMilestone `graphql:"milestone10: milestone(number: $number10) @include(if: $has10)"` 28 | Milestone11 *graphqlMilestone `graphql:"milestone11: milestone(number: $number11) @include(if: $has11)"` 29 | Milestone12 *graphqlMilestone `graphql:"milestone12: milestone(number: $number12) @include(if: $has12)"` 30 | Milestone13 *graphqlMilestone `graphql:"milestone13: milestone(number: $number13) @include(if: $has13)"` 31 | Milestone14 *graphqlMilestone `graphql:"milestone14: milestone(number: $number14) @include(if: $has14)"` 32 | Milestone15 *graphqlMilestone `graphql:"milestone15: milestone(number: $number15) @include(if: $has15)"` 33 | Milestone16 *graphqlMilestone `graphql:"milestone16: milestone(number: $number16) @include(if: $has16)"` 34 | Milestone17 *graphqlMilestone `graphql:"milestone17: milestone(number: $number17) @include(if: $has17)"` 35 | Milestone18 *graphqlMilestone `graphql:"milestone18: milestone(number: $number18) @include(if: $has18)"` 36 | Milestone19 *graphqlMilestone `graphql:"milestone19: milestone(number: $number19) @include(if: $has19)"` 37 | Milestone20 *graphqlMilestone `graphql:"milestone20: milestone(number: $number20) @include(if: $has20)"` 38 | Milestone21 *graphqlMilestone `graphql:"milestone21: milestone(number: $number21) @include(if: $has21)"` 39 | Milestone22 *graphqlMilestone `graphql:"milestone22: milestone(number: $number22) @include(if: $has22)"` 40 | Milestone23 *graphqlMilestone `graphql:"milestone23: milestone(number: $number23) @include(if: $has23)"` 41 | Milestone24 *graphqlMilestone `graphql:"milestone24: milestone(number: $number24) @include(if: $has24)"` 42 | Milestone25 *graphqlMilestone `graphql:"milestone25: milestone(number: $number25) @include(if: $has25)"` 43 | Milestone26 *graphqlMilestone `graphql:"milestone26: milestone(number: $number26) @include(if: $has26)"` 44 | Milestone27 *graphqlMilestone `graphql:"milestone27: milestone(number: $number27) @include(if: $has27)"` 45 | Milestone28 *graphqlMilestone `graphql:"milestone28: milestone(number: $number28) @include(if: $has28)"` 46 | Milestone29 *graphqlMilestone `graphql:"milestone29: milestone(number: $number29) @include(if: $has29)"` 47 | Milestone30 *graphqlMilestone `graphql:"milestone30: milestone(number: $number30) @include(if: $has30)"` 48 | Milestone31 *graphqlMilestone `graphql:"milestone31: milestone(number: $number31) @include(if: $has31)"` 49 | Milestone32 *graphqlMilestone `graphql:"milestone32: milestone(number: $number32) @include(if: $has32)"` 50 | Milestone33 *graphqlMilestone `graphql:"milestone33: milestone(number: $number33) @include(if: $has33)"` 51 | Milestone34 *graphqlMilestone `graphql:"milestone34: milestone(number: $number34) @include(if: $has34)"` 52 | Milestone35 *graphqlMilestone `graphql:"milestone35: milestone(number: $number35) @include(if: $has35)"` 53 | Milestone36 *graphqlMilestone `graphql:"milestone36: milestone(number: $number36) @include(if: $has36)"` 54 | Milestone37 *graphqlMilestone `graphql:"milestone37: milestone(number: $number37) @include(if: $has37)"` 55 | Milestone38 *graphqlMilestone `graphql:"milestone38: milestone(number: $number38) @include(if: $has38)"` 56 | Milestone39 *graphqlMilestone `graphql:"milestone39: milestone(number: $number39) @include(if: $has39)"` 57 | Milestone40 *graphqlMilestone `graphql:"milestone40: milestone(number: $number40) @include(if: $has40)"` 58 | Milestone41 *graphqlMilestone `graphql:"milestone41: milestone(number: $number41) @include(if: $has41)"` 59 | Milestone42 *graphqlMilestone `graphql:"milestone42: milestone(number: $number42) @include(if: $has42)"` 60 | Milestone43 *graphqlMilestone `graphql:"milestone43: milestone(number: $number43) @include(if: $has43)"` 61 | Milestone44 *graphqlMilestone `graphql:"milestone44: milestone(number: $number44) @include(if: $has44)"` 62 | Milestone45 *graphqlMilestone `graphql:"milestone45: milestone(number: $number45) @include(if: $has45)"` 63 | Milestone46 *graphqlMilestone `graphql:"milestone46: milestone(number: $number46) @include(if: $has46)"` 64 | Milestone47 *graphqlMilestone `graphql:"milestone47: milestone(number: $number47) @include(if: $has47)"` 65 | Milestone48 *graphqlMilestone `graphql:"milestone48: milestone(number: $number48) @include(if: $has48)"` 66 | Milestone49 *graphqlMilestone `graphql:"milestone49: milestone(number: $number49) @include(if: $has49)"` 67 | Milestone50 *graphqlMilestone `graphql:"milestone50: milestone(number: $number50) @include(if: $has50)"` 68 | Milestone51 *graphqlMilestone `graphql:"milestone51: milestone(number: $number51) @include(if: $has51)"` 69 | Milestone52 *graphqlMilestone `graphql:"milestone52: milestone(number: $number52) @include(if: $has52)"` 70 | Milestone53 *graphqlMilestone `graphql:"milestone53: milestone(number: $number53) @include(if: $has53)"` 71 | Milestone54 *graphqlMilestone `graphql:"milestone54: milestone(number: $number54) @include(if: $has54)"` 72 | Milestone55 *graphqlMilestone `graphql:"milestone55: milestone(number: $number55) @include(if: $has55)"` 73 | Milestone56 *graphqlMilestone `graphql:"milestone56: milestone(number: $number56) @include(if: $has56)"` 74 | Milestone57 *graphqlMilestone `graphql:"milestone57: milestone(number: $number57) @include(if: $has57)"` 75 | Milestone58 *graphqlMilestone `graphql:"milestone58: milestone(number: $number58) @include(if: $has58)"` 76 | Milestone59 *graphqlMilestone `graphql:"milestone59: milestone(number: $number59) @include(if: $has59)"` 77 | Milestone60 *graphqlMilestone `graphql:"milestone60: milestone(number: $number60) @include(if: $has60)"` 78 | Milestone61 *graphqlMilestone `graphql:"milestone61: milestone(number: $number61) @include(if: $has61)"` 79 | Milestone62 *graphqlMilestone `graphql:"milestone62: milestone(number: $number62) @include(if: $has62)"` 80 | Milestone63 *graphqlMilestone `graphql:"milestone63: milestone(number: $number63) @include(if: $has63)"` 81 | Milestone64 *graphqlMilestone `graphql:"milestone64: milestone(number: $number64) @include(if: $has64)"` 82 | Milestone65 *graphqlMilestone `graphql:"milestone65: milestone(number: $number65) @include(if: $has65)"` 83 | Milestone66 *graphqlMilestone `graphql:"milestone66: milestone(number: $number66) @include(if: $has66)"` 84 | Milestone67 *graphqlMilestone `graphql:"milestone67: milestone(number: $number67) @include(if: $has67)"` 85 | Milestone68 *graphqlMilestone `graphql:"milestone68: milestone(number: $number68) @include(if: $has68)"` 86 | Milestone69 *graphqlMilestone `graphql:"milestone69: milestone(number: $number69) @include(if: $has69)"` 87 | Milestone70 *graphqlMilestone `graphql:"milestone70: milestone(number: $number70) @include(if: $has70)"` 88 | Milestone71 *graphqlMilestone `graphql:"milestone71: milestone(number: $number71) @include(if: $has71)"` 89 | Milestone72 *graphqlMilestone `graphql:"milestone72: milestone(number: $number72) @include(if: $has72)"` 90 | Milestone73 *graphqlMilestone `graphql:"milestone73: milestone(number: $number73) @include(if: $has73)"` 91 | Milestone74 *graphqlMilestone `graphql:"milestone74: milestone(number: $number74) @include(if: $has74)"` 92 | Milestone75 *graphqlMilestone `graphql:"milestone75: milestone(number: $number75) @include(if: $has75)"` 93 | Milestone76 *graphqlMilestone `graphql:"milestone76: milestone(number: $number76) @include(if: $has76)"` 94 | Milestone77 *graphqlMilestone `graphql:"milestone77: milestone(number: $number77) @include(if: $has77)"` 95 | Milestone78 *graphqlMilestone `graphql:"milestone78: milestone(number: $number78) @include(if: $has78)"` 96 | Milestone79 *graphqlMilestone `graphql:"milestone79: milestone(number: $number79) @include(if: $has79)"` 97 | Milestone80 *graphqlMilestone `graphql:"milestone80: milestone(number: $number80) @include(if: $has80)"` 98 | Milestone81 *graphqlMilestone `graphql:"milestone81: milestone(number: $number81) @include(if: $has81)"` 99 | Milestone82 *graphqlMilestone `graphql:"milestone82: milestone(number: $number82) @include(if: $has82)"` 100 | Milestone83 *graphqlMilestone `graphql:"milestone83: milestone(number: $number83) @include(if: $has83)"` 101 | Milestone84 *graphqlMilestone `graphql:"milestone84: milestone(number: $number84) @include(if: $has84)"` 102 | Milestone85 *graphqlMilestone `graphql:"milestone85: milestone(number: $number85) @include(if: $has85)"` 103 | Milestone86 *graphqlMilestone `graphql:"milestone86: milestone(number: $number86) @include(if: $has86)"` 104 | Milestone87 *graphqlMilestone `graphql:"milestone87: milestone(number: $number87) @include(if: $has87)"` 105 | Milestone88 *graphqlMilestone `graphql:"milestone88: milestone(number: $number88) @include(if: $has88)"` 106 | Milestone89 *graphqlMilestone `graphql:"milestone89: milestone(number: $number89) @include(if: $has89)"` 107 | Milestone90 *graphqlMilestone `graphql:"milestone90: milestone(number: $number90) @include(if: $has90)"` 108 | Milestone91 *graphqlMilestone `graphql:"milestone91: milestone(number: $number91) @include(if: $has91)"` 109 | Milestone92 *graphqlMilestone `graphql:"milestone92: milestone(number: $number92) @include(if: $has92)"` 110 | Milestone93 *graphqlMilestone `graphql:"milestone93: milestone(number: $number93) @include(if: $has93)"` 111 | Milestone94 *graphqlMilestone `graphql:"milestone94: milestone(number: $number94) @include(if: $has94)"` 112 | Milestone95 *graphqlMilestone `graphql:"milestone95: milestone(number: $number95) @include(if: $has95)"` 113 | Milestone96 *graphqlMilestone `graphql:"milestone96: milestone(number: $number96) @include(if: $has96)"` 114 | Milestone97 *graphqlMilestone `graphql:"milestone97: milestone(number: $number97) @include(if: $has97)"` 115 | Milestone98 *graphqlMilestone `graphql:"milestone98: milestone(number: $number98) @include(if: $has98)"` 116 | Milestone99 *graphqlMilestone `graphql:"milestone99: milestone(number: $number99) @include(if: $has99)"` 117 | } `graphql:"repository(owner: $owner, name: $name)"` 118 | } 119 | 120 | func (r *numberedMilestoneQuery) GetAll() []graphqlMilestone { 121 | result := []graphqlMilestone{} 122 | 123 | for i := 0; i < MaxMilestonesPerQuery; i++ { 124 | if milestone := r.Get(i); milestone != nil { 125 | result = append(result, *milestone) 126 | } 127 | } 128 | 129 | return result 130 | } 131 | 132 | func (r *numberedMilestoneQuery) Get(index int) *graphqlMilestone { 133 | switch index { 134 | case 0: 135 | return r.Repository.Milestone0 136 | case 1: 137 | return r.Repository.Milestone1 138 | case 2: 139 | return r.Repository.Milestone2 140 | case 3: 141 | return r.Repository.Milestone3 142 | case 4: 143 | return r.Repository.Milestone4 144 | case 5: 145 | return r.Repository.Milestone5 146 | case 6: 147 | return r.Repository.Milestone6 148 | case 7: 149 | return r.Repository.Milestone7 150 | case 8: 151 | return r.Repository.Milestone8 152 | case 9: 153 | return r.Repository.Milestone9 154 | case 10: 155 | return r.Repository.Milestone10 156 | case 11: 157 | return r.Repository.Milestone11 158 | case 12: 159 | return r.Repository.Milestone12 160 | case 13: 161 | return r.Repository.Milestone13 162 | case 14: 163 | return r.Repository.Milestone14 164 | case 15: 165 | return r.Repository.Milestone15 166 | case 16: 167 | return r.Repository.Milestone16 168 | case 17: 169 | return r.Repository.Milestone17 170 | case 18: 171 | return r.Repository.Milestone18 172 | case 19: 173 | return r.Repository.Milestone19 174 | case 20: 175 | return r.Repository.Milestone20 176 | case 21: 177 | return r.Repository.Milestone21 178 | case 22: 179 | return r.Repository.Milestone22 180 | case 23: 181 | return r.Repository.Milestone23 182 | case 24: 183 | return r.Repository.Milestone24 184 | case 25: 185 | return r.Repository.Milestone25 186 | case 26: 187 | return r.Repository.Milestone26 188 | case 27: 189 | return r.Repository.Milestone27 190 | case 28: 191 | return r.Repository.Milestone28 192 | case 29: 193 | return r.Repository.Milestone29 194 | case 30: 195 | return r.Repository.Milestone30 196 | case 31: 197 | return r.Repository.Milestone31 198 | case 32: 199 | return r.Repository.Milestone32 200 | case 33: 201 | return r.Repository.Milestone33 202 | case 34: 203 | return r.Repository.Milestone34 204 | case 35: 205 | return r.Repository.Milestone35 206 | case 36: 207 | return r.Repository.Milestone36 208 | case 37: 209 | return r.Repository.Milestone37 210 | case 38: 211 | return r.Repository.Milestone38 212 | case 39: 213 | return r.Repository.Milestone39 214 | case 40: 215 | return r.Repository.Milestone40 216 | case 41: 217 | return r.Repository.Milestone41 218 | case 42: 219 | return r.Repository.Milestone42 220 | case 43: 221 | return r.Repository.Milestone43 222 | case 44: 223 | return r.Repository.Milestone44 224 | case 45: 225 | return r.Repository.Milestone45 226 | case 46: 227 | return r.Repository.Milestone46 228 | case 47: 229 | return r.Repository.Milestone47 230 | case 48: 231 | return r.Repository.Milestone48 232 | case 49: 233 | return r.Repository.Milestone49 234 | case 50: 235 | return r.Repository.Milestone50 236 | case 51: 237 | return r.Repository.Milestone51 238 | case 52: 239 | return r.Repository.Milestone52 240 | case 53: 241 | return r.Repository.Milestone53 242 | case 54: 243 | return r.Repository.Milestone54 244 | case 55: 245 | return r.Repository.Milestone55 246 | case 56: 247 | return r.Repository.Milestone56 248 | case 57: 249 | return r.Repository.Milestone57 250 | case 58: 251 | return r.Repository.Milestone58 252 | case 59: 253 | return r.Repository.Milestone59 254 | case 60: 255 | return r.Repository.Milestone60 256 | case 61: 257 | return r.Repository.Milestone61 258 | case 62: 259 | return r.Repository.Milestone62 260 | case 63: 261 | return r.Repository.Milestone63 262 | case 64: 263 | return r.Repository.Milestone64 264 | case 65: 265 | return r.Repository.Milestone65 266 | case 66: 267 | return r.Repository.Milestone66 268 | case 67: 269 | return r.Repository.Milestone67 270 | case 68: 271 | return r.Repository.Milestone68 272 | case 69: 273 | return r.Repository.Milestone69 274 | case 70: 275 | return r.Repository.Milestone70 276 | case 71: 277 | return r.Repository.Milestone71 278 | case 72: 279 | return r.Repository.Milestone72 280 | case 73: 281 | return r.Repository.Milestone73 282 | case 74: 283 | return r.Repository.Milestone74 284 | case 75: 285 | return r.Repository.Milestone75 286 | case 76: 287 | return r.Repository.Milestone76 288 | case 77: 289 | return r.Repository.Milestone77 290 | case 78: 291 | return r.Repository.Milestone78 292 | case 79: 293 | return r.Repository.Milestone79 294 | case 80: 295 | return r.Repository.Milestone80 296 | case 81: 297 | return r.Repository.Milestone81 298 | case 82: 299 | return r.Repository.Milestone82 300 | case 83: 301 | return r.Repository.Milestone83 302 | case 84: 303 | return r.Repository.Milestone84 304 | case 85: 305 | return r.Repository.Milestone85 306 | case 86: 307 | return r.Repository.Milestone86 308 | case 87: 309 | return r.Repository.Milestone87 310 | case 88: 311 | return r.Repository.Milestone88 312 | case 89: 313 | return r.Repository.Milestone89 314 | case 90: 315 | return r.Repository.Milestone90 316 | case 91: 317 | return r.Repository.Milestone91 318 | case 92: 319 | return r.Repository.Milestone92 320 | case 93: 321 | return r.Repository.Milestone93 322 | case 94: 323 | return r.Repository.Milestone94 324 | case 95: 325 | return r.Repository.Milestone95 326 | case 96: 327 | return r.Repository.Milestone96 328 | case 97: 329 | return r.Repository.Milestone97 330 | case 98: 331 | return r.Repository.Milestone98 332 | case 99: 333 | return r.Repository.Milestone99 334 | } 335 | 336 | panic(fmt.Sprintf("Index %d out of range [0,%d] when accessing milestone request", index, MaxMilestonesPerQuery-1)) 337 | } 338 | --------------------------------------------------------------------------------