├── .github └── workflows │ ├── contribution.yaml │ ├── lint.yaml │ └── test.yaml ├── LICENSE ├── README.md ├── e2e_test ├── e2e_test.go ├── go.mod └── go.sum ├── example ├── advanced.go ├── basic.go ├── go.mod └── go.sum ├── github_ratelimit ├── combined_ratelimit_test.go ├── github_primary_ratelimit │ ├── callback.go │ ├── category.go │ ├── category_test.go │ ├── config.go │ ├── http_response.go │ ├── options.go │ ├── primary_rate_limit.go │ └── ratelimit_state.go ├── github_ratelimit_test │ ├── ratelimit_injecter.go │ └── test_utils.go ├── github_ratelimiter.go ├── github_secondary_ratelimit │ ├── callback.go │ ├── config.go │ ├── detect.go │ ├── options.go │ ├── secondary_rate_limit.go │ ├── sleep.go │ └── sleep_test.go ├── primary_ratelimit_test.go └── secondary_ratelimit_test.go ├── go.mod └── go.sum /.github/workflows/contribution.yaml: -------------------------------------------------------------------------------- 1 | name: Auto Comment on PRs and Issues 2 | 3 | on: 4 | issues: 5 | types: [opened] 6 | pull_request: 7 | types: [opened] 8 | 9 | permissions: 10 | issues: write 11 | pull-requests: write 12 | 13 | jobs: 14 | auto-comment: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Post Comment 18 | uses: actions/github-script@v7 19 | with: 20 | github-token: ${{ secrets.GITHUB_TOKEN }} 21 | script: | 22 | const issue_number = context.payload.issue ? context.payload.issue.number : context.payload.pull_request.number; 23 | const repo = context.repo; 24 | const message = `Thank you for your contribution! 25 | I am maintaining this repository as a side thing, so my availability varies with time. 26 | No coffee is expected, but please leave a star if you find this repository useful :) 27 | Please feel free to ping this issue/PR if I don't respond for a while.`; 28 | 29 | github.rest.issues.createComment({ 30 | owner: repo.owner, 31 | repo: repo.repo, 32 | issue_number: issue_number, 33 | body: message 34 | }); -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - main 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | lint: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # v3 17 | with: 18 | go-version: 1.23.1 19 | - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3 20 | - name: Lint 21 | uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # v3.4.0 22 | with: 23 | version: v1.64.6 24 | args: --timeout=3m 25 | 26 | lint-test: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # v3 30 | with: 31 | go-version: 1.23.1 32 | - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3 33 | - name: Lint 34 | uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # v3.4.0 35 | with: 36 | version: v1.64.6 37 | args: --timeout=3m 38 | working-directory: github_ratelimit/github_ratelimit_test -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - main 8 | pull_request: 9 | branches: 10 | - main 11 | schedule: 12 | - cron: '0 0 * * 0' # Runs at 00:00 UTC every Sunday 13 | 14 | jobs: 15 | build_and_test: 16 | runs-on: ubuntu-latest 17 | env: 18 | TEST_DIR: e2e_test 19 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 20 | steps: 21 | - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # v3 22 | with: 23 | go-version: 1.23.1 24 | - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3 25 | - name: Verify dependencies 26 | run: go mod verify 27 | - name: Build 28 | run: go build -v ./... 29 | - name: Vet 30 | run: go vet -v ./... 31 | - name: Test 32 | run: go test -v -count=1 -shuffle=on -timeout=30m -race ./... 33 | - name: E2E-Vet 34 | run: cd "$TEST_DIR" && go vet -v ./... 35 | - name: Test 36 | run: cd "$TEST_DIR" && go test -v -count=1 -shuffle=on -timeout=30m -race ./... 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Gal Ofri 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-github-ratelimit 2 | 3 | [![Go Report Card](https://goreportcard.com/badge/github.com/gofri/go-github-ratelimit)](https://goreportcard.com/report/github.com/gofri/go-github-ratelimit) 4 | 5 | Package `go-github-ratelimit` provides a middleware (http.RoundTripper) that handles both [Primary Rate Limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?#about-primary-rate-limits) and [Secondary Rate Limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?#about-secondary-rate-limits) for the GitHub API. 6 | 7 | * Primary rate limits are handled by returning a detailed error. 8 | * Secondary rate limits are handled by waiting in blocking mode (sleep) and then issuing/retrying requests. 9 | * There is support for callbacks to be triggered when rate limits are detected/exceeded/etc. - see below. 10 | 11 | The module can be used with any HTTP client communicating with GitHub API. It is designed to have low overhead during good path. 12 | It is meant to complement [go-github](https://github.com/google/go-github), but there is no association between this repository and the go-github repository nor Google. 13 | 14 | ## Recommended: Pagination Handling 15 | 16 | If you like this package, please check out [go-github-pagination](https://github.com/gofri/go-github-pagination). 17 | It supports pagination out of the box, and plays well with the rate limit round-tripper. 18 | It is best to stack the pagination round-tripper on top of the rate limit round-tripper. 19 | 20 | 21 | ## Installation 22 | 23 | ```go get github.com/gofri/go-github-ratelimit/v2``` 24 | 25 | ## Usage Example (with [go-github](https://github.com/google/go-github)) 26 | 27 | see [example/basic.go](example/basic.go) for a runnable example. 28 | ```go 29 | rateLimiter := github_ratelimit.NewClient(nil) 30 | client := github.NewClient(rateLimiter) // .WithAuthToken("your personal access token") 31 | 32 | // disable go-github's built-in rate limiting 33 | ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true) 34 | 35 | tags, _, err := client.Repositories.ListTags(ctx, "gofri", "go-github-ratelimit", nil) 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | for _, tag := range tags { 41 | fmt.Printf("- %v\n", *tag.Name) 42 | } 43 | ``` 44 | 45 | ## Client Options 46 | 47 | Both RoundTrippers support a set of options to configure their behavior and set callbacks. 48 | nil callbacks are treated as no-op. 49 | 50 | ### Primary Rate Limit Options (see [options.go](github_ratelimit/github_primary_ratelimit/options.go)): 51 | 52 | - `WithLimitDetectedCallback(callback)`: the callback is triggered when any primary rate limit is detected. 53 | - `WithRequestPreventedCallback(callback)`: the callback is triggered when a request is prevented due to an active rate limit. 54 | - `WithLimitResetCallback(callback)`: the callback is triggered when the rate limit is reset (deactived). 55 | - `WithUnknownCategoryCallback`: the callback is triggered when the rate limit category in the response is unknown. note: please open an issue if it happens. 56 | - `WithSharedState(state)`: share state between multiple clients (e.g., for a single user running concurrently). 57 | - `WithBypassLimit()`: bypass the rate limit mechanism, i.e., do not prevent requests when a rate limit is active. 58 | 59 | ### Secondary Rate Limit Options (see [options.go](github_ratelimit/github_secondary_ratelimit/options.go)): 60 | 61 | - `WithLimitDetectedCallback(callback)`: the callback is triggered before a sleep. 62 | - `WithSingleSleepLimit(duration, callback)`: limit the sleep duration for a single secondary rate limit & trigger a callback when the limit is exceeded. 63 | - `WithTotalSleepLimit(duration, callback)`: limit the accumulated sleep duration for all secondary rate limits & trigger a callback when the limit is exceeded. 64 | - `WithNoSleep(callback)`: disable sleep for secondary rate limits & trigger a callback upon any secondary rate limit. 65 | 66 | ## Per-Request Options 67 | 68 | Use `WithOverrideConfig(opts...)` to override the configuration for a specific request (using the request context). 69 | Per-request overrides may be useful for special cases of user requests, 70 | as well as fine-grained policy control (e.g., for a sophisticated pagination mechanism). 71 | 72 | ## Advanced Example 73 | 74 | See [example/advanced.go](example/advanced.go) for a runnable example. 75 | ```go 76 | rateLimiter := github_ratelimit.New(nil, 77 | github_primary_ratelimit.WithLimitDetectedCallback(func(ctx *github_primary_ratelimit.CallbackContext) { 78 | fmt.Printf("Primary rate limit detected: category %s, reset time: %v\n", ctx.Category, ctx.ResetTime) 79 | }), 80 | github_secondary_ratelimit.WithLimitDetectedCallback(func(ctx *github_secondary_ratelimit.CallbackContext) { 81 | fmt.Printf("Secondary rate limit detected: reset time: %v, total sleep time: %v\n", ctx.ResetTime, ctx.TotalSleepTime) 82 | }), 83 | ) 84 | 85 | paginator := githubpagination.NewClient(rateLimiter, 86 | githubpagination.WithPerPage(100), // default to 100 results per page 87 | ) 88 | client := github.NewClient(paginator) // .WithAuthToken("your personal access token") 89 | 90 | // disable go-github's built-in rate limiting 91 | ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true) 92 | 93 | // list repository tags 94 | tags, _, err := client.Repositories.ListTags(ctx, "gofri", "go-github-ratelimit", nil) 95 | if err != nil { 96 | panic(err) 97 | } 98 | 99 | for _, tag := range tags { 100 | fmt.Printf("- %v\n", *tag.Name) 101 | } 102 | ``` 103 | 104 | ## Migration (V1 => V2) 105 | 106 | The migraiton from v1 to v2 is relatively straight-forward once you check out the examples. 107 | Please open an issue if you have any trouble - 108 | I'd be glad to help and add documetation per need. 109 | 110 | ## Github Rate Limit References 111 | 112 | - [Primary Rate Limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?#about-primary-rate-limits) 113 | - [Secondary Rate Limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?#about-secondary-rate-limits) 114 | 115 | ## License 116 | 117 | This package is distributed under the MIT license found in the LICENSE file. 118 | 119 | ## Contribution 120 | Contribution and feedback is welcome. 121 | I'm trying to drink less coffee so don't buy me one, but please do star this repository if you find it useful. 122 | 123 | [![Star History Chart](https://api.star-history.com/svg?repos=gofri/go-github-ratelimit&type=Date)](https://www.star-history.com/#gofri/go-github-ratelimit&Date) 124 | -------------------------------------------------------------------------------- /e2e_test/e2e_test.go: -------------------------------------------------------------------------------- 1 | package e2e_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "io" 8 | "net/http" 9 | "testing" 10 | "time" 11 | 12 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_ratelimit_test" 13 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" 14 | "github.com/google/go-github/v64/github" 15 | ) 16 | 17 | type orgLister struct { 18 | } 19 | 20 | func (o *orgLister) GetOrgName() string { 21 | return "org" 22 | } 23 | 24 | func (o *orgLister) RoundTrip(r *http.Request) (*http.Response, error) { 25 | org := github.Organization{ 26 | Login: github.String(o.GetOrgName()), 27 | } 28 | 29 | body, err := json.Marshal([]*github.Organization{&org}) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return &http.Response{ 35 | Body: io.NopCloser(bytes.NewReader(body)), 36 | Header: http.Header{}, 37 | StatusCode: http.StatusOK, 38 | }, nil 39 | } 40 | 41 | // TestGoGithubClient is a test that uses the go-github client. 42 | func TestGoGithubClientCompatability(t *testing.T) { 43 | t.Parallel() 44 | const every = 5 * time.Second 45 | const sleep = 1 * time.Second 46 | 47 | print := func(context *github_secondary_ratelimit.CallbackContext) { 48 | t.Logf("Secondary rate limit reached! Sleeping for %.2f seconds [%v --> %v]", 49 | time.Until(*context.ResetTime).Seconds(), time.Now(), *context.ResetTime) 50 | } 51 | 52 | orgLister := &orgLister{} 53 | options := github_ratelimit_test.RateLimitInjecterOptions{ 54 | Every: every, 55 | InjectionDuration: sleep, 56 | } 57 | 58 | i := github_ratelimit_test.SetupInjecterWithOptions(t, options, orgLister) 59 | rateLimiter := github_ratelimit_test.NewSecondaryClient(i, github_secondary_ratelimit.WithLimitDetectedCallback(print)) 60 | 61 | client := github.NewClient(rateLimiter) 62 | orgs, resp, err := client.Organizations.List(context.Background(), "", nil) 63 | if err != nil { 64 | t.Fatalf("unexpected error response: %v", err) 65 | } 66 | 67 | if resp.StatusCode != http.StatusOK { 68 | t.Fatalf("unexpected status code: %v", resp.StatusCode) 69 | } 70 | 71 | if len(orgs) != 1 { 72 | t.Fatalf("unexpected number of orgs: %v", len(orgs)) 73 | } 74 | 75 | if orgs[0].GetLogin() != orgLister.GetOrgName() { 76 | t.Fatalf("unexpected org name: %v", orgs[0].GetLogin()) 77 | } 78 | 79 | // TODO add tests for: 80 | // - WithSingleSleepLimit(0, ...) => expect AbuseError 81 | // - WithSingleSleepLimit(>0, ...) => expect sleeping 82 | } 83 | -------------------------------------------------------------------------------- /e2e_test/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gofri/go-github-ratelimit-e2e 2 | 3 | replace github.com/gofri/go-github-ratelimit/v2 => ../ 4 | 5 | go 1.23.1 6 | 7 | require ( 8 | github.com/gofri/go-github-ratelimit/v2 v2.0.0-00010101000000-000000000000 9 | github.com/google/go-github/v64 v64.0.0 10 | ) 11 | 12 | require github.com/google/go-querystring v1.1.0 // indirect 13 | -------------------------------------------------------------------------------- /e2e_test/go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 2 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 3 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 4 | github.com/google/go-github/v64 v64.0.0 h1:4G61sozmY3eiPAjjoOHponXDBONm+utovTKbyUb2Qdg= 5 | github.com/google/go-github/v64 v64.0.0/go.mod h1:xB3vqMQNdHzilXBiO2I+M7iEFtHf+DP/omBOv6tQzVo= 6 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 7 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 8 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 9 | -------------------------------------------------------------------------------- /example/advanced.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/gofri/go-github-pagination/githubpagination" 8 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit" 9 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" 10 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" 11 | "github.com/google/go-github/v69/github" 12 | ) 13 | 14 | func main() { 15 | rateLimiter := github_ratelimit.New(nil, 16 | github_primary_ratelimit.WithLimitDetectedCallback(func(ctx *github_primary_ratelimit.CallbackContext) { 17 | fmt.Printf("Primary rate limit detected: category %s, reset time: %v\n", ctx.Category, ctx.ResetTime) 18 | }), 19 | github_secondary_ratelimit.WithLimitDetectedCallback(func(ctx *github_secondary_ratelimit.CallbackContext) { 20 | fmt.Printf("Secondary rate limit detected: reset time: %v, total sleep time: %v\n", ctx.ResetTime, ctx.TotalSleepTime) 21 | }), 22 | ) 23 | 24 | paginator := githubpagination.NewClient(rateLimiter, 25 | githubpagination.WithPerPage(100), // default to 100 results per page 26 | ) 27 | client := github.NewClient(paginator) // .WithAuthToken("your personal access token") 28 | 29 | // disable go-github's built-in rate limiting 30 | ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true) 31 | 32 | // list repository tags 33 | tags, _, err := client.Repositories.ListTags(ctx, "gofri", "go-github-ratelimit", nil) 34 | if err != nil { 35 | panic(err) 36 | } 37 | 38 | for _, tag := range tags { 39 | fmt.Printf("- %v\n", *tag.Name) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /example/basic.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit" 8 | "github.com/google/go-github/v69/github" 9 | ) 10 | 11 | func main() { 12 | // use the plain ratelimiter, without options / callbacks / underlying http.RoundTripper. 13 | rateLimiter := github_ratelimit.NewClient(nil) 14 | client := github.NewClient(rateLimiter) // .WithAuthToken("your personal access token") 15 | 16 | // disable go-github's built-in rate limiting 17 | ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true) 18 | 19 | tags, _, err := client.Repositories.ListTags(ctx, "gofri", "go-github-ratelimit", nil) 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | for _, tag := range tags { 25 | fmt.Printf("- %v\n", *tag.Name) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /example/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gofri/go-github-ratelimit/v2/example 2 | 3 | replace github.com/gofri/go-github-ratelimit/v2 => ../ 4 | 5 | go 1.23.1 6 | 7 | require ( 8 | github.com/gofri/go-github-pagination v1.0.0 9 | github.com/gofri/go-github-ratelimit/v2 v2.0.1 10 | github.com/google/go-github/v69 v69.2.0 11 | ) 12 | 13 | require github.com/google/go-querystring v1.1.0 // indirect 14 | -------------------------------------------------------------------------------- /example/go.sum: -------------------------------------------------------------------------------- 1 | github.com/gofri/go-github-pagination v1.0.0 h1:nnCi+1xT5ybqY/plctISgiQPWZOtfSciVQlbx/hM/Yw= 2 | github.com/gofri/go-github-pagination v1.0.0/go.mod h1:Qij55Fb4fNPjam3SB+8cLnqp4pgR8RGMyIspYXcyHX0= 3 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 4 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 5 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 6 | github.com/google/go-github/v69 v69.2.0 h1:wR+Wi/fN2zdUx9YxSmYE0ktiX9IAR/BeePzeaUUbEHE= 7 | github.com/google/go-github/v69 v69.2.0/go.mod h1:xne4jymxLR6Uj9b7J7PyTpkMYstEMMwGZa0Aehh1azM= 8 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 9 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 10 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 11 | -------------------------------------------------------------------------------- /github_ratelimit/combined_ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package github_ratelimit 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | "time" 7 | 8 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" 9 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_ratelimit_test" 10 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" 11 | ) 12 | 13 | func TestCombinedRateLimiter(t *testing.T) { 14 | t.Parallel() 15 | 16 | everySecondary := 300 * time.Millisecond 17 | everyPrimary := 500 * time.Millisecond 18 | sleepTime := 100 * time.Millisecond 19 | 20 | injecter := github_ratelimit_test.SetupSecondaryInjecter(t, everySecondary, sleepTime).(*github_ratelimit_test.RateLimitInjecter) 21 | injecter.Base = github_ratelimit_test.SetupPrimaryInjecter(t, 22 | everyPrimary, sleepTime, 23 | github_primary_ratelimit.ResourceCategoryCore, 24 | ) 25 | 26 | primaryCalled := false 27 | secondaryCalled := false 28 | c := &http.Client{ 29 | Transport: New(injecter, 30 | github_primary_ratelimit.WithLimitDetectedCallback(func(context *github_primary_ratelimit.CallbackContext) { 31 | t.Logf("primary rate limit reached!") 32 | primaryCalled = true 33 | }), 34 | github_secondary_ratelimit.WithLimitDetectedCallback(func(context *github_secondary_ratelimit.CallbackContext) { 35 | t.Logf("secondary rate limit reached!") 36 | secondaryCalled = true 37 | }), 38 | github_secondary_ratelimit.WithNoSleep(nil), 39 | ), 40 | } 41 | 42 | _, err := c.Get("/initial-no-called") 43 | if err != nil { 44 | t.Fatalf("expecting first request to succeed, got %v", err) 45 | } 46 | if primaryCalled || secondaryCalled { 47 | t.Fatalf("expecting primary=false and secondary=false, got primary=%v, secondary=%v", primaryCalled, secondaryCalled) 48 | } 49 | 50 | // wait until secondary rate limit is triggered 51 | injecter.WaitForNextInjection() 52 | _, err = c.Get("/only-secondary") 53 | if primaryCalled || !secondaryCalled { 54 | t.Fatalf("expecting primary=false, and secondary=true, got primary=%v, secondary=%v. err: %v", primaryCalled, secondaryCalled, err) 55 | } 56 | 57 | // wait until primary rate limit is triggered 58 | injecter.Base.(*github_ratelimit_test.RateLimitInjecter).WaitForNextInjection() 59 | _, err = c.Get("/only-primary") 60 | if !primaryCalled { 61 | t.Fatalf("expecting primary=true, got primary=%v. err: %v", primaryCalled, err) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /github_ratelimit/github_primary_ratelimit/callback.go: -------------------------------------------------------------------------------- 1 | package github_primary_ratelimit 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | ) 7 | 8 | // CallbackContext is passed to all callbacks. 9 | // Fields might be nillable, depending on the specific callback and field. 10 | type CallbackContext struct { 11 | RoundTripper *PrimaryRateLimiter 12 | Request *http.Request 13 | Response *http.Response 14 | ResetTime *time.Time 15 | Category ResourceCategory 16 | } 17 | 18 | // OnLimitReached is called when a new rate limit is detected. 19 | type OnLimitReached func(*CallbackContext) 20 | 21 | // OnRequestPrevented is called when an existing rate limit is detected, 22 | // such that the current request is not sent. 23 | type OnRequestPrevented func(*CallbackContext) 24 | 25 | // OnLimitReset is called when a rate limit reset time is reached, 26 | // which means that the category is available for use again. 27 | type OnLimitReset func(*CallbackContext) 28 | 29 | // OnUnknownCategory is called when an unknown category is detected at the response, 30 | // which means that the rate limiter does not handle it. 31 | type OnUnknownCategory func(*CallbackContext) 32 | 33 | func (ctx *CallbackContext) AsError() *RateLimitReachedError { 34 | return &RateLimitReachedError{ 35 | Request: ctx.Request, 36 | Response: ctx.Response, 37 | Category: ctx.Category, 38 | ResetTime: ctx.ResetTime, 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /github_ratelimit/github_primary_ratelimit/category.go: -------------------------------------------------------------------------------- 1 | package github_primary_ratelimit 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | // General references (note there are some inconsistencies between them): 9 | // https://docs.github.com/en/rest/rate-limit/rate-limit#about-rate-limits 10 | // https://docs.github.com/en/rest/rate-limit/rate-limit#get-rate-limit-status-for-the-authenticated-user 11 | type ResourceCategory string 12 | 13 | const ( 14 | // The default category 15 | // used for all HTTP method/url with no other match. 16 | ResourceCategoryCore ResourceCategory = "core" 17 | 18 | // https://docs.github.com/en/rest/search/search#about-search 19 | // * /search (except for /search/code) 20 | ResourceCategorySearch ResourceCategory = "search" 21 | 22 | // https://docs.github.com/en/rest/search/search#search-code 23 | // * /search/code 24 | ResourceCategoryCodeSearch ResourceCategory = "code_search" 25 | 26 | // https://docs.github.com/en/graphql 27 | // * /graphql 28 | ResourceCategoryGraphQL ResourceCategory = "graphql" 29 | 30 | // https://docs.github.com/en/rest/migrations/source-imports#start-an-import 31 | // deprecated endpoint; still applicable 32 | // * /repos/{OWNER}/{REPO}/import 33 | ResourceCategorySourceImport ResourceCategory = "source_import" 34 | 35 | // https://docs.github.com/en/enterprise-cloud@latest/rest/enterprise-admin/audit-log#get-the-audit-log-for-an-enterprise 36 | // * /enterprises/{ENTERPRISE}/audit-log 37 | ResourceCategoryAuditLog ResourceCategory = "audit_log" 38 | 39 | // https://docs.github.com/en/rest/dependency-graph/dependency-submission 40 | // POST /app/manfiests/{code}/conversions 41 | ResourceCategoryIntegrationManifest ResourceCategory = "integration_manifest" 42 | 43 | // https://docs.github.com/en/rest/dependency-graph/dependency-submission#create-a-snapshot-of-dependencies-for-a-repository 44 | // POST /repos/{OWNER}/{REPO}/dependency-graph/snapshots 45 | ResourceCategoryDependencySnapshots ResourceCategory = "dependency_snapshots" 46 | 47 | // https://docs.github.com/en/rest/code-scanning/code-scanning#upload-an-analysis-as-sarif-data 48 | // POST /repos/{OWNER}/{REPO}/code-scanning/sarifs 49 | ResourceCategoryCodeScanningUpload ResourceCategory = "code_scanning_upload" 50 | 51 | // https://docs.github.com/en/rest/actions/self-hosted-runners#about-self-hosted-runners-in-github-actions 52 | // "... for registring self-hosted runners"; assuming only POST requests are counted 53 | // POST /orgs/{ORG}/actions/runners 54 | ResourceCategoryActionsRunnerRegistration ResourceCategory = "actions_runner_registration" 55 | 56 | // https://docs.github.com/en/enterprise-cloud@latest/rest/scim/scim 57 | // no explicit documentation; assuming only POST requests are counted 58 | // POST /scim 59 | ResourceCategoryScim ResourceCategory = "scim" 60 | 61 | // https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise 62 | // no API endpoints 63 | ResourceCategoryAuditLogStreaming ResourceCategory = "audit_log_streaming" 64 | ) 65 | 66 | func GetAllCategories() []ResourceCategory { 67 | return []ResourceCategory{ 68 | ResourceCategoryCore, 69 | ResourceCategorySearch, 70 | ResourceCategoryCodeSearch, 71 | ResourceCategoryGraphQL, 72 | ResourceCategorySourceImport, 73 | ResourceCategoryAuditLog, 74 | ResourceCategoryIntegrationManifest, 75 | ResourceCategoryDependencySnapshots, 76 | ResourceCategoryCodeScanningUpload, 77 | ResourceCategoryActionsRunnerRegistration, 78 | ResourceCategoryScim, 79 | } 80 | } 81 | 82 | func parseRequestCategory(request *http.Request) ResourceCategory { 83 | return parseCategory(request.Method, request.URL.RawPath) 84 | } 85 | 86 | func parseCategory(method string, path string) ResourceCategory { 87 | switch { // method-agnostic checks: 88 | case strings.HasPrefix(path, "/search/code"): 89 | return ResourceCategoryCodeSearch 90 | case strings.HasPrefix(path, "/search"): 91 | return ResourceCategorySearch 92 | case strings.HasPrefix(path, "/graphql"): 93 | return ResourceCategoryGraphQL 94 | case strings.HasPrefix(path, "/repos/") && strings.HasSuffix(path, "/import"): 95 | return ResourceCategorySourceImport 96 | case strings.HasSuffix(path, "/audit_log"): 97 | return ResourceCategoryAuditLog 98 | } 99 | 100 | if method == http.MethodPost { 101 | switch { 102 | case strings.HasPrefix(path, "/app/manfiests/") && strings.HasSuffix(path, "/conversions"): 103 | return ResourceCategoryIntegrationManifest 104 | case strings.HasPrefix(path, "/repos/") && strings.HasSuffix(path, "/dependency-graph/snapshots"): 105 | return ResourceCategoryDependencySnapshots 106 | case strings.HasPrefix(path, "/repos/") && strings.HasSuffix(path, "/code-scanning/sarifs"): 107 | return ResourceCategoryCodeScanningUpload 108 | case strings.HasPrefix(path, "/orgs/") && strings.HasSuffix(path, "/actions/runners"): 109 | return ResourceCategoryActionsRunnerRegistration 110 | case strings.HasPrefix(path, "/scim"): 111 | return ResourceCategoryScim 112 | } 113 | } 114 | 115 | // default to core 116 | return ResourceCategoryCore 117 | } 118 | -------------------------------------------------------------------------------- /github_ratelimit/github_primary_ratelimit/category_test.go: -------------------------------------------------------------------------------- 1 | package github_primary_ratelimit 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | ) 7 | 8 | func TestCategory(t *testing.T) { 9 | t.Parallel() 10 | tests := []struct { 11 | Path string 12 | Category ResourceCategory 13 | Method string 14 | }{ 15 | { 16 | Path: "/search/code/xxx", 17 | Category: ResourceCategoryCodeSearch, 18 | Method: http.MethodGet, 19 | }, 20 | { 21 | Path: "/search?xxx=yyy", 22 | Category: ResourceCategorySearch, 23 | Method: http.MethodGet, 24 | }, 25 | { 26 | Path: "/graphql?xxx=yyy", 27 | Category: ResourceCategoryGraphQL, 28 | Method: http.MethodGet, 29 | }, 30 | { 31 | Path: "xxx/audit_log", 32 | Category: ResourceCategoryAuditLog, 33 | Method: http.MethodGet, 34 | }, 35 | { 36 | Path: "/app/manfiests/xxx/conversions", 37 | Category: ResourceCategoryIntegrationManifest, 38 | Method: http.MethodPost, 39 | }, 40 | { 41 | Path: "/repos/xxx/dependency-graph/snapshots", 42 | Category: ResourceCategoryDependencySnapshots, 43 | Method: http.MethodPost, 44 | }, 45 | { 46 | Path: "/repos/xxx/code-scanning/sarifs", 47 | Category: ResourceCategoryCodeScanningUpload, 48 | Method: http.MethodPost, 49 | }, 50 | { 51 | Path: "/orgs/xxx/actions/runners", 52 | Category: ResourceCategoryActionsRunnerRegistration, 53 | Method: http.MethodPost, 54 | }, 55 | { 56 | Path: "/scim", 57 | Category: ResourceCategoryScim, 58 | Method: http.MethodPost, 59 | }, 60 | { 61 | Path: "/xxx", 62 | Category: ResourceCategoryCore, 63 | Method: http.MethodPost, 64 | }, 65 | } 66 | 67 | for _, test := range tests { 68 | if got, want := parseCategory(test.Method, test.Path), test.Category; got != want { 69 | t.Fatalf("category mismatch for path: '%v' with method %v: got %v, expected %v", test.Path, test.Method, got, want) 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /github_ratelimit/github_primary_ratelimit/config.go: -------------------------------------------------------------------------------- 1 | package github_primary_ratelimit 2 | 3 | import "context" 4 | 5 | // Config is the configuration for the rate limiter. 6 | // It is used internally and generated from the options. 7 | // It holds the state of the rate limiter in order to enable state sharing. 8 | type Config struct { 9 | state *RateLimitState 10 | bypassLimit bool 11 | 12 | // callbacks 13 | onLimitReached OnLimitReached 14 | onReuqestPrevented OnRequestPrevented 15 | onLimitReset OnLimitReset 16 | onUnknownCategory OnUnknownCategory 17 | } 18 | 19 | // newConfig creates a new config with the given options. 20 | func newConfig(opts ...Option) *Config { 21 | var config Config 22 | config.ApplyOptions(opts...) 23 | 24 | if config.state == nil { 25 | config.state = NewRateLimitState(GetAllCategories()) 26 | } 27 | 28 | return &config 29 | } 30 | 31 | // ApplyOptions applies the options to the config. 32 | func (c *Config) ApplyOptions(opts ...Option) { 33 | for _, o := range opts { 34 | if o == nil { 35 | continue 36 | } 37 | o(c) 38 | } 39 | } 40 | 41 | type ConfigOverridesKey struct{} 42 | 43 | // WithOverrideConfig adds config overrides to the context. 44 | // The overrides are applied on top of the existing config. 45 | // Allows for request-specific overrides. 46 | func WithOverrideConfig(ctx context.Context, opts ...Option) context.Context { 47 | return context.WithValue(ctx, ConfigOverridesKey{}, opts) 48 | } 49 | 50 | // GetConfigOverrides returns the config overrides from the context, if any. 51 | func GetConfigOverrides(ctx context.Context) []Option { 52 | cfg := ctx.Value(ConfigOverridesKey{}) 53 | if cfg == nil { 54 | return nil 55 | } 56 | return cfg.([]Option) 57 | } 58 | 59 | func (c *Config) TriggerLimitReached(ctx *CallbackContext) { 60 | if c.onLimitReached == nil { 61 | return 62 | } 63 | c.onLimitReached(ctx) 64 | } 65 | 66 | func (c *Config) TriggerRequestPrevented(ctx *CallbackContext) { 67 | if c.onReuqestPrevented == nil { 68 | return 69 | } 70 | c.onReuqestPrevented(ctx) 71 | } 72 | 73 | func (c *Config) TriggerLimitReset(ctx *CallbackContext) { 74 | if c.onLimitReset == nil { 75 | return 76 | } 77 | c.onLimitReset(ctx) 78 | } 79 | 80 | func (c *Config) TriggerUnknownCategory(ctx *CallbackContext) { 81 | if c.onUnknownCategory == nil { 82 | return 83 | } 84 | c.onUnknownCategory(ctx) 85 | } 86 | -------------------------------------------------------------------------------- /github_ratelimit/github_primary_ratelimit/http_response.go: -------------------------------------------------------------------------------- 1 | package github_primary_ratelimit 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "net/http" 7 | "slices" 8 | "strconv" 9 | ) 10 | 11 | // ParsedResponse is a wrapper around http.Response that provides additional functionality. 12 | // It is used to parse the response and extract rate limit information. 13 | type ParsedResponse struct { 14 | resp *http.Response 15 | } 16 | 17 | // https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api#checking-the-status-of-your-rate-limit 18 | type ResponseHeaderKey string 19 | 20 | // https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api#exceeding-the-rate-limit 21 | const ( 22 | ResponseHeaderKeyRemaining ResponseHeaderKey = "x-ratelimit-remaining" 23 | ResponseHeaderKeyReset ResponseHeaderKey = "x-ratelimit-reset" 24 | ResponseHeaderKeyCategory ResponseHeaderKey = "x-ratelimit-resource" 25 | ) 26 | 27 | func (k ResponseHeaderKey) Get(response *http.Response) string { 28 | return response.Header.Get(string(k)) 29 | } 30 | 31 | var PrimaryLimitStatusCodes = []int{ 32 | http.StatusForbidden, 33 | http.StatusTooManyRequests, 34 | } 35 | 36 | func (p ParsedResponse) GetCatgory() ResourceCategory { 37 | category := p.getHeader(ResponseHeaderKeyCategory) 38 | return ResourceCategory(category) 39 | } 40 | 41 | func (p ParsedResponse) GetResetTime() *SecondsSinceEpoch { 42 | if !p.limitReached() { 43 | return nil 44 | } 45 | 46 | reset := p.getHeader(ResponseHeaderKeyReset) 47 | seconds, _ := strconv.Atoi(reset) 48 | s := SecondsSinceEpoch(seconds) 49 | return &s 50 | } 51 | 52 | func (p ParsedResponse) limitReached() bool { 53 | if !slices.Contains(PrimaryLimitStatusCodes, p.resp.StatusCode) { 54 | return false 55 | } 56 | if remaining := p.getHeader(ResponseHeaderKeyRemaining); remaining != "0" { 57 | return false 58 | } 59 | return true 60 | } 61 | 62 | func (p ParsedResponse) getHeader(key ResponseHeaderKey) string { 63 | return p.resp.Header.Get(string(key)) 64 | } 65 | 66 | func NewErrorResponse(request *http.Request, category ResourceCategory) *http.Response { 67 | header := make(http.Header) 68 | header.Set(string(ResponseHeaderKeyRemaining), "0") 69 | header.Set(string(ResponseHeaderKeyCategory), string(category)) 70 | return &http.Response{ 71 | Status: http.StatusText(http.StatusForbidden), 72 | StatusCode: http.StatusForbidden, 73 | Request: request, 74 | Header: header, 75 | Body: io.NopCloser(bytes.NewReader(nil)), 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /github_ratelimit/github_primary_ratelimit/options.go: -------------------------------------------------------------------------------- 1 | package github_primary_ratelimit 2 | 3 | import "time" 4 | 5 | type Option func(*Config) 6 | 7 | // WithLimitDetectedCallback adds a callback to be called when a new active rate limit is detected. 8 | func WithLimitDetectedCallback(callback OnLimitReached) Option { 9 | return func(c *Config) { 10 | c.onLimitReached = callback 11 | } 12 | } 13 | 14 | // WithRequestPreventedCallback adds a callback to be called when a request is prevented, 15 | // i.e., when the rate limit is active. 16 | // note: this callback is not called when the limit is first detected. 17 | func WithRequestPreventedCallback(callback OnRequestPrevented) Option { 18 | return func(c *Config) { 19 | c.onReuqestPrevented = callback 20 | } 21 | } 22 | 23 | // WithLimitResetCallback adds a callback to be called when a rate limit is reset, 24 | // i.e., when an ongoing rate limit is no longer active. 25 | func WithLimitResetCallback(callback OnLimitReset) Option { 26 | return func(c *Config) { 27 | c.onLimitReset = callback 28 | } 29 | } 30 | 31 | // WithUnknownCategoryCallback adds a callback to be called when a response from Github contains an unknown category. 32 | // please open an issue if you encounter this to help improve the handling. 33 | func WithUnknownCategoryCallback(callback OnUnknownCategory) Option { 34 | return func(c *Config) { 35 | c.onUnknownCategory = callback 36 | } 37 | } 38 | 39 | // WithSharedState is used to set the rate limiter state from an external source. 40 | // Specifically, it is used to share the state between multiple rate limiters. 41 | // e.g., 42 | // `rateLimiterB := New(nil, WithSharedState(rateLimiterA.GetState()))` 43 | func WithSharedState(state *RateLimitState) Option { 44 | return func(c *Config) { 45 | c.state = state 46 | } 47 | } 48 | 49 | // WithBypassLimit is used to flag that no requests shall be prevented. 50 | // Callbacks are still called regardless of this flag. 51 | // This is useful for testing, out-of-band token switching, etc. 52 | func WithBypassLimit() Option { 53 | return func(c *Config) { 54 | c.bypassLimit = true 55 | } 56 | } 57 | 58 | // WithSleepUntilReset is used to flag that the rate limiter shall sleep until the reset time. 59 | // This is useful for testing, long-running offline applications, etc. 60 | // Note: it is using the LimitDetectedCallback, so it will not be otherwise called. 61 | func WithSleepUntilReset() Option { 62 | return WithLimitDetectedCallback(func(ctx *CallbackContext) { 63 | time.Sleep(time.Until(*ctx.ResetTime)) 64 | }) 65 | } 66 | -------------------------------------------------------------------------------- /github_ratelimit/github_primary_ratelimit/primary_rate_limit.go: -------------------------------------------------------------------------------- 1 | package github_primary_ratelimit 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | // PrimaryRateLimiter is a RoundTripper for avoiding GitHub primary rate limits. 10 | // see notes @ ratelimit_state.go for design considerations. 11 | type PrimaryRateLimiter struct { 12 | Base http.RoundTripper 13 | config *Config 14 | } 15 | 16 | // RateLimitReachedError is an error type for when the primary rate limit is reached. 17 | type RateLimitReachedError struct { 18 | ResetTime *time.Time 19 | Request *http.Request 20 | Response *http.Response 21 | Category ResourceCategory 22 | } 23 | 24 | func (e *RateLimitReachedError) Error() string { 25 | return fmt.Sprintf( 26 | "primary rate limit reached on request to %v with category: %s. wait until %v before sending more requests.", 27 | e.Request.URL, 28 | e.Category, 29 | e.ResetTime, 30 | ) 31 | } 32 | 33 | func New(base http.RoundTripper, opts ...Option) *PrimaryRateLimiter { 34 | if base == nil { 35 | base = http.DefaultTransport 36 | } 37 | config := newConfig(opts...) 38 | return &PrimaryRateLimiter{ 39 | Base: base, 40 | config: config, 41 | } 42 | } 43 | 44 | func (l *PrimaryRateLimiter) RoundTrip(request *http.Request) (*http.Response, error) { 45 | config := l.getRequestConfig(request) 46 | category := parseRequestCategory(request) 47 | resetTime := config.state.GetResetTime(category) 48 | if resetTime != nil { 49 | resp := NewErrorResponse(request, category) 50 | ctx := &CallbackContext{ 51 | RoundTripper: l, 52 | Request: request, 53 | Response: resp, 54 | Category: category, 55 | ResetTime: resetTime.AsTime(), 56 | } 57 | config.TriggerRequestPrevented(ctx) 58 | if !config.bypassLimit { 59 | return nil, ctx.AsError() 60 | } 61 | } 62 | 63 | resp, err := l.Base.RoundTrip(request) 64 | if err != nil { 65 | return resp, err 66 | } 67 | callbackContext := &CallbackContext{ 68 | RoundTripper: l, 69 | Request: request, 70 | Response: resp, 71 | } 72 | 73 | // update and check 74 | resetTime = config.state.Update(config, ParsedResponse{resp}, callbackContext) 75 | if resetTime == nil { 76 | return resp, nil 77 | } 78 | config.TriggerLimitReached(callbackContext) 79 | 80 | return nil, callbackContext.AsError() 81 | } 82 | 83 | // GetState can be used to share the primary rate limit knowledge - 84 | // when multiple clients are involved. 85 | // TODO add tests for state sharing 86 | func (l *PrimaryRateLimiter) GetState() *RateLimitState { 87 | return l.config.state 88 | } 89 | 90 | func (r *PrimaryRateLimiter) getRequestConfig(request *http.Request) *Config { 91 | overrides := GetConfigOverrides(request.Context()) 92 | if overrides == nil { 93 | // no config override - use the default config (zero-copy) 94 | return r.config 95 | } 96 | reqConfig := *r.config 97 | reqConfig.ApplyOptions(overrides...) 98 | return &reqConfig 99 | } 100 | -------------------------------------------------------------------------------- /github_ratelimit/github_primary_ratelimit/ratelimit_state.go: -------------------------------------------------------------------------------- 1 | package github_primary_ratelimit 2 | 3 | import ( 4 | "log" 5 | "sync/atomic" 6 | "time" 7 | ) 8 | 9 | // XXX: Synchronization Notes 10 | // We need to deal with weak consistency guarantees to start with, 11 | // i.e., resources are allowed to be exausted by different clients concurrently. 12 | // In addition, we allow internal concurrency of requests and responses. 13 | // So, eventual consistency it is. 14 | // 15 | // Performance Considerations: 16 | // 1. Clients assume low cpu utilization around network transmission. 17 | // 2. Clients care about the latency of the overhead, 18 | // in terms of both networking and CPU. 19 | // 3. Clients are expected to stop sending requests once the limit is reached. 20 | // As a result, overhead when usage>=limit is preferred over when usage atomic timestamp pointer (atomic.Pointer[SecondsSinceEpoch]). 38 | // on request: block the request if timestamp != nil. 39 | // on response: if limit is reached, set the timestamp and trigger a timer. 40 | // on timer expiration: reset the timestamp back to nil. 41 | // note: in principle, we could use an atomic bool instead of the atomic timestamp, 42 | // but we want to regenerate the bad response during the blockage time. 43 | 44 | type SecondsSinceEpoch int64 45 | 46 | func (s SecondsSinceEpoch) StartTimer() *time.Timer { 47 | timeLeft := time.Until(*s.AsTime()) 48 | return time.NewTimer(timeLeft) 49 | } 50 | 51 | func (s SecondsSinceEpoch) AsTime() *time.Time { 52 | t := time.Unix(int64(s), 0) 53 | return &t 54 | } 55 | 56 | // ------------------------- 57 | type AtomicTime = atomic.Pointer[SecondsSinceEpoch] 58 | 59 | // UpdateContainer is a simple abstraction over HTTP response, 60 | // to isolate the perf-centric state management domain from the rest of the logic. 61 | // It retains the wider-domain terminology of categories, 62 | // but it is just a key-string as far as RateLimitState is concerned. 63 | type UpdateContainer interface { 64 | GetCatgory() ResourceCategory 65 | GetResetTime() *SecondsSinceEpoch 66 | } 67 | 68 | type RateLimitState struct { 69 | resetTimeMap map[ResourceCategory]*AtomicTime 70 | } 71 | 72 | func NewRateLimitState(categories []ResourceCategory) *RateLimitState { 73 | resetTimeMap := make(map[ResourceCategory]*AtomicTime) 74 | for _, category := range categories { 75 | resetTimeMap[category] = &AtomicTime{} 76 | } 77 | return &RateLimitState{ 78 | resetTimeMap: resetTimeMap, 79 | } 80 | } 81 | 82 | func (s *RateLimitState) GetResetTime(category ResourceCategory) *SecondsSinceEpoch { 83 | resetTime, exists := s.resetTimeMap[category] 84 | if !exists { 85 | log.Printf("unexpected category detected: %v. Please open an issue @ go-github-ratelimit", category) 86 | return nil 87 | } 88 | return resetTime.Load() 89 | } 90 | 91 | func (s *RateLimitState) Update(config *Config, update UpdateContainer, callbackContext *CallbackContext) *SecondsSinceEpoch { 92 | category := update.GetCatgory() // TODO detect req-resp category mismatch (and do what?) 93 | callbackContext.Category = category 94 | 95 | newResetTime := update.GetResetTime() 96 | if newResetTime == nil { 97 | // nothing to update on a successful request 98 | return nil 99 | } 100 | callbackContext.ResetTime = newResetTime.AsTime() 101 | 102 | sharedResetTime, exists := s.resetTimeMap[category] 103 | if !exists { 104 | // XXX: there is no point in adding it as a new category to the map, 105 | // because we will not detect it anyway. so just trigger and continue. 106 | config.TriggerUnknownCategory(callbackContext) 107 | return nil 108 | } 109 | 110 | // XXX: should hold a ref to the timer to free resources early on-demand. 111 | // please open an issue if you actually need it. 112 | sharedResetTime.Store(newResetTime) 113 | timer := newResetTime.StartTimer() 114 | go func(timer *time.Timer, callbackContext CallbackContext) { 115 | <-timer.C 116 | sharedResetTime.Store(nil) 117 | cbContext := &CallbackContext{ 118 | Category: callbackContext.Category, 119 | ResetTime: callbackContext.ResetTime, 120 | } 121 | config.TriggerLimitReset(cbContext) 122 | }(timer, *callbackContext) 123 | 124 | return newResetTime 125 | } 126 | -------------------------------------------------------------------------------- /github_ratelimit/github_ratelimit_test/ratelimit_injecter.go: -------------------------------------------------------------------------------- 1 | package github_ratelimit_test 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "math/rand" 9 | "net/http" 10 | "strconv" 11 | "sync" 12 | "time" 13 | 14 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" 15 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" 16 | github_ratelimit "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" 17 | ) 18 | 19 | const ( 20 | InvalidBodyContent = `{"message": "not as expected"}` 21 | ) 22 | 23 | const ( 24 | SecondaryRateLimitMessage = `You have exceeded a secondary rate limit. Please wait a few minutes before you try again.` 25 | ) 26 | 27 | var SecondaryRateLimitDocumentationURLs = []string{ 28 | `https://docs.github.com/rest/overview/resources-in-the-rest-api#secondary-rate-limits`, 29 | `https://docs.github.com/free-pro-team@latest/rest/overview/resources-in-the-rest-api#secondary-rate-limits`, 30 | `https://docs.github.com/en/free-pro-team@latest/rest/overview/rate-limits-for-the-rest-api#about-secondary-rate-limits`, 31 | `https://docs.github.com/en/some-other-option#abuse-rate-limits`, 32 | } 33 | 34 | // RateLimitInjecterOptions provide options for the injection. 35 | // note: 36 | // Every is the interval between the start of two rate limit injections. 37 | // It is first counted from the first request, 38 | // then counted from the end of the last injection. 39 | type RateLimitInjecterOptions struct { 40 | Every time.Duration 41 | InjectionDuration time.Duration 42 | InvalidBody bool 43 | UseXRateLimit bool 44 | UsePrimaryRateLimit bool 45 | DocumentationURL string 46 | HttpStatusCode int 47 | PrimaryRateLimitCategory github_primary_ratelimit.ResourceCategory 48 | } 49 | 50 | func NewRateLimitInjecter(base http.RoundTripper, options *RateLimitInjecterOptions) (http.RoundTripper, error) { 51 | if options.IsNoop() { 52 | return base, nil 53 | } 54 | if err := options.Validate(); err != nil { 55 | return nil, err 56 | } 57 | 58 | injecter := &RateLimitInjecter{ 59 | Base: base, 60 | options: options, 61 | } 62 | return injecter, nil 63 | } 64 | 65 | func (r *RateLimitInjecterOptions) IsNoop() bool { 66 | return r.InjectionDuration == 0 67 | } 68 | 69 | func (r *RateLimitInjecterOptions) Validate() error { 70 | if r.Every < 0 { 71 | return fmt.Errorf("injecter expects a positive trigger interval") 72 | } 73 | if r.InjectionDuration < 0 { 74 | return fmt.Errorf("injecter expects a positive sleep interval") 75 | } 76 | return nil 77 | } 78 | 79 | type RateLimitInjecter struct { 80 | Base http.RoundTripper 81 | options *RateLimitInjecterOptions 82 | blockUntil time.Time 83 | lock sync.Mutex 84 | AbuseAttempts int 85 | } 86 | 87 | func (t *RateLimitInjecter) RoundTrip(request *http.Request) (*http.Response, error) { 88 | resp, err := t.Base.RoundTrip(request) 89 | if err != nil { 90 | return resp, err 91 | } 92 | 93 | t.lock.Lock() 94 | defer t.lock.Unlock() 95 | 96 | now := time.Now() 97 | 98 | // initialize on first use 99 | if t.blockUntil.IsZero() { 100 | t.blockUntil = now 101 | return resp, nil 102 | } 103 | 104 | // on-going rate limit 105 | if t.blockUntil.After(now) { 106 | t.AbuseAttempts++ 107 | return t.inject(resp) 108 | } 109 | 110 | nextStart := t.nextInjectionStart(now) 111 | 112 | // no-injection period 113 | if now.Before(nextStart) { 114 | return resp, nil 115 | } 116 | 117 | // start a new injection period 118 | t.blockUntil = nextStart.Add(t.options.InjectionDuration) 119 | 120 | return t.inject(resp) 121 | } 122 | 123 | // nextInjectionStart returns the time when the next injection starts. 124 | // note that we use blockUntil as the origin, 125 | // because we want it to be after the last injection. 126 | // we need to handle the case of multiple-cycle gap, 127 | // because the user might have waited for a long time between requests. 128 | func (r *RateLimitInjecter) nextInjectionStart(now time.Time) time.Time { 129 | cycleSize := r.options.Every + r.options.InjectionDuration 130 | sinceLastBlock := now.Sub(r.blockUntil) 131 | 132 | numOfCyclesGap := int(sinceLastBlock / cycleSize) 133 | gap := cycleSize * time.Duration(numOfCyclesGap) 134 | endOfLastBlock := r.blockUntil.Add(gap) 135 | 136 | return endOfLastBlock.Add(r.options.Every) 137 | 138 | } 139 | 140 | func (r *RateLimitInjecter) WaitForNextInjection() { 141 | r.lock.Lock() 142 | defer r.lock.Unlock() 143 | time.Sleep(time.Until(r.nextInjectionStart(time.Now()))) 144 | } 145 | 146 | func getSecondaryRateLimitBody(documentationURL string) (io.ReadCloser, error) { 147 | if len(documentationURL) == 0 { 148 | documentationURL = SecondaryRateLimitDocumentationURLs[rand.Intn(len(SecondaryRateLimitDocumentationURLs))] 149 | } 150 | 151 | body := github_secondary_ratelimit.SecondaryRateLimitBody{ 152 | Message: SecondaryRateLimitMessage, 153 | DocumentURL: documentationURL, 154 | } 155 | bodyBytes, err := json.Marshal(body) 156 | if err != nil { 157 | return nil, err 158 | } 159 | 160 | return io.NopCloser(bytes.NewReader(bodyBytes)), nil 161 | } 162 | 163 | func getHttpStatusCode(statusCode int) int { 164 | if statusCode == 0 { 165 | // XXX: not perfect, but luckily primary & secondary share status codes, 166 | // so let's keep it at that for now. 167 | codes := github_secondary_ratelimit.LimitStatusCodes 168 | return codes[rand.Intn(len(codes))] 169 | } 170 | return statusCode 171 | } 172 | 173 | func (t *RateLimitInjecter) inject(resp *http.Response) (*http.Response, error) { 174 | if t.options.UsePrimaryRateLimit { 175 | return t.toPrimaryRateLimitResponse(resp), nil 176 | } else { 177 | body, err := getSecondaryRateLimitBody(t.options.DocumentationURL) 178 | if err != nil { 179 | return nil, err 180 | } 181 | if t.options.InvalidBody { 182 | body = io.NopCloser(bytes.NewReader([]byte(InvalidBodyContent))) 183 | } 184 | 185 | resp.StatusCode = getHttpStatusCode(t.options.HttpStatusCode) 186 | resp.Body = body 187 | if t.options.UseXRateLimit { 188 | return t.toXRateLimitResponse(resp), nil 189 | } else { 190 | return t.toRetryResponse(resp), nil 191 | } 192 | } 193 | } 194 | 195 | func (t *RateLimitInjecter) toRetryResponse(resp *http.Response) *http.Response { 196 | secondsToBlock := t.getTimeToBlock() 197 | httpHeaderSetIntValue(resp, github_ratelimit.HeaderRetryAfter, int(secondsToBlock.Seconds())) 198 | return resp 199 | } 200 | 201 | func (t *RateLimitInjecter) toXRateLimitResponse(resp *http.Response) *http.Response { 202 | endOfBlock := time.Now().Add(t.getTimeToBlock()) 203 | httpHeaderSetIntValue(resp, github_ratelimit.HeaderXRateLimitReset, int(endOfBlock.Unix())) 204 | return resp 205 | } 206 | 207 | func (t *RateLimitInjecter) toPrimaryRateLimitResponse(resp *http.Response) *http.Response { 208 | httpHeaderSetIntValue(resp, github_ratelimit.HeaderXRateLimitRemaining, 0) 209 | if category := t.options.PrimaryRateLimitCategory; category != "" { 210 | resp.Header.Set(string(github_primary_ratelimit.ResponseHeaderKeyCategory), string(category)) 211 | } 212 | resp.StatusCode = GetRandomStatusCode() 213 | return t.toXRateLimitResponse(resp) 214 | } 215 | 216 | func (t *RateLimitInjecter) getTimeToBlock() time.Duration { 217 | timeUntil := time.Until(t.blockUntil) 218 | if timeUntil.Nanoseconds()%int64(time.Second) > 0 { 219 | timeUntil += time.Second 220 | } 221 | return timeUntil 222 | } 223 | 224 | func httpHeaderSetIntValue(resp *http.Response, key string, value int) { 225 | resp.Header.Set(key, strconv.Itoa(value)) 226 | } 227 | 228 | func IsInvalidBody(resp *http.Response) (bool, error) { 229 | defer resp.Body.Close() 230 | body, err := io.ReadAll(resp.Body) 231 | if err != nil { 232 | return false, err 233 | } 234 | 235 | return string(body) == InvalidBodyContent, nil 236 | } 237 | 238 | func GetRandomStatusCode() int { 239 | codes := github_primary_ratelimit.PrimaryLimitStatusCodes 240 | return codes[rand.Intn(len(codes))] 241 | } 242 | -------------------------------------------------------------------------------- /github_ratelimit/github_ratelimit_test/test_utils.go: -------------------------------------------------------------------------------- 1 | package github_ratelimit_test 2 | 3 | import ( 4 | "io" 5 | "math/rand" 6 | "net/http" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" 12 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" 13 | ) 14 | 15 | func NewSecondaryClient(base http.RoundTripper, opts ...github_secondary_ratelimit.Option) *http.Client { 16 | w := github_secondary_ratelimit.New(base, opts...) 17 | return &http.Client{ 18 | Transport: w, 19 | } 20 | } 21 | 22 | func NewPrimaryClient(base http.RoundTripper, opts ...github_primary_ratelimit.Option) *http.Client { 23 | w := github_primary_ratelimit.New(base, opts...) 24 | return &http.Client{ 25 | Transport: w, 26 | } 27 | } 28 | 29 | type nopServer struct { 30 | } 31 | 32 | func UpTo1SecDelay() time.Duration { 33 | return time.Duration(int(time.Millisecond) * (rand.Int() % 1000)) 34 | } 35 | 36 | func (n *nopServer) RoundTrip(r *http.Request) (*http.Response, error) { 37 | time.Sleep(UpTo1SecDelay() / 100) 38 | return &http.Response{ 39 | Body: io.NopCloser(strings.NewReader("some response")), 40 | Header: http.Header{}, 41 | }, nil 42 | } 43 | 44 | func SetupSecondaryInjecter(t *testing.T, every time.Duration, sleep time.Duration) http.RoundTripper { 45 | options := RateLimitInjecterOptions{ 46 | Every: every, 47 | InjectionDuration: sleep, 48 | } 49 | return SetupInjecterWithOptions(t, options, nil) 50 | } 51 | 52 | func SetupPrimaryInjecter(t *testing.T, every time.Duration, sleep time.Duration, category github_primary_ratelimit.ResourceCategory) http.RoundTripper { 53 | options := RateLimitInjecterOptions{ 54 | Every: every, 55 | InjectionDuration: sleep, 56 | UsePrimaryRateLimit: true, 57 | PrimaryRateLimitCategory: category, 58 | } 59 | return SetupInjecterWithOptions(t, options, nil) 60 | } 61 | 62 | func SetupInjecterWithOptions(t *testing.T, options RateLimitInjecterOptions, roundTrippger http.RoundTripper) http.RoundTripper { 63 | if roundTrippger == nil { 64 | roundTrippger = &nopServer{} 65 | } 66 | i, err := NewRateLimitInjecter(roundTrippger, &options) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | 71 | return i 72 | } 73 | 74 | func WaitForNextSleep(injecter http.RoundTripper) { 75 | injecter.(*RateLimitInjecter).WaitForNextInjection() 76 | } 77 | -------------------------------------------------------------------------------- /github_ratelimit/github_ratelimiter.go: -------------------------------------------------------------------------------- 1 | package github_ratelimit 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" 9 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" 10 | ) 11 | 12 | type PrimaryRateLimiter = github_primary_ratelimit.PrimaryRateLimiter 13 | type PrimaryRateLimiterOption = github_primary_ratelimit.Option 14 | 15 | // NewPrimaryLimiter is an alias for github_primary_ratelimit.New. 16 | // Check out options.go @ github_primary_ratelimit for available options. 17 | func NewPrimaryLimiter(base http.RoundTripper, opts ...PrimaryRateLimiterOption) *PrimaryRateLimiter { 18 | return github_primary_ratelimit.New(base, opts...) 19 | } 20 | 21 | type SecondaryRateLimiter = github_secondary_ratelimit.SecondaryRateLimiter 22 | type SecondaryRateLimiterOption = github_secondary_ratelimit.Option 23 | 24 | // NewSecondaryLimiter is an alias for github_secondary_ratelimit.New. 25 | // Check out options.go @ github_secondary_ratelimit for available options. 26 | func NewSecondaryLimiter(base http.RoundTripper, opts ...SecondaryRateLimiterOption) *SecondaryRateLimiter { 27 | return github_secondary_ratelimit.New(base, opts...) 28 | } 29 | 30 | // New creates a combined limiter by stacking a SecondaryRateLimiter on top of a PrimaryRateLimiterOption. 31 | // It accepts options of both types and creates the RoundTrippers. 32 | // Check out options.go @ github_primary_ratelimit / github_secondary_ratelimit for available options. 33 | func New(base http.RoundTripper, opts ...any) http.RoundTripper { 34 | primaryOpts, secondaryOpts := gatherOptions(opts...) 35 | primary := NewPrimaryLimiter(base, primaryOpts...) 36 | secondary := NewSecondaryLimiter(primary, secondaryOpts...) 37 | 38 | return secondary 39 | } 40 | 41 | // NewClient creates a new HTTP client with the combined rate limiter. 42 | func NewClient(base http.RoundTripper, opts ...any) *http.Client { 43 | return &http.Client{ 44 | Transport: New(base, opts...), 45 | } 46 | } 47 | 48 | // WithOverrideConfig adds config overrides to the context. 49 | // The overrides are applied on top of the existing config. 50 | // Allows for request-specific overrides. 51 | // It accepts options of both types and overrides accordingly. 52 | func WithOverrideConfig(ctx context.Context, opts ...any) context.Context { 53 | primaryOpts, secondaryOpts := gatherOptions(opts...) 54 | if len(primaryOpts) > 0 { 55 | ctx = github_primary_ratelimit.WithOverrideConfig(ctx, primaryOpts...) 56 | } 57 | if len(secondaryOpts) > 0 { 58 | ctx = github_secondary_ratelimit.WithOverrideConfig(ctx, secondaryOpts...) 59 | } 60 | return ctx 61 | } 62 | 63 | func gatherOptions(opts ...any) ([]PrimaryRateLimiterOption, []SecondaryRateLimiterOption) { 64 | primaryOpts := []PrimaryRateLimiterOption{} 65 | secondaryOpts := []SecondaryRateLimiterOption{} 66 | for _, opt := range opts { 67 | if o, ok := opt.(PrimaryRateLimiterOption); ok { 68 | primaryOpts = append(primaryOpts, o) 69 | } else if o, ok := opt.(SecondaryRateLimiterOption); ok { 70 | secondaryOpts = append(secondaryOpts, o) 71 | } else { 72 | panic(fmt.Sprintf("unexpected option of type %T: %v", opt, opt)) 73 | } 74 | } 75 | return primaryOpts, secondaryOpts 76 | } 77 | -------------------------------------------------------------------------------- /github_ratelimit/github_secondary_ratelimit/callback.go: -------------------------------------------------------------------------------- 1 | package github_secondary_ratelimit 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | ) 7 | 8 | // CallbackContext is passed to all callbacks. 9 | // Fields might be nillable, depending on the specific callback and field. 10 | type CallbackContext struct { 11 | RoundTripper *SecondaryRateLimiter 12 | ResetTime *time.Time 13 | TotalSleepTime *time.Duration 14 | Request *http.Request 15 | Response *http.Response 16 | } 17 | 18 | // OnLimitDetected is a callback to be called when a new rate limit is detected (before the sleep). 19 | // The totalSleepTime includes the sleep duration for the upcoming sleep. 20 | // Note: called while holding the lock. 21 | type OnLimitDetected func(*CallbackContext) 22 | 23 | // OnSingleLimitPassed is a callback to be called when a rate limit is exceeding the limit for a single sleep. 24 | // The ResetTime represents the end of sleep duration if the limit was not exceeded. 25 | // The totalSleepTime does not include the sleep (that is not going to happen). 26 | // Note: called while holding the lock. 27 | type OnSingleLimitExceeded func(*CallbackContext) 28 | 29 | // OnTotalLimitExceeded is a callback to be called when a rate limit is exceeding the limit for the total sleep. 30 | // The ResetTime represents the end of sleep duration if the limit was not exceeded. 31 | // The totalSleepTime does not include the sleep (that is not going to happen). 32 | // Note: called while holding the lock. 33 | type OnTotalLimitExceeded func(*CallbackContext) 34 | -------------------------------------------------------------------------------- /github_ratelimit/github_secondary_ratelimit/config.go: -------------------------------------------------------------------------------- 1 | package github_secondary_ratelimit 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // Config is the config for the secondary rate limit waiter. 9 | // Use the options to set the config. 10 | type Config struct { 11 | // limits 12 | singleSleepLimit *time.Duration 13 | totalSleepLimit *time.Duration 14 | 15 | // callbacks 16 | onLimitDetected OnLimitDetected 17 | onSingleLimitExceeded OnSingleLimitExceeded 18 | onTotalLimitExceeded OnTotalLimitExceeded 19 | } 20 | 21 | // newConfig creates a new config with the given options. 22 | func newConfig(opts ...Option) *Config { 23 | var config Config 24 | config.ApplyOptions(opts...) 25 | return &config 26 | } 27 | 28 | // ApplyOptions applies the options to the config. 29 | func (c *Config) ApplyOptions(opts ...Option) { 30 | for _, o := range opts { 31 | if o == nil { 32 | continue 33 | } 34 | o(c) 35 | } 36 | } 37 | 38 | // IsAboveSingleSleepLimit returns true if the single sleep duration is above the limit. 39 | func (c *Config) IsAboveSingleSleepLimit(sleepTime time.Duration) bool { 40 | return c.singleSleepLimit != nil && sleepTime > *c.singleSleepLimit 41 | } 42 | 43 | // IsAboveTotalSleepLimit returns true if the total sleep duration is above the limit. 44 | func (c *Config) IsAboveTotalSleepLimit(sleepTime time.Duration, totalSleepTime time.Duration) bool { 45 | return c.totalSleepLimit != nil && totalSleepTime+sleepTime > *c.totalSleepLimit 46 | } 47 | 48 | type ConfigOverridesKey struct{} 49 | 50 | // WithOverrideConfig adds config overrides to the context. 51 | // The overrides are applied on top of the existing config. 52 | // Allows for request-specific overrides. 53 | func WithOverrideConfig(ctx context.Context, opts ...Option) context.Context { 54 | return context.WithValue(ctx, ConfigOverridesKey{}, opts) 55 | } 56 | 57 | // GetConfigOverrides returns the config overrides from the context, if any. 58 | func GetConfigOverrides(ctx context.Context) []Option { 59 | cfg := ctx.Value(ConfigOverridesKey{}) 60 | if cfg == nil { 61 | return nil 62 | } 63 | return cfg.([]Option) 64 | } 65 | -------------------------------------------------------------------------------- /github_ratelimit/github_secondary_ratelimit/detect.go: -------------------------------------------------------------------------------- 1 | package github_secondary_ratelimit 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "net/http" 8 | "slices" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | const ( 15 | HeaderRetryAfter = "retry-after" 16 | HeaderXRateLimitReset = "x-ratelimit-reset" 17 | HeaderXRateLimitRemaining = "x-ratelimit-remaining" 18 | ) 19 | 20 | type SecondaryRateLimitBody struct { 21 | Message string `json:"message"` 22 | DocumentURL string `json:"documentation_url"` 23 | } 24 | 25 | const ( 26 | SecondaryRateLimitMessage = `You have exceeded a secondary rate limit` 27 | ) 28 | 29 | var DocumentationSuffixes = []string{ 30 | `secondary-rate-limits`, 31 | `#abuse-rate-limits`, 32 | } 33 | 34 | func HasSecondaryRateLimitSuffix(documentation_url string) bool { 35 | return slices.ContainsFunc(DocumentationSuffixes, func(suffix string) bool { 36 | return strings.HasSuffix(documentation_url, suffix) 37 | }) 38 | } 39 | 40 | // IsSecondaryRateLimit checks whether the response is a legitimate secondary rate limit. 41 | // It checks the prefix of the message and the suffix of the documentation URL in the response body in case 42 | // the message or documentation URL is modified in the future. 43 | // https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api#about-secondary-rate-limits 44 | func (s SecondaryRateLimitBody) IsSecondaryRateLimit() bool { 45 | return strings.HasPrefix(s.Message, SecondaryRateLimitMessage) || 46 | HasSecondaryRateLimitSuffix(s.DocumentURL) 47 | } 48 | 49 | // see https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api#exceeding-the-rate-limit 50 | var LimitStatusCodes = []int{ 51 | http.StatusForbidden, 52 | http.StatusTooManyRequests, 53 | } 54 | 55 | // isSecondaryRateLimit checks whether the response is a legitimate secondary rate limit. 56 | func isSecondaryRateLimit(resp *http.Response) bool { 57 | if !slices.Contains(LimitStatusCodes, resp.StatusCode) { 58 | return false 59 | } 60 | 61 | if resp.Header == nil { 62 | return false 63 | } 64 | 65 | // a primary rate limit 66 | if remaining, ok := httpHeaderIntValue(resp.Header, HeaderXRateLimitRemaining); ok && remaining == 0 { 67 | return false 68 | } 69 | 70 | // an authentic HTTP response (not a primary rate limit) 71 | defer resp.Body.Close() 72 | rawBody, err := io.ReadAll(resp.Body) 73 | if err != nil { 74 | return false // unexpected error 75 | } 76 | 77 | // restore original body 78 | resp.Body = io.NopCloser(bytes.NewReader(rawBody)) 79 | 80 | var body SecondaryRateLimitBody 81 | if err := json.Unmarshal(rawBody, &body); err != nil { 82 | return false // unexpected error 83 | } 84 | if !body.IsSecondaryRateLimit() { 85 | return false 86 | } 87 | 88 | return true 89 | } 90 | 91 | // parseSecondaryLimitTime parses the GitHub API response header, 92 | // looking for the secondary rate limit as defined by GitHub API documentation. 93 | // https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits 94 | func parseSecondaryLimitTime(resp *http.Response) *time.Time { 95 | if !isSecondaryRateLimit(resp) { 96 | return nil 97 | } 98 | 99 | if resetTime := parseRetryAfter(resp.Header); resetTime != nil { 100 | return resetTime 101 | } 102 | 103 | if resetTime := parseXRateLimitReset(resp); resetTime != nil { 104 | return resetTime 105 | } 106 | 107 | // XXX: per GitHub API docs, we should default to a 60 seconds sleep duration in case the header is missing, 108 | // with an exponential backoff mechanism. 109 | // we may want to implement this in the future (with configurable limits), 110 | // but let's avoid it while there are no known cases of missing headers. 111 | return nil 112 | } 113 | 114 | // parseRetryAfter parses the GitHub API response header in case a Retry-After is returned. 115 | func parseRetryAfter(header http.Header) *time.Time { 116 | retryAfterSeconds, ok := httpHeaderIntValue(header, "retry-after") 117 | if !ok || retryAfterSeconds <= 0 { 118 | return nil 119 | } 120 | 121 | // per GitHub API, the header is set to the number of seconds to wait 122 | resetTime := time.Now().Add(time.Duration(retryAfterSeconds) * time.Second) 123 | 124 | return &resetTime 125 | } 126 | 127 | // parseXRateLimitReset parses the GitHub API response header in case a x-ratelimit-reset is returned. 128 | // to avoid handling primary rate limits (which are categorized), 129 | // we only handle x-ratelimit-reset in case the primary rate limit is not reached. 130 | func parseXRateLimitReset(resp *http.Response) *time.Time { 131 | secondsSinceEpoch, ok := httpHeaderIntValue(resp.Header, HeaderXRateLimitReset) 132 | if !ok || secondsSinceEpoch <= 0 { 133 | return nil 134 | } 135 | 136 | // per GitHub API, the header is set to the number of seconds since epoch (UTC) 137 | resetTime := time.Unix(secondsSinceEpoch, 0) 138 | 139 | return &resetTime 140 | } 141 | 142 | // httpHeaderIntValue parses an integer value from the given HTTP header. 143 | func httpHeaderIntValue(header http.Header, key string) (int64, bool) { 144 | val := header.Get(key) 145 | if val == "" { 146 | return 0, false 147 | } 148 | asInt, err := strconv.ParseInt(val, 10, 64) 149 | if err != nil { 150 | return 0, false 151 | } 152 | return asInt, true 153 | } 154 | -------------------------------------------------------------------------------- /github_ratelimit/github_secondary_ratelimit/options.go: -------------------------------------------------------------------------------- 1 | package github_secondary_ratelimit 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type Option func(*Config) 8 | 9 | // WithLimitDetectedCallback adds a callback to be called when a new active rate limit is detected. 10 | func WithLimitDetectedCallback(callback OnLimitDetected) Option { 11 | return func(c *Config) { 12 | c.onLimitDetected = callback 13 | } 14 | } 15 | 16 | // WithSingleSleepLimit adds a limit to the duration allowed to wait for a single sleep (rate limit). 17 | // The callback parameter is nillable. 18 | func WithSingleSleepLimit(limit time.Duration, callback OnSingleLimitExceeded) Option { 19 | return func(c *Config) { 20 | c.singleSleepLimit = &limit 21 | c.onSingleLimitExceeded = callback 22 | } 23 | } 24 | 25 | // WithNoSleep avoid sleeping during secondary rate limits. 26 | // it can be used to detect the limit but handle it out-of-band. 27 | // It is a helper function around WithSingleSleepLimit. 28 | // The callback parameter is nillable. 29 | func WithNoSleep(callback OnSingleLimitExceeded) Option { 30 | return func(c *Config) { 31 | WithSingleSleepLimit(0, callback) 32 | } 33 | } 34 | 35 | // WithTotalSleepLimit adds a limit to the accumulated duration allowed to wait for all sleeps (one or more rate limits). 36 | // The callback parameter is nillable. 37 | func WithTotalSleepLimit(limit time.Duration, callback OnTotalLimitExceeded) Option { 38 | return func(c *Config) { 39 | c.totalSleepLimit = &limit 40 | c.onTotalLimitExceeded = callback 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /github_ratelimit/github_secondary_ratelimit/secondary_rate_limit.go: -------------------------------------------------------------------------------- 1 | package github_secondary_ratelimit 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | // SecondaryRateLimiter is a RoundTripper for handling GitHub secondary rate limits. 11 | type SecondaryRateLimiter struct { 12 | Base http.RoundTripper 13 | resetTime *time.Time 14 | lock sync.RWMutex 15 | totalSleepTime time.Duration 16 | config *Config 17 | } 18 | 19 | // New creates a new SecondaryRateLimiter with the given base RoundTripper and options. 20 | // see optins.go for available options. 21 | // see RoundTrip() for the actual rate limit handling. 22 | func New(base http.RoundTripper, opts ...Option) *SecondaryRateLimiter { 23 | if base == nil { 24 | base = http.DefaultTransport 25 | } 26 | 27 | waiter := SecondaryRateLimiter{ 28 | Base: base, 29 | config: newConfig(opts...), 30 | } 31 | 32 | return &waiter 33 | } 34 | 35 | // RoundTrip handles the secondary rate limit by waiting for it to finish before issuing new requests. 36 | // If a request got a secondary rate limit error as a response, we retry the request after waiting. 37 | // Issuing more requests during a secondary rate limit may cause a ban from the server side, 38 | // so we want to prevent these requests, not just for the sake of cpu/network utilization. 39 | // Nonetheless, there is no way to prevent subtle race conditions without completely serializing the requests, 40 | // so we prefer to let some slip in case of a race condition, i.e., 41 | // after a retry-after response is received and before it is processed, 42 | // a few other (concurrent) requests may be issued. 43 | func (t *SecondaryRateLimiter) RoundTrip(request *http.Request) (*http.Response, error) { 44 | t.waitForRateLimit(request.Context()) 45 | 46 | resp, err := t.Base.RoundTrip(request) 47 | if err != nil { 48 | return resp, err 49 | } 50 | 51 | secondaryLimit := parseSecondaryLimitTime(resp) 52 | if secondaryLimit == nil { 53 | return resp, nil 54 | } 55 | 56 | callbackContext := CallbackContext{ 57 | Request: request, 58 | Response: resp, 59 | } 60 | 61 | shouldRetry := t.updateRateLimit(*secondaryLimit, &callbackContext) 62 | if !shouldRetry { 63 | return resp, nil 64 | } 65 | 66 | return t.RoundTrip(request) 67 | } 68 | 69 | func (t *SecondaryRateLimiter) getRequestConfig(request *http.Request) *Config { 70 | overrides := GetConfigOverrides(request.Context()) 71 | if overrides == nil { 72 | // no config override - use the default config (zero-copy) 73 | return t.config 74 | } 75 | reqConfig := *t.config 76 | reqConfig.ApplyOptions(overrides...) 77 | return &reqConfig 78 | } 79 | 80 | // waitForRateLimit waits for the cooldown time to finish if a secondary rate limit is active. 81 | func (t *SecondaryRateLimiter) waitForRateLimit(ctx context.Context) { 82 | t.lock.RLock() 83 | sleepDuration := t.currentSleepDurationUnlocked() 84 | t.lock.RUnlock() 85 | 86 | _ = sleepWithContext(ctx, sleepDuration) 87 | } 88 | 89 | // updateRateLimit updates the active rate limit and triggers user callbacks if needed. 90 | // the rate limit is not updated if there is already an active rate limit. 91 | // it never waits because the retry handles sleeping anyway. 92 | // returns whether or not to retry the request. 93 | func (t *SecondaryRateLimiter) updateRateLimit(secondaryLimit time.Time, callbackContext *CallbackContext) (needRetry bool) { 94 | // quick check without the lock: maybe the secondary limit just passed 95 | if time.Now().After(secondaryLimit) { 96 | return true 97 | } 98 | 99 | t.lock.Lock() 100 | defer t.lock.Unlock() 101 | 102 | // check before update if there is already an active rate limit 103 | if t.currentSleepDurationUnlocked() > 0 { 104 | return true 105 | } 106 | 107 | // check if the secondary rate limit happened to have passed while we waited for the lock 108 | sleepDuration := time.Until(secondaryLimit) 109 | if sleepDuration <= 0 { 110 | return true 111 | } 112 | 113 | config := t.getRequestConfig(callbackContext.Request) 114 | 115 | // do not sleep in case it is above the single sleep limit 116 | if config.IsAboveSingleSleepLimit(sleepDuration) { 117 | t.triggerCallback(config.onSingleLimitExceeded, callbackContext, secondaryLimit) 118 | return false 119 | } 120 | 121 | // do not sleep in case it is above the total sleep limit 122 | if config.IsAboveTotalSleepLimit(sleepDuration, t.totalSleepTime) { 123 | t.triggerCallback(config.onTotalLimitExceeded, callbackContext, secondaryLimit) 124 | return false 125 | } 126 | 127 | // a legitimate new limit 128 | t.resetTime = &secondaryLimit 129 | t.totalSleepTime += smoothSleepTime(sleepDuration) 130 | t.triggerCallback(config.onLimitDetected, callbackContext, secondaryLimit) 131 | 132 | return true 133 | } 134 | 135 | func (t *SecondaryRateLimiter) currentSleepDurationUnlocked() time.Duration { 136 | if t.resetTime == nil { 137 | return 0 138 | } 139 | return time.Until(*t.resetTime) 140 | } 141 | 142 | func (t *SecondaryRateLimiter) triggerCallback(callback func(*CallbackContext), callbackContext *CallbackContext, newResetTime time.Time) { 143 | if callback == nil { 144 | return 145 | } 146 | 147 | callbackContext.RoundTripper = t 148 | callbackContext.ResetTime = &newResetTime 149 | callbackContext.TotalSleepTime = &t.totalSleepTime 150 | 151 | callback(callbackContext) 152 | } 153 | 154 | // smoothSleepTime rounds up the sleep duration to whole seconds. 155 | // github only uses seconds to indicate the time to sleep, 156 | // but we sleep for less time because internal processing delay is taken into account. 157 | // round up the duration to get the original value. 158 | func smoothSleepTime(sleepTime time.Duration) time.Duration { 159 | if sleepTime.Milliseconds() == 0 { 160 | return sleepTime 161 | } else { 162 | seconds := sleepTime.Seconds() + 1 163 | return time.Duration(seconds) * time.Second 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /github_ratelimit/github_secondary_ratelimit/sleep.go: -------------------------------------------------------------------------------- 1 | package github_secondary_ratelimit 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // sleepWithContext sleeps for d duration or until ctx is done. 9 | // Returns nil if the sleep completes successfully, or the error from ctx. 10 | // special thanks to Cedric Bail (@cedric-appdirect) for the original cancellation-aware code. 11 | func sleepWithContext(ctx context.Context, d time.Duration) error { 12 | timer := time.NewTimer(d) 13 | select { 14 | case <-ctx.Done(): 15 | if !timer.Stop() { 16 | <-timer.C 17 | } 18 | return ctx.Err() 19 | case <-timer.C: 20 | return nil 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /github_ratelimit/github_secondary_ratelimit/sleep_test.go: -------------------------------------------------------------------------------- 1 | package github_secondary_ratelimit 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func Test_SleepWithContextCancel(t *testing.T) { 12 | ctx, cancel := context.WithCancel(context.Background()) 13 | 14 | wg := &sync.WaitGroup{} 15 | wg.Add(1) 16 | 17 | errChan := make(chan error, 1) 18 | start := time.Now() 19 | go func() { 20 | err := sleepWithContext(ctx, 1*time.Second) 21 | if err == nil { 22 | errChan <- fmt.Errorf("expected error, got nil") 23 | } else if err != context.Canceled { 24 | errChan <- fmt.Errorf("expected context.Canceled, got %v", err) 25 | } else { 26 | errChan <- nil 27 | } 28 | close(errChan) 29 | wg.Done() 30 | }() 31 | 32 | time.Sleep(50 * time.Millisecond) 33 | cancel() 34 | 35 | wg.Wait() 36 | if err := <-errChan; err != nil { 37 | t.Fatal(err.Error()) 38 | } 39 | 40 | elapsed := time.Since(start) 41 | if elapsed > 100*time.Millisecond { 42 | t.Fatalf("expected elapsed time to be less than 100ms, got %v", elapsed) 43 | } 44 | } 45 | 46 | func Test_SleepWithContext(t *testing.T) { 47 | ctx := context.Background() 48 | 49 | wg := &sync.WaitGroup{} 50 | wg.Add(1) 51 | 52 | errChan := make(chan error, 1) 53 | start := time.Now() 54 | go func() { 55 | err := sleepWithContext(ctx, 50*time.Millisecond) 56 | if err != nil { 57 | errChan <- fmt.Errorf("expected nil, got %v", err) 58 | } else { 59 | errChan <- nil 60 | } 61 | wg.Done() 62 | }() 63 | 64 | wg.Wait() 65 | if err := <-errChan; err != nil { 66 | t.Fatal(err.Error()) 67 | } 68 | 69 | elapsed := time.Since(start) 70 | if elapsed > 100*time.Millisecond { 71 | t.Fatalf("expected elapsed time to be less than 100ms, got %v", elapsed) 72 | } 73 | } 74 | 75 | func Test_SleepWithContextTimeout(t *testing.T) { 76 | ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) 77 | defer cancel() 78 | 79 | wg := &sync.WaitGroup{} 80 | wg.Add(1) 81 | 82 | errChan := make(chan error, 1) 83 | start := time.Now() 84 | go func() { 85 | err := sleepWithContext(ctx, 1*time.Second) 86 | if err == nil { 87 | errChan <- fmt.Errorf("expected error, got nil") 88 | } else if err != context.DeadlineExceeded { 89 | errChan <- fmt.Errorf("expected context.DeadlineExceeded, got %v", err) 90 | } else { 91 | errChan <- nil 92 | } 93 | close(errChan) 94 | wg.Done() 95 | }() 96 | 97 | wg.Wait() 98 | if err := <-errChan; err != nil { 99 | t.Fatal(err.Error()) 100 | } 101 | 102 | elapsed := time.Since(start) 103 | if elapsed > 100*time.Millisecond { 104 | t.Fatalf("expected elapsed time to be less than 100ms, got %v", elapsed) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /github_ratelimit/primary_ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package github_ratelimit 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math/rand/v2" 7 | "net/http" 8 | "sync" 9 | "sync/atomic" 10 | "testing" 11 | "time" 12 | 13 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" 14 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_ratelimit_test" 15 | ) 16 | 17 | var AllCategories = github_primary_ratelimit.GetAllCategories() 18 | 19 | func getRandomCategory() github_primary_ratelimit.ResourceCategory { 20 | return AllCategories[rand.IntN(len(AllCategories))] 21 | } 22 | 23 | func TestPrimaryRateLimit(t *testing.T) { 24 | t.Parallel() 25 | for _, determenistic := range []bool{true, false} { 26 | t.Logf("primary with determenistic = %v", determenistic) 27 | const requests = 1000 28 | const every = 1 * time.Second 29 | const sleep = 99999999 * time.Second 30 | 31 | var preventionCount atomic.Bool 32 | countPrevented := func(ctx *github_primary_ratelimit.CallbackContext) { 33 | preventionCount.Store(true) 34 | } 35 | 36 | print := func(context *github_primary_ratelimit.CallbackContext) { 37 | t.Logf("primary rate limit reached! %v", *context.ResetTime) 38 | } 39 | 40 | category := github_primary_ratelimit.ResourceCategoryCore 41 | i := github_ratelimit_test.SetupPrimaryInjecter(t, every, sleep, category) 42 | c := github_ratelimit_test.NewPrimaryClient(i, 43 | github_primary_ratelimit.WithLimitDetectedCallback(print), 44 | github_primary_ratelimit.WithRequestPreventedCallback(countPrevented), 45 | ) 46 | 47 | _, err := c.Get("/trigger-core-category") 48 | if err != nil { 49 | t.Fatalf("expecting first request to succeed, got %v", err) 50 | } 51 | 52 | github_ratelimit_test.WaitForNextSleep(i) 53 | if determenistic { 54 | _, err = c.Get("/trigger-core-category") 55 | if err == nil { 56 | t.Fatalf("expecting second request to fail, got %v", err) 57 | } 58 | } 59 | 60 | var gw sync.WaitGroup 61 | gw.Add(requests) 62 | var maxAbuseAttempts = 10 63 | if determenistic { 64 | maxAbuseAttempts = 0 65 | } 66 | tooQuickSuccesses := 0 67 | for i := 0; i < requests; i++ { 68 | // sleep some time between requests 69 | sleepTime := github_ratelimit_test.UpTo1SecDelay() / 150 70 | if sleepTime.Milliseconds()%2 == 0 { 71 | sleepTime = 0 // bias towards no-sleep for high parallelism 72 | } 73 | time.Sleep(sleepTime) 74 | 75 | go func(i int) { 76 | defer gw.Done() 77 | _, err := c.Get("/trigger-core-category") 78 | if err == nil { 79 | tooQuickSuccesses += 1 80 | if tooQuickSuccesses > maxAbuseAttempts { 81 | // TODO channel for errors 82 | t.Logf("expecting error, got nil") 83 | } 84 | } 85 | }(i) 86 | } 87 | gw.Wait() 88 | 89 | // expect a low number of abuse attempts, i.e., 90 | // not a lot of "slipped" messages due to race conditions. 91 | asInjecter, ok := i.(*github_ratelimit_test.RateLimitInjecter) 92 | if !ok { 93 | t.Fatal() 94 | } 95 | if real, max := asInjecter.AbuseAttempts, maxAbuseAttempts; real > max { 96 | t.Fatalf("got %v abusing requests, expected up to %v", real, max) 97 | } 98 | abuseAttempts := asInjecter.AbuseAttempts 99 | abusePrecent := float64(abuseAttempts) / requests * 100 100 | t.Logf("abuse requests: %v/%v (%v%%)\n", asInjecter.AbuseAttempts, requests, abusePrecent) 101 | 102 | if !preventionCount.Load() { 103 | t.Fatal("no prevention was happening") 104 | } 105 | } 106 | } 107 | 108 | func getChannelErrorNonblocking(errChan chan error) (error, bool) { 109 | select { 110 | case err := <-errChan: 111 | return err, true 112 | default: 113 | return nil, false 114 | } 115 | } 116 | 117 | func TestPrimaryCallbacks(t *testing.T) { 118 | t.Parallel() 119 | 120 | const ( 121 | validCategory = github_primary_ratelimit.ResourceCategoryCore 122 | invalidCategory = github_primary_ratelimit.ResourceCategory("rubbish") 123 | sleep = 1 * time.Second 124 | ) 125 | 126 | t.Run("limit and prevention", func(t *testing.T) { 127 | limitChan := make(chan error, 1) 128 | limitCB := func(ctx *github_primary_ratelimit.CallbackContext) { 129 | defer close(limitChan) 130 | if ctx.Request == nil || ctx.Response == nil || ctx.ResetTime == nil { 131 | limitChan <- fmt.Errorf("expected all fields, got %p, %p, %p", 132 | ctx.Request, ctx.Response, ctx.ResetTime) 133 | return 134 | } 135 | if ctx.Category != validCategory { 136 | limitChan <- fmt.Errorf("expected category %v, got %v", validCategory, ctx.Category) 137 | return 138 | } 139 | limitChan <- nil 140 | } 141 | 142 | preventionChan := make(chan error, 1) 143 | preventionCB := func(ctx *github_primary_ratelimit.CallbackContext) { 144 | defer close(preventionChan) 145 | if ctx.Request == nil || ctx.Response == nil || ctx.ResetTime == nil { 146 | preventionChan <- fmt.Errorf("expected all fields, got %p, %p, %p", 147 | ctx.Request, ctx.Response, ctx.ResetTime) 148 | return 149 | } 150 | if ctx.Category != validCategory { 151 | preventionChan <- fmt.Errorf("expected category %v, got %v", validCategory, ctx.Category) 152 | return 153 | } 154 | if remaining := github_primary_ratelimit.ResponseHeaderKeyRemaining.Get(ctx.Response); remaining != "0" { 155 | preventionChan <- fmt.Errorf("expected remaining 0, got %v", remaining) 156 | return 157 | } 158 | preventionChan <- nil 159 | } 160 | 161 | resetChan := make(chan error, 1) 162 | resetCB := func(ctx *github_primary_ratelimit.CallbackContext) { 163 | defer close(resetChan) 164 | if ctx.Request != nil || ctx.Response != nil { 165 | resetChan <- fmt.Errorf("expected empty request & response, got %v %v", 166 | ctx.Request, ctx.Response) 167 | return 168 | } 169 | if ctx.ResetTime == nil { 170 | resetChan <- fmt.Errorf("expected reset time, got nil") 171 | return 172 | } 173 | if ctx.Category != validCategory { 174 | resetChan <- fmt.Errorf("expected category %v, got %v", validCategory, ctx.Category) 175 | return 176 | } 177 | resetChan <- nil 178 | } 179 | 180 | every := 500 * time.Millisecond 181 | i := github_ratelimit_test.SetupPrimaryInjecter(t, every, sleep, validCategory) 182 | c := github_ratelimit_test.NewPrimaryClient(i, 183 | github_primary_ratelimit.WithLimitDetectedCallback(limitCB), 184 | github_primary_ratelimit.WithRequestPreventedCallback(preventionCB), 185 | github_primary_ratelimit.WithLimitResetCallback(resetCB), 186 | ) 187 | 188 | // initiate to wait for first 189 | _, _ = c.Get("/") 190 | time.Sleep(every) 191 | 192 | // limited request 193 | resp, err := c.Get("/trigger-limit") 194 | if err == nil { 195 | t.Fatalf("expecting limit, got nil: %v", resp) 196 | } 197 | err, ok := getChannelErrorNonblocking(limitChan) 198 | if !ok || err != nil { 199 | t.Fatalf("limit check failed: %v, %v", ok, err) 200 | } 201 | err, ok = getChannelErrorNonblocking(preventionChan) 202 | if ok || err != nil { 203 | t.Fatalf("not expected prevention, got: %v, %v", ok, err) 204 | } 205 | 206 | // prevented request 207 | resp, err = c.Get("/trigger-prevent") 208 | if err == nil { 209 | t.Fatalf("expecting request to fail, got nil: %v", resp) 210 | } 211 | var typedErr *github_primary_ratelimit.RateLimitReachedError 212 | if ok := errors.As(err, &typedErr); !ok { 213 | t.Fatalf("unexpected error type: %T - %v", err, err) 214 | } 215 | resetTime := *typedErr.ResetTime 216 | err, ok = getChannelErrorNonblocking(preventionChan) 217 | if !ok || err != nil { 218 | t.Fatalf("prevention check failed: %v, %v", ok, err) 219 | } 220 | 221 | // reset check 222 | err, ok = getChannelErrorNonblocking(resetChan) 223 | if ok || err != nil { 224 | t.Fatalf("not expected reset, got: %v, %v", ok, err) 225 | } 226 | 227 | extraTime := 500 * time.Millisecond // make "sure" the timer goes first 228 | time.Sleep(time.Until(resetTime) + extraTime) 229 | err, ok = getChannelErrorNonblocking(resetChan) 230 | if !ok || err != nil { 231 | t.Fatalf("reset check failed: %v, %v", ok, err) 232 | } 233 | }) 234 | 235 | t.Run("unknown category", func(t *testing.T) { 236 | unknownChan := make(chan error, 1) 237 | unknownCB := func(ctx *github_primary_ratelimit.CallbackContext) { 238 | defer close(unknownChan) 239 | if ctx.Request == nil || ctx.Response == nil || ctx.ResetTime == nil { 240 | unknownChan <- fmt.Errorf("expected all fields, got %p, %p, %p", 241 | ctx.Request, ctx.Response, ctx.ResetTime) 242 | return 243 | } 244 | if ctx.Category != invalidCategory { 245 | unknownChan <- fmt.Errorf("expected category %v, got %v", invalidCategory, ctx.Category) 246 | return 247 | } 248 | unknownChan <- nil 249 | } 250 | 251 | every := 0 * time.Second 252 | i := github_ratelimit_test.SetupPrimaryInjecter(t, every, sleep, invalidCategory) 253 | c := github_ratelimit_test.NewPrimaryClient(i, 254 | github_primary_ratelimit.WithUnknownCategoryCallback(unknownCB), 255 | ) 256 | 257 | _, err := c.Get("/") 258 | if err != nil { 259 | t.Fatalf("expecting first request to succeed, got: %v", err) 260 | } 261 | err, ok := getChannelErrorNonblocking(unknownChan) 262 | if ok || err != nil { 263 | t.Fatalf("unknown category check failed: %v, %v", ok, err) 264 | } 265 | 266 | _, err = c.Get("/") 267 | if err != nil { 268 | t.Fatalf("expecting second request to succeed, got: %v", err) 269 | } 270 | err, ok = getChannelErrorNonblocking(unknownChan) 271 | if !ok || err != nil { 272 | t.Fatalf("unknown category check failed: %v, %v", ok, err) 273 | } 274 | }) 275 | } 276 | 277 | func TestConfigOverride(t *testing.T) { 278 | const ( 279 | category = github_primary_ratelimit.ResourceCategoryCore 280 | sleep = 1 * time.Second 281 | every = 1 * time.Second 282 | ) 283 | 284 | var preOverride, postOverride atomic.Bool 285 | 286 | defaultCB := func(ctx *github_primary_ratelimit.CallbackContext) { 287 | preOverride.Store(true) 288 | } 289 | 290 | overrideCB := func(ctx *github_primary_ratelimit.CallbackContext) { 291 | postOverride.Store(true) 292 | } 293 | 294 | i := github_ratelimit_test.SetupPrimaryInjecter(t, every, sleep, category) 295 | c := github_ratelimit_test.NewPrimaryClient(i, 296 | github_primary_ratelimit.WithLimitDetectedCallback(defaultCB), 297 | ) 298 | 299 | // initiate to wait for first 300 | _, _ = c.Get("/") 301 | time.Sleep(every) 302 | 303 | // request with overridden callback 304 | req, _ := http.NewRequest(http.MethodGet, "/trigger-limit", nil) 305 | req = req.WithContext( 306 | github_primary_ratelimit.WithOverrideConfig( 307 | req.Context(), 308 | github_primary_ratelimit.WithLimitDetectedCallback(overrideCB), 309 | ), 310 | ) 311 | _, err := c.Do(req) 312 | if err == nil { 313 | t.Fatalf("expecting limit, got nil") 314 | } 315 | 316 | if !postOverride.Load() { 317 | t.Fatal("override callback was not called") 318 | } 319 | if preOverride.Load() { 320 | t.Fatal("default callback was called instead of override") 321 | } 322 | } 323 | -------------------------------------------------------------------------------- /github_ratelimit/secondary_ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package github_ratelimit 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "sync" 8 | "sync/atomic" 9 | "testing" 10 | "time" 11 | 12 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_ratelimit_test" 13 | "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" 14 | ) 15 | 16 | func TestSecondaryRateLimit(t *testing.T) { 17 | t.Parallel() 18 | const requests = 10000 19 | const every = 5 * time.Second 20 | const sleep = 1 * time.Second 21 | 22 | print := func(context *github_secondary_ratelimit.CallbackContext) { 23 | t.Logf("Secondary rate limit reached! Sleeping for %.2f seconds [%v --> %v]", 24 | time.Until(*context.ResetTime).Seconds(), time.Now(), *context.ResetTime) 25 | } 26 | 27 | i := github_ratelimit_test.SetupSecondaryInjecter(t, every, sleep) 28 | c := github_ratelimit_test.NewSecondaryClient(i, github_secondary_ratelimit.WithLimitDetectedCallback(print)) 29 | 30 | var gw sync.WaitGroup 31 | gw.Add(requests) 32 | for i := 0; i < requests; i++ { 33 | // sleep some time between requests 34 | sleepTime := github_ratelimit_test.UpTo1SecDelay() / 150 35 | if sleepTime.Milliseconds()%2 == 0 { 36 | sleepTime = 0 // bias towards no-sleep for high parallelism 37 | } 38 | time.Sleep(sleepTime) 39 | 40 | go func() { 41 | defer gw.Done() 42 | _, _ = c.Get("/") 43 | }() 44 | } 45 | gw.Wait() 46 | 47 | // expect a low number of abuse attempts, i.e., 48 | // not a lot of "slipped" messages due to race conditions. 49 | asInjecter, ok := i.(*github_ratelimit_test.RateLimitInjecter) 50 | if !ok { 51 | t.Fatal() 52 | } 53 | const maxAbuseAttempts = requests / 200 // 0.5% sounds good 54 | if real, max := asInjecter.AbuseAttempts, maxAbuseAttempts; real > max { 55 | t.Fatal(real, max) 56 | } 57 | abusePrecent := float64(asInjecter.AbuseAttempts) / requests * 100 58 | t.Logf("abuse requests: %v/%v (%v%%)\n", asInjecter.AbuseAttempts, requests, abusePrecent) 59 | } 60 | 61 | func TestSecondaryRateLimitCombinations(t *testing.T) { 62 | t.Parallel() 63 | const every = 1 * time.Second 64 | const sleep = 1 * time.Second 65 | 66 | for i, docURL := range github_ratelimit_test.SecondaryRateLimitDocumentationURLs { 67 | docURL := docURL 68 | for j, statusCode := range github_secondary_ratelimit.LimitStatusCodes { 69 | statusCode := statusCode 70 | t.Run(fmt.Sprintf("docURL_%d_%d", i, j), func(t *testing.T) { 71 | t.Parallel() 72 | 73 | slept := false 74 | callback := func(*github_secondary_ratelimit.CallbackContext) { 75 | slept = true 76 | } 77 | 78 | // test documentation URL 79 | i := github_ratelimit_test.SetupInjecterWithOptions(t, github_ratelimit_test.RateLimitInjecterOptions{ 80 | Every: every, 81 | InjectionDuration: sleep, 82 | DocumentationURL: docURL, 83 | HttpStatusCode: statusCode, 84 | }, nil) 85 | c := github_ratelimit_test.NewSecondaryClient( 86 | i, 87 | github_secondary_ratelimit.WithLimitDetectedCallback(callback), 88 | ) 89 | 90 | // initialize injecter timing 91 | _, _ = c.Get("/") 92 | github_ratelimit_test.WaitForNextSleep(i) 93 | 94 | // attempt during rate limit 95 | _, err := c.Get("/") 96 | if err != nil { 97 | t.Fatal(err) 98 | } 99 | if !slept { 100 | t.Fatal(slept) 101 | } 102 | }) 103 | } 104 | } 105 | } 106 | 107 | func TestSingleSleepLimit(t *testing.T) { 108 | t.Parallel() 109 | const every = 1 * time.Second 110 | const sleep = 1 * time.Second 111 | 112 | slept := false 113 | callback := func(*github_secondary_ratelimit.CallbackContext) { 114 | slept = true 115 | } 116 | exceeded := false 117 | onLimitExceeded := func(*github_secondary_ratelimit.CallbackContext) { 118 | exceeded = true 119 | } 120 | 121 | // test sleep is short enough 122 | i := github_ratelimit_test.SetupSecondaryInjecter(t, every, sleep) 123 | c := github_ratelimit_test.NewSecondaryClient(i, 124 | github_secondary_ratelimit.WithLimitDetectedCallback(callback), 125 | github_secondary_ratelimit.WithSingleSleepLimit(5*time.Second, onLimitExceeded)) 126 | 127 | // initialize injecter timing 128 | _, _ = c.Get("/") 129 | github_ratelimit_test.WaitForNextSleep(i) 130 | 131 | // attempt during rate limit 132 | _, err := c.Get("/") 133 | if err != nil { 134 | t.Fatal(err) 135 | } 136 | if !slept || exceeded { 137 | t.Fatal(slept, exceeded) 138 | } 139 | 140 | // test sleep is too long 141 | slept = false 142 | i = github_ratelimit_test.SetupSecondaryInjecter(t, every, sleep) 143 | c = github_ratelimit_test.NewSecondaryClient(i, 144 | github_secondary_ratelimit.WithLimitDetectedCallback(callback), 145 | github_secondary_ratelimit.WithSingleSleepLimit(sleep/2, onLimitExceeded)) 146 | 147 | // initialize injecter timing 148 | _, _ = c.Get("/") 149 | github_ratelimit_test.WaitForNextSleep(i) 150 | 151 | // attempt during rate limit 152 | resp, err := c.Get("/") 153 | if err != nil { 154 | t.Fatal(err) 155 | } 156 | if slept || !exceeded { 157 | t.Fatal(err) 158 | } 159 | if got, want := resp.Header.Get(github_secondary_ratelimit.HeaderRetryAfter), fmt.Sprintf("%v", sleep.Seconds()); got != want { 160 | t.Fatal(got, want) 161 | } 162 | // try again - make sure that triggering the callback does not cause it to sleep next time 163 | tBefore := time.Now() 164 | _, err = c.Get("/") 165 | if err != nil { 166 | t.Fatal(err) 167 | } 168 | tAfter := time.Now() 169 | // choose sleep 2 arbitrarily - should be much less (but should be close to almost sleep if error) 170 | if got, limit := tAfter.Sub(tBefore), sleep/2; got >= limit { 171 | t.Fatal(got, limit) 172 | } 173 | } 174 | 175 | func TestTotalSleepLimit(t *testing.T) { 176 | t.Parallel() 177 | const every = 1 * time.Second 178 | const sleep = 1 * time.Second 179 | 180 | slept := false 181 | callback := func(*github_secondary_ratelimit.CallbackContext) { 182 | slept = true 183 | } 184 | exceeded := false 185 | onLimitExceeded := func(*github_secondary_ratelimit.CallbackContext) { 186 | exceeded = true 187 | } 188 | 189 | // test sleep is short enough 190 | i := github_ratelimit_test.SetupSecondaryInjecter(t, every, sleep) 191 | c := github_ratelimit_test.NewSecondaryClient(i, 192 | github_secondary_ratelimit.WithLimitDetectedCallback(callback), 193 | github_secondary_ratelimit.WithTotalSleepLimit(time.Second+time.Second/2, onLimitExceeded)) 194 | 195 | // initialize injecter timing 196 | _, _ = c.Get("/") 197 | github_ratelimit_test.WaitForNextSleep(i) 198 | 199 | // attempt during rate limit - sleep is still short enough 200 | _, err := c.Get("/") 201 | if err != nil { 202 | t.Fatal(err) 203 | } 204 | if !slept || exceeded { 205 | t.Fatal(slept, exceeded) 206 | } 207 | 208 | // test sleep is too long 209 | slept = false 210 | github_ratelimit_test.WaitForNextSleep(i) 211 | resp, err := c.Get("/") 212 | if err != nil { 213 | t.Fatal(err) 214 | } 215 | if slept || !exceeded { 216 | t.Fatal(slept, exceeded) 217 | } 218 | if got, want := resp.Header.Get(github_secondary_ratelimit.HeaderRetryAfter), fmt.Sprintf("%v", sleep.Seconds()); got != want { 219 | t.Fatal(got, want) 220 | } 221 | // try again - make sure that triggering the callback does not cause it to sleep next time 222 | tBefore := time.Now() 223 | _, err = c.Get("/") 224 | if err != nil { 225 | t.Fatal(err) 226 | } 227 | tAfter := time.Now() 228 | // choose sleep 2 arbitrarily - should be much less (but should be close to almost sleep if error) 229 | if got, limit := tAfter.Sub(tBefore), sleep/2; got >= limit { 230 | t.Fatal(got, limit) 231 | } 232 | } 233 | 234 | func TestXRateLimit(t *testing.T) { 235 | t.Parallel() 236 | const every = 1 * time.Second 237 | const sleep = 1 * time.Second 238 | 239 | slept := false 240 | callback := func(*github_secondary_ratelimit.CallbackContext) { 241 | slept = true 242 | } 243 | 244 | // test sleep is short enough 245 | i := github_ratelimit_test.SetupInjecterWithOptions(t, github_ratelimit_test.RateLimitInjecterOptions{ 246 | Every: every, 247 | InjectionDuration: sleep, 248 | UseXRateLimit: true, 249 | }, nil) 250 | c := github_ratelimit_test.NewSecondaryClient(i, github_secondary_ratelimit.WithLimitDetectedCallback(callback)) 251 | 252 | // initialize injecter timing 253 | _, _ = c.Get("/") 254 | github_ratelimit_test.WaitForNextSleep(i) 255 | 256 | // attempt during rate limit 257 | _, err := c.Get("/") 258 | if err != nil { 259 | t.Fatal(err) 260 | } 261 | if !slept { 262 | t.Fatal(slept) 263 | } 264 | } 265 | 266 | func TestPrimaryRateLimitIgnored(t *testing.T) { 267 | t.Parallel() 268 | const every = 1 * time.Second 269 | const sleep = 1 * time.Second 270 | 271 | slept := false 272 | callback := func(*github_secondary_ratelimit.CallbackContext) { 273 | slept = true 274 | } 275 | 276 | // test sleep is short enough 277 | i := github_ratelimit_test.SetupPrimaryInjecter(t, every, sleep, getRandomCategory()) 278 | c := github_ratelimit_test.NewSecondaryClient(i, github_secondary_ratelimit.WithLimitDetectedCallback(callback)) 279 | 280 | // initialize injecter timing 281 | _, _ = c.Get("/") 282 | github_ratelimit_test.WaitForNextSleep(i) 283 | 284 | // attempt during rate limit 285 | _, err := c.Get("/") 286 | if err != nil { 287 | t.Fatal(err) 288 | } 289 | if slept { 290 | t.Fatal(slept) 291 | } 292 | } 293 | 294 | func TestHTTPForbiddenIgnored(t *testing.T) { 295 | t.Parallel() 296 | const every = 1 * time.Second 297 | const sleep = 1 * time.Second 298 | 299 | slept := false 300 | callback := func(*github_secondary_ratelimit.CallbackContext) { 301 | slept = true 302 | } 303 | 304 | // test sleep is short enough 305 | i := github_ratelimit_test.SetupInjecterWithOptions(t, github_ratelimit_test.RateLimitInjecterOptions{ 306 | Every: every, 307 | InjectionDuration: sleep, 308 | InvalidBody: true, 309 | }, nil) 310 | 311 | c := github_ratelimit_test.NewSecondaryClient(i, github_secondary_ratelimit.WithLimitDetectedCallback(callback)) 312 | 313 | // initialize injecter timing 314 | _, _ = c.Get("/") 315 | github_ratelimit_test.WaitForNextSleep(i) 316 | 317 | // attempt during rate limit (using invalid body, so the injection is of HTTP Forbidden) 318 | resp, err := c.Get("/") 319 | if err != nil { 320 | t.Fatal(err) 321 | } 322 | if slept { 323 | t.Fatal(slept) 324 | } 325 | 326 | if invalidBody, err := github_ratelimit_test.IsInvalidBody(resp); err != nil { 327 | t.Fatal(err) 328 | } else if !invalidBody { 329 | t.Fatalf("expected invalid body") 330 | } 331 | } 332 | 333 | func TestCallbackContext(t *testing.T) { 334 | t.Parallel() 335 | const every = 1 * time.Second 336 | const sleep = 1 * time.Second 337 | i := github_ratelimit_test.SetupSecondaryInjecter(t, every, sleep) 338 | 339 | var roundTripper http.RoundTripper = nil 340 | var requestNum atomic.Int64 341 | requestNum.Add(1) 342 | requestsCycle := 1 343 | 344 | callback := func(ctx *github_secondary_ratelimit.CallbackContext) { 345 | if got, want := ctx.RoundTripper, roundTripper; got != want { 346 | t.Fatalf("roundtripper mismatch: %v != %v", got, want) 347 | } 348 | if ctx.Request == nil || ctx.Response == nil { 349 | t.Fatalf("missing request / response: %v / %v:", ctx.Request, ctx.Response) 350 | } 351 | if got, min, max := time.Until(*ctx.ResetTime), time.Duration(0), sleep*time.Duration(requestNum.Load()); got <= min || got > max { 352 | t.Fatalf("unexpected sleep until time: %v < %v <= %v", min, got, max) 353 | } 354 | if got, want := *ctx.TotalSleepTime, sleep*time.Duration(requestsCycle); got != want { 355 | t.Fatalf("unexpected total sleep duration: %v != %v", got, want) 356 | } 357 | requestNum.Add(1) 358 | } 359 | 360 | c := github_ratelimit_test.NewSecondaryClient(i, 361 | github_secondary_ratelimit.WithLimitDetectedCallback(callback), 362 | ) 363 | roundTripper = c.Transport 364 | 365 | // initialize injecter timing 366 | _, _ = c.Get("/") 367 | github_ratelimit_test.WaitForNextSleep(i) 368 | 369 | // attempt during rate limit 370 | _, err := c.Get("/") 371 | if err != nil { 372 | t.Fatal(err) 373 | } 374 | 375 | github_ratelimit_test.WaitForNextSleep(i) 376 | requestsCycle++ 377 | errChan := make(chan error) 378 | parallelReqs := 10 379 | for index := 0; index < parallelReqs; index++ { 380 | go func() { 381 | _, err := c.Get("/") 382 | errChan <- err 383 | }() 384 | } 385 | for index := 0; index < parallelReqs; index++ { 386 | if err := <-errChan; err != nil { 387 | t.Fatal(err) 388 | } 389 | } 390 | close(errChan) 391 | } 392 | 393 | func TestRequestConfigOverride(t *testing.T) { 394 | t.Parallel() 395 | const every = 1 * time.Second 396 | const sleep = 1 * time.Second 397 | 398 | slept := false 399 | callback := func(*github_secondary_ratelimit.CallbackContext) { 400 | slept = true 401 | } 402 | exceeded := false 403 | onLimitExceeded := func(*github_secondary_ratelimit.CallbackContext) { 404 | exceeded = true 405 | } 406 | 407 | // test sleep is short enough 408 | i := github_ratelimit_test.SetupSecondaryInjecter(t, every, sleep) 409 | c := github_ratelimit_test.NewSecondaryClient(i, 410 | github_secondary_ratelimit.WithLimitDetectedCallback(callback), 411 | github_secondary_ratelimit.WithSingleSleepLimit(5*time.Second, onLimitExceeded)) 412 | 413 | // initialize injecter timing 414 | _, _ = c.Get("/") 415 | 416 | // prepare an override - force sleep duration to be 0, 417 | // so that it will not sleep at all regardless of the original config. 418 | limit := github_secondary_ratelimit.WithSingleSleepLimit(0, onLimitExceeded) 419 | ctx := github_secondary_ratelimit.WithOverrideConfig(context.Background(), limit) 420 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil) 421 | if err != nil { 422 | t.Fatal(err) 423 | } 424 | 425 | // wait for next sleep to kick in, but issue the request with the override 426 | github_ratelimit_test.WaitForNextSleep(i) 427 | 428 | // attempt during rate limit 429 | slept = false 430 | exceeded = false 431 | _, err = c.Do(req) 432 | if err != nil { 433 | t.Fatal(err) 434 | } 435 | // expect no sleep because the override is set to 0 436 | if slept || !exceeded { 437 | t.Fatal(slept, exceeded) 438 | } 439 | 440 | // prepare an override with a different nature (extra safety check) 441 | exceeded = false 442 | usedAltCallback := false 443 | onSleepAlt := func(*github_secondary_ratelimit.CallbackContext) { 444 | usedAltCallback = true 445 | } 446 | 447 | limit = github_secondary_ratelimit.WithSingleSleepLimit(10*time.Second, onLimitExceeded) 448 | sleepCB := github_secondary_ratelimit.WithLimitDetectedCallback(onSleepAlt) 449 | ctx = github_secondary_ratelimit.WithOverrideConfig(context.Background(), limit, sleepCB) 450 | req, err = http.NewRequestWithContext(ctx, http.MethodGet, "/", nil) 451 | if err != nil { 452 | t.Fatal(err) 453 | } 454 | 455 | // attempt during rate limit 456 | _, err = c.Do(req) 457 | if err != nil { 458 | t.Fatal(err) 459 | } 460 | if !usedAltCallback || exceeded { 461 | t.Fatal(slept, exceeded) 462 | } 463 | 464 | } 465 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gofri/go-github-ratelimit/v2 2 | 3 | go 1.23.0 4 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gofri/go-github-ratelimit/483ce26b8f82c066ba491da40cce700cf790c9bc/go.sum --------------------------------------------------------------------------------