├── .dockerignore ├── .github ├── buf-logo.svg ├── dependabot.yml └── workflows │ ├── add-to-project.yaml │ ├── ci.yaml │ ├── emergency-review-bypass.yaml │ ├── notify-approval-bypass.yaml │ └── pr-title.yaml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── cache.go ├── cache ├── filecache │ ├── filecache.go │ └── filecache_test.go ├── internal │ └── cachetesting │ │ └── cachetesting.go ├── memcache │ ├── memcache.go │ └── memcache_test.go └── rediscache │ ├── rediscache.go │ └── rediscache_test.go ├── cache_test.go ├── config.go ├── converter.go ├── converter_test.go ├── doc.go ├── example_test.go ├── filter.go ├── filter_test.go ├── format.go ├── go.mod ├── go.sum ├── internal ├── proto │ ├── buf.gen.yaml │ ├── buf.yaml │ ├── buf │ │ └── prototransform │ │ │ └── v1alpha1 │ │ │ ├── cache.proto │ │ │ └── lease.proto │ └── gen │ │ └── buf │ │ └── prototransform │ │ └── v1alpha1 │ │ ├── cache.pb.go │ │ └── lease.pb.go └── testdata │ ├── buf.gen.yaml │ ├── foo │ └── v1 │ │ └── test.proto │ └── gen │ └── foo │ └── v1 │ └── test.pb.go ├── jitter.go ├── jitter_test.go ├── leaser.go ├── leaser ├── internal │ └── leasertesting │ │ └── leasertesting.go ├── memcacheleaser │ ├── memcacheleaser.go │ └── memcacheleaser_test.go ├── polling_leaser.go └── redisleaser │ ├── redisleaser.go │ └── redisleaser_test.go ├── reflect_client.go ├── reflect_client_test.go ├── resolver.go ├── schema_poller.go ├── schema_watcher.go └── schema_watcher_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .build/ 2 | .ctrlp 3 | .env/ 4 | .idea/ 5 | .tmp/ 6 | .vscode/ 7 | -------------------------------------------------------------------------------- /.github/buf-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | - package-ecosystem: "gomod" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" 11 | -------------------------------------------------------------------------------- /.github/workflows/add-to-project.yaml: -------------------------------------------------------------------------------- 1 | name: Add issues and PRs to project 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | - reopened 8 | - transferred 9 | pull_request_target: 10 | types: 11 | - opened 12 | - reopened 13 | issue_comment: 14 | types: 15 | - created 16 | 17 | jobs: 18 | call-workflow-add-to-project: 19 | name: Call workflow to add issue to project 20 | uses: bufbuild/base-workflows/.github/workflows/add-to-project.yaml@main 21 | secrets: inherit 22 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: ci 2 | on: push 3 | permissions: 4 | contents: read 5 | jobs: 6 | ci: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | go-version: 11 | - name: latest 12 | version: 1.24.x 13 | - name: previous 14 | version: 1.23.x 15 | steps: 16 | - name: checkout 17 | uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 20 | - name: setup-go 21 | uses: actions/setup-go@v5 22 | with: 23 | go-version: ${{ matrix.go-version.version }} 24 | - name: setup-buf 25 | uses: bufbuild/buf-setup-action@v1 26 | - name: cache 27 | uses: actions/cache@v4 28 | with: 29 | path: ~/go/pkg/mod 30 | key: ${{ runner.os }}-connect-ci-${{ hashFiles('**/go.sum') }} 31 | restore-keys: ${{ runner.os }}-connect-ci- 32 | - name: generate 33 | run: make generate && make checkgenerate 34 | - name: test 35 | env: 36 | BUF_TOKEN: ${{ secrets.BUF_TOKEN }} 37 | run: make test 38 | - name: lint 39 | if: matrix.go-version.name == 'latest' 40 | run: make lint 41 | -------------------------------------------------------------------------------- /.github/workflows/emergency-review-bypass.yaml: -------------------------------------------------------------------------------- 1 | name: Bypass review in case of emergency 2 | on: 3 | pull_request: 4 | types: 5 | - labeled 6 | permissions: 7 | pull-requests: write 8 | jobs: 9 | approve: 10 | if: github.event.label.name == 'Emergency Bypass Review' 11 | uses: bufbuild/base-workflows/.github/workflows/emergency-review-bypass.yaml@main 12 | secrets: inherit 13 | -------------------------------------------------------------------------------- /.github/workflows/notify-approval-bypass.yaml: -------------------------------------------------------------------------------- 1 | name: PR Approval Bypass Notifier 2 | on: 3 | pull_request: 4 | types: 5 | - closed 6 | branches: 7 | - main 8 | permissions: 9 | pull-requests: read 10 | jobs: 11 | approval: 12 | uses: bufbuild/base-workflows/.github/workflows/notify-approval-bypass.yaml@main 13 | secrets: inherit 14 | -------------------------------------------------------------------------------- /.github/workflows/pr-title.yaml: -------------------------------------------------------------------------------- 1 | name: Lint PR Title 2 | # Prevent writing to the repository using the CI token. 3 | # Ref: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#permissions 4 | permissions: 5 | pull-requests: read 6 | on: 7 | pull_request: 8 | # By default, a workflow only runs when a pull_request's activity type is opened, 9 | # synchronize, or reopened. We explicity override here so that PR titles are 10 | # re-linted when the PR text content is edited. 11 | types: 12 | - opened 13 | - edited 14 | - reopened 15 | - synchronize 16 | jobs: 17 | lint: 18 | uses: bufbuild/base-workflows/.github/workflows/pr-title.yaml@main 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.build/ 2 | /.ctrlp 3 | /.env/ 4 | /.idea/ 5 | /.tmp/ 6 | /.vscode/ 7 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | errcheck: 3 | check-type-assertions: true 4 | forbidigo: 5 | forbid: 6 | - '^fmt\.Print' 7 | - '^log\.' 8 | - '^print$' 9 | - '^println$' 10 | - '^panic$' 11 | godox: 12 | # TODO, OPT, etc. comments are fine to commit. Use FIXME comments for 13 | # temporary hacks, and use godox to prevent committing them. 14 | keywords: [FIXME] 15 | varnamelen: 16 | ignore-decls: 17 | - T any 18 | - i int 19 | - wg sync.WaitGroup 20 | - ok bool 21 | linters: 22 | enable-all: true 23 | disable: 24 | - cyclop # covered by gocyclo 25 | - depguard # requires custom config in newer versions to use non-stdlib deps 26 | - exhaustruct # super-spammy and doesn't like idiomatic Go (especially w/ protos) 27 | - funlen # rely on code review to limit function length 28 | - gocognit # dubious "cognitive overhead" quantification 29 | - gofumpt # prefer standard gofmt 30 | - goimports # rely on gci instead 31 | - inamedparam # convention is not followed 32 | - ireturn # "accept interfaces, return structs" isn't ironclad 33 | - lll # don't want hard limits for line length 34 | - maintidx # covered by gocyclo 35 | - mnd # some unnamed constants are okay 36 | - nlreturn # generous whitespace violates house style 37 | - nonamedreturns # no bare returns is really what we care about 38 | - tenv # replaced by usetesting 39 | - testpackage # internal tests are fine 40 | - wrapcheck # don't _always_ need to wrap errors 41 | - wsl # generous whitespace violates house style 42 | issues: 43 | exclude-dirs-use-default: false 44 | exclude: 45 | # Don't ban use of fmt.Errorf to create new errors, but the remaining 46 | # checks from err113 are useful. 47 | - "do not define dynamic errors.*" 48 | exclude-rules: 49 | - path: example_test\.go 50 | linters: 51 | - gocritic 52 | - gochecknoglobals 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2023-2024 Buf Technologies, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # See https://tech.davis-hansson.com/p/make/ 2 | SHELL := bash 3 | .DELETE_ON_ERROR: 4 | .SHELLFLAGS := -eu -o pipefail -c 5 | .DEFAULT_GOAL := all 6 | MAKEFLAGS += --warn-undefined-variables 7 | MAKEFLAGS += --no-builtin-rules 8 | MAKEFLAGS += --no-print-directory 9 | BIN := .tmp/bin 10 | COPYRIGHT_YEARS := 2023-2025 11 | LICENSE_IGNORE := -e internal/testdata/ 12 | # Set to use a different compiler. For example, `GO=go1.18rc1 make test`. 13 | GO ?= go 14 | BUF_VERSION ?= v1.50.0 15 | GOLANGCI_LINT_VERSION ?= v1.64.5 16 | 17 | .PHONY: help 18 | help: ## Describe useful make targets 19 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "%-30s %s\n", $$1, $$2}' 20 | 21 | .PHONY: all 22 | all: ## Build, test, and lint (default) 23 | $(MAKE) test 24 | $(MAKE) lint 25 | 26 | .PHONY: clean 27 | clean: ## Delete intermediate build artifacts 28 | @# -X only removes untracked files, -d recurses into directories, -f actually removes files/dirs 29 | git clean -Xdf 30 | 31 | .PHONY: test 32 | test: build ## Run unit tests 33 | $(GO) test -vet=off -race -cover ./... 34 | 35 | .PHONY: build 36 | build: generate ## Build all packages 37 | $(GO) build ./... 38 | 39 | .PHONY: install 40 | install: ## Install all binaries 41 | $(GO) install ./... 42 | 43 | .PHONY: lint 44 | lint: $(BIN)/golangci-lint ## Lint Go and protobuf 45 | $(GO) vet ./... 46 | $(BIN)/golangci-lint run 47 | 48 | .PHONY: lintfix 49 | lintfix: $(BIN)/golangci-lint ## Automatically fix some lint errors 50 | $(BIN)/golangci-lint run --fix 51 | 52 | .PHONY: generate 53 | generate: $(BIN)/buf $(BIN)/license-header ## Regenerate code and licenses 54 | rm -rf internal/testdata/gen internal/proto/gen 55 | PATH=$(abspath $(BIN)) cd internal/testdata && buf generate && cd ../proto && buf generate 56 | @# We want to operate on a list of modified and new files, excluding 57 | @# deleted and ignored files. git-ls-files can't do this alone. comm -23 takes 58 | @# two files and prints the union, dropping lines common to both (-3) and 59 | @# those only in the second file (-2). We make one git-ls-files call for 60 | @# the modified, cached, and new (--others) files, and a second for the 61 | @# deleted files. 62 | comm -23 \ 63 | <(git ls-files --cached --modified --others --no-empty-directory --exclude-standard | sort -u | grep -v $(LICENSE_IGNORE) ) \ 64 | <(git ls-files --deleted | sort -u) | \ 65 | xargs $(BIN)/license-header \ 66 | --license-type apache \ 67 | --copyright-holder "Buf Technologies, Inc." \ 68 | --year-range "$(COPYRIGHT_YEARS)" 69 | 70 | .PHONY: upgrade 71 | upgrade: ## Upgrade dependencies 72 | go get -u -t ./... && go mod tidy -v 73 | 74 | .PHONY: checkgenerate 75 | checkgenerate: 76 | @# Used in CI to verify that `make generate` doesn't produce a diff. 77 | test -z "$$(git status --porcelain | tee /dev/stderr)" 78 | 79 | $(BIN)/buf: Makefile 80 | @mkdir -p $(@D) 81 | GOBIN=$(abspath $(@D)) $(GO) install github.com/bufbuild/buf/cmd/buf@$(BUF_VERSION) 82 | 83 | $(BIN)/license-header: Makefile 84 | @mkdir -p $(@D) 85 | GOBIN=$(abspath $(@D)) $(GO) install \ 86 | github.com/bufbuild/buf/private/pkg/licenseheader/cmd/license-header@$(BUF_VERSION) 87 | 88 | $(BIN)/golangci-lint: Makefile 89 | @mkdir -p $(@D) 90 | GOBIN=$(abspath $(@D)) $(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![The Buf logo](./.github/buf-logo.svg) 2 | 3 | # Prototransform 4 | 5 | [![Build](https://github.com/bufbuild/prototransform/actions/workflows/ci.yaml/badge.svg?branch=main)][badges_ci] 6 | [![Report Card](https://goreportcard.com/badge/github.com/bufbuild/prototransform)][badges_goreportcard] 7 | [![GoDoc](https://pkg.go.dev/badge/github.com/bufbuild/prototransform.svg)][badges_godoc] 8 | 9 | ### Convert protobuf message data to alternate formats 10 | 11 | Use the `prototransform` library to simplify your data transformation & 12 | collection. Our simple package allows the caller to convert a given message data 13 | blob from one format to another by referring to a type schema on the Buf Schema 14 | Registry. 15 | 16 | * No need to bake in proto files 17 | * Supports Binary, JSON and Text formats 18 | * Extensible for other/custom formats 19 | 20 | ## Getting started 21 | 22 | `prototransform` is designed to be flexible enough to fit quickly into your 23 | development environment. 24 | 25 | Here's an example of how you could use `prototransform` to transform messages 26 | received from a PubSub topic... 27 | 28 | ### Transform Messages from a Topic 29 | 30 | Whilst `prototransform` has various applications, converting messages off some 31 | kind of message queue is a primary use-case. This can take many forms, for the 32 | purposes of simplicity we will look at this abstractly in a pub/sub model where 33 | we want to: 34 | 35 | 1. Open a subscription to a topic with the Pub/Sub service of your choice 36 | 2. Start a `SchemaWatcher` to fetch a module from the Buf Schema Registry 37 | 3. Receive, Transform and Acknowledge messages from the topic 38 | 39 | #### Opening a Subscription & Schema Watcher 40 | 41 | ```go 42 | import ( 43 | "context" 44 | "fmt" 45 | 46 | "github.com/bufbuild/prototransform" 47 | "gocloud.dev/pubsub" 48 | _ "gocloud.dev/pubsub/" 49 | ) 50 | ... 51 | subs, err := pubsub.OpenSubscription(ctx, "") 52 | if err != nil { 53 | return fmt.Errorf("could not open topic subscription: %v", err) 54 | } 55 | defer subs.Shutdown(ctx) 56 | // Supply auth credentials to the BSR 57 | client := prototransform.NewDefaultFileDescriptorSetServiceClient("") 58 | // Configure the module for schema watcher 59 | cfg := &prototransform.SchemaWatcherConfig{ 60 | SchemaPoller: prototransform.NewSchemaPoller( 61 | client, 62 | "buf.build/someuser/somerepo", // BSR module 63 | "some-tag", // tag or draft name or leave blank for "latest" 64 | ), 65 | } 66 | watcher, err := prototransform.NewSchemaWatcher(ctx, cfg) 67 | if err != nil { 68 | return fmt.Errorf("failed to create schema watcher: %v", err) 69 | } 70 | defer watcher.Stop() 71 | // before we start processing messages, make sure the schema has been 72 | // downloaded 73 | ctx, cancel := context.WithTimeout(ctx, 10*time.Second) 74 | defer cancel() 75 | if err := watcher.AwaitReady(ctx); err != nil { 76 | return fmt.Errorf("schema watcher never became ready: %v", err) 77 | } 78 | ... 79 | ``` 80 | 81 | A `SchemaWatcher` is the entrypoint of `prototransform`. This is created first 82 | so your code can connect to the Buf Schema Registry and fetch a schema to be used 83 | to transform and/or filter payloads. 84 | 85 | #### Prepare a converter 86 | 87 | A `Converter` implements the functionality to convert payloads to different formats 88 | and optionally filter/mutate messages during this transformation. In the following 89 | example, we have initialized a `*prototransform.Converter` which expects a binary 90 | input and will return JSON. 91 | 92 | ```go 93 | ... 94 | converter := &prototransform.Converter{ 95 | Resolver: schemaWatcher, 96 | InputFormat: prototransform.BinaryInputFormat(proto.UnmarshalOptions{}), 97 | OutputFormat: prototransform.JSONOutputFormat(protojson.MarshalOptions{}), 98 | } 99 | ... 100 | ``` 101 | 102 | Out of the box, you can supply `proto`, `protojson` and `prototext` here but 103 | feel free to supply your own custom formats as-well. 104 | 105 | | FORMAT | InputFormat | OutputFormat | 106 | |-----------------------------------------------------------------------------------------|--------------------------------------|---------------------------------------| 107 | | [JSON](https://pkg.go.dev/google.golang.org/protobuf/encoding/protojson#MarshalOptions) | `prototransform.JSONInputFormat()` | `prototransform.JSONOutputFormat()` | 108 | | [TEXT](https://pkg.go.dev/google.golang.org/protobuf/encoding/prototext#MarshalOptions) | `prototransform.TEXTInputFormat()` | `prototransform.TEXTOutputFormat()` | 109 | | [Binary](https://pkg.go.dev/google.golang.org/protobuf/proto#MarshalOptions) | `prototransform.BinaryInputFormat()` | `prototransform.BinaryOutputFormat()` | 110 | 111 | #### Receiving and Transforming Messages 112 | 113 | Now that we have an active subscription, schema watcher, and converter, we can 114 | start processing messages. A simple subscriber that transforms received messages 115 | looks like this: 116 | 117 | ```go 118 | ... 119 | // Loop on received messages. 120 | for { 121 | msg, err := subscription.Receive(ctx) 122 | if err != nil { 123 | log.Printf("Receiving message: %v", err) 124 | break 125 | } 126 | // Do transformation based on the message name 127 | convertedMessage, err := converter.ConvertMessage("", msg.Body) 128 | if err != nil { 129 | log.Printf("Converting message: %v", err) 130 | break 131 | } 132 | fmt.Printf("Converted message: %q\n", convertedMessage) 133 | 134 | msg.Ack() 135 | } 136 | ... 137 | ``` 138 | 139 | For illustrative purposes, let's assume that the topic we have subscribed to is 140 | `buf.connect.demo.eliza.v1`, we have the module stored on the BSR [here](https://buf.build/bufbuild/eliza). 141 | We would configure the message name as `buf.connect.demo.eliza.v1.ConverseRequest`. 142 | 143 | ## Options 144 | 145 | ### Cache 146 | 147 | A `SchemaWatcher` can be configured with a user-supplied cache 148 | implementation, to act as a fallback when fetching schemas. The interface is of 149 | the form: 150 | 151 | ```go 152 | type Cache interface { 153 | Load(ctx context.Context, key string) ([]byte, error) 154 | Save(ctx context.Context, key string, data []byte) error 155 | } 156 | ``` 157 | 158 | This repo provides three implementations that you can use: 159 | 1. [`filecache`](https://pkg.go.dev/github.com/bufbuild/prototransform/cache/filecache): Cache schemas in local files. 160 | 2. [`rediscache`](https://pkg.go.dev/github.com/bufbuild/prototransform/cache/rediscache): Cache schemas in a shared Redis server. 161 | 3. [`memcache`](https://pkg.go.dev/github.com/bufbuild/prototransform/cache/memcache): Cache schemas in a shared memcached server. 162 | 163 | ### Filters 164 | 165 | A use-case exists where the values within the output message should differ from 166 | the input given some set of defined rules. For example, Personally Identifiable 167 | Information(PII) may want to be removed from a message before it is piped into a 168 | sink. For this reason, we have supplied Filters. 169 | 170 | Here's an example where we have defined a custom annotation to mark fields 171 | as `sensitive`: 172 | 173 | ```protobuf 174 | syntax = "proto3"; 175 | package foo.v1; 176 | // ... 177 | extend google.protobuf.FieldOptions { 178 | bool sensitive = 30000; 179 | } 180 | // ... 181 | message FooMessage { 182 | string name = 1 [(sensitive) = true]; 183 | } 184 | ``` 185 | 186 | We then use `prototransform.Redact()` to create a filter and 187 | supply it to our converter via its `Filters` field: 188 | 189 | ```go 190 | ... 191 | isSensitive := func (in protoreflect.FieldDescriptor) bool { 192 | return proto.GetExtension(in.Options(), foov1.E_Sensitive).(bool) 193 | } 194 | filter := prototransform.Redact(isSensitive) 195 | converter.Filters = prototransform.Filters{filter} 196 | ... 197 | ``` 198 | 199 | Now, any attribute marked as "sensitive" will be omitted from the output 200 | produced by the converter. 201 | 202 | This package also includes a predicate named `HasDebugRedactOption` that 203 | can be used to redact data for fields that have the `debug_redact` standard 204 | option set (this option was introduced in `protoc` v22.0). 205 | 206 | ## Community 207 | 208 | For help and discussion around Protobuf, best practices, and more, join us 209 | on [Slack][badges_slack]. 210 | 211 | ## Status 212 | 213 | This project is currently in **alpha**. The API should be considered unstable and likely to change. 214 | 215 | ## Legal 216 | 217 | Offered under the [Apache 2 license][license]. 218 | 219 | [badges_ci]: https://github.com/bufbuild/prototransform/actions/workflows/ci.yaml 220 | [badges_goreportcard]: https://goreportcard.com/report/github.com/bufbuild/prototransform 221 | [badges_godoc]: https://pkg.go.dev/github.com/bufbuild/prototransform 222 | [badges_slack]: https://join.slack.com/t/bufbuild/shared_invite/zt-f5k547ki-dW9LjSwEnl6qTzbyZtPojw 223 | [license]: https://github.com/bufbuild/prototransform/blob/main/LICENSE.txt 224 | -------------------------------------------------------------------------------- /cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "context" 19 | "time" 20 | 21 | prototransformv1alpha1 "github.com/bufbuild/prototransform/internal/proto/gen/buf/prototransform/v1alpha1" 22 | "google.golang.org/protobuf/proto" 23 | "google.golang.org/protobuf/types/descriptorpb" 24 | "google.golang.org/protobuf/types/known/timestamppb" 25 | ) 26 | 27 | // Cache can be implemented and supplied to prototransform for added 28 | // guarantees in environments where uptime is critical. If present and the API 29 | // call to retrieve a schema fails, the schema will instead be loaded from this 30 | // cache. Whenever a new schema is downloaded from the BSR, it will be saved to 31 | // the cache. Cache can be used from multiple goroutines and thus must be 32 | // thread-safe. 33 | type Cache interface { 34 | Load(ctx context.Context, key string) ([]byte, error) 35 | Save(ctx context.Context, key string, data []byte) error 36 | } 37 | 38 | func encodeForCache( 39 | schemaID string, 40 | syms []string, 41 | descriptors *descriptorpb.FileDescriptorSet, 42 | version string, 43 | timestamp time.Time, 44 | ) ([]byte, error) { 45 | entry := &prototransformv1alpha1.CacheEntry{ 46 | Schema: &prototransformv1alpha1.Schema{ 47 | Descriptors: descriptors, 48 | Version: version, 49 | }, 50 | SchemaTimestamp: timestamppb.New(timestamp), 51 | Id: schemaID, 52 | IncludedSymbols: syms, 53 | } 54 | return proto.Marshal(entry) 55 | } 56 | 57 | func decodeForCache(data []byte) (*prototransformv1alpha1.CacheEntry, error) { 58 | var entry prototransformv1alpha1.CacheEntry 59 | if err := proto.Unmarshal(data, &entry); err != nil { 60 | return nil, err 61 | } 62 | return &entry, nil 63 | } 64 | 65 | func isCorrectCacheEntry(entry *prototransformv1alpha1.CacheEntry, schemaID string, syms []string) bool { 66 | return entry.GetId() == schemaID && isSuperSet(entry.GetIncludedSymbols(), syms) 67 | } 68 | 69 | func isSuperSet(have, want []string) bool { 70 | if len(want) == 0 { 71 | return len(have) == 0 72 | } 73 | if len(have) < len(want) { 74 | // Technically, len(have) == 0 means it should be the full 75 | // schema and thus we should possibly return true. But there 76 | // are possible cases where "full schema, no filtering" could 77 | // actually return something other than a superset, such as 78 | // if the schema poller impl can't authoritatively enumerate 79 | // the entire schema (like for gRPC server reflection). 80 | return false 81 | } 82 | j := 0 83 | for i := range have { 84 | if have[i] == want[j] { 85 | j++ 86 | if j == len(want) { 87 | // matched them all 88 | return true 89 | } 90 | } 91 | } 92 | return false 93 | } 94 | -------------------------------------------------------------------------------- /cache/filecache/filecache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package filecache provides an implementation of prototransform.Cache 16 | // that is based on the file system. Cached data is stored in and loaded 17 | // from files, with cache keys being used to form the file names. 18 | // 19 | // This is the simplest form of caching when sharing cache results is 20 | // not needed and the workload has a persistent volume (e.g. if the 21 | // application restarts, it will still have access to the same cache 22 | // files written before it restarted). 23 | package filecache 24 | 25 | import ( 26 | "context" 27 | "encoding/hex" 28 | "errors" 29 | "fmt" 30 | "io/fs" 31 | "os" 32 | "path/filepath" 33 | "strings" 34 | 35 | "github.com/bufbuild/prototransform" 36 | ) 37 | 38 | // Config represents the configuration parameters used to 39 | // create a new file-system-backed cache. 40 | type Config struct { 41 | // Required: the folder in which cached files live. 42 | Path string 43 | // Defaults to "cache_" if left empty. This is added to the 44 | // cache key and the extension below to form a file name. 45 | // A trailing underscore is not necessary and will be added 46 | // if not present (to separate prefix from the rest of the 47 | // cache key). 48 | FilenamePrefix string 49 | // Defaults to ".bin" if left empty. This is added to the 50 | // cache key and prefix above to form a file name. 51 | FilenameExtension string 52 | // The mode to use when creating new files in the cache 53 | // directory. Defaults to 0600 if left zero. If not left 54 | // as default, the mode must have at least bits 0400 and 55 | // 0200 (read and write permissions for owner) set. 56 | FileMode fs.FileMode 57 | } 58 | 59 | // New creates a new file-system-backed cache with the given 60 | // configuration. 61 | func New(config Config) (prototransform.Cache, error) { 62 | // validate config 63 | if config.Path == "" { 64 | return nil, errors.New("path cannot be empty") 65 | } 66 | path, err := filepath.Abs(config.Path) 67 | if err != nil { 68 | return nil, err 69 | } 70 | config.Path = path 71 | if config.FilenamePrefix == "" { 72 | config.FilenamePrefix = "cache" 73 | } else { 74 | config.FilenamePrefix = strings.TrimSuffix(config.FilenamePrefix, "_") 75 | } 76 | if config.FilenameExtension == "" { 77 | config.FilenameExtension = ".bin" 78 | } else if !strings.HasPrefix(config.FilenameExtension, ".") { 79 | config.FilenameExtension = "." + config.FilenameExtension 80 | } 81 | if config.FileMode == 0 { 82 | config.FileMode = 0600 83 | } else if (config.FileMode & 0600) != 0600 { 84 | return nil, fmt.Errorf("mode %#o must include bits 0600", config.FileMode) 85 | } 86 | 87 | // make sure we can write files to cache directory 88 | info, err := os.Stat(path) 89 | if err != nil { 90 | return nil, err 91 | } 92 | if !info.IsDir() { 93 | return nil, fmt.Errorf("%s is not a directory", path) 94 | } 95 | testFile := filepath.Join(path, ".test") 96 | file, err := os.OpenFile(testFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) 97 | if err != nil { 98 | if os.IsPermission(err) { 99 | return nil, fmt.Errorf("insufficient permission to create file in %s", path) 100 | } 101 | return nil, fmt.Errorf("failed to create file in %s: %w", path, err) 102 | } 103 | closeErr := file.Close() 104 | rmErr := os.Remove(testFile) 105 | if closeErr != nil { 106 | return nil, closeErr 107 | } else if rmErr != nil { 108 | return nil, rmErr 109 | } 110 | 111 | return (*cache)(&config), nil 112 | } 113 | 114 | type cache Config 115 | 116 | func (c *cache) Load(_ context.Context, key string) ([]byte, error) { 117 | fileName := filepath.Join(c.Path, c.fileNameForKey(key)) 118 | return os.ReadFile(fileName) 119 | } 120 | 121 | func (c *cache) Save(_ context.Context, key string, data []byte) error { 122 | fileName := filepath.Join(c.Path, c.fileNameForKey(key)) 123 | return os.WriteFile(fileName, data, c.FileMode) 124 | } 125 | 126 | func (c *cache) fileNameForKey(key string) string { 127 | if key != "" { 128 | key = "_" + sanitize(key) 129 | } 130 | return c.FilenamePrefix + key + c.FilenameExtension 131 | } 132 | 133 | func sanitize(s string) string { 134 | var builder strings.Builder 135 | hexWriter := hex.NewEncoder(&builder) 136 | var buf [1]byte 137 | for i, length := 0, len(s); i < length; i++ { 138 | char := s[i] 139 | switch { 140 | case char >= 'a' && char <= 'z', 141 | char >= 'A' && char <= 'Z', 142 | char >= '0' && char <= '9', 143 | char == '.' || char == '-' || char == '_': 144 | builder.WriteByte(char) 145 | default: 146 | builder.WriteByte('%') 147 | buf[0] = char 148 | _, _ = hexWriter.Write(buf[:]) 149 | } 150 | } 151 | return builder.String() 152 | } 153 | -------------------------------------------------------------------------------- /cache/filecache/filecache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package filecache 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "io/fs" 21 | "os" 22 | "testing" 23 | 24 | "github.com/bufbuild/prototransform/cache/internal/cachetesting" 25 | "github.com/stretchr/testify/assert" 26 | "github.com/stretchr/testify/require" 27 | ) 28 | 29 | func TestFileCache(t *testing.T) { 30 | t.Parallel() 31 | 32 | testCases := []struct { 33 | name string 34 | config Config 35 | expectPrefix, expectExtension string 36 | expectMode fs.FileMode 37 | }{ 38 | { 39 | name: "default config", 40 | expectPrefix: "cache", 41 | expectExtension: "bin", 42 | expectMode: 0600, 43 | }, 44 | { 45 | name: "custom prefix with underscore", 46 | config: Config{FilenamePrefix: "abc_"}, 47 | expectPrefix: "abc", 48 | expectExtension: "bin", 49 | expectMode: 0600, 50 | }, 51 | { 52 | name: "custom prefix without underscore", 53 | config: Config{FilenamePrefix: "abc"}, 54 | expectPrefix: "abc", 55 | expectExtension: "bin", 56 | expectMode: 0600, 57 | }, 58 | { 59 | name: "custom extension", 60 | config: Config{FilenameExtension: "cdb"}, 61 | expectPrefix: "cache", 62 | expectExtension: "cdb", 63 | expectMode: 0600, 64 | }, 65 | { 66 | name: "custom mode", 67 | config: Config{FileMode: 0740}, 68 | expectPrefix: "cache", 69 | expectExtension: "bin", 70 | expectMode: 0740, 71 | }, 72 | } 73 | for _, testCase := range testCases { 74 | t.Run(testCase.name, func(t *testing.T) { 75 | t.Parallel() 76 | tmpDir := t.TempDir() 77 | testCase.config.Path = tmpDir 78 | cache, err := New(testCase.config) 79 | require.NoError(t, err) 80 | ctx := context.Background() 81 | 82 | entries := cachetesting.RunSimpleCacheTests(t, ctx, cache) 83 | files := make(map[string]struct{}, len(entries)) 84 | for k := range entries { 85 | if k == "" { 86 | files[fmt.Sprintf("%s.%s", testCase.expectPrefix, testCase.expectExtension)] = struct{}{} 87 | } else { 88 | files[fmt.Sprintf("%s_%s.%s", testCase.expectPrefix, k, testCase.expectExtension)] = struct{}{} 89 | } 90 | } 91 | 92 | // check the actual files 93 | checkFiles(t, tmpDir, testCase.expectMode, files) 94 | }) 95 | } 96 | } 97 | 98 | func TestFileCache_ConfigValidation(t *testing.T) { 99 | t.Parallel() 100 | testCases := []struct { 101 | name string 102 | config Config 103 | expectErr string 104 | }{ 105 | { 106 | name: "no path", 107 | expectErr: "path cannot be empty", 108 | }, 109 | { 110 | name: "bad path", 111 | config: Config{Path: "/some/path/that/certainly/does/not/exist/anywhere"}, 112 | expectErr: "no such file or directory", 113 | }, 114 | { 115 | name: "bad mode", 116 | config: Config{Path: "./", FileMode: 0111}, 117 | expectErr: "mode 0111 must include bits 0600", 118 | }, 119 | } 120 | for _, testCase := range testCases { 121 | t.Run(testCase.name, func(t *testing.T) { 122 | t.Parallel() 123 | _, err := New(testCase.config) 124 | require.ErrorContains(t, err, testCase.expectErr) 125 | }) 126 | } 127 | } 128 | 129 | func TestSanitize(t *testing.T) { 130 | t.Parallel() 131 | testCases := []struct { 132 | key string 133 | sanitized string 134 | }{ 135 | { 136 | key: "only-valid-chars", 137 | sanitized: "only-valid-chars", 138 | }, 139 | { 140 | key: "buf.build/foo/bar:draft-abc_23489abcf123400de", 141 | sanitized: "buf.build%2ffoo%2fbar%3adraft-abc_23489abcf123400de", 142 | }, 143 | { 144 | key: "has whitespace", 145 | sanitized: "has%20whitespace", 146 | }, 147 | { 148 | key: "other &!@#$ funny chars!", 149 | sanitized: "other%20%26%21%40%23%24%20funny%20chars%21", 150 | }, 151 | { 152 | key: "even unicode!! ↩ ↯ ψ 𝄞 😍 🌍", 153 | sanitized: "even%20unicode%21%21%20%e2%86%a9%20%e2%86%af%20%cf%88%20%f0%9d%84%9e%20%f0%9f%98%8d%20%f0%9f%8c%8d", 154 | }, 155 | } 156 | for _, testCase := range testCases { 157 | t.Run(testCase.key, func(t *testing.T) { 158 | t.Parallel() 159 | val := sanitize(testCase.key) 160 | require.Equal(t, testCase.sanitized, val) 161 | }) 162 | } 163 | } 164 | 165 | func checkFiles(t *testing.T, dir string, mode fs.FileMode, names map[string]struct{}) { 166 | t.Helper() 167 | err := fs.WalkDir(os.DirFS(dir), ".", func(path string, dirEntry fs.DirEntry, err error) error { 168 | if !assert.NoError(t, err) { //nolint:testifylint 169 | return nil 170 | } 171 | if dirEntry.IsDir() { 172 | if path == "." { 173 | return nil 174 | } 175 | t.Errorf("not expecting any sub-directories, found %s", path) 176 | return fs.SkipDir 177 | } 178 | _, ok := names[path] 179 | if !assert.Truef(t, ok, "not expecting file named %s", path) { 180 | return nil 181 | } 182 | delete(names, path) 183 | info, err := dirEntry.Info() 184 | if !assert.NoErrorf(t, err, "failed to get file info for %s", path) { //nolint:testifylint 185 | return nil 186 | } 187 | assert.Equal(t, mode, info.Mode()) 188 | return nil 189 | }) 190 | require.NoError(t, err) 191 | require.Equal(t, map[string]struct{}{}, names, "some files expected but not found") 192 | } 193 | -------------------------------------------------------------------------------- /cache/internal/cachetesting/cachetesting.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cachetesting 16 | 17 | import ( 18 | "context" 19 | "crypto/rand" 20 | "testing" 21 | 22 | "github.com/bufbuild/prototransform" 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | //nolint:revive // okay that ctx is second; prefer t to be first 27 | func RunSimpleCacheTests(t *testing.T, ctx context.Context, cache prototransform.Cache) map[string][]byte { 28 | t.Helper() 29 | 30 | // In case tests are run concurrently, we want to make sure we aren't reading 31 | // values stored to a cache by another concurrent test. While the caller of this 32 | // function should arrange for that (using different server/directory/keyspace, etc) 33 | // we want to catch accidental misuse. So for that, we generate random data so that 34 | // the values we expect are different from one call to another. 35 | 36 | const ( 37 | keyFoo = "foo" 38 | keyBar = "bar" 39 | keyEmpty = "" 40 | ) 41 | 42 | entries := make(map[string][]byte, 3) 43 | for _, k := range []string{keyFoo, keyBar, keyEmpty} { 44 | val := make([]byte, 100) 45 | _, err := rand.Read(val) 46 | require.NoError(t, err) 47 | entries[k] = val 48 | } 49 | valFoo, valBar, valEmpty := entries[keyFoo], entries[keyBar], entries[keyEmpty] 50 | 51 | // load fails since nothing exists 52 | _, err := cache.Load(ctx, keyFoo) 53 | require.Error(t, err) 54 | err = cache.Save(ctx, keyFoo, valFoo) 55 | require.NoError(t, err) 56 | loaded, err := cache.Load(ctx, keyFoo) 57 | require.NoError(t, err) 58 | require.Equal(t, valFoo, loaded) 59 | 60 | // another key 61 | _, err = cache.Load(ctx, keyBar) 62 | require.Error(t, err) 63 | err = cache.Save(ctx, keyBar, valBar) 64 | require.NoError(t, err) 65 | loaded, err = cache.Load(ctx, keyBar) 66 | require.NoError(t, err) 67 | require.Equal(t, valBar, loaded) 68 | 69 | // original key unchanged 70 | loaded, err = cache.Load(ctx, keyFoo) 71 | require.NoError(t, err) 72 | require.Equal(t, valFoo, loaded) 73 | 74 | // empty key 75 | err = cache.Save(ctx, keyEmpty, valEmpty) 76 | require.NoError(t, err) 77 | loaded, err = cache.Load(ctx, keyEmpty) 78 | require.NoError(t, err) 79 | require.Equal(t, valEmpty, loaded) 80 | 81 | return entries 82 | } 83 | -------------------------------------------------------------------------------- /cache/memcache/memcache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package memcache provides an implementation of prototransform.Cache 16 | // that is backed by a memcached instance: https://memcached.org/. 17 | package memcache 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "fmt" 23 | 24 | "github.com/bradfitz/gomemcache/memcache" 25 | "github.com/bufbuild/prototransform" 26 | ) 27 | 28 | // Config represents the configuration parameters used to 29 | // create a new memcached-backed cache. 30 | type Config struct { 31 | Client *memcache.Client 32 | KeyPrefix string 33 | ExpirationSeconds int32 34 | } 35 | 36 | // New creates a new memcached-backed cache with the given configuration. 37 | func New(config Config) (prototransform.Cache, error) { 38 | // validate config 39 | if config.Client == nil { 40 | return nil, errors.New("client cannot be nil") 41 | } 42 | if config.ExpirationSeconds < 0 { 43 | return nil, fmt.Errorf("expiration seconds (%d) cannot be negative", config.ExpirationSeconds) 44 | } 45 | return (*cache)(&config), nil 46 | } 47 | 48 | type cache Config 49 | 50 | func (c *cache) Load(_ context.Context, key string) ([]byte, error) { 51 | item, err := c.Client.Get(c.KeyPrefix + key) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return item.Value, nil 56 | } 57 | 58 | func (c *cache) Save(_ context.Context, key string, data []byte) error { 59 | item := &memcache.Item{ 60 | Key: c.KeyPrefix + key, 61 | Value: data, 62 | Expiration: c.ExpirationSeconds, 63 | } 64 | return c.Client.Set(item) 65 | } 66 | -------------------------------------------------------------------------------- /cache/memcache/memcache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // We use a build tag since memcached may not be running. If it is 16 | // running, for use with the test, then pass flag "-tags with_servers" 17 | // when running tests to enable these tests. 18 | //go:build with_servers 19 | 20 | package memcache 21 | 22 | import ( 23 | "context" 24 | "crypto/rand" 25 | "encoding/base64" 26 | "testing" 27 | "time" 28 | 29 | "github.com/bradfitz/gomemcache/memcache" 30 | "github.com/bufbuild/prototransform/cache/internal/cachetesting" 31 | "github.com/stretchr/testify/require" 32 | ) 33 | 34 | func TestMemcache(t *testing.T) { 35 | t.Parallel() 36 | 37 | client := memcache.New("localhost:11211") 38 | testCases := []struct { 39 | name string 40 | expirySeconds int32 41 | }{ 42 | { 43 | name: "without expiry", 44 | }, 45 | { 46 | name: "with expiry", 47 | // this needs to be small, because we actually sleep this long to 48 | // verify that keys expire (since there's no way in memcache to 49 | // just query for the ttl) 50 | expirySeconds: 2, 51 | }, 52 | } 53 | for _, testCase := range testCases { 54 | testCase := testCase 55 | t.Run(testCase.name, func(t *testing.T) { 56 | t.Parallel() 57 | 58 | prefix := make([]byte, 24) 59 | _, err := rand.Read(prefix) 60 | require.NoError(t, err) 61 | keyPrefix := base64.StdEncoding.EncodeToString(prefix) + ":" 62 | 63 | cache, err := New(Config{ 64 | Client: client, 65 | KeyPrefix: keyPrefix, 66 | ExpirationSeconds: testCase.expirySeconds, 67 | }) 68 | require.NoError(t, err) 69 | ctx := context.Background() 70 | 71 | entries := cachetesting.RunSimpleCacheTests(t, ctx, cache) 72 | 73 | // check the actual keys in memcache 74 | checkKeys(t, client, keyPrefix, true, entries) 75 | if testCase.expirySeconds != 0 { 76 | time.Sleep(time.Duration(testCase.expirySeconds) * time.Second) // let them all expire 77 | checkKeys(t, client, keyPrefix, false, entries) 78 | } 79 | }) 80 | } 81 | } 82 | 83 | func checkKeys(t *testing.T, client *memcache.Client, keyPrefix string, present bool, entries map[string][]byte) { 84 | for k, v := range entries { 85 | item, err := client.Get(keyPrefix + k) 86 | if present { 87 | require.NoError(t, err) 88 | require.Equal(t, v, item.Value) 89 | } else { 90 | require.ErrorIs(t, err, memcache.ErrCacheMiss) 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /cache/rediscache/rediscache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package rediscache provides an implementation of prototransform.Cache 16 | // that is backed by a Redis instance: https://redis.io/. 17 | package rediscache 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "fmt" 23 | "time" 24 | 25 | "github.com/bufbuild/prototransform" 26 | "github.com/gomodule/redigo/redis" 27 | ) 28 | 29 | // Config represents the configuration parameters used to 30 | // create a new Redis-backed cache. 31 | type Config struct { 32 | Client *redis.Pool 33 | KeyPrefix string 34 | Expiration time.Duration 35 | } 36 | 37 | // New creates a new Redis-backed cache with the given configuration. 38 | func New(config Config) (prototransform.Cache, error) { 39 | // validate config 40 | if config.Client == nil { 41 | return nil, errors.New("client cannot be nil") 42 | } 43 | if config.Expiration < 0 { 44 | return nil, fmt.Errorf("expiration (%v) cannot be negative", config.Expiration) 45 | } 46 | return (*cache)(&config), nil 47 | } 48 | 49 | type cache Config 50 | 51 | func (c *cache) Load(ctx context.Context, key string) ([]byte, error) { 52 | conn, err := c.Client.GetContext(ctx) 53 | if err != nil { 54 | return nil, err 55 | } 56 | defer func() { 57 | _ = conn.Close() 58 | }() 59 | return redis.Bytes(redis.DoContext(conn, ctx, "get", c.KeyPrefix+key)) 60 | } 61 | 62 | func (c *cache) Save(ctx context.Context, key string, data []byte) error { 63 | conn, err := c.Client.GetContext(ctx) 64 | if err != nil { 65 | return err 66 | } 67 | defer func() { 68 | _ = conn.Close() 69 | }() 70 | 71 | args := []any{c.KeyPrefix + key, data} 72 | if c.Expiration != 0 { 73 | millis := int(c.Expiration.Milliseconds()) 74 | if millis > 0 { 75 | args = append(args, "px", millis) 76 | } 77 | } 78 | _, err = redis.DoContext(conn, ctx, "set", args...) 79 | return err 80 | } 81 | -------------------------------------------------------------------------------- /cache/rediscache/rediscache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // We use a build tag since redis may not be running. If it is 16 | // running, for use with the test, then pass flag "-tags with_servers" 17 | // when running tests to enable these tests. 18 | //go:build with_servers 19 | 20 | package rediscache 21 | 22 | import ( 23 | "context" 24 | "crypto/rand" 25 | "encoding/base64" 26 | "testing" 27 | "time" 28 | 29 | "github.com/bufbuild/prototransform/cache/internal/cachetesting" 30 | "github.com/gomodule/redigo/redis" 31 | "github.com/stretchr/testify/require" 32 | ) 33 | 34 | func TestRedisCache(t *testing.T) { 35 | t.Parallel() 36 | 37 | pool := &redis.Pool{ 38 | DialContext: func(ctx context.Context) (redis.Conn, error) { 39 | return redis.Dial("tcp", "localhost:6379") 40 | }, 41 | } 42 | t.Cleanup(func() { 43 | err := pool.Close() 44 | require.NoError(t, err) 45 | }) 46 | 47 | testCases := []struct { 48 | name string 49 | expiry time.Duration 50 | }{ 51 | { 52 | name: "without expiry", 53 | }, 54 | { 55 | name: "with expiry", 56 | expiry: 60 * time.Second, 57 | }, 58 | } 59 | for _, testCase := range testCases { 60 | testCase := testCase 61 | t.Run(testCase.name, func(t *testing.T) { 62 | t.Parallel() 63 | 64 | prefix := make([]byte, 24) 65 | _, err := rand.Read(prefix) 66 | require.NoError(t, err) 67 | keyPrefix := base64.StdEncoding.EncodeToString(prefix) + ":" 68 | 69 | cache, err := New(Config{ 70 | Client: pool, 71 | KeyPrefix: keyPrefix, 72 | Expiration: testCase.expiry, 73 | }) 74 | require.NoError(t, err) 75 | ctx := context.Background() 76 | 77 | entries := cachetesting.RunSimpleCacheTests(t, ctx, cache) 78 | 79 | // check the actual keys in memcache 80 | checkKeys(t, pool.Get(), keyPrefix, testCase.expiry, entries) 81 | }) 82 | } 83 | } 84 | 85 | func checkKeys(t *testing.T, client redis.Conn, keyPrefix string, ttl time.Duration, entries map[string][]byte) { 86 | for k, v := range entries { 87 | data, err := redis.Bytes(client.Do("get", keyPrefix+k)) 88 | require.NoError(t, err) 89 | require.Equal(t, v, data) 90 | if ttl != 0 { 91 | reply, err := client.Do("pttl", keyPrefix+k) 92 | require.NoError(t, err) 93 | ttlMillis, ok := reply.(int64) 94 | require.True(t, ok) 95 | actualTTL := time.Duration(ttlMillis) * time.Millisecond 96 | require.LessOrEqual(t, actualTTL, ttl) 97 | // 2 second window for TTL to have decreased: generous to accommodate slow/flaky CI machines 98 | require.Greater(t, actualTTL, ttl-2*time.Second) 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /cache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | prototransformv1alpha1 "github.com/bufbuild/prototransform/internal/proto/gen/buf/prototransform/v1alpha1" 22 | "github.com/stretchr/testify/assert" 23 | "github.com/stretchr/testify/require" 24 | "google.golang.org/protobuf/proto" 25 | "google.golang.org/protobuf/types/descriptorpb" 26 | "google.golang.org/protobuf/types/known/timestamppb" 27 | ) 28 | 29 | func TestCacheEntryRoundTrip(t *testing.T) { 30 | t.Parallel() 31 | descriptors := &descriptorpb.FileDescriptorSet{ 32 | File: []*descriptorpb.FileDescriptorProto{ 33 | { 34 | Name: proto.String("test.proto"), 35 | Package: proto.String("test"), 36 | }, 37 | { 38 | Name: proto.String("foo.proto"), 39 | Package: proto.String("foo"), 40 | Dependency: []string{"test.proto"}, 41 | MessageType: []*descriptorpb.DescriptorProto{ 42 | { 43 | Name: proto.String("Foo"), 44 | }, 45 | }, 46 | }, 47 | }, 48 | } 49 | version := "abcdefg" 50 | respTime := time.Date(2023, time.January, 1, 12, 0, 0, 0, time.UTC) 51 | data, err := encodeForCache("123", []string{"a1", "b2", "c3"}, descriptors, version, respTime) 52 | require.NoError(t, err) 53 | entry, err := decodeForCache(data) 54 | require.NoError(t, err) 55 | expectedEntry := &prototransformv1alpha1.CacheEntry{ 56 | Schema: &prototransformv1alpha1.Schema{ 57 | Descriptors: descriptors, 58 | Version: version, 59 | }, 60 | Id: "123", 61 | IncludedSymbols: []string{"a1", "b2", "c3"}, 62 | SchemaTimestamp: timestamppb.New(respTime), 63 | } 64 | assert.True(t, proto.Equal(expectedEntry, entry)) 65 | } 66 | 67 | func TestIsSuperSet(t *testing.T) { 68 | t.Parallel() 69 | t.Run("both empty", func(t *testing.T) { 70 | t.Parallel() 71 | assert.True(t, isSuperSet(nil, nil)) 72 | assert.True(t, isSuperSet([]string{}, nil)) 73 | assert.True(t, isSuperSet(nil, make([]string, 0, 10))) 74 | }) 75 | t.Run("superset is empty", func(t *testing.T) { 76 | t.Parallel() 77 | assert.False(t, isSuperSet(nil, []string{"abc"})) 78 | }) 79 | t.Run("subset is empty", func(t *testing.T) { 80 | t.Parallel() 81 | assert.False(t, isSuperSet([]string{"abc"}, nil)) 82 | }) 83 | t.Run("same", func(t *testing.T) { 84 | t.Parallel() 85 | assert.True(t, isSuperSet([]string{"abc", "def", "ghi", "xyz"}, []string{"abc", "def", "ghi", "xyz"})) 86 | }) 87 | t.Run("is superset (1)", func(t *testing.T) { 88 | t.Parallel() 89 | assert.True(t, isSuperSet([]string{"abc", "def", "ghi", "xyz"}, []string{"abc"})) 90 | }) 91 | t.Run("is superset (2)", func(t *testing.T) { 92 | t.Parallel() 93 | assert.True(t, isSuperSet([]string{"abc", "def", "ghi", "xyz"}, []string{"abc", "ghi"})) 94 | }) 95 | t.Run("is superset (3)", func(t *testing.T) { 96 | t.Parallel() 97 | assert.True(t, isSuperSet([]string{"abc", "def", "ghi", "xyz"}, []string{"abc", "xyz"})) 98 | }) 99 | t.Run("is superset (4)", func(t *testing.T) { 100 | t.Parallel() 101 | assert.True(t, isSuperSet([]string{"abc", "def", "ghi", "xyz"}, []string{"xyz"})) 102 | }) 103 | t.Run("is subset (1)", func(t *testing.T) { 104 | t.Parallel() 105 | assert.False(t, isSuperSet([]string{"abc"}, []string{"abc", "def", "ghi", "xyz"})) 106 | }) 107 | t.Run("is subset (2)", func(t *testing.T) { 108 | t.Parallel() 109 | assert.False(t, isSuperSet([]string{"abc", "ghi"}, []string{"abc", "def", "ghi", "xyz"})) 110 | }) 111 | t.Run("is subset (3)", func(t *testing.T) { 112 | t.Parallel() 113 | assert.False(t, isSuperSet([]string{"abc", "xyz"}, []string{"abc", "def", "ghi", "xyz"})) 114 | }) 115 | t.Run("is subset (4)", func(t *testing.T) { 116 | t.Parallel() 117 | assert.False(t, isSuperSet([]string{"xyz"}, []string{"abc", "def", "ghi", "xyz"})) 118 | }) 119 | } 120 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "time" 21 | 22 | "google.golang.org/protobuf/reflect/protoreflect" 23 | ) 24 | 25 | const defaultPollingPeriod = 5 * time.Minute 26 | 27 | // SchemaWatcherConfig contains the configurable attributes of the [SchemaWatcher]. 28 | type SchemaWatcherConfig struct { 29 | // The downloader of descriptors. See [NewSchemaPoller]. 30 | SchemaPoller SchemaPoller 31 | // The symbols that should be included in the downloaded schema. These must be 32 | // the fully-qualified names of elements in the schema, which can include 33 | // packages, messages, enums, extensions, services, and methods. If specified, 34 | // the downloaded schema will only include descriptors to describe these symbols. 35 | // If left empty, the entire schema will be downloaded. 36 | IncludeSymbols []string 37 | // The period of the polling the BSR for new versions is specified by the 38 | // PollingPeriod argument. The PollingPeriod will adjust the time interval. 39 | // The duration must be greater than zero; if not, [NewSchemaWatcher] will 40 | // return an error. If unset and left zero, a default period of 5 minutes 41 | // is used. 42 | PollingPeriod time.Duration 43 | // A number between 0 and 1 that represents the amount of jitter to add to 44 | // the polling period. A value of zero means no jitter. A value of one means 45 | // up to 100% jitter, so the actual period would be between 0 and 2*PollingPeriod. 46 | // To prevent self-synchronization (and thus thundering herds) when there are 47 | // multiple pollers, a value of 0.1 to 0.3 is typical. 48 | Jitter float64 49 | // If Cache is non-nil, it is used for increased robustness, even in the 50 | // face of the remote schema registry being unavailable. If non-nil and the 51 | // API call to initially retrieve a schema fails, the schema will instead 52 | // be loaded from this cache. Whenever a new schema is downloaded from the 53 | // remote registry, it will be saved to the cache. So if the process is 54 | // restarted and the remote registry is unavailable, the latest cached schema 55 | // can still be used. 56 | Cache Cache 57 | // If Leaser is non-nil, it is used to decide whether the current process 58 | // can poll for the schema. Cache must be non-nil. This is useful when the 59 | // schema source is a remote process, and the current process is replicated 60 | // (e.g. many instances running the same workload, for redundancy and/or 61 | // capacity). This prevents all the replicas from polling. Instead, a single 62 | // replica will "own" the lease and poll for the schema. It will then store 63 | // the downloaded schema in the shared cache. A replica that does not have 64 | // the lease will look only in the cache instead of polling the remote 65 | // source. 66 | Leaser Leaser 67 | // CurrentProcess is an optional identifier for the current process. This 68 | // is only used if Leaser is non-nil. If present, this value is used to 69 | // identify the current process as the leaseholder. If not present, a 70 | // default value will be computed using the current process's PID and the 71 | // host name and network addresses of the current host. If present, this 72 | // value must be unique for all other processes that might try to acquire 73 | // the same lease. 74 | CurrentProcess []byte 75 | // OnUpdate is an optional callback that will be invoked when a new schema 76 | // is fetched. This can be used by an application to take action when a new 77 | // schema becomes available. 78 | OnUpdate func(*SchemaWatcher) 79 | // OnError is an optional callback that will be invoked when a schema cannot 80 | // be fetched. This could be due to the SchemaPoller returning an error or 81 | // failure to convert the fetched descriptors into a resolver. 82 | OnError func(*SchemaWatcher, error) 83 | } 84 | 85 | func (c *SchemaWatcherConfig) validate() error { 86 | if c.SchemaPoller == nil { 87 | return errors.New("schema poller not provided") 88 | } 89 | if c.PollingPeriod < 0 { 90 | return errors.New("polling period duration cannot be negative") 91 | } 92 | if c.Jitter < 0 { 93 | return errors.New("jitter cannot be negative") 94 | } 95 | if c.Jitter > 1.0 { 96 | return errors.New("jitter cannot be greater than 1.0 (100%)") 97 | } 98 | if c.Leaser != nil && c.Cache == nil { 99 | return errors.New("leaser config should only be present when cache config also present") 100 | } 101 | if c.Leaser != nil && c.CurrentProcess != nil && len(c.CurrentProcess) == 0 { 102 | return errors.New("current process is empty but not nil; leave nil or set to non-empty value") 103 | } 104 | for _, sym := range c.IncludeSymbols { 105 | if sym == "" { 106 | // Counter-intuitively, empty string is valid in this context as it 107 | // indicates the default/unnamed package. Requesting it will include 108 | // all files in the module that are defined without a package. 109 | continue 110 | } 111 | if !protoreflect.FullName(sym).IsValid() { 112 | return fmt.Errorf("%q is not a valid symbol name", sym) 113 | } 114 | } 115 | return nil 116 | } 117 | -------------------------------------------------------------------------------- /converter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/pkg/errors" 21 | "google.golang.org/protobuf/reflect/protoreflect" 22 | "google.golang.org/protobuf/types/dynamicpb" 23 | ) 24 | 25 | // Converter allows callers to convert byte payloads from one format to another. 26 | type Converter struct { 27 | // A custom [Resolver] can be supplied with the InputFormat [Unmarshaler] and 28 | // OutputFormat [Marshaler] for looking up types when expanding 29 | // google.protobuf.Any messages. As such, this is likely only needed in cases 30 | // where extensions may be present. For [proto], [protojson], and [prototext] 31 | // marshalers and unmarshalers are already handled so there is no need to 32 | // provide a WithResolver method. If nil, this defaults to using 33 | // protoregistry.GlobalTypes. 34 | Resolver Resolver 35 | // InputFormat handles unmarshaling bytes from the expected input format. 36 | // You can use a [proto.Unmarshaler], [protojson.Unmarshaler], or 37 | // [prototext.Unmarshaler] as a value for this field. You can also supply 38 | // your own custom format that implements the [Unmarshaler] interface. If 39 | // your custom format needs a [Resolver] (e.g. to resolve types in a 40 | // google.protobuf.Any message or to resolve extensions), then your custom 41 | // type should provide a method with the following signature: 42 | // WithResolver(Resolver) Unmarshaler 43 | // This method should return a new unmarshaler that will make use of the 44 | // given resolver. 45 | InputFormat InputFormat 46 | // OutputFormat handles marshaling to bytes in the desired output format. 47 | // You can use a [proto.Marshaler], [protojson.Marshaler], or 48 | // [prototext.Marshaler] as a value for this field. You can also supply 49 | // your own custom format that implements the [Marshaler] interface. If 50 | // your custom format needs a [Resolver] (e.g. to format types in a 51 | // google.protobuf.Any message), then your custom type should provide 52 | // a method with the following signature: 53 | // WithResolver(Resolver) Marshaler 54 | // This method should return a new marshaler that will make use of the 55 | // given resolver. 56 | OutputFormat OutputFormat 57 | // Filters are a set of user-supplied actions which will be performed on a 58 | // [ConvertMessage] call before the conversion takes place, meaning the 59 | // output value can be modified according to some set of rules. 60 | Filters Filters 61 | } 62 | 63 | // ConvertMessage allows the caller to convert a given message data blob from 64 | // one format to another by referring to a type schema for the blob. 65 | func (c *Converter) ConvertMessage(messageName string, inputData []byte) ([]byte, error) { 66 | md, err := c.Resolver.FindMessageByName(protoreflect.FullName(messageName)) 67 | if err != nil { 68 | return nil, errors.Wrapf(err, "message_name '%s' is not found in proto", messageName) 69 | } 70 | msg := dynamicpb.NewMessage(md.Descriptor()) 71 | if err := c.InputFormat.WithResolver(c.Resolver).Unmarshal(inputData, msg); err != nil { 72 | return nil, fmt.Errorf("input_data cannot be unmarshaled to %s in %s: %w", messageName, c.InputFormat, err) 73 | } 74 | 75 | // apply filters 76 | c.Filters.do(msg) 77 | 78 | data, err := c.OutputFormat.WithResolver(c.Resolver).Marshal(msg) 79 | if err != nil { 80 | return nil, errors.Wrapf(err, "input message cannot be unmarshaled to %s", c.InputFormat) 81 | } 82 | return data, nil 83 | } 84 | -------------------------------------------------------------------------------- /converter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/google/go-cmp/cmp" 22 | "github.com/stretchr/testify/require" 23 | "google.golang.org/protobuf/encoding/protojson" 24 | "google.golang.org/protobuf/encoding/prototext" 25 | "google.golang.org/protobuf/proto" 26 | "google.golang.org/protobuf/reflect/protodesc" 27 | "google.golang.org/protobuf/reflect/protoreflect" 28 | "google.golang.org/protobuf/reflect/protoregistry" 29 | "google.golang.org/protobuf/testing/protocmp" 30 | "google.golang.org/protobuf/types/dynamicpb" 31 | ) 32 | 33 | func TestSchemaService_ConvertMessage(t *testing.T) { 34 | t.Parallel() 35 | // create schema for message to convert 36 | sourceFile := fakeFileDescriptorSet().GetFile()[0] 37 | 38 | // create test message 39 | fileDescriptor, err := protodesc.NewFile(sourceFile, nil) 40 | require.NoError(t, err) 41 | messageDescriptor := fileDescriptor.Messages().Get(0) 42 | message := dynamicpb.NewMessage(messageDescriptor) 43 | message.Set(messageDescriptor.Fields().ByNumber(1), protoreflect.ValueOfString("abcdef")) 44 | message.Set(messageDescriptor.Fields().ByNumber(2), protoreflect.ValueOfInt64(12345678)) 45 | list := message.Mutable(messageDescriptor.Fields().ByNumber(3)).List() 46 | list.Append(protoreflect.ValueOfMessage(message.New())) 47 | list.Append(protoreflect.ValueOfMessage(message.New())) 48 | list.Append(protoreflect.ValueOfMessage(message.New())) 49 | message.Set(messageDescriptor.Fields().ByNumber(4), protoreflect.ValueOfEnum(3)) 50 | 51 | formats := []struct { 52 | name string 53 | outputFormat OutputFormat 54 | inputFormat InputFormat 55 | }{ 56 | { 57 | name: "binary", 58 | outputFormat: BinaryOutputFormat(proto.MarshalOptions{}), 59 | inputFormat: BinaryInputFormat(proto.UnmarshalOptions{}), 60 | }, 61 | { 62 | name: "json", 63 | outputFormat: JSONOutputFormat(protojson.MarshalOptions{}), 64 | inputFormat: JSONInputFormat(protojson.UnmarshalOptions{}), 65 | }, 66 | { 67 | name: "text", 68 | outputFormat: TextOutputFormat(prototext.MarshalOptions{}), 69 | inputFormat: TextInputFormat(prototext.UnmarshalOptions{}), 70 | }, 71 | { 72 | name: "TextWithoutResolver", 73 | outputFormat: OutputFormatWithoutResolver(prototext.MarshalOptions{}), 74 | inputFormat: InputFormatWithoutResolver(prototext.UnmarshalOptions{}), 75 | }, 76 | { 77 | name: "custom", 78 | outputFormat: marshalProtoJSONWithResolver{}, 79 | inputFormat: unmarshalProtoJSONWithResolver{}, 80 | }, 81 | } 82 | 83 | resolver := &protoregistry.Types{} 84 | for i := range fileDescriptor.Messages().Len() { 85 | message := dynamicpb.NewMessageType(fileDescriptor.Messages().Get(i)) 86 | require.NoError(t, resolver.RegisterMessage(message)) 87 | } 88 | for i := range fileDescriptor.Enums().Len() { 89 | enum := dynamicpb.NewEnumType(fileDescriptor.Enums().Get(i)) 90 | require.NoError(t, resolver.RegisterEnum(enum)) 91 | } 92 | for i := range fileDescriptor.Extensions().Len() { 93 | extension := dynamicpb.NewExtensionType(fileDescriptor.Extensions().Get(i)) 94 | require.NoError(t, resolver.RegisterExtension(extension)) 95 | } 96 | 97 | for _, inFormat := range formats { 98 | inputFormat := inFormat 99 | for _, outFormat := range formats { 100 | outputFormat := outFormat 101 | t.Run(fmt.Sprintf("%v_to_%v", inputFormat.name, outputFormat.name), func(t *testing.T) { 102 | t.Parallel() 103 | data, err := inputFormat.outputFormat.WithResolver(nil).Marshal(message) 104 | require.NoError(t, err) 105 | 106 | converter := Converter{ 107 | Resolver: resolver, 108 | InputFormat: inputFormat.inputFormat, 109 | OutputFormat: outputFormat.outputFormat, 110 | } 111 | require.NoError(t, err) 112 | resp, err := converter.ConvertMessage("foo.bar.Message", data) 113 | require.NoError(t, err) 114 | clone := message.New().Interface() 115 | err = outputFormat.inputFormat.WithResolver(nil).Unmarshal(resp, clone) 116 | require.NoError(t, err) 117 | diff := cmp.Diff(message, clone, protocmp.Transform()) 118 | if diff != "" { 119 | t.Errorf("round-trip failure (-want +got):\n%s", diff) 120 | } 121 | }) 122 | } 123 | } 124 | } 125 | 126 | type marshalProtoJSONWithResolver struct { 127 | protojson.MarshalOptions 128 | } 129 | 130 | func (p marshalProtoJSONWithResolver) WithResolver(r Resolver) Marshaler { 131 | return protojson.MarshalOptions{ 132 | Resolver: r, 133 | } 134 | } 135 | 136 | type unmarshalProtoJSONWithResolver struct { 137 | protojson.UnmarshalOptions 138 | } 139 | 140 | func (p unmarshalProtoJSONWithResolver) WithResolver(r Resolver) Unmarshaler { 141 | return protojson.UnmarshalOptions{ 142 | Resolver: r, 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package prototransform will fetch purpose-built descriptor sets on the run 16 | // and easily converting protobuf messages into human-readable formats. 17 | // 18 | // Use the [prototransform] library to simplify your data transformation & 19 | // collection. Our simple package allows the caller to convert a given message data 20 | // blob from one format to another by referring to a type schema on the Buf Schema 21 | // Registry. 22 | // 23 | // The package supports to and from Binary, JSON and Text formats out of the box, 24 | // extensible for other/custom formats also. 25 | // 26 | // The Buf Schema Registry Schema API builds an integration that can easily make 27 | // use of your protobuf messages in new ways. This package will reduce your 28 | // serialization down to exactly what you need and forget about everything else 29 | // 30 | // Some advantages of using [prototransform] include: Automatic version handling, 31 | // No baking proto files into containers, No flaky fetching logic, get only the 32 | // descriptors you need. 33 | package prototransform 34 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform_test 16 | 17 | import ( 18 | "context" 19 | "encoding/hex" 20 | "fmt" 21 | "log" 22 | "time" 23 | 24 | "github.com/bufbuild/prototransform" 25 | "google.golang.org/protobuf/encoding/protojson" 26 | "google.golang.org/protobuf/proto" 27 | ) 28 | 29 | var inputData = []byte(`{"sentence": "I feel happy."}`) 30 | 31 | const ( 32 | messageName = "buf.connect.demo.eliza.v1.SayRequest" 33 | moduleName = "buf.build/bufbuild/eliza" 34 | ) 35 | 36 | func Example() { 37 | token, err := prototransform.BufTokenFromEnvironment(moduleName) 38 | if err != nil { 39 | log.Fatalf("Failed to get token from environment: %v\n"+ 40 | "For help with authenticating with the Buf Schema Registry visit: https://docs.buf.build/bsr/authentication", 41 | err) 42 | } 43 | // Supply auth credentials to the BSR 44 | client := prototransform.NewDefaultFileDescriptorSetServiceClient(token) 45 | // Configure the module for schema watcher 46 | cfg := &prototransform.SchemaWatcherConfig{ 47 | SchemaPoller: prototransform.NewSchemaPoller( 48 | client, 49 | moduleName, // BSR module 50 | "main", // tag or draft name or leave blank for "latest" 51 | ), 52 | } 53 | ctx := context.Background() 54 | schemaWatcher, err := prototransform.NewSchemaWatcher(ctx, cfg) 55 | if err != nil { 56 | log.Fatalf("failed to create schema watcher: %v", err) 57 | return 58 | } 59 | // The BSR imposes a rate limit, so that multiple concurrent CI jobs can tickle it 60 | // and then cause this next call to fail because all calls get rejected with a 61 | // "resource exhausted" error. So that's why we have a large timeout of a whole 62 | // minute: eventually, it will succeed, even if we get rate-limited due to other 63 | // concurrent CI jobs hitting the same API with the same token. 64 | ctx, cancel := context.WithTimeout(ctx, time.Minute) 65 | defer cancel() 66 | if err := schemaWatcher.AwaitReady(ctx); err != nil { 67 | log.Fatalf("schema watcher never became ready: %v", err) 68 | return 69 | } 70 | converter := &prototransform.Converter{ 71 | Resolver: schemaWatcher, 72 | InputFormat: prototransform.JSONInputFormat(protojson.UnmarshalOptions{}), 73 | OutputFormat: prototransform.BinaryOutputFormat(proto.MarshalOptions{}), 74 | } 75 | convertedMessage, err := converter.ConvertMessage(messageName, inputData) 76 | if err != nil { 77 | log.Fatalf("Converting message: %v\n", err) 78 | return 79 | } 80 | fmt.Printf("Converted message: 0x%s\n", hex.EncodeToString(convertedMessage)) 81 | // Output: Converted message: 0x0a0d49206665656c2068617070792e 82 | } 83 | -------------------------------------------------------------------------------- /filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "google.golang.org/protobuf/reflect/protoreflect" 19 | "google.golang.org/protobuf/types/descriptorpb" 20 | ) 21 | 22 | // Filters is a slice of filters. When there is more than one element, they 23 | // are applied in order. In other words, the first filter is evaluated first. 24 | // The result of that is then provided as input to the second, and so on. 25 | type Filters []Filter 26 | 27 | func (f Filters) do(message protoreflect.Message) { 28 | for _, filter := range f { 29 | filter(message) 30 | } 31 | } 32 | 33 | // Filter provides a way for user-provided logic to alter the message being converted. It can 34 | // return a derived message (which could even be a different type), or it can mutate the given 35 | // message and return it. 36 | type Filter func(protoreflect.Message) protoreflect.Message 37 | 38 | // Redact returns a Filter that will remove information from a message. It invokes 39 | // the given predicate for each field in the message (including in any nested 40 | // messages) and _removes_ the field and corresponding value if the predicate 41 | // returns true. This can be used to remove sensitive data from a message, for example. 42 | func Redact(predicate func(protoreflect.FieldDescriptor) bool) Filter { 43 | return func(msg protoreflect.Message) protoreflect.Message { 44 | redactMessage(msg, predicate) 45 | return msg 46 | } 47 | } 48 | 49 | // HasDebugRedactOption returns a function that can be used as a predicate, with 50 | // [Redact], to omit fields where the `debug_redact` field option is set to true. 51 | // 52 | // message UserDetails { 53 | // int64 user_id = 1; 54 | // string name = 2; 55 | // string email = 4; 56 | // string ssn = 3 [debug_redact=true]; // social security number is sensitive 57 | // } 58 | func HasDebugRedactOption(fd protoreflect.FieldDescriptor) bool { 59 | opts, ok := fd.Options().(*descriptorpb.FieldOptions) 60 | return ok && opts.GetDebugRedact() 61 | } 62 | 63 | func redactMessage(message protoreflect.Message, redaction func(protoreflect.FieldDescriptor) bool) { 64 | message.Range( 65 | func(descriptor protoreflect.FieldDescriptor, value protoreflect.Value) bool { 66 | if redaction(descriptor) { 67 | message.Clear(descriptor) 68 | return true 69 | } 70 | switch { 71 | case descriptor.IsMap() && isMessage(descriptor.MapValue()): 72 | redactMap(value, redaction) 73 | case descriptor.IsList() && isMessage(descriptor): 74 | redactList(value, redaction) 75 | case !descriptor.IsMap() && isMessage(descriptor): 76 | // isMessage by itself returns true for maps, since the type is 77 | // a synthetic map entry message, so we also need !IsMap in 78 | // this case's criteria. 79 | redactMessage(value.Message(), redaction) 80 | } 81 | return true 82 | }, 83 | ) 84 | } 85 | 86 | func redactList(value protoreflect.Value, redaction func(protoreflect.FieldDescriptor) bool) { 87 | for i := range value.List().Len() { 88 | redactMessage(value.List().Get(i).Message(), redaction) 89 | } 90 | } 91 | 92 | func redactMap(value protoreflect.Value, redaction func(protoreflect.FieldDescriptor) bool) { 93 | value.Map().Range(func(_ protoreflect.MapKey, mapValue protoreflect.Value) bool { 94 | redactMessage(mapValue.Message(), redaction) 95 | return true 96 | }) 97 | } 98 | 99 | func isMessage(descriptor protoreflect.FieldDescriptor) bool { 100 | return descriptor.Kind() == protoreflect.MessageKind || 101 | descriptor.Kind() == protoreflect.GroupKind 102 | } 103 | -------------------------------------------------------------------------------- /filter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "testing" 19 | 20 | foov1 "github.com/bufbuild/prototransform/internal/testdata/gen/foo/v1" 21 | "github.com/stretchr/testify/assert" 22 | "google.golang.org/protobuf/proto" 23 | "google.golang.org/protobuf/reflect/protoreflect" 24 | ) 25 | 26 | func TestRedact(t *testing.T) { 27 | t.Parallel() 28 | t.Run("RedactedMessage", func(t *testing.T) { 29 | t.Parallel() 30 | input := &foov1.RedactedMessage{ 31 | Name: "sensitiveInformation", 32 | } 33 | got := Redact(removeSensitiveData())(input.ProtoReflect()) 34 | want := &foov1.RedactedMessage{} 35 | assert.True(t, proto.Equal(want, got.Interface())) 36 | }) 37 | t.Run("RedactedMessageField", func(t *testing.T) { 38 | t.Parallel() 39 | input := &foov1.RedactedMessageField{ 40 | Name: &foov1.RedactedMessage{ 41 | Name: "sensitiveInformation", 42 | }, 43 | } 44 | got := Redact(removeSensitiveData())(input.ProtoReflect()) 45 | want := &foov1.RedactedMessageField{ 46 | Name: &foov1.RedactedMessage{}, 47 | } 48 | assert.True(t, proto.Equal(want, got.Interface())) 49 | }) 50 | t.Run("RedactedRepeatedField", func(t *testing.T) { 51 | t.Parallel() 52 | input := &foov1.RedactedRepeatedField{ 53 | Name: []string{"sensitiveInformation"}, 54 | } 55 | got := Redact(removeSensitiveData())(input.ProtoReflect()) 56 | want := &foov1.RedactedRepeatedField{} 57 | assert.True(t, proto.Equal(want, got.Interface())) 58 | }) 59 | t.Run("RedactedMap", func(t *testing.T) { 60 | t.Parallel() 61 | input := &foov1.RedactedMap{ 62 | Name: map[string]*foov1.RedactedMessage{ 63 | "foo": { 64 | Name: "sensitiveInformation", 65 | }, 66 | }, 67 | NotRedacted: map[string]string{ 68 | "foo": "bar", 69 | "frob": "nitz", 70 | }, 71 | } 72 | got := Redact(removeSensitiveData())(input.ProtoReflect()) 73 | want := &foov1.RedactedMap{ 74 | Name: map[string]*foov1.RedactedMessage{ 75 | "foo": {}, 76 | }, 77 | NotRedacted: map[string]string{ 78 | "foo": "bar", 79 | "frob": "nitz", 80 | }, 81 | } 82 | assert.True(t, proto.Equal(want, got.Interface())) 83 | }) 84 | t.Run("RedactedOneOf", func(t *testing.T) { 85 | t.Parallel() 86 | input := &foov1.RedactedOneOf{ 87 | OneofField: &foov1.RedactedOneOf_Foo1{Foo1: 64}, 88 | } 89 | got := Redact(removeSensitiveData())(input.ProtoReflect()) 90 | want := &foov1.RedactedOneOf{} 91 | assert.True(t, proto.Equal(want, got.Interface())) 92 | }) 93 | t.Run("RedactedEnum", func(t *testing.T) { 94 | t.Parallel() 95 | input := &foov1.RedactedEnum{ 96 | Name: foov1.Enum_ENUM_FIRST, 97 | } 98 | got := Redact(removeSensitiveData())(input.ProtoReflect()) 99 | want := &foov1.RedactedEnum{} 100 | assert.True(t, proto.Equal(want, got.Interface())) 101 | }) 102 | t.Run("RedactedRepeatedEnum", func(t *testing.T) { 103 | t.Parallel() 104 | input := &foov1.RedactedRepeatedEnum{ 105 | Name: []foov1.Enum{ 106 | foov1.Enum_ENUM_FIRST, 107 | foov1.Enum_ENUM_SECOND, 108 | foov1.Enum_ENUM_THIRD, 109 | }, 110 | NotRedacted: []int32{1, 2, 3}, 111 | } 112 | got := Redact(removeSensitiveData())(input.ProtoReflect()) 113 | want := &foov1.RedactedRepeatedEnum{NotRedacted: []int32{1, 2, 3}} 114 | assert.True(t, proto.Equal(want, got.Interface())) 115 | }) 116 | t.Run("NotRedactedField", func(t *testing.T) { 117 | t.Parallel() 118 | input := &foov1.NotRedactedField{ 119 | Name: "NotSensitiveData", 120 | } 121 | got := Redact(removeSensitiveData())(input.ProtoReflect()) 122 | assert.True(t, proto.Equal(input, got.Interface())) 123 | }) 124 | } 125 | 126 | func removeSensitiveData() func(in protoreflect.FieldDescriptor) bool { 127 | return func(in protoreflect.FieldDescriptor) bool { 128 | isSensitive, ok := proto.GetExtension(in.Options(), foov1.E_Sensitive).(bool) 129 | if !ok { 130 | return false 131 | } 132 | return isSensitive 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /format.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "google.golang.org/protobuf/encoding/protojson" 19 | "google.golang.org/protobuf/encoding/prototext" 20 | "google.golang.org/protobuf/proto" 21 | ) 22 | 23 | // InputFormat provides the interface to supply the [Converter] with an input 24 | // composition. The format provided must accept a [Resolver]. Includes 25 | // WithResolver() method to return the [Unmarshaler] with [Resolver] Supplied. 26 | type InputFormat interface { 27 | WithResolver(Resolver) Unmarshaler 28 | } 29 | 30 | // Unmarshaler is a [Unmarshaler]. 31 | type Unmarshaler interface { 32 | Unmarshal([]byte, proto.Message) error 33 | } 34 | 35 | // OutputFormat provides the interface to supply the [Converter] with an output 36 | // composition. The format provided must accept a [Resolver]. Includes 37 | // WithResolver() method to return the [Marshaler] with [Resolver] Supplied. 38 | type OutputFormat interface { 39 | WithResolver(Resolver) Marshaler 40 | } 41 | 42 | // Marshaler is a [Marshaler]. 43 | type Marshaler interface { 44 | Marshal(proto.Message) ([]byte, error) 45 | } 46 | 47 | type binaryInputFormat struct { 48 | proto.UnmarshalOptions 49 | } 50 | 51 | // BinaryInputFormat convenience method for binary input format. 52 | func BinaryInputFormat(in proto.UnmarshalOptions) InputFormat { 53 | return &binaryInputFormat{ 54 | UnmarshalOptions: in, 55 | } 56 | } 57 | 58 | // WithResolver to supply binary input format with a prebuilt types [Resolver]. 59 | func (x binaryInputFormat) WithResolver(in Resolver) Unmarshaler { 60 | x.Resolver = in 61 | return x 62 | } 63 | 64 | // BinaryOutputFormat convenience method for binary output format. 65 | func BinaryOutputFormat(in proto.MarshalOptions) OutputFormat { 66 | return outputFormatWithoutResolver{Marshaler: in} 67 | } 68 | 69 | type jSONInputFormat struct { 70 | protojson.UnmarshalOptions 71 | } 72 | 73 | // JSONInputFormat convenience method for JSON input format. 74 | func JSONInputFormat(in protojson.UnmarshalOptions) InputFormat { 75 | return jSONInputFormat{ 76 | UnmarshalOptions: in, 77 | } 78 | } 79 | 80 | // WithResolver to supply j s o n input format with a prebuilt types [Resolver]. 81 | func (x jSONInputFormat) WithResolver(in Resolver) Unmarshaler { 82 | x.Resolver = in 83 | return x 84 | } 85 | 86 | type jSONOutputFormat struct { 87 | protojson.MarshalOptions 88 | } 89 | 90 | // JSONOutputFormat convenience method for JSON output format. 91 | func JSONOutputFormat(in protojson.MarshalOptions) OutputFormat { 92 | return jSONOutputFormat{ 93 | MarshalOptions: in, 94 | } 95 | } 96 | 97 | // WithResolver to supply j s o n output format with a prebuilt types [Resolver]. 98 | func (x jSONOutputFormat) WithResolver(in Resolver) Marshaler { 99 | x.Resolver = in 100 | return x 101 | } 102 | 103 | type textInputFormat struct { 104 | prototext.UnmarshalOptions 105 | } 106 | 107 | // TextInputFormat convenience method for text input format. 108 | func TextInputFormat(in prototext.UnmarshalOptions) InputFormat { 109 | return textInputFormat{ 110 | UnmarshalOptions: in, 111 | } 112 | } 113 | 114 | // WithResolver to supply text input format with a prebuilt types [Resolver]. 115 | func (x textInputFormat) WithResolver(in Resolver) Unmarshaler { 116 | x.Resolver = in 117 | return x 118 | } 119 | 120 | type textOutputFormat struct { 121 | prototext.MarshalOptions 122 | } 123 | 124 | // TextOutputFormat convenience method for text output format. 125 | func TextOutputFormat(in prototext.MarshalOptions) OutputFormat { 126 | return textOutputFormat{ 127 | MarshalOptions: in, 128 | } 129 | } 130 | 131 | // WithResolver to supply text output format with a prebuilt types [Resolver]. 132 | func (x textOutputFormat) WithResolver(in Resolver) Marshaler { 133 | x.Resolver = in 134 | return x 135 | } 136 | 137 | type inputFormatWithoutResolver struct { 138 | Unmarshaler 139 | } 140 | 141 | // InputFormatWithoutResolver convenience method for input format without resolver. 142 | func InputFormatWithoutResolver(in Unmarshaler) InputFormat { 143 | return inputFormatWithoutResolver{ 144 | Unmarshaler: in, 145 | } 146 | } 147 | 148 | // WithResolver to supply input format without resolver. 149 | func (x inputFormatWithoutResolver) WithResolver(_ Resolver) Unmarshaler { 150 | return x 151 | } 152 | 153 | type outputFormatWithoutResolver struct { 154 | Marshaler 155 | } 156 | 157 | // OutputFormatWithoutResolver convenience method for output format without resolver. 158 | func OutputFormatWithoutResolver(in Marshaler) OutputFormat { 159 | return outputFormatWithoutResolver{ 160 | Marshaler: in, 161 | } 162 | } 163 | 164 | // WithResolver to supply output format without resolver. 165 | func (x outputFormatWithoutResolver) WithResolver(_ Resolver) Marshaler { 166 | return x 167 | } 168 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/bufbuild/prototransform 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | buf.build/gen/go/bufbuild/reflect/connectrpc/go v1.18.1-20240117202343-bf8f65e8876c.1 7 | buf.build/gen/go/bufbuild/reflect/protocolbuffers/go v1.36.6-20240117202343-bf8f65e8876c.1 8 | connectrpc.com/connect v1.18.1 9 | github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 10 | github.com/gomodule/redigo v1.9.2 11 | github.com/google/go-cmp v0.7.0 12 | github.com/pkg/errors v0.9.1 13 | github.com/stretchr/testify v1.10.0 14 | google.golang.org/protobuf v1.36.6 15 | ) 16 | 17 | require ( 18 | github.com/davecgh/go-spew v1.1.1 // indirect 19 | github.com/kr/pretty v0.3.0 // indirect 20 | github.com/pmezard/go-difflib v1.0.0 // indirect 21 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 22 | gopkg.in/yaml.v3 v3.0.1 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | buf.build/gen/go/bufbuild/reflect/connectrpc/go v1.18.1-20240117202343-bf8f65e8876c.1 h1:tHI0A2R8edX+Prm22Jb08kCtSy5dB6HvE/0AlaqDFEY= 2 | buf.build/gen/go/bufbuild/reflect/connectrpc/go v1.18.1-20240117202343-bf8f65e8876c.1/go.mod h1:tM/OH72d+hwRXLM6xJwGPhmStju+bA2dLNi19sVLed8= 3 | buf.build/gen/go/bufbuild/reflect/protocolbuffers/go v1.36.6-20240117202343-bf8f65e8876c.1 h1:95Cxx6URdkrsi/WyMeeHNlu7YcYaegu8qXblD76Ypko= 4 | buf.build/gen/go/bufbuild/reflect/protocolbuffers/go v1.36.6-20240117202343-bf8f65e8876c.1/go.mod h1:NDZi60swKc0m+69O4N/xYR/ia9+GHEjgRyG9ooAZXjs= 5 | connectrpc.com/connect v1.18.1 h1:PAg7CjSAGvscaf6YZKUefjoih5Z/qYkyaTrBW8xvYPw= 6 | connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2prWDO8= 7 | github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous= 8 | github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= 9 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= 13 | github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= 14 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 15 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 16 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 17 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 18 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= 19 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 20 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 21 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 22 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 23 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 24 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 25 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 26 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 27 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 28 | github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= 29 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 30 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 31 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 32 | golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= 33 | golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= 34 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 35 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 36 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 37 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 38 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 39 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 40 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 41 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 42 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 43 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 44 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 45 | -------------------------------------------------------------------------------- /internal/proto/buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v1 2 | managed: 3 | enabled: true 4 | go_package_prefix: 5 | default: github.com/bufbuild/prototransform/internal/proto/gen 6 | plugins: 7 | - plugin: buf.build/protocolbuffers/go:v1.32.0 8 | out: gen 9 | opt: paths=source_relative 10 | -------------------------------------------------------------------------------- /internal/proto/buf.yaml: -------------------------------------------------------------------------------- 1 | version: v1 2 | breaking: 3 | use: 4 | - FILE 5 | lint: 6 | use: 7 | - DEFAULT 8 | -------------------------------------------------------------------------------- /internal/proto/buf/prototransform/v1alpha1/cache.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package buf.prototransform.v1alpha1; 18 | 19 | import "google/protobuf/descriptor.proto"; 20 | import "google/protobuf/timestamp.proto"; 21 | 22 | // CacheEntry represents the serialized form of a cached schema. 23 | message CacheEntry { 24 | Schema schema = 1; 25 | google.protobuf.Timestamp schema_timestamp = 2; 26 | 27 | // An identifier for the schema. This allows a cache to verify 28 | // that the cached data is for the correct schema. This can be 29 | // useful in cases where cache keys must be shortened, to prevent 30 | // possible collisions from leading to the wrong schema being used. 31 | string id = 3; 32 | // If the schema was fetched for specified symbols, this is the 33 | // list of those symbols. If empty, this represents the entire 34 | // schema identified by id. But if non-empty, it could be filtered 35 | // with elements irrelevant to these symbols being omitted. 36 | repeated string included_symbols = 4; 37 | } 38 | 39 | // The actual cached schema. This is a separate message, instead of 40 | // inlined into CacheEntry, for backwards compatibility with cache 41 | // entries generated by early versions of this library. 42 | message Schema { 43 | google.protobuf.FileDescriptorSet descriptors = 1; 44 | string version = 2; 45 | } 46 | -------------------------------------------------------------------------------- /internal/proto/buf/prototransform/v1alpha1/lease.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package buf.prototransform.v1alpha1; 18 | 19 | // LeaseEntry represents the serialized form of a lease. 20 | message LeaseEntry { 21 | // Description of the process that holds the lease. 22 | oneof holder { 23 | LeaseHolder computed = 1; 24 | bytes user_provided = 2; 25 | } 26 | } 27 | 28 | // LeaseHolder is a computed leaseholder ID for a 29 | // client process. These attributes together should 30 | // uniquely identify any process. 31 | message LeaseHolder { 32 | string hostname = 1; 33 | bytes ip_address = 2; 34 | bytes mac_address = 3; 35 | uint64 pid = 4; 36 | uint64 start_nanos = 5; 37 | } 38 | -------------------------------------------------------------------------------- /internal/proto/gen/buf/prototransform/v1alpha1/cache.pb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Code generated by protoc-gen-go. DO NOT EDIT. 16 | // versions: 17 | // protoc-gen-go v1.32.0 18 | // protoc (unknown) 19 | // source: buf/prototransform/v1alpha1/cache.proto 20 | 21 | package prototransformv1alpha1 22 | 23 | import ( 24 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 25 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 26 | descriptorpb "google.golang.org/protobuf/types/descriptorpb" 27 | timestamppb "google.golang.org/protobuf/types/known/timestamppb" 28 | reflect "reflect" 29 | sync "sync" 30 | ) 31 | 32 | const ( 33 | // Verify that this generated code is sufficiently up-to-date. 34 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 35 | // Verify that runtime/protoimpl is sufficiently up-to-date. 36 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 37 | ) 38 | 39 | // CacheEntry represents the serialized form of a cached schema. 40 | type CacheEntry struct { 41 | state protoimpl.MessageState 42 | sizeCache protoimpl.SizeCache 43 | unknownFields protoimpl.UnknownFields 44 | 45 | Schema *Schema `protobuf:"bytes,1,opt,name=schema,proto3" json:"schema,omitempty"` 46 | SchemaTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=schema_timestamp,json=schemaTimestamp,proto3" json:"schema_timestamp,omitempty"` 47 | // An identifier for the schema. This allows a cache to verify 48 | // that the cached data is for the correct schema. This can be 49 | // useful in cases where cache keys must be shortened, to prevent 50 | // possible collisions from leading to the wrong schema being used. 51 | Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` 52 | // If the schema was fetched for specified symbols, this is the 53 | // list of those symbols. If empty, this represents the entire 54 | // schema identified by id. But if non-empty, it could be filtered 55 | // with elements irrelevant to these symbols being omitted. 56 | IncludedSymbols []string `protobuf:"bytes,4,rep,name=included_symbols,json=includedSymbols,proto3" json:"included_symbols,omitempty"` 57 | } 58 | 59 | func (x *CacheEntry) Reset() { 60 | *x = CacheEntry{} 61 | if protoimpl.UnsafeEnabled { 62 | mi := &file_buf_prototransform_v1alpha1_cache_proto_msgTypes[0] 63 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 64 | ms.StoreMessageInfo(mi) 65 | } 66 | } 67 | 68 | func (x *CacheEntry) String() string { 69 | return protoimpl.X.MessageStringOf(x) 70 | } 71 | 72 | func (*CacheEntry) ProtoMessage() {} 73 | 74 | func (x *CacheEntry) ProtoReflect() protoreflect.Message { 75 | mi := &file_buf_prototransform_v1alpha1_cache_proto_msgTypes[0] 76 | if protoimpl.UnsafeEnabled && x != nil { 77 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 78 | if ms.LoadMessageInfo() == nil { 79 | ms.StoreMessageInfo(mi) 80 | } 81 | return ms 82 | } 83 | return mi.MessageOf(x) 84 | } 85 | 86 | // Deprecated: Use CacheEntry.ProtoReflect.Descriptor instead. 87 | func (*CacheEntry) Descriptor() ([]byte, []int) { 88 | return file_buf_prototransform_v1alpha1_cache_proto_rawDescGZIP(), []int{0} 89 | } 90 | 91 | func (x *CacheEntry) GetSchema() *Schema { 92 | if x != nil { 93 | return x.Schema 94 | } 95 | return nil 96 | } 97 | 98 | func (x *CacheEntry) GetSchemaTimestamp() *timestamppb.Timestamp { 99 | if x != nil { 100 | return x.SchemaTimestamp 101 | } 102 | return nil 103 | } 104 | 105 | func (x *CacheEntry) GetId() string { 106 | if x != nil { 107 | return x.Id 108 | } 109 | return "" 110 | } 111 | 112 | func (x *CacheEntry) GetIncludedSymbols() []string { 113 | if x != nil { 114 | return x.IncludedSymbols 115 | } 116 | return nil 117 | } 118 | 119 | // The actual cached schema. This is a separate message, instead of 120 | // inlined into CacheEntry, for backwards compatibility with cache 121 | // entries generated by early versions of this library. 122 | type Schema struct { 123 | state protoimpl.MessageState 124 | sizeCache protoimpl.SizeCache 125 | unknownFields protoimpl.UnknownFields 126 | 127 | Descriptors *descriptorpb.FileDescriptorSet `protobuf:"bytes,1,opt,name=descriptors,proto3" json:"descriptors,omitempty"` 128 | Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` 129 | } 130 | 131 | func (x *Schema) Reset() { 132 | *x = Schema{} 133 | if protoimpl.UnsafeEnabled { 134 | mi := &file_buf_prototransform_v1alpha1_cache_proto_msgTypes[1] 135 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 136 | ms.StoreMessageInfo(mi) 137 | } 138 | } 139 | 140 | func (x *Schema) String() string { 141 | return protoimpl.X.MessageStringOf(x) 142 | } 143 | 144 | func (*Schema) ProtoMessage() {} 145 | 146 | func (x *Schema) ProtoReflect() protoreflect.Message { 147 | mi := &file_buf_prototransform_v1alpha1_cache_proto_msgTypes[1] 148 | if protoimpl.UnsafeEnabled && x != nil { 149 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 150 | if ms.LoadMessageInfo() == nil { 151 | ms.StoreMessageInfo(mi) 152 | } 153 | return ms 154 | } 155 | return mi.MessageOf(x) 156 | } 157 | 158 | // Deprecated: Use Schema.ProtoReflect.Descriptor instead. 159 | func (*Schema) Descriptor() ([]byte, []int) { 160 | return file_buf_prototransform_v1alpha1_cache_proto_rawDescGZIP(), []int{1} 161 | } 162 | 163 | func (x *Schema) GetDescriptors() *descriptorpb.FileDescriptorSet { 164 | if x != nil { 165 | return x.Descriptors 166 | } 167 | return nil 168 | } 169 | 170 | func (x *Schema) GetVersion() string { 171 | if x != nil { 172 | return x.Version 173 | } 174 | return "" 175 | } 176 | 177 | var File_buf_prototransform_v1alpha1_cache_proto protoreflect.FileDescriptor 178 | 179 | var file_buf_prototransform_v1alpha1_cache_proto_rawDesc = []byte{ 180 | 0x0a, 0x27, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 181 | 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x63, 0x61, 182 | 0x63, 0x68, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1b, 0x62, 0x75, 0x66, 0x2e, 0x70, 183 | 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x76, 0x31, 184 | 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 185 | 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 186 | 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 187 | 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 188 | 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcb, 0x01, 0x0a, 0x0a, 0x43, 0x61, 189 | 0x63, 0x68, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x3b, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 190 | 0x6d, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x62, 0x75, 0x66, 0x2e, 0x70, 191 | 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x76, 0x31, 192 | 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x06, 0x73, 193 | 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x45, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 194 | 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 195 | 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 196 | 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x73, 0x63, 0x68, 197 | 0x65, 0x6d, 0x61, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x0e, 0x0a, 0x02, 198 | 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x29, 0x0a, 0x10, 199 | 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 200 | 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 201 | 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x22, 0x68, 0x0a, 0x06, 0x53, 0x63, 0x68, 0x65, 0x6d, 202 | 0x61, 0x12, 0x44, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x73, 203 | 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 204 | 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 205 | 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x53, 0x65, 0x74, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 206 | 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 207 | 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 208 | 0x6e, 0x42, 0xa5, 0x02, 0x0a, 0x1f, 0x63, 0x6f, 0x6d, 0x2e, 0x62, 0x75, 0x66, 0x2e, 0x70, 0x72, 209 | 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x76, 0x31, 0x61, 210 | 0x6c, 0x70, 0x68, 0x61, 0x31, 0x42, 0x0a, 0x43, 0x61, 0x63, 0x68, 0x65, 0x50, 0x72, 0x6f, 0x74, 211 | 0x6f, 0x50, 0x01, 0x5a, 0x68, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 212 | 0x62, 0x75, 0x66, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 213 | 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 214 | 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x75, 0x66, 0x2f, 0x70, 215 | 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x76, 0x31, 216 | 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 217 | 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xa2, 0x02, 0x03, 218 | 0x42, 0x50, 0x58, 0xaa, 0x02, 0x1b, 0x42, 0x75, 0x66, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x74, 219 | 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 220 | 0x31, 0xca, 0x02, 0x1b, 0x42, 0x75, 0x66, 0x5c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 221 | 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xe2, 222 | 0x02, 0x27, 0x42, 0x75, 0x66, 0x5c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 223 | 0x66, 0x6f, 0x72, 0x6d, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x47, 0x50, 224 | 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x1d, 0x42, 0x75, 0x66, 0x3a, 225 | 0x3a, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x3a, 226 | 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 227 | 0x33, 228 | } 229 | 230 | var ( 231 | file_buf_prototransform_v1alpha1_cache_proto_rawDescOnce sync.Once 232 | file_buf_prototransform_v1alpha1_cache_proto_rawDescData = file_buf_prototransform_v1alpha1_cache_proto_rawDesc 233 | ) 234 | 235 | func file_buf_prototransform_v1alpha1_cache_proto_rawDescGZIP() []byte { 236 | file_buf_prototransform_v1alpha1_cache_proto_rawDescOnce.Do(func() { 237 | file_buf_prototransform_v1alpha1_cache_proto_rawDescData = protoimpl.X.CompressGZIP(file_buf_prototransform_v1alpha1_cache_proto_rawDescData) 238 | }) 239 | return file_buf_prototransform_v1alpha1_cache_proto_rawDescData 240 | } 241 | 242 | var file_buf_prototransform_v1alpha1_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) 243 | var file_buf_prototransform_v1alpha1_cache_proto_goTypes = []interface{}{ 244 | (*CacheEntry)(nil), // 0: buf.prototransform.v1alpha1.CacheEntry 245 | (*Schema)(nil), // 1: buf.prototransform.v1alpha1.Schema 246 | (*timestamppb.Timestamp)(nil), // 2: google.protobuf.Timestamp 247 | (*descriptorpb.FileDescriptorSet)(nil), // 3: google.protobuf.FileDescriptorSet 248 | } 249 | var file_buf_prototransform_v1alpha1_cache_proto_depIdxs = []int32{ 250 | 1, // 0: buf.prototransform.v1alpha1.CacheEntry.schema:type_name -> buf.prototransform.v1alpha1.Schema 251 | 2, // 1: buf.prototransform.v1alpha1.CacheEntry.schema_timestamp:type_name -> google.protobuf.Timestamp 252 | 3, // 2: buf.prototransform.v1alpha1.Schema.descriptors:type_name -> google.protobuf.FileDescriptorSet 253 | 3, // [3:3] is the sub-list for method output_type 254 | 3, // [3:3] is the sub-list for method input_type 255 | 3, // [3:3] is the sub-list for extension type_name 256 | 3, // [3:3] is the sub-list for extension extendee 257 | 0, // [0:3] is the sub-list for field type_name 258 | } 259 | 260 | func init() { file_buf_prototransform_v1alpha1_cache_proto_init() } 261 | func file_buf_prototransform_v1alpha1_cache_proto_init() { 262 | if File_buf_prototransform_v1alpha1_cache_proto != nil { 263 | return 264 | } 265 | if !protoimpl.UnsafeEnabled { 266 | file_buf_prototransform_v1alpha1_cache_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 267 | switch v := v.(*CacheEntry); i { 268 | case 0: 269 | return &v.state 270 | case 1: 271 | return &v.sizeCache 272 | case 2: 273 | return &v.unknownFields 274 | default: 275 | return nil 276 | } 277 | } 278 | file_buf_prototransform_v1alpha1_cache_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 279 | switch v := v.(*Schema); i { 280 | case 0: 281 | return &v.state 282 | case 1: 283 | return &v.sizeCache 284 | case 2: 285 | return &v.unknownFields 286 | default: 287 | return nil 288 | } 289 | } 290 | } 291 | type x struct{} 292 | out := protoimpl.TypeBuilder{ 293 | File: protoimpl.DescBuilder{ 294 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 295 | RawDescriptor: file_buf_prototransform_v1alpha1_cache_proto_rawDesc, 296 | NumEnums: 0, 297 | NumMessages: 2, 298 | NumExtensions: 0, 299 | NumServices: 0, 300 | }, 301 | GoTypes: file_buf_prototransform_v1alpha1_cache_proto_goTypes, 302 | DependencyIndexes: file_buf_prototransform_v1alpha1_cache_proto_depIdxs, 303 | MessageInfos: file_buf_prototransform_v1alpha1_cache_proto_msgTypes, 304 | }.Build() 305 | File_buf_prototransform_v1alpha1_cache_proto = out.File 306 | file_buf_prototransform_v1alpha1_cache_proto_rawDesc = nil 307 | file_buf_prototransform_v1alpha1_cache_proto_goTypes = nil 308 | file_buf_prototransform_v1alpha1_cache_proto_depIdxs = nil 309 | } 310 | -------------------------------------------------------------------------------- /internal/proto/gen/buf/prototransform/v1alpha1/lease.pb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Code generated by protoc-gen-go. DO NOT EDIT. 16 | // versions: 17 | // protoc-gen-go v1.32.0 18 | // protoc (unknown) 19 | // source: buf/prototransform/v1alpha1/lease.proto 20 | 21 | package prototransformv1alpha1 22 | 23 | import ( 24 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 25 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 26 | reflect "reflect" 27 | sync "sync" 28 | ) 29 | 30 | const ( 31 | // Verify that this generated code is sufficiently up-to-date. 32 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 33 | // Verify that runtime/protoimpl is sufficiently up-to-date. 34 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 35 | ) 36 | 37 | // LeaseEntry represents the serialized form of a lease. 38 | type LeaseEntry struct { 39 | state protoimpl.MessageState 40 | sizeCache protoimpl.SizeCache 41 | unknownFields protoimpl.UnknownFields 42 | 43 | // Description of the process that holds the lease. 44 | // 45 | // Types that are assignable to Holder: 46 | // 47 | // *LeaseEntry_Computed 48 | // *LeaseEntry_UserProvided 49 | Holder isLeaseEntry_Holder `protobuf_oneof:"holder"` 50 | } 51 | 52 | func (x *LeaseEntry) Reset() { 53 | *x = LeaseEntry{} 54 | if protoimpl.UnsafeEnabled { 55 | mi := &file_buf_prototransform_v1alpha1_lease_proto_msgTypes[0] 56 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 57 | ms.StoreMessageInfo(mi) 58 | } 59 | } 60 | 61 | func (x *LeaseEntry) String() string { 62 | return protoimpl.X.MessageStringOf(x) 63 | } 64 | 65 | func (*LeaseEntry) ProtoMessage() {} 66 | 67 | func (x *LeaseEntry) ProtoReflect() protoreflect.Message { 68 | mi := &file_buf_prototransform_v1alpha1_lease_proto_msgTypes[0] 69 | if protoimpl.UnsafeEnabled && x != nil { 70 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 71 | if ms.LoadMessageInfo() == nil { 72 | ms.StoreMessageInfo(mi) 73 | } 74 | return ms 75 | } 76 | return mi.MessageOf(x) 77 | } 78 | 79 | // Deprecated: Use LeaseEntry.ProtoReflect.Descriptor instead. 80 | func (*LeaseEntry) Descriptor() ([]byte, []int) { 81 | return file_buf_prototransform_v1alpha1_lease_proto_rawDescGZIP(), []int{0} 82 | } 83 | 84 | func (m *LeaseEntry) GetHolder() isLeaseEntry_Holder { 85 | if m != nil { 86 | return m.Holder 87 | } 88 | return nil 89 | } 90 | 91 | func (x *LeaseEntry) GetComputed() *LeaseHolder { 92 | if x, ok := x.GetHolder().(*LeaseEntry_Computed); ok { 93 | return x.Computed 94 | } 95 | return nil 96 | } 97 | 98 | func (x *LeaseEntry) GetUserProvided() []byte { 99 | if x, ok := x.GetHolder().(*LeaseEntry_UserProvided); ok { 100 | return x.UserProvided 101 | } 102 | return nil 103 | } 104 | 105 | type isLeaseEntry_Holder interface { 106 | isLeaseEntry_Holder() 107 | } 108 | 109 | type LeaseEntry_Computed struct { 110 | Computed *LeaseHolder `protobuf:"bytes,1,opt,name=computed,proto3,oneof"` 111 | } 112 | 113 | type LeaseEntry_UserProvided struct { 114 | UserProvided []byte `protobuf:"bytes,2,opt,name=user_provided,json=userProvided,proto3,oneof"` 115 | } 116 | 117 | func (*LeaseEntry_Computed) isLeaseEntry_Holder() {} 118 | 119 | func (*LeaseEntry_UserProvided) isLeaseEntry_Holder() {} 120 | 121 | // LeaseHolder is a computed leaseholder ID for a 122 | // client process. These attributes together should 123 | // uniquely identify any process. 124 | type LeaseHolder struct { 125 | state protoimpl.MessageState 126 | sizeCache protoimpl.SizeCache 127 | unknownFields protoimpl.UnknownFields 128 | 129 | Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` 130 | IpAddress []byte `protobuf:"bytes,2,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` 131 | MacAddress []byte `protobuf:"bytes,3,opt,name=mac_address,json=macAddress,proto3" json:"mac_address,omitempty"` 132 | Pid uint64 `protobuf:"varint,4,opt,name=pid,proto3" json:"pid,omitempty"` 133 | StartNanos uint64 `protobuf:"varint,5,opt,name=start_nanos,json=startNanos,proto3" json:"start_nanos,omitempty"` 134 | } 135 | 136 | func (x *LeaseHolder) Reset() { 137 | *x = LeaseHolder{} 138 | if protoimpl.UnsafeEnabled { 139 | mi := &file_buf_prototransform_v1alpha1_lease_proto_msgTypes[1] 140 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 141 | ms.StoreMessageInfo(mi) 142 | } 143 | } 144 | 145 | func (x *LeaseHolder) String() string { 146 | return protoimpl.X.MessageStringOf(x) 147 | } 148 | 149 | func (*LeaseHolder) ProtoMessage() {} 150 | 151 | func (x *LeaseHolder) ProtoReflect() protoreflect.Message { 152 | mi := &file_buf_prototransform_v1alpha1_lease_proto_msgTypes[1] 153 | if protoimpl.UnsafeEnabled && x != nil { 154 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 155 | if ms.LoadMessageInfo() == nil { 156 | ms.StoreMessageInfo(mi) 157 | } 158 | return ms 159 | } 160 | return mi.MessageOf(x) 161 | } 162 | 163 | // Deprecated: Use LeaseHolder.ProtoReflect.Descriptor instead. 164 | func (*LeaseHolder) Descriptor() ([]byte, []int) { 165 | return file_buf_prototransform_v1alpha1_lease_proto_rawDescGZIP(), []int{1} 166 | } 167 | 168 | func (x *LeaseHolder) GetHostname() string { 169 | if x != nil { 170 | return x.Hostname 171 | } 172 | return "" 173 | } 174 | 175 | func (x *LeaseHolder) GetIpAddress() []byte { 176 | if x != nil { 177 | return x.IpAddress 178 | } 179 | return nil 180 | } 181 | 182 | func (x *LeaseHolder) GetMacAddress() []byte { 183 | if x != nil { 184 | return x.MacAddress 185 | } 186 | return nil 187 | } 188 | 189 | func (x *LeaseHolder) GetPid() uint64 { 190 | if x != nil { 191 | return x.Pid 192 | } 193 | return 0 194 | } 195 | 196 | func (x *LeaseHolder) GetStartNanos() uint64 { 197 | if x != nil { 198 | return x.StartNanos 199 | } 200 | return 0 201 | } 202 | 203 | var File_buf_prototransform_v1alpha1_lease_proto protoreflect.FileDescriptor 204 | 205 | var file_buf_prototransform_v1alpha1_lease_proto_rawDesc = []byte{ 206 | 0x0a, 0x27, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 207 | 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x6c, 0x65, 208 | 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1b, 0x62, 0x75, 0x66, 0x2e, 0x70, 209 | 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x76, 0x31, 210 | 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x22, 0x85, 0x01, 0x0a, 0x0a, 0x4c, 0x65, 0x61, 0x73, 0x65, 211 | 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x46, 0x0a, 0x08, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 212 | 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x62, 0x75, 0x66, 0x2e, 0x70, 0x72, 213 | 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x76, 0x31, 0x61, 214 | 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x48, 0x6f, 0x6c, 0x64, 0x65, 215 | 0x72, 0x48, 0x00, 0x52, 0x08, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x64, 0x12, 0x25, 0x0a, 216 | 0x0d, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x64, 0x18, 0x02, 217 | 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x75, 0x73, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x76, 218 | 0x69, 0x64, 0x65, 0x64, 0x42, 0x08, 0x0a, 0x06, 0x68, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x22, 0x9c, 219 | 0x01, 0x0a, 0x0b, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x48, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x1a, 220 | 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 221 | 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x70, 222 | 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 223 | 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x63, 224 | 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 225 | 0x6d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 226 | 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 227 | 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 228 | 0x04, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4e, 0x61, 0x6e, 0x6f, 0x73, 0x42, 0xa5, 0x02, 229 | 0x0a, 0x1f, 0x63, 0x6f, 0x6d, 0x2e, 0x62, 0x75, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 230 | 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 231 | 0x31, 0x42, 0x0a, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 232 | 0x68, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x62, 0x75, 0x66, 0x62, 233 | 0x75, 0x69, 0x6c, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 234 | 0x6f, 0x72, 0x6d, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 235 | 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 236 | 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 237 | 0x61, 0x31, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 238 | 0x6d, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xa2, 0x02, 0x03, 0x42, 0x50, 0x58, 0xaa, 239 | 0x02, 0x1b, 0x42, 0x75, 0x66, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 240 | 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xca, 0x02, 0x1b, 241 | 0x42, 0x75, 0x66, 0x5c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 242 | 0x72, 0x6d, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xe2, 0x02, 0x27, 0x42, 0x75, 243 | 0x66, 0x5c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 244 | 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 245 | 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x1d, 0x42, 0x75, 0x66, 0x3a, 0x3a, 0x50, 0x72, 0x6f, 246 | 0x74, 0x6f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x3a, 0x3a, 0x56, 0x31, 0x61, 247 | 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 248 | } 249 | 250 | var ( 251 | file_buf_prototransform_v1alpha1_lease_proto_rawDescOnce sync.Once 252 | file_buf_prototransform_v1alpha1_lease_proto_rawDescData = file_buf_prototransform_v1alpha1_lease_proto_rawDesc 253 | ) 254 | 255 | func file_buf_prototransform_v1alpha1_lease_proto_rawDescGZIP() []byte { 256 | file_buf_prototransform_v1alpha1_lease_proto_rawDescOnce.Do(func() { 257 | file_buf_prototransform_v1alpha1_lease_proto_rawDescData = protoimpl.X.CompressGZIP(file_buf_prototransform_v1alpha1_lease_proto_rawDescData) 258 | }) 259 | return file_buf_prototransform_v1alpha1_lease_proto_rawDescData 260 | } 261 | 262 | var file_buf_prototransform_v1alpha1_lease_proto_msgTypes = make([]protoimpl.MessageInfo, 2) 263 | var file_buf_prototransform_v1alpha1_lease_proto_goTypes = []interface{}{ 264 | (*LeaseEntry)(nil), // 0: buf.prototransform.v1alpha1.LeaseEntry 265 | (*LeaseHolder)(nil), // 1: buf.prototransform.v1alpha1.LeaseHolder 266 | } 267 | var file_buf_prototransform_v1alpha1_lease_proto_depIdxs = []int32{ 268 | 1, // 0: buf.prototransform.v1alpha1.LeaseEntry.computed:type_name -> buf.prototransform.v1alpha1.LeaseHolder 269 | 1, // [1:1] is the sub-list for method output_type 270 | 1, // [1:1] is the sub-list for method input_type 271 | 1, // [1:1] is the sub-list for extension type_name 272 | 1, // [1:1] is the sub-list for extension extendee 273 | 0, // [0:1] is the sub-list for field type_name 274 | } 275 | 276 | func init() { file_buf_prototransform_v1alpha1_lease_proto_init() } 277 | func file_buf_prototransform_v1alpha1_lease_proto_init() { 278 | if File_buf_prototransform_v1alpha1_lease_proto != nil { 279 | return 280 | } 281 | if !protoimpl.UnsafeEnabled { 282 | file_buf_prototransform_v1alpha1_lease_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 283 | switch v := v.(*LeaseEntry); i { 284 | case 0: 285 | return &v.state 286 | case 1: 287 | return &v.sizeCache 288 | case 2: 289 | return &v.unknownFields 290 | default: 291 | return nil 292 | } 293 | } 294 | file_buf_prototransform_v1alpha1_lease_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 295 | switch v := v.(*LeaseHolder); i { 296 | case 0: 297 | return &v.state 298 | case 1: 299 | return &v.sizeCache 300 | case 2: 301 | return &v.unknownFields 302 | default: 303 | return nil 304 | } 305 | } 306 | } 307 | file_buf_prototransform_v1alpha1_lease_proto_msgTypes[0].OneofWrappers = []interface{}{ 308 | (*LeaseEntry_Computed)(nil), 309 | (*LeaseEntry_UserProvided)(nil), 310 | } 311 | type x struct{} 312 | out := protoimpl.TypeBuilder{ 313 | File: protoimpl.DescBuilder{ 314 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 315 | RawDescriptor: file_buf_prototransform_v1alpha1_lease_proto_rawDesc, 316 | NumEnums: 0, 317 | NumMessages: 2, 318 | NumExtensions: 0, 319 | NumServices: 0, 320 | }, 321 | GoTypes: file_buf_prototransform_v1alpha1_lease_proto_goTypes, 322 | DependencyIndexes: file_buf_prototransform_v1alpha1_lease_proto_depIdxs, 323 | MessageInfos: file_buf_prototransform_v1alpha1_lease_proto_msgTypes, 324 | }.Build() 325 | File_buf_prototransform_v1alpha1_lease_proto = out.File 326 | file_buf_prototransform_v1alpha1_lease_proto_rawDesc = nil 327 | file_buf_prototransform_v1alpha1_lease_proto_goTypes = nil 328 | file_buf_prototransform_v1alpha1_lease_proto_depIdxs = nil 329 | } 330 | -------------------------------------------------------------------------------- /internal/testdata/buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v1 2 | managed: 3 | enabled: true 4 | go_package_prefix: 5 | default: github.com/bufbuild/prototransform/internal/testdata/gen 6 | plugins: 7 | - plugin: buf.build/protocolbuffers/go:v1.30.0 8 | out: gen 9 | opt: paths=source_relative 10 | -------------------------------------------------------------------------------- /internal/testdata/foo/v1/test.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package foo.v1; 4 | 5 | import "google/protobuf/descriptor.proto"; 6 | 7 | option go_package = "foopb"; 8 | 9 | extend google.protobuf.FieldOptions { 10 | bool sensitive = 30000; 11 | } 12 | 13 | message RedactedMessage { 14 | string name = 1 [(sensitive) = true]; 15 | } 16 | 17 | message RedactedMessageField { 18 | RedactedMessage name = 1; 19 | } 20 | 21 | message RedactedRepeatedField { 22 | repeated string name = 1 [(sensitive) = true]; 23 | } 24 | 25 | message RedactedMap { 26 | map name = 1; 27 | map not_redacted = 2; 28 | } 29 | 30 | message RedactedOneOf { 31 | oneof oneof_field { 32 | int64 foo1 = 1 [(sensitive) = true]; 33 | string foo2 = 2 [(sensitive) = true]; 34 | uint32 foo3 = 3 [(sensitive) = true]; 35 | } 36 | } 37 | 38 | message RedactedEnum { 39 | Enum name = 1 [(sensitive) = true]; 40 | } 41 | 42 | message RedactedRepeatedEnum { 43 | repeated Enum name = 1 [(sensitive) = true]; 44 | repeated int32 not_redacted = 2; 45 | } 46 | 47 | message NotRedactedField { 48 | string name = 1; 49 | } 50 | 51 | enum Enum { 52 | ENUM_UNSPECIFIED = 0; 53 | ENUM_FIRST = 1; 54 | ENUM_SECOND = 2; 55 | ENUM_THIRD = 3; 56 | } 57 | -------------------------------------------------------------------------------- /jitter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "hash/maphash" 19 | "math/rand" 20 | "sync" 21 | "time" 22 | ) 23 | 24 | // We create our own locked RNG so we won't have lock contention with the global 25 | // RNG that package "math/rand" creates. Also, that way we are not at the mercy 26 | // of other packages that might be seeding "math/rand"'s global RNG in a bad way. 27 | var rnd = newLockedRand() //nolint:gochecknoglobals 28 | 29 | func addJitter(period time.Duration, jitter float64) time.Duration { 30 | factor := (rnd.Float64()*2 - 1) * jitter // produces a number between -jitter and jitter 31 | period = time.Duration(float64(period) * (factor + 1)) 32 | if period == 0 { 33 | period = 1 // ticker.Reset panics if duration is zero 34 | } 35 | return period 36 | } 37 | 38 | type lockedSource struct { 39 | mu sync.Mutex 40 | src rand.Source 41 | } 42 | 43 | func (s *lockedSource) Int63() int64 { 44 | s.mu.Lock() 45 | i := s.src.Int63() 46 | s.mu.Unlock() 47 | return i 48 | } 49 | 50 | func (s *lockedSource) Seed(seed int64) { 51 | s.mu.Lock() 52 | s.src.Seed(seed) 53 | s.mu.Unlock() 54 | } 55 | 56 | func newLockedRand() *rand.Rand { 57 | //nolint:gosec // don't need secure RNG for this and prefer something that can't exhaust entropy 58 | return rand.New(&lockedSource{src: rand.NewSource(int64(seed()))}) 59 | } 60 | 61 | func seed() uint64 { 62 | // lock-free and fast; under-the-hood calls runtime.fastrand 63 | // https://www.reddit.com/r/golang/comments/ntyi7i/comment/h0w0tu7/ 64 | return new(maphash.Hash).Sum64() 65 | } 66 | -------------------------------------------------------------------------------- /jitter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func TestAddJitter(t *testing.T) { 25 | t.Parallel() 26 | testCases := []struct { 27 | name string 28 | pollingPeriod time.Duration 29 | jitter float64 30 | min, max time.Duration 31 | }{ 32 | { 33 | name: "no jitter", 34 | pollingPeriod: time.Minute, 35 | jitter: 0, 36 | min: time.Minute, 37 | max: time.Minute, 38 | }, 39 | { 40 | name: "100% jitter", 41 | pollingPeriod: 3 * time.Minute, 42 | jitter: 1, 43 | min: 1, 44 | max: 6 * time.Minute, 45 | }, 46 | { 47 | name: "25% jitter", 48 | pollingPeriod: 4 * time.Second, 49 | jitter: 0.25, 50 | min: 3 * time.Second, 51 | max: 5 * time.Second, 52 | }, 53 | } 54 | for _, testCase := range testCases { 55 | t.Run(testCase.name, func(t *testing.T) { 56 | t.Parallel() 57 | var minPeriod, maxPeriod time.Duration 58 | for i := range 10_000 { 59 | period := addJitter(testCase.pollingPeriod, testCase.jitter) 60 | require.GreaterOrEqual(t, period, testCase.min) 61 | require.LessOrEqual(t, period, testCase.max) 62 | if i == 0 || period < minPeriod { 63 | minPeriod = period 64 | } 65 | if i == 0 || period > maxPeriod { 66 | maxPeriod = period 67 | } 68 | } 69 | // After 10k iterations, with uniform distribution RNG, we could 70 | // probably put much tighter bound on this; but we're a bit lenient 71 | // to make sure we don't see flaky failures in CI. We want to observe 72 | // at least 90% of the effective jitter range. 73 | minVariation := time.Duration(0.9 * float64(testCase.max-testCase.min)) 74 | require.GreaterOrEqual(t, maxPeriod-minPeriod, minVariation) 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /leaser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "errors" 21 | "fmt" 22 | "net" 23 | "os" 24 | "sync" 25 | "time" 26 | 27 | prototransformv1alpha1 "github.com/bufbuild/prototransform/internal/proto/gen/buf/prototransform/v1alpha1" 28 | "google.golang.org/protobuf/proto" 29 | ) 30 | 31 | // ErrLeaseStateNotYetKnown is an error that may be returned by Lease.IsHeld to 32 | // indicate that the leaser has not yet completed querying for the lease's initial 33 | // state. 34 | var ErrLeaseStateNotYetKnown = errors.New("haven't completed initial lease check yet") 35 | 36 | //nolint:gochecknoglobals 37 | var ( 38 | startNanos = uint64(time.Now().UnixNano()) 39 | 40 | currentProcessInit sync.Once 41 | currentProcessVal *prototransformv1alpha1.LeaseHolder 42 | currentProcessErr error //nolint:errname 43 | ) 44 | 45 | // Leaser provides access to long-lived distributed leases, for 46 | // leader election or distributed locking. This can be used by a 47 | // SchemaWatcher so that only a single "leader" process polls the 48 | // remote source for a schema and the others ("followers") just 49 | // get the latest schema from a shared cache. 50 | type Leaser interface { 51 | // NewLease tries to acquire the given lease name. This 52 | // returns a lease object, which represents the state of 53 | // the new lease, and whether the current process holds 54 | // it or not. 55 | // 56 | // Implementations should monitor the lease store so that 57 | // if the lease is not held but suddenly becomes available 58 | // (e.g. current leaseholder releases it or crashes), 59 | // another process can immediately pick it up. The given 60 | // leaseHolder bytes represent the current process and may 61 | // be persisted in the lease store if necessary. This is 62 | // particularly useful if the lease store has no other way 63 | // to identify connected clients or entry "owners", in which 64 | // case a lease implementation can compare the persisted 65 | // lease state to this value to determine if the current 66 | // client holds the lease. 67 | // 68 | // This may start background goroutines. In order to release 69 | // any such resources associated with the lease, callers must 70 | // call Lease.Cancel() or cancel the given context. 71 | NewLease(ctx context.Context, leaseName string, leaseHolder []byte) Lease 72 | } 73 | 74 | // Lease represents a long-lived distributed lease. This allows 75 | // the current process to query if the lease is currently held 76 | // or not as well as to configure callbacks for when the lease 77 | // is acquired or released. 78 | type Lease interface { 79 | // IsHeld returns whether the current process holds the 80 | // lease. If it returns an error, then it is not known who 81 | // holds the lease, and the error indicates why not. Polling 82 | // for a schema will be suspended unless/until this method 83 | // returns (true, nil). 84 | IsHeld() (bool, error) 85 | // SetCallbacks configures the given functions to be called 86 | // when the lease is acquired or released. The initial state 87 | // of a lease is "not held". So if the lease is not held at 88 | // the time this method is invoked, neither callback is 89 | // invoked. But if the lease IS held at the time this method 90 | // is invoked, the onAcquire callback will be immediately 91 | // invoked. A lease must synchronize invocations of the 92 | // callbacks so that there will never be multiple concurrent 93 | // calls. 94 | SetCallbacks(onAcquire, onRelease func()) 95 | // Cancel cancels this lease and frees any associated 96 | // resources (which may include background goroutines). If 97 | // the lease is currently held, it will be immediately 98 | // released, and any onRelease callback will be invoked. 99 | // IsHeld will return false from that moment. If the 100 | // same lease needs to be re-acquired later, use the 101 | // Leaser to create a new lease with the same name. 102 | Cancel() 103 | } 104 | 105 | func getLeaseHolder(userProvidedData []byte) ([]byte, error) { 106 | var leaseEntry prototransformv1alpha1.LeaseEntry 107 | if userProvidedData != nil { 108 | leaseEntry.Holder = &prototransformv1alpha1.LeaseEntry_UserProvided{ 109 | UserProvided: userProvidedData, 110 | } 111 | } else { 112 | leaseHolder, err := currentProcess() 113 | if err != nil { 114 | return nil, fmt.Errorf("failed to compute current process bytes for lease: %w", err) 115 | } 116 | leaseEntry.Holder = &prototransformv1alpha1.LeaseEntry_Computed{ 117 | Computed: leaseHolder, 118 | } 119 | } 120 | leaseData, err := proto.Marshal(&leaseEntry) 121 | if err != nil { 122 | return nil, fmt.Errorf("failed to marshal current process info to bytes for lease: %w", err) 123 | } 124 | return leaseData, nil 125 | } 126 | 127 | func currentProcess() (*prototransformv1alpha1.LeaseHolder, error) { 128 | currentProcessInit.Do(func() { 129 | var errs []error 130 | hostname, hostnameErr := os.Hostname() 131 | if hostnameErr != nil { 132 | errs = append(errs, hostnameErr) 133 | } 134 | // UDP isn't stateful, so this does not actually connect to anything. 135 | // But this is a reliable way to see the preferred network interface 136 | // and IP of the host, by examining the client IP of the socket. 137 | conn, connErr := net.Dial("udp", "8.8.8.8:53") 138 | if connErr != nil { 139 | errs = append(errs, connErr) 140 | } 141 | 142 | var ipAddress net.IP 143 | var macAddress net.HardwareAddr 144 | if connErr == nil { //nolint:nestif 145 | if udpAddr, ok := conn.LocalAddr().(*net.UDPAddr); ok { 146 | ipAddress = udpAddr.IP 147 | } 148 | if len(ipAddress) == 0 || ipAddress.IsLoopback() { 149 | ipAddress = nil // don't use loopback addresses! 150 | } else { 151 | // look at network interfaces to find the MAC address 152 | // associated with this IP address 153 | ifaces, ifaceErr := net.Interfaces() 154 | if ifaceErr != nil { 155 | errs = append(errs, ifaceErr) 156 | } 157 | var macAddrErr error 158 | for _, iface := range ifaces { 159 | if len(iface.HardwareAddr) == 0 { 160 | // no MAC address on this one 161 | continue 162 | } 163 | addrs, err := iface.Addrs() 164 | if err != nil { 165 | // remember one of the address errors to report 166 | // in case we can't find the IP address on any 167 | // of the interfaces 168 | macAddrErr = err 169 | continue 170 | } 171 | for _, addr := range addrs { 172 | ipNet, ok := addr.(*net.IPNet) 173 | if !ok { 174 | continue 175 | } 176 | if ipNet.IP.Equal(ipAddress) { 177 | macAddress = iface.HardwareAddr 178 | break 179 | } 180 | } 181 | if len(macAddress) > 0 { 182 | // found it 183 | break 184 | } 185 | } 186 | if len(macAddress) == 0 && macAddrErr != nil { 187 | errs = append(errs, macAddrErr) 188 | } 189 | } 190 | } 191 | 192 | // We need at least the host name or the IP address. If we have neither, then 193 | // we report all errors. 194 | if hostname == "" && len(ipAddress) == 0 { 195 | switch len(errs) { 196 | case 0: 197 | currentProcessErr = errors.New("internal: could not compute non-empty hostname or IP address for client process ID") 198 | case 1: 199 | currentProcessErr = errs[0] 200 | default: 201 | currentProcessErr = multiErr(errs) 202 | } 203 | return 204 | } 205 | 206 | currentProcessVal = &prototransformv1alpha1.LeaseHolder{ 207 | Hostname: hostname, 208 | IpAddress: ipAddress, 209 | MacAddress: macAddress, 210 | Pid: uint64(os.Getpid()), //nolint:gosec 211 | StartNanos: startNanos, 212 | } 213 | }) 214 | return currentProcessVal, currentProcessErr 215 | } 216 | 217 | type multiErr []error //nolint:errname 218 | 219 | func (m multiErr) Error() string { 220 | var buf bytes.Buffer 221 | for i, err := range m { 222 | if i > 0 { 223 | buf.WriteRune('\n') 224 | } 225 | buf.WriteString(err.Error()) 226 | } 227 | return buf.String() 228 | } 229 | 230 | func (m multiErr) Unwrap() error { 231 | return m[0] 232 | } 233 | -------------------------------------------------------------------------------- /leaser/internal/leasertesting/leasertesting.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package leasertesting 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | "testing" 21 | "time" 22 | 23 | "github.com/bufbuild/prototransform" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | //nolint:revive // okay that ctx is second; prefer t to be first 28 | func RunSimpleLeaserTests(t *testing.T, ctx context.Context, leaser prototransform.Leaser, forceOwner func(key string, val []byte) error) map[string][]byte { 29 | t.Helper() 30 | 31 | var mutex sync.Mutex 32 | acquired, released := map[string]int{}, map[string]int{} 33 | track := func(key string, channel chan struct{}, acq bool) func() { 34 | return func() { 35 | mutex.Lock() 36 | defer mutex.Unlock() 37 | var trackingMap map[string]int 38 | msg := key + " " 39 | if acq { 40 | msg += "acquired" 41 | trackingMap = acquired 42 | } else { 43 | msg += "released" 44 | trackingMap = released 45 | } 46 | trackingMap[key]++ 47 | select { 48 | case channel <- struct{}{}: 49 | default: 50 | msg += "(could not notify; channel full)" 51 | } 52 | t.Log(msg) 53 | } 54 | } 55 | 56 | aAcquiredByX := make(chan struct{}, 1) 57 | aReleasedByX := make(chan struct{}, 1) 58 | aAcquiredByY := make(chan struct{}, 1) 59 | aReleasedByY := make(chan struct{}, 1) 60 | bAcquiredByX := make(chan struct{}, 1) 61 | bReleasedByX := make(chan struct{}, 1) 62 | cAcquiredByX := make(chan struct{}, 1) 63 | cReleasedByX := make(chan struct{}, 1) 64 | dAcquiredByZ := make(chan struct{}, 1) 65 | dReleasedByZ := make(chan struct{}, 1) 66 | 67 | checkLeaseInitial := func(lease prototransform.Lease, success bool) { 68 | // initial check should complete in 200 millis 69 | for i := range 4 { 70 | held, err := lease.IsHeld() 71 | if err == nil { 72 | require.Equal(t, held, success) 73 | return 74 | } 75 | require.ErrorIs(t, err, prototransform.ErrLeaseStateNotYetKnown) 76 | if i < 3 { 77 | time.Sleep(50 * time.Millisecond) 78 | } 79 | } 80 | require.Fail(t, "lease never checked successfully") 81 | } 82 | awaitSignal := func(key string, channel chan struct{}, expectAcquired, expectReleased int) { 83 | select { 84 | case <-time.After(500 * time.Millisecond): 85 | require.Fail(t, "callback never invoked") 86 | case <-channel: 87 | } 88 | mutex.Lock() 89 | defer mutex.Unlock() 90 | require.Equal(t, expectAcquired, acquired[key]) 91 | require.Equal(t, expectReleased, released[key]) 92 | } 93 | noSignal := func(key string, channel chan struct{}, expectAcquired, expectReleased int) { 94 | select { 95 | case <-time.After(500 * time.Millisecond): 96 | case <-channel: 97 | require.Fail(t, "callback invoked but should not have been") 98 | } 99 | mutex.Lock() 100 | defer mutex.Unlock() 101 | require.Equal(t, expectAcquired, acquired[key]) 102 | require.Equal(t, expectReleased, released[key]) 103 | } 104 | noSignalsPending := func(key string, acqCh, relCh chan struct{}, expectAcquired, expectReleased int) { 105 | select { 106 | case <-acqCh: 107 | require.Fail(t, "acquired callback invoked but should not have been") 108 | case <-relCh: 109 | require.Fail(t, "released callback invoked but should not have been") 110 | default: 111 | } 112 | mutex.Lock() 113 | defer mutex.Unlock() 114 | require.Equal(t, expectAcquired, acquired[key]) 115 | require.Equal(t, expectReleased, released[key]) 116 | } 117 | aLeaseForX := leaser.NewLease(ctx, "a", []byte{'x'}) 118 | checkLeaseInitial(aLeaseForX, true) 119 | aLeaseForX.SetCallbacks(track("a:x", aAcquiredByX, true), track("a:x", aReleasedByX, false)) 120 | awaitSignal("a:x", aAcquiredByX, 1, 0) 121 | 122 | aLeaseForY := leaser.NewLease(ctx, "a", []byte{'y'}) 123 | checkLeaseInitial(aLeaseForY, false) 124 | aLeaseForY.SetCallbacks(track("a:y", aAcquiredByY, true), track("a:y", aReleasedByY, false)) 125 | noSignal("a:y", aReleasedByY, 0, 0) 126 | 127 | bLeaseForX := leaser.NewLease(ctx, "b", []byte{'x'}) 128 | checkLeaseInitial(bLeaseForX, true) 129 | bLeaseForX.SetCallbacks(track("b", bAcquiredByX, true), track("b", bReleasedByX, false)) 130 | awaitSignal("b", bAcquiredByX, 1, 0) 131 | 132 | cLeaseForX := leaser.NewLease(ctx, "c", []byte{'x'}) 133 | checkLeaseInitial(cLeaseForX, true) 134 | cLeaseForX.SetCallbacks(track("c", cAcquiredByX, true), track("c", cReleasedByX, false)) 135 | awaitSignal("c", cAcquiredByX, 1, 0) 136 | 137 | dCtx, dCancel := context.WithCancel(ctx) 138 | dLeaseForZ := leaser.NewLease(dCtx, "d", []byte{'z'}) 139 | checkLeaseInitial(dLeaseForZ, true) 140 | dLeaseForZ.SetCallbacks(track("d", dAcquiredByZ, true), track("d", dReleasedByZ, false)) 141 | awaitSignal("d", dAcquiredByZ, 1, 0) 142 | 143 | aLeaseForX.Cancel() 144 | awaitSignal("a:x", aReleasedByX, 1, 1) 145 | awaitSignal("a:y", aAcquiredByY, 1, 0) 146 | 147 | dCancel() 148 | awaitSignal("d", dReleasedByZ, 1, 1) 149 | 150 | finalOwners := map[string][]byte{ 151 | "a": {'y'}, 152 | "b": {'x'}, 153 | "c": {'x'}, 154 | } 155 | 156 | cExpectReleased := 0 157 | if forceOwner != nil { 158 | err := forceOwner("c", []byte{'x', 'y', 'z'}) 159 | require.NoError(t, err) 160 | awaitSignal("c", cReleasedByX, 1, 1) 161 | 162 | finalOwners["c"] = []byte{'x', 'y', 'z'} 163 | cExpectReleased = 1 164 | } 165 | 166 | // final check of stats and signals 167 | time.Sleep(200 * time.Millisecond) 168 | noSignalsPending("a:x", aAcquiredByX, aReleasedByX, 1, 1) 169 | noSignalsPending("a:y", aAcquiredByY, aReleasedByY, 1, 0) 170 | noSignalsPending("b", bAcquiredByX, bReleasedByX, 1, 0) 171 | noSignalsPending("c", cAcquiredByX, cReleasedByX, 1, cExpectReleased) 172 | noSignalsPending("d", dAcquiredByZ, dReleasedByZ, 1, 1) 173 | 174 | return finalOwners 175 | } 176 | -------------------------------------------------------------------------------- /leaser/memcacheleaser/memcacheleaser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package memcacheleaser provides an implementation of prototransform.Leaser 16 | // that is backed by a memcached instance: https://memcached.org/. 17 | package memcacheleaser 18 | 19 | import ( 20 | "bytes" 21 | "context" 22 | "errors" 23 | "fmt" 24 | "time" 25 | 26 | "github.com/bradfitz/gomemcache/memcache" 27 | "github.com/bufbuild/prototransform" 28 | "github.com/bufbuild/prototransform/leaser" 29 | ) 30 | 31 | // Config represents the configuration parameters used to 32 | // create a new memcached-backed leaser. 33 | type Config struct { 34 | Client *memcache.Client 35 | KeyPrefix string 36 | LeaseTTLSeconds int32 37 | PollingPeriod time.Duration 38 | } 39 | 40 | // New creates a new memcached-backed leaser with the given configuration. 41 | func New(config Config) (prototransform.Leaser, error) { 42 | // validate config 43 | if config.Client == nil { 44 | return nil, errors.New("client cannot be nil") 45 | } 46 | if config.LeaseTTLSeconds < 0 { 47 | return nil, fmt.Errorf("lease TTL seconds (%d) cannot be negative", config.LeaseTTLSeconds) 48 | } 49 | if config.LeaseTTLSeconds == 0 { 50 | config.LeaseTTLSeconds = 30 51 | } 52 | if config.PollingPeriod < 0 { 53 | return nil, fmt.Errorf("polling period (%v) cannot be negative", config.PollingPeriod) 54 | } 55 | if config.PollingPeriod == 0 { 56 | config.PollingPeriod = 10 * time.Second 57 | } 58 | leaseDuration := time.Duration(config.LeaseTTLSeconds) * time.Second 59 | if config.PollingPeriod > leaseDuration { 60 | return nil, fmt.Errorf("polling period (%v) should be <= lease TTL (%v)", config.PollingPeriod, leaseDuration) 61 | } 62 | return &leaser.PollingLeaser{ 63 | LeaseStore: &leaseStore{ 64 | client: config.Client, 65 | keyPrefix: config.KeyPrefix, 66 | }, 67 | LeaseTTL: leaseDuration, 68 | PollingPeriod: config.PollingPeriod, 69 | }, nil 70 | } 71 | 72 | type leaseStore struct { 73 | client *memcache.Client 74 | keyPrefix string 75 | } 76 | 77 | func (l *leaseStore) TryAcquire(_ context.Context, leaseName string, leaseHolder []byte, ttl time.Duration) (created bool, holder []byte, err error) { 78 | key := l.keyPrefix + leaseName 79 | expire := int32(ttl.Seconds()) 80 | if expire == 0 { 81 | expire = 1 // minimum expiry is 1 second 82 | } 83 | for { 84 | // Optimize for the normal/expected case: lease exists. 85 | // So we first do a GET and then ADD if it doesn't exist. 86 | item, err := l.client.Get(key) 87 | if err != nil && !errors.Is(err, memcache.ErrCacheMiss) { 88 | return false, nil, err 89 | } 90 | if errors.Is(err, memcache.ErrCacheMiss) { 91 | // no such key; try to add it 92 | item = &memcache.Item{ 93 | Key: key, 94 | Value: leaseHolder, 95 | Expiration: expire, 96 | } 97 | err := l.client.Add(item) 98 | if errors.Is(err, memcache.ErrNotStored) { 99 | // we lost race to create item, so re-query and try again 100 | continue 101 | } 102 | if err != nil { 103 | return false, nil, err 104 | } 105 | // success! 106 | return true, leaseHolder, nil 107 | } 108 | if len(item.Value) == 0 { 109 | // no current leaseholder; let's try to take it 110 | item.Value = leaseHolder 111 | item.Expiration = expire 112 | err := l.client.CompareAndSwap(item) 113 | if errors.Is(err, memcache.ErrCASConflict) || errors.Is(err, memcache.ErrNotStored) { 114 | // CAS failed, so re-query and try again 115 | continue 116 | } 117 | if err != nil { 118 | return false, nil, err 119 | } 120 | // success! 121 | return true, leaseHolder, nil 122 | } 123 | if bytes.Equal(item.Value, leaseHolder) { 124 | // we are the leaseholder; bump the expiry (best effort only) 125 | item.Expiration = expire 126 | _ = l.client.CompareAndSwap(item) 127 | } 128 | return false, item.Value, nil 129 | } 130 | } 131 | 132 | func (l *leaseStore) Release(_ context.Context, leaseName string, leaseHolder []byte) error { 133 | // memcached doesn't have a way to conditionally delete, so if we read 134 | // the lease value and then deleted if it has the right value, we could 135 | // end up racing with a concurrent writer. So instead of deleting, we 136 | // CAS the value to empty bytes and set the shortest possible TTL (which 137 | // is one second). So empty bytes value means "no leaseholder". 138 | key := l.keyPrefix + leaseName 139 | for { 140 | item, err := l.client.Get(key) 141 | if err != nil { 142 | if errors.Is(err, memcache.ErrCacheMiss) { 143 | // not there anymore so nothing to delete 144 | return nil 145 | } 146 | return err 147 | } 148 | if !bytes.Equal(item.Value, leaseHolder) { 149 | // lease is not ours; nothing to do 150 | return nil 151 | } 152 | item.Value = nil 153 | item.Expiration = 1 154 | err = l.client.CompareAndSwap(item) 155 | if errors.Is(err, memcache.ErrCASConflict) || errors.Is(err, memcache.ErrNotStored) { 156 | // CAS failed, so re-query and try again 157 | continue 158 | } 159 | return err 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /leaser/memcacheleaser/memcacheleaser_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // We use a build tag since memcached may not be running. If it is 16 | // running, for use with the test, then pass flag "-tags with_servers" 17 | // when running tests to enable these tests. 18 | //go:build with_servers 19 | 20 | package memcacheleaser 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | "time" 26 | 27 | "github.com/bradfitz/gomemcache/memcache" 28 | "github.com/bufbuild/prototransform/leaser" 29 | "github.com/bufbuild/prototransform/leaser/internal/leasertesting" 30 | "github.com/stretchr/testify/assert" 31 | "github.com/stretchr/testify/require" 32 | ) 33 | 34 | func TestMemcacheLeaser(t *testing.T) { 35 | t.Parallel() 36 | ctx, cancel := context.WithCancel(context.Background()) 37 | t.Cleanup(cancel) 38 | 39 | client := memcache.New("localhost:11211") 40 | t.Cleanup(func() { 41 | err := client.Close() 42 | require.NoError(t, err) 43 | }) 44 | 45 | l, err := New(Config{ 46 | Client: client, 47 | }) 48 | require.NoError(t, err) 49 | // check defaults 50 | assert.Equal(t, 10*time.Second, l.(*leaser.PollingLeaser).PollingPeriod) 51 | assert.Equal(t, 30*time.Second, l.(*leaser.PollingLeaser).LeaseTTL) 52 | 53 | keyPrefix := "abc:" 54 | l, err = New(Config{ 55 | Client: client, 56 | LeaseTTLSeconds: 1, 57 | PollingPeriod: 25 * time.Millisecond, 58 | KeyPrefix: keyPrefix, 59 | }) 60 | require.NoError(t, err) 61 | 62 | forceOwner := func(key string, owner []byte) error { 63 | item := &memcache.Item{ 64 | Key: keyPrefix + key, 65 | Value: owner, 66 | Expiration: 5, 67 | } 68 | err = client.Set(item) 69 | t.Cleanup(func() { 70 | // delete when test done 71 | err = client.Delete(keyPrefix + key) 72 | require.NoError(t, err) 73 | }) 74 | return err 75 | } 76 | 77 | leasertesting.RunSimpleLeaserTests(t, ctx, l, forceOwner) 78 | } 79 | -------------------------------------------------------------------------------- /leaser/polling_leaser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package leaser 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "sync" 21 | "time" 22 | 23 | "github.com/bufbuild/prototransform" 24 | ) 25 | 26 | // PollingLeaser implements prototransform.Leaser by polling a lease 27 | // store, periodically trying to create a lease or query for the 28 | // current owner. Leaser implementations need only provide a lease 29 | // store implementation. This is suitable for many types of stores, 30 | // including Redis, memcached, or an RDBMS. It is not necessarily 31 | // suitable for stores that have better primitives for distributed 32 | // locking or leader election, such as ZooKeeper. 33 | type PollingLeaser struct { 34 | LeaseStore LeaseStore 35 | LeaseTTL time.Duration 36 | PollingPeriod time.Duration 37 | } 38 | 39 | // LeaseStore is the interface used to try to acquire and release leases. 40 | type LeaseStore interface { 41 | // TryAcquire tries to create a lease with the given leaseName and leaseHolder. If it is 42 | // successful (implying the given leaseHolder is the active holder), then it should return 43 | // true for the first value. If creation fails, the store should query for the leaseholder 44 | // value of the existing lease. The second returned value is the actual current leaseholder, 45 | // which can only differ from the given value if creation failed. If creation failed, but 46 | // the current process is the holder (i.e. the given leaseHolder matches the value in the 47 | // store), the store should bump the lease's TTL, so that the current process will continue 48 | // to hold it. 49 | // 50 | // This method is invoked regardless of whether the leaser believes that the current process 51 | // is the leaseholder. This is used to both try to acquire a lease, bump the TTL on a lease 52 | // if already held, and check the owner of the lease if not held. The store must be able to 53 | // do all of this atomically so that it is safe and correct in the face of concurrent 54 | // processes all trying to manage the same leaseName. 55 | TryAcquire(ctx context.Context, leaseName string, leaseHolder []byte, ttl time.Duration) (created bool, holder []byte, err error) 56 | // Release tries to delete a lease with the given leaseName and leaseHolder. If the lease 57 | // exists but is held by a different owner, it should not be deleted. 58 | // 59 | // This method will only be called when the leaser believes that the current process holds 60 | // the lease, for cleanup. 61 | Release(ctx context.Context, leaseName string, leaseHolder []byte) error 62 | } 63 | 64 | // NewLease implements the prototransform.Leaser interface. 65 | func (l *PollingLeaser) NewLease(ctx context.Context, leaseName string, leaseHolder []byte) prototransform.Lease { 66 | ctx, cancel := context.WithCancel(ctx) 67 | done := make(chan struct{}) 68 | newLease := &lease{ 69 | cancel: cancel, 70 | done: done, 71 | err: prototransform.ErrLeaseStateNotYetKnown, 72 | } 73 | go newLease.run(ctx, l, leaseName, leaseHolder, done) 74 | return newLease 75 | } 76 | 77 | type lease struct { 78 | cancel context.CancelFunc 79 | done <-chan struct{} 80 | 81 | mu sync.Mutex 82 | isHeld bool 83 | err error 84 | 85 | notifyMu sync.Mutex 86 | onAcquire, onRelease func() 87 | } 88 | 89 | func (l *lease) IsHeld() (bool, error) { 90 | l.mu.Lock() 91 | isHeld, err := l.isHeld, l.err 92 | l.mu.Unlock() 93 | return isHeld, err 94 | } 95 | 96 | func (l *lease) SetCallbacks(onAcquire, onRelease func()) { 97 | l.mu.Lock() 98 | defer l.mu.Unlock() 99 | l.onAcquire, l.onRelease = onAcquire, onRelease 100 | if l.isHeld && l.onAcquire != nil { 101 | go func() { 102 | l.notifyMu.Lock() 103 | defer l.notifyMu.Unlock() 104 | l.onAcquire() 105 | }() 106 | } 107 | } 108 | 109 | func (l *lease) Cancel() { 110 | l.cancel() 111 | <-l.done 112 | } 113 | 114 | func (l *lease) run(ctx context.Context, leaser *PollingLeaser, key string, value []byte, done chan<- struct{}) { 115 | defer close(done) 116 | ticker := time.NewTicker(leaser.PollingPeriod) 117 | defer ticker.Stop() 118 | l.poll(ctx, leaser, key, value) 119 | for { 120 | select { 121 | case <-ctx.Done(): 122 | l.releaseNow(ctx, leaser, key, value) 123 | return 124 | case <-ticker.C: 125 | if ctx.Err() != nil { 126 | // skip polling if context is done 127 | l.releaseNow(ctx, leaser, key, value) 128 | return 129 | } 130 | l.poll(ctx, leaser, key, value) 131 | } 132 | } 133 | } 134 | 135 | func (l *lease) poll(ctx context.Context, leaser *PollingLeaser, key string, value []byte) { 136 | created, holder, err := leaser.LeaseStore.TryAcquire(ctx, key, value, leaser.LeaseTTL) 137 | if err != nil { 138 | l.released(err) 139 | return 140 | } 141 | if created { 142 | l.acquired() 143 | return 144 | } 145 | if bytes.Equal(holder, value) { 146 | // The existing lease is ours 147 | l.acquired() 148 | return 149 | } 150 | // The existing lease is not ours 151 | l.released(nil) 152 | } 153 | 154 | func (l *lease) releaseNow(ctx context.Context, leaser *PollingLeaser, key string, value []byte) { 155 | l.mu.Lock() 156 | isHeld := l.isHeld 157 | l.mu.Unlock() 158 | if isHeld { 159 | // best effort: immediately release if we hold it 160 | _ = leaser.LeaseStore.Release(ctx, key, value) 161 | } 162 | l.released(nil) 163 | } 164 | 165 | func (l *lease) acquired() { 166 | l.mu.Lock() 167 | defer l.mu.Unlock() 168 | if !l.isHeld && l.onAcquire != nil { 169 | go func() { 170 | l.notifyMu.Lock() 171 | defer l.notifyMu.Unlock() 172 | l.onAcquire() 173 | }() 174 | } 175 | l.isHeld = true 176 | l.err = nil 177 | } 178 | 179 | func (l *lease) released(err error) { 180 | l.mu.Lock() 181 | defer l.mu.Unlock() 182 | if l.isHeld && l.onRelease != nil { 183 | go func() { 184 | l.notifyMu.Lock() 185 | defer l.notifyMu.Unlock() 186 | l.onRelease() 187 | }() 188 | } 189 | l.isHeld = false 190 | l.err = err 191 | } 192 | -------------------------------------------------------------------------------- /leaser/redisleaser/redisleaser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package redisleaser provides an implementation of prototransform.Leaser 16 | // that is backed by a Redis instance: https://redis.io/. 17 | package redisleaser 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "fmt" 23 | "strings" 24 | "sync" 25 | "time" 26 | 27 | "github.com/bufbuild/prototransform" 28 | "github.com/bufbuild/prototransform/leaser" 29 | "github.com/gomodule/redigo/redis" 30 | ) 31 | 32 | const ( 33 | // This script atomically tries to acquire a lease by creating a new key. 34 | // If creation fails, but the existing key has a value that indicates the 35 | // current process holds the lease, it bumps the lease's TTL. 36 | // 37 | // Input: 38 | // KEYS[1]: the lease key 39 | // ARGV[1]: the desired leaseholder value (current process) 40 | // ARGV[2]: desired lease TTL in milliseconds 41 | // 42 | // Output: 43 | // TABLE: {bool, bulk string} 44 | // bool is true if key was created; bulk string is value of key. 45 | tryAcquireLUAScript = ` 46 | redis.setresp(3) 47 | local resp = redis.call('set', KEYS[1], ARGV[1], 'get', 'nx', 'px', ARGV[2]) 48 | if resp == nil then 49 | return {true, ARGV[1]} 50 | elseif resp == ARGV[1] then 51 | redis.call('pexpire', KEYS[1], ARGV[2], 'lt') 52 | end 53 | return {false, resp} 54 | ` 55 | 56 | // This script atomically releases a lease by deleting the key if 57 | // and only if the current process is the holder. 58 | // 59 | // Input: 60 | // KEYS[1]: the lease key 61 | // ARGV[1]: the leaseholder value of the current process 62 | // 63 | // Output: 64 | // None 65 | releaseLUAScript = ` 66 | if redis.call('get', KEYS[1]) == ARGV[1] then redis.call('del', KEYS[1]) end 67 | ` 68 | ) 69 | 70 | // Config represents the configuration parameters used to 71 | // create a new Redis-backed leaser. 72 | type Config struct { 73 | Client *redis.Pool 74 | KeyPrefix string 75 | LeaseTTL time.Duration 76 | PollingPeriod time.Duration 77 | } 78 | 79 | // New creates a new Redis-backed leaser with the given configuration. 80 | func New(config Config) (prototransform.Leaser, error) { 81 | // validate config 82 | if config.Client == nil { 83 | return nil, errors.New("client cannot be nil") 84 | } 85 | if config.LeaseTTL < 0 { 86 | return nil, fmt.Errorf("lease TTL (%v) cannot be negative", config.LeaseTTL) 87 | } 88 | if config.LeaseTTL == 0 { 89 | config.LeaseTTL = 30 * time.Second 90 | } 91 | if config.PollingPeriod < 0 { 92 | return nil, fmt.Errorf("polling period (%v) cannot be negative", config.PollingPeriod) 93 | } 94 | if config.PollingPeriod == 0 { 95 | config.PollingPeriod = 10 * time.Second 96 | } 97 | if config.PollingPeriod > config.LeaseTTL { 98 | return nil, fmt.Errorf("polling period (%v) should be <= lease TTL (%v)", config.PollingPeriod, config.LeaseTTL) 99 | } 100 | return &leaser.PollingLeaser{ 101 | LeaseStore: &leaseStore{ 102 | pool: config.Client, 103 | keyPrefix: config.KeyPrefix, 104 | }, 105 | LeaseTTL: config.LeaseTTL, 106 | PollingPeriod: config.PollingPeriod, 107 | }, nil 108 | } 109 | 110 | type leaseStore struct { 111 | pool *redis.Pool 112 | keyPrefix string 113 | 114 | scriptsMu sync.RWMutex 115 | tryAcquireSHA string 116 | releaseSHA string 117 | } 118 | 119 | func (l *leaseStore) TryAcquire(ctx context.Context, leaseName string, leaseHolder []byte, ttl time.Duration) (created bool, holder []byte, err error) { 120 | for { 121 | tryAcquire, _, err := l.getScriptSHAs(ctx) 122 | if tryAcquire == "" { 123 | return false, nil, err 124 | } 125 | conn, err := l.pool.GetContext(ctx) 126 | if err != nil { 127 | return false, nil, err 128 | } 129 | defer func() { 130 | _ = conn.Close() 131 | }() 132 | ttlMillis := ttl.Milliseconds() 133 | if ttlMillis == 0 { 134 | ttlMillis = 1 // must not be zero 135 | } 136 | resp, err := redis.Values(redis.DoContext(conn, ctx, "evalsha", tryAcquire, 1, l.keyPrefix+leaseName, leaseHolder, ttlMillis)) 137 | if err != nil && strings.Contains(err.Error(), "NOSCRIPT") { 138 | // script cache was flushed; reload scripts and try again 139 | l.scriptsMu.Lock() 140 | l.tryAcquireSHA, l.releaseSHA = "", "" 141 | l.scriptsMu.Unlock() 142 | continue 143 | } 144 | if err != nil { 145 | return false, nil, err 146 | } 147 | if len(resp) != 2 { 148 | return false, nil, errors.New("tryacquire script returned wrong number of values") 149 | } 150 | boolVal, err := redis.Bool(resp[0], nil) 151 | if err != nil { 152 | return false, nil, fmt.Errorf("tryacquire script returned non-bool for first value: %w", err) 153 | } 154 | bytesVal, err := redis.Bytes(resp[1], nil) 155 | if err != nil { 156 | return false, nil, fmt.Errorf("tryacquire script returned non-bytes for second value: %w", err) 157 | } 158 | return boolVal, bytesVal, nil 159 | } 160 | } 161 | 162 | func (l *leaseStore) Release(ctx context.Context, leaseName string, leaseHolder []byte) error { 163 | for { 164 | _, release, err := l.getScriptSHAs(ctx) 165 | if release == "" { 166 | return err 167 | } 168 | conn, err := l.pool.GetContext(ctx) 169 | if err != nil { 170 | return err 171 | } 172 | defer func() { 173 | _ = conn.Close() 174 | }() 175 | _, err = redis.DoContext(conn, ctx, "evalsha", release, 1, l.keyPrefix+leaseName, leaseHolder) 176 | if err != nil && strings.Contains(err.Error(), "NOSCRIPT") { 177 | // script cache was flushed; reload scripts and try again 178 | l.scriptsMu.Lock() 179 | l.tryAcquireSHA, l.releaseSHA = "", "" 180 | l.scriptsMu.Unlock() 181 | continue 182 | } 183 | return err 184 | } 185 | } 186 | 187 | func (l *leaseStore) getScriptSHAs(ctx context.Context) (tryAcquire, release string, err error) { 188 | l.scriptsMu.RLock() 189 | tryAcquire, release = l.tryAcquireSHA, l.releaseSHA 190 | l.scriptsMu.RUnlock() 191 | if tryAcquire != "" && release != "" { 192 | return tryAcquire, release, nil 193 | } 194 | 195 | l.scriptsMu.Lock() 196 | defer l.scriptsMu.Unlock() 197 | // check again, in case they were concurrently added while upgrading lock 198 | tryAcquire, release = l.tryAcquireSHA, l.releaseSHA 199 | if tryAcquire != "" && release != "" { 200 | return tryAcquire, release, nil 201 | } 202 | conn, err := l.pool.GetContext(ctx) 203 | if err != nil { 204 | return tryAcquire, release, err 205 | } 206 | defer func() { 207 | _ = conn.Close() 208 | }() 209 | 210 | if tryAcquire == "" { 211 | tryAcquire, err = redis.String(redis.DoContext(conn, ctx, "script", "load", tryAcquireLUAScript)) 212 | } 213 | if release == "" { 214 | var releaseErr error 215 | release, releaseErr = redis.String(redis.DoContext(conn, ctx, "script", "load", releaseLUAScript)) 216 | if releaseErr != nil && err == nil { 217 | err = releaseErr 218 | } 219 | } 220 | l.tryAcquireSHA, l.releaseSHA = tryAcquire, release 221 | return tryAcquire, release, err 222 | } 223 | -------------------------------------------------------------------------------- /leaser/redisleaser/redisleaser_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // We use a build tag since redis may not be running. If it is 16 | // running, for use with the test, then pass flag "-tags with_servers" 17 | // when running tests to enable these tests. 18 | //go:build with_servers 19 | 20 | package redisleaser 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | "time" 26 | 27 | "github.com/bufbuild/prototransform/leaser" 28 | "github.com/bufbuild/prototransform/leaser/internal/leasertesting" 29 | "github.com/gomodule/redigo/redis" 30 | "github.com/stretchr/testify/assert" 31 | "github.com/stretchr/testify/require" 32 | ) 33 | 34 | func TestRedisLeaser(t *testing.T) { 35 | t.Parallel() 36 | ctx, cancel := context.WithCancel(context.Background()) 37 | t.Cleanup(cancel) 38 | 39 | pool := &redis.Pool{ 40 | DialContext: func(ctx context.Context) (redis.Conn, error) { 41 | return redis.Dial("tcp", "localhost:6379") 42 | }, 43 | } 44 | t.Cleanup(func() { 45 | err := pool.Close() 46 | require.NoError(t, err) 47 | }) 48 | 49 | l, err := New(Config{ 50 | Client: pool, 51 | }) 52 | require.NoError(t, err) 53 | // check defaults 54 | assert.Equal(t, 10*time.Second, l.(*leaser.PollingLeaser).PollingPeriod) 55 | assert.Equal(t, 30*time.Second, l.(*leaser.PollingLeaser).LeaseTTL) 56 | 57 | keyPrefix := "abc:" 58 | l, err = New(Config{ 59 | Client: pool, 60 | LeaseTTL: time.Second, 61 | PollingPeriod: 25 * time.Millisecond, 62 | KeyPrefix: keyPrefix, 63 | }) 64 | require.NoError(t, err) 65 | 66 | forceOwner := func(key string, owner []byte) error { 67 | conn, err := pool.GetContext(ctx) 68 | if err != nil { 69 | return err 70 | } 71 | defer func() { 72 | _ = conn.Close() 73 | }() 74 | _, err = redis.DoContext(conn, ctx, "set", keyPrefix+key, owner, "ex", 5) 75 | t.Cleanup(func() { 76 | // delete when test done 77 | conn, err := pool.GetContext(ctx) 78 | require.NoError(t, err) 79 | defer func() { 80 | _ = conn.Close() 81 | }() 82 | _, err = redis.DoContext(conn, ctx, "del", keyPrefix+key) 83 | require.NoError(t, err) 84 | }) 85 | return err 86 | } 87 | 88 | leasertesting.RunSimpleLeaserTests(t, ctx, l, forceOwner) 89 | } 90 | -------------------------------------------------------------------------------- /reflect_client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "fmt" 21 | "net/http" 22 | "os" 23 | "strings" 24 | 25 | "buf.build/gen/go/bufbuild/reflect/connectrpc/go/buf/reflect/v1beta1/reflectv1beta1connect" 26 | "connectrpc.com/connect" 27 | ) 28 | 29 | // NewDefaultFileDescriptorSetServiceClient will create an authenticated connection to the 30 | // public Buf Schema Registry (BSR) at https://buf.build. If the given token is empty, 31 | // the BUF_TOKEN environment variable will be consulted. 32 | // 33 | // If you require a connection to a different BSR instance, create your own 34 | // [reflectv1beta1connect.FileDescriptorSetServiceClient]. You can use [NewAuthInterceptor] 35 | // to configure authentication credentials. Also keep in mind that BSR instances support 36 | // conditional GET requests for the endpoint in question, so also use [connect.WithHTTPGet] 37 | // to enable that, which will typically eliminate unnecessary re-downloads of a schema. 38 | // (It may not eliminate them if you are filtering the schema by a large number of types 39 | // such that the entire request cannot fit in the URL of a GET request.) 40 | // 41 | // For help with authenticating with the Buf Schema Registry visit: https://docs.buf.build/bsr/authentication 42 | func NewDefaultFileDescriptorSetServiceClient(token string) reflectv1beta1connect.FileDescriptorSetServiceClient { 43 | if token == "" { 44 | token, _ = BufTokenFromEnvironment("buf.build") 45 | } 46 | return reflectv1beta1connect.NewFileDescriptorSetServiceClient( 47 | http.DefaultClient, "https://buf.build", 48 | connect.WithInterceptors( 49 | NewAuthInterceptor(token), 50 | connect.UnaryInterceptorFunc(func(call connect.UnaryFunc) connect.UnaryFunc { 51 | return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) { 52 | // decorate user-agent with the name of the package 53 | userAgent := req.Header().Get("User-Agent") + " prototransform-go" 54 | req.Header().Set("User-Agent", userAgent) 55 | return call(ctx, req) 56 | } 57 | }), 58 | ), 59 | connect.WithHTTPGet(), 60 | connect.WithHTTPGetMaxURLSize(8192, true), 61 | ) 62 | } 63 | 64 | // NewAuthInterceptor accepts a token for a Buf Schema Registry (BSR) and returns an 65 | // interceptor which can be used when creating a Connect client so that every RPC 66 | // to the BSR is correctly authenticated. 67 | // 68 | // To understand more about authenticating with the BSR visit: https://docs.buf.build/bsr/authentication 69 | // 70 | // To get a token from the environment (e.g. BUF_TOKEN env var), see BufTokenFromEnvironment. 71 | func NewAuthInterceptor(token string) connect.Interceptor { 72 | bearerAuthValue := "Bearer " + token 73 | return connect.UnaryInterceptorFunc(func(next connect.UnaryFunc) connect.UnaryFunc { 74 | return func(ctx context.Context, request connect.AnyRequest) (connect.AnyResponse, error) { 75 | request.Header().Set("Authorization", bearerAuthValue) 76 | return next(ctx, request) 77 | } 78 | }) 79 | } 80 | 81 | // BufTokenFromEnvironment returns a token that can be used to download the given module from 82 | // the BSR by inspecting the BUF_TOKEN environment variable. The given moduleRef can be a full 83 | // module reference, with or without a version, or it can just be the domain of the BSR. 84 | func BufTokenFromEnvironment(moduleRef string) (string, error) { 85 | parts := strings.SplitN(moduleRef, "/", 2) 86 | envBufToken := os.Getenv("BUF_TOKEN") 87 | if envBufToken == "" { 88 | return "", errors.New("no BUF_TOKEN environment variable set") 89 | } 90 | tok := parseBufToken(envBufToken, parts[0]) 91 | if tok == "" { 92 | return "", fmt.Errorf("BUF_TOKEN environment variable did not include a token for remote %q", parts[0]) 93 | } 94 | return tok, nil 95 | } 96 | 97 | func parseBufToken(envVar, remote string) string { 98 | isMultiToken := strings.ContainsAny(envVar, "@,") 99 | if !isMultiToken { 100 | return envVar 101 | } 102 | tokenConfigs := strings.Split(envVar, ",") 103 | suffix := "@" + remote 104 | for _, tokenConfig := range tokenConfigs { 105 | token := strings.TrimSuffix(tokenConfig, suffix) 106 | if token == tokenConfig { 107 | // did not have the right suffix 108 | continue 109 | } 110 | return token 111 | } 112 | return "" 113 | } 114 | -------------------------------------------------------------------------------- /reflect_client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "context" 19 | "testing" 20 | "time" 21 | 22 | "buf.build/gen/go/bufbuild/reflect/connectrpc/go/buf/reflect/v1beta1/reflectv1beta1connect" 23 | reflectv1beta1 "buf.build/gen/go/bufbuild/reflect/protocolbuffers/go/buf/reflect/v1beta1" 24 | "connectrpc.com/connect" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | const ( 29 | moduleName = "buf.build/googleapis/googleapis" 30 | moduleVersion = "cc916c31859748a68fd229a3c8d7a2e8" 31 | ) 32 | 33 | func TestReflectClientLargeRequest(t *testing.T) { 34 | t.Parallel() 35 | token, err := BufTokenFromEnvironment(moduleName) 36 | require.NoError(t, err) 37 | // Supply auth credentials to the BSR 38 | client := NewDefaultFileDescriptorSetServiceClient(token) 39 | 40 | // Max URL size is 8192. Subtract the preamble from that. 41 | maxRequestSz := 8192 - len("https://buf.build"+ 42 | reflectv1beta1connect.FileDescriptorSetServiceGetFileDescriptorSetProcedure+ 43 | "?encoding=proto&base64=1&connect=v1&message=") 44 | // The message will be base64-encoded, so we really only have 3/4 this length 45 | // due to expansion from the encoding. 46 | maxRequestSz = maxRequestSz * 3 / 4 47 | // Finally subtract the module and version request fields. 48 | maxRequestSz -= 2 + len(moduleName) + 2 + len(moduleVersion) 49 | // What remains is what we can use to enumerate types to fill up the message size. 50 | 51 | var types []string 52 | for maxRequestSz >= 0 { 53 | for _, name := range symbolNames { 54 | types = append(types, name) 55 | maxRequestSz -= 2 + len(name) 56 | if maxRequestSz < 0 { 57 | // that pushed us over the edge, so types slice has one extra item 58 | break 59 | } 60 | } 61 | } 62 | 63 | ctx := context.Background() 64 | req := connect.NewRequest(&reflectv1beta1.GetFileDescriptorSetRequest{ 65 | Module: moduleName, 66 | Version: moduleVersion, 67 | Symbols: types, 68 | }) 69 | req.Header().Set("If-None-Match", moduleVersion) 70 | // The full types slice should be too large for a GET, which means we end up 71 | // downloading the entire response. 72 | resp, err := getFileDescriptorSet(ctx, client, req) 73 | require.NoError(t, err) 74 | require.GreaterOrEqual(t, len(resp.Msg.GetFileDescriptorSet().GetFile()), 42) 75 | 76 | // If we ask for one fewer type, we should be below the limit and get back 77 | // a "not modified" response code instead of a response body. 78 | req.Msg.Symbols = types[:len(types)-1] 79 | _, err = getFileDescriptorSet(ctx, client, req) 80 | require.True(t, connect.IsNotModifiedError(err)) 81 | } 82 | 83 | func getFileDescriptorSet( 84 | ctx context.Context, 85 | client reflectv1beta1connect.FileDescriptorSetServiceClient, 86 | req *connect.Request[reflectv1beta1.GetFileDescriptorSetRequest], 87 | ) (*connect.Response[reflectv1beta1.GetFileDescriptorSetResponse], error) { 88 | // We are hitting the real buf.build endpoint. To work-around spurious errors 89 | // from a temporary network partition or encountering the endpoint rate limit, 90 | // we will retry for up to a minute. (The long retry window is to allow 91 | // exponential back-off delays between attempts and to be resilient to cases 92 | // in CI where multiple concurrent jobs are hitting the endpoint and exceeding 93 | // the rate limit.) 94 | var lastErr error 95 | delay := 250 * time.Millisecond 96 | start := time.Now() 97 | for { 98 | if lastErr != nil { 99 | // delay between attempts 100 | time.Sleep(delay) 101 | delay *= 2 102 | } 103 | resp, err := client.GetFileDescriptorSet(ctx, req) 104 | if err == nil { 105 | return resp, nil 106 | } 107 | code := connect.CodeOf(err) 108 | if code != connect.CodeUnavailable && code != connect.CodeResourceExhausted { 109 | return nil, err 110 | } 111 | if time.Since(start) > time.Minute { 112 | // Took too long. Fail. 113 | return nil, err 114 | } 115 | // On "unavailable" (could be transient network issue) or 116 | // "resource exhausted" (rate limited), we loop and try again. 117 | lastErr = err 118 | } 119 | } 120 | 121 | // Message types in buf.build/googleapis/googleapis. Generated via the following: 122 | // 123 | // buf build -o -#format=json buf.build/googleapis/googleapis \ 124 | // | jq '.file[] | .package as $pkg | .messageType[]? | ( $pkg + "." + .name )' \ 125 | // | sort \ 126 | // | jq -s . 127 | // 128 | //nolint:gochecknoglobals 129 | var symbolNames = []string{ 130 | "google.api.ClientLibrarySettings", 131 | "google.api.CommonLanguageSettings", 132 | "google.api.CppSettings", 133 | "google.api.CustomHttpPattern", 134 | "google.api.DotnetSettings", 135 | "google.api.GoSettings", 136 | "google.api.Http", 137 | "google.api.HttpBody", 138 | "google.api.HttpRule", 139 | "google.api.JavaSettings", 140 | "google.api.MethodSettings", 141 | "google.api.NodeSettings", 142 | "google.api.PhpSettings", 143 | "google.api.Publishing", 144 | "google.api.PythonSettings", 145 | "google.api.ResourceDescriptor", 146 | "google.api.ResourceReference", 147 | "google.api.RubySettings", 148 | "google.api.Visibility", 149 | "google.api.VisibilityRule", 150 | "google.api.expr.v1alpha1.CheckedExpr", 151 | "google.api.expr.v1alpha1.Constant", 152 | "google.api.expr.v1alpha1.Decl", 153 | "google.api.expr.v1alpha1.EnumValue", 154 | "google.api.expr.v1alpha1.ErrorSet", 155 | "google.api.expr.v1alpha1.EvalState", 156 | "google.api.expr.v1alpha1.Explain", 157 | "google.api.expr.v1alpha1.Expr", 158 | "google.api.expr.v1alpha1.ExprValue", 159 | "google.api.expr.v1alpha1.ListValue", 160 | "google.api.expr.v1alpha1.MapValue", 161 | "google.api.expr.v1alpha1.ParsedExpr", 162 | "google.api.expr.v1alpha1.Reference", 163 | "google.api.expr.v1alpha1.SourceInfo", 164 | "google.api.expr.v1alpha1.SourcePosition", 165 | "google.api.expr.v1alpha1.Type", 166 | "google.api.expr.v1alpha1.UnknownSet", 167 | "google.api.expr.v1alpha1.Value", 168 | "google.api.expr.v1beta1.Decl", 169 | "google.api.expr.v1beta1.DeclType", 170 | "google.api.expr.v1beta1.EnumValue", 171 | "google.api.expr.v1beta1.ErrorSet", 172 | "google.api.expr.v1beta1.EvalState", 173 | "google.api.expr.v1beta1.Expr", 174 | "google.api.expr.v1beta1.ExprValue", 175 | "google.api.expr.v1beta1.FunctionDecl", 176 | "google.api.expr.v1beta1.IdRef", 177 | "google.api.expr.v1beta1.IdentDecl", 178 | "google.api.expr.v1beta1.ListValue", 179 | "google.api.expr.v1beta1.Literal", 180 | "google.api.expr.v1beta1.MapValue", 181 | "google.api.expr.v1beta1.ParsedExpr", 182 | "google.api.expr.v1beta1.SourceInfo", 183 | "google.api.expr.v1beta1.SourcePosition", 184 | "google.api.expr.v1beta1.UnknownSet", 185 | "google.api.expr.v1beta1.Value", 186 | "google.bytestream.QueryWriteStatusRequest", 187 | "google.bytestream.QueryWriteStatusResponse", 188 | "google.bytestream.ReadRequest", 189 | "google.bytestream.ReadResponse", 190 | "google.bytestream.WriteRequest", 191 | "google.bytestream.WriteResponse", 192 | "google.geo.type.Viewport", 193 | "google.longrunning.CancelOperationRequest", 194 | "google.longrunning.DeleteOperationRequest", 195 | "google.longrunning.GetOperationRequest", 196 | "google.longrunning.ListOperationsRequest", 197 | "google.longrunning.ListOperationsResponse", 198 | "google.longrunning.Operation", 199 | "google.longrunning.OperationInfo", 200 | "google.longrunning.WaitOperationRequest", 201 | "google.protobuf.Any", 202 | "google.protobuf.BoolValue", 203 | "google.protobuf.BytesValue", 204 | "google.protobuf.DescriptorProto", 205 | "google.protobuf.DoubleValue", 206 | "google.protobuf.Duration", 207 | "google.protobuf.Empty", 208 | "google.protobuf.EnumDescriptorProto", 209 | "google.protobuf.EnumOptions", 210 | "google.protobuf.EnumValueDescriptorProto", 211 | "google.protobuf.EnumValueOptions", 212 | "google.protobuf.ExtensionRangeOptions", 213 | "google.protobuf.FieldDescriptorProto", 214 | "google.protobuf.FieldOptions", 215 | "google.protobuf.FileDescriptorProto", 216 | "google.protobuf.FileDescriptorSet", 217 | "google.protobuf.FileOptions", 218 | "google.protobuf.FloatValue", 219 | "google.protobuf.GeneratedCodeInfo", 220 | "google.protobuf.Int32Value", 221 | "google.protobuf.Int64Value", 222 | "google.protobuf.ListValue", 223 | "google.protobuf.MessageOptions", 224 | "google.protobuf.MethodDescriptorProto", 225 | "google.protobuf.MethodOptions", 226 | "google.protobuf.OneofDescriptorProto", 227 | "google.protobuf.OneofOptions", 228 | "google.protobuf.ServiceDescriptorProto", 229 | "google.protobuf.ServiceOptions", 230 | "google.protobuf.SourceCodeInfo", 231 | "google.protobuf.StringValue", 232 | "google.protobuf.Struct", 233 | "google.protobuf.Timestamp", 234 | "google.protobuf.UInt32Value", 235 | "google.protobuf.UInt64Value", 236 | "google.protobuf.UninterpretedOption", 237 | "google.protobuf.Value", 238 | "google.rpc.BadRequest", 239 | "google.rpc.DebugInfo", 240 | "google.rpc.ErrorInfo", 241 | "google.rpc.Help", 242 | "google.rpc.LocalizedMessage", 243 | "google.rpc.PreconditionFailure", 244 | "google.rpc.QuotaFailure", 245 | "google.rpc.RequestInfo", 246 | "google.rpc.ResourceInfo", 247 | "google.rpc.RetryInfo", 248 | "google.rpc.Status", 249 | "google.rpc.context.AttributeContext", 250 | "google.type.Color", 251 | "google.type.Date", 252 | "google.type.DateTime", 253 | "google.type.Decimal", 254 | "google.type.Expr", 255 | "google.type.Fraction", 256 | "google.type.Interval", 257 | "google.type.LatLng", 258 | "google.type.LocalizedText", 259 | "google.type.Money", 260 | "google.type.PhoneNumber", 261 | "google.type.PostalAddress", 262 | "google.type.Quaternion", 263 | "google.type.TimeOfDay", 264 | "google.type.TimeZone", 265 | } 266 | -------------------------------------------------------------------------------- /resolver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "google.golang.org/protobuf/reflect/protodesc" 19 | "google.golang.org/protobuf/reflect/protoreflect" 20 | "google.golang.org/protobuf/reflect/protoregistry" 21 | "google.golang.org/protobuf/types/descriptorpb" 22 | "google.golang.org/protobuf/types/dynamicpb" 23 | ) 24 | 25 | // Resolver is used to resolve symbol names and numbers into schema definitions. 26 | type Resolver interface { 27 | protoregistry.ExtensionTypeResolver 28 | protoregistry.MessageTypeResolver 29 | 30 | // FindEnumByName looks up an enum by its full name. 31 | // E.g., "google.protobuf.Field.Kind". 32 | // 33 | // This returns (nil, NotFound) if not found. 34 | FindEnumByName(enum protoreflect.FullName) (protoreflect.EnumType, error) 35 | } 36 | 37 | type resolver struct { 38 | *protoregistry.Files 39 | *protoregistry.Types 40 | } 41 | 42 | // newResolver creates a new Resolver. 43 | // 44 | // If the input slice is empty, this returns nil 45 | // The given FileDescriptors must be self-contained, that is they must contain all imports. 46 | // This can NOT be guaranteed for FileDescriptorSets given over the wire, and can only be guaranteed from builds. 47 | func newResolver(fileDescriptors *descriptorpb.FileDescriptorSet) (*resolver, error) { 48 | var result resolver 49 | // TODO(TCN-925): maybe should reparse unrecognized fields in fileDescriptors after creating resolver? 50 | if len(fileDescriptors.GetFile()) == 0 { 51 | return &result, nil 52 | } 53 | files, err := protodesc.FileOptions{AllowUnresolvable: true}.NewFiles(fileDescriptors) 54 | if err != nil { 55 | return nil, err 56 | } 57 | result.Files = files 58 | result.Types = &protoregistry.Types{} 59 | var rangeErr error 60 | files.RangeFiles(func(fileDescriptor protoreflect.FileDescriptor) bool { 61 | if err := registerTypes(result.Types, fileDescriptor); err != nil { 62 | rangeErr = err 63 | return false 64 | } 65 | return true 66 | }) 67 | if rangeErr != nil { 68 | return nil, rangeErr 69 | } 70 | return &result, nil 71 | } 72 | 73 | type typeContainer interface { 74 | Enums() protoreflect.EnumDescriptors 75 | Messages() protoreflect.MessageDescriptors 76 | Extensions() protoreflect.ExtensionDescriptors 77 | } 78 | 79 | func registerTypes(types *protoregistry.Types, container typeContainer) error { 80 | for i := range container.Enums().Len() { 81 | if err := types.RegisterEnum(dynamicpb.NewEnumType(container.Enums().Get(i))); err != nil { 82 | return err 83 | } 84 | } 85 | for i := range container.Messages().Len() { 86 | msg := container.Messages().Get(i) 87 | if err := types.RegisterMessage(dynamicpb.NewMessageType(msg)); err != nil { 88 | return err 89 | } 90 | if err := registerTypes(types, msg); err != nil { 91 | return err 92 | } 93 | } 94 | for i := range container.Extensions().Len() { 95 | if err := types.RegisterExtension(dynamicpb.NewExtensionType(container.Extensions().Get(i))); err != nil { 96 | return err 97 | } 98 | } 99 | return nil 100 | } 101 | -------------------------------------------------------------------------------- /schema_poller.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | 21 | "buf.build/gen/go/bufbuild/reflect/connectrpc/go/buf/reflect/v1beta1/reflectv1beta1connect" 22 | reflectv1beta1 "buf.build/gen/go/bufbuild/reflect/protocolbuffers/go/buf/reflect/v1beta1" 23 | "connectrpc.com/connect" 24 | "google.golang.org/protobuf/types/descriptorpb" 25 | ) 26 | 27 | var ( 28 | // ErrSchemaNotModified is an error that may be returned by a SchemaPoller to 29 | // indicate that the poller did not return any descriptors because the caller's 30 | // cached version is still sufficiently fresh. 31 | ErrSchemaNotModified = errors.New("no response because schema not modified") 32 | ) 33 | 34 | // SchemaPoller polls for descriptors from a remote source. 35 | // See [NewSchemaPoller]. 36 | type SchemaPoller interface { 37 | // GetSchema polls for a schema. The given symbols may be used to filter 38 | // the schema to return a smaller result. The given currentVersion, if not 39 | // empty, indicates the version that the caller already has fetched and 40 | // cached. So if that is still the current version of the schema (nothing 41 | // newer to download), the implementation may return an ErrSchemaNotModified 42 | // error. 43 | GetSchema(ctx context.Context, symbols []string, currentVersion string) (descriptors *descriptorpb.FileDescriptorSet, version string, err error) 44 | // GetSchemaID returns a string that identifies the schema that it fetches. 45 | // For a BSR module, for example, this might be "buf.build/owner/module:version". 46 | GetSchemaID() string 47 | } 48 | 49 | // NewSchemaPoller returns a SchemaPoller that uses the given Buf Reflection 50 | // API client to download descriptors for the given module. If the given version is 51 | // non-empty, the descriptors will be downloaded from that version of the module. 52 | // 53 | // The version should either be blank or indicate a tag that may change over time, 54 | // such as a draft name. If a fixed tag or commit is provided, then the periodic 55 | // polling is unnecessary since the schema for such a version is immutable. 56 | // 57 | // To create a client that can download descriptors from the buf.build public BSR, 58 | // see [NewDefaultFileDescriptorSetServiceClient]. 59 | func NewSchemaPoller( 60 | client reflectv1beta1connect.FileDescriptorSetServiceClient, 61 | module string, 62 | version string, 63 | ) SchemaPoller { 64 | return &bufReflectPoller{ 65 | client: client, 66 | module: module, 67 | version: version, 68 | } 69 | } 70 | 71 | type bufReflectPoller struct { 72 | client reflectv1beta1connect.FileDescriptorSetServiceClient 73 | module, version string 74 | } 75 | 76 | func (b *bufReflectPoller) GetSchema(ctx context.Context, symbols []string, currentVersion string) (*descriptorpb.FileDescriptorSet, string, error) { 77 | req := connect.NewRequest(&reflectv1beta1.GetFileDescriptorSetRequest{ 78 | Module: b.module, 79 | Version: b.version, 80 | Symbols: symbols, 81 | }) 82 | if currentVersion != "" { 83 | req.Header().Set("If-None-Match", currentVersion) 84 | } 85 | resp, err := b.client.GetFileDescriptorSet(ctx, req) 86 | if err != nil { 87 | if currentVersion != "" && connect.IsNotModifiedError(err) { 88 | return nil, "", ErrSchemaNotModified 89 | } 90 | return nil, "", err 91 | } 92 | return resp.Msg.GetFileDescriptorSet(), resp.Msg.GetVersion(), err 93 | } 94 | 95 | func (b *bufReflectPoller) GetSchemaID() string { 96 | if b.version == "" { 97 | return b.module 98 | } 99 | return b.module + ":" + b.version 100 | } 101 | -------------------------------------------------------------------------------- /schema_watcher.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023-2025 Buf Technologies, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prototransform 16 | 17 | import ( 18 | "context" 19 | "crypto/sha256" 20 | "encoding/hex" 21 | "errors" 22 | "fmt" 23 | "sort" 24 | "strconv" 25 | "strings" 26 | "sync" 27 | "time" 28 | 29 | "google.golang.org/protobuf/proto" 30 | "google.golang.org/protobuf/reflect/protoreflect" 31 | "google.golang.org/protobuf/types/descriptorpb" 32 | ) 33 | 34 | var ( 35 | // ErrSchemaWatcherStopped is an error returned from the AwaitReady method 36 | // that indicates the schema watcher was stopped before it ever became ready. 37 | ErrSchemaWatcherStopped = errors.New("SchemaWatcher was stopped") 38 | // ErrSchemaWatcherNotReady is an error returned from the various Find* 39 | // methods of SchemaWatcher an initial schema has not yet been downloaded (or 40 | // loaded from cache). 41 | ErrSchemaWatcherNotReady = errors.New("SchemaWatcher not ready") 42 | ) 43 | 44 | // SchemaWatcher watches a schema in a remote registry by periodically polling. 45 | // It implements the [Resolver] interface using the most recently downloaded 46 | // schema. As schema changes are pushed to the remote registry, the watcher 47 | // will incorporate the changes by downloading each change via regular polling. 48 | type SchemaWatcher struct { 49 | poller SchemaPoller 50 | schemaID string 51 | includeSymbols []string 52 | cacheKey string 53 | resolveNow chan struct{} 54 | lease Lease 55 | 56 | // used to prevent concurrent calls to cache.Save, which could 57 | // otherwise potentially result in a known-stale value in the cache. 58 | cacheMu sync.Mutex 59 | cache Cache 60 | 61 | callbackMu sync.Mutex 62 | callback func(*SchemaWatcher) 63 | errCallback func(*SchemaWatcher, error) 64 | 65 | resolverMu sync.RWMutex 66 | resolver *resolver 67 | resolvedSchema *descriptorpb.FileDescriptorSet 68 | resolveTime time.Time 69 | resolvedVersion string 70 | // if nil, watcher has been stopped; if not nil, will be called 71 | // when watcher is stopped 72 | stop context.CancelFunc 73 | // If nil, resolver is ready; if not nil, will be closed 74 | // once resolver is ready. 75 | resolverReady chan struct{} 76 | // set to most recent resolver error until resolver is ready 77 | resolverErr error 78 | } 79 | 80 | // NewSchemaWatcher creates a new [SchemaWatcher] for the given 81 | // [SchemaWatcherConfig]. 82 | // 83 | // The config is first validated to ensure all required attributes are provided. 84 | // A non-nil error is returned if the configuration is not valid. 85 | // 86 | // If the configuration is valid, a [SchemaWatcher] is returned, and the configured 87 | // SchemaPoller is used to download a schema. The schema will then be periodically 88 | // re-fetched based on the configured polling period. Either the Stop() method of the 89 | // [SchemaWatcher] must be called or the given ctx must be cancelled to release 90 | // resources and stop the periodic polling. 91 | // 92 | // This function returns immediately, even before a schema has been initially 93 | // downloaded. If the Find* methods on the returned watcher are called before an 94 | // initial schema has been downloaded, they will return ErrSchemaWatcherNotReady. 95 | // Use the [SchemaWatcher.AwaitReady] method to make sure the watcher is ready 96 | // before use. 97 | // 98 | // If the [SchemaWatcher.Stop]() method is called or the given ctx is cancelled, 99 | // polling for an updated schema aborts. The SchemaWatcher may still be used after 100 | // this, but it will be "frozen" using its most recently downloaded schema. If no 101 | // schema was ever successfully downloaded, it will be frozen in a bad state and 102 | // methods will return ErrSchemaWatcherNotReady. 103 | func NewSchemaWatcher(ctx context.Context, config *SchemaWatcherConfig) (*SchemaWatcher, error) { 104 | if err := config.validate(); err != nil { 105 | return nil, err 106 | } 107 | pollingPeriod := config.PollingPeriod 108 | if pollingPeriod == 0 { 109 | pollingPeriod = defaultPollingPeriod 110 | } 111 | 112 | // canonicalize symbols: remove duplicates and sort 113 | symSet := map[string]struct{}{} 114 | for _, sym := range config.IncludeSymbols { 115 | symSet[sym] = struct{}{} 116 | } 117 | syms := make([]string, 0, len(symSet)) 118 | for sym := range symSet { 119 | syms = append(syms, sym) 120 | } 121 | sort.Strings(syms) 122 | schemaID := config.SchemaPoller.GetSchemaID() 123 | 124 | // compute cache key 125 | var cacheKey string 126 | if config.Cache != nil { 127 | cacheKey = schemaID 128 | if len(syms) > 0 { 129 | // Add a strong hash of symbols to the end. 130 | var builder strings.Builder 131 | builder.WriteString(cacheKey) 132 | builder.WriteByte('_') 133 | sha := sha256.New() 134 | for _, sym := range syms { 135 | sha.Write(([]byte)(sym)) 136 | } 137 | hx := hex.NewEncoder(&builder) 138 | if _, err := hx.Write(sha.Sum(nil)); err != nil { 139 | // should never happen... 140 | return nil, fmt.Errorf("failed to generate hash of symbols for cache key: %w", err) 141 | } 142 | cacheKey = builder.String() 143 | } 144 | } 145 | 146 | ctx, cancel := context.WithCancel(ctx) 147 | var lease Lease 148 | if config.Leaser != nil { 149 | leaseHolder, err := getLeaseHolder(config.CurrentProcess) 150 | if err != nil { 151 | cancel() 152 | return nil, err 153 | } 154 | lease = config.Leaser.NewLease(ctx, schemaID, leaseHolder) 155 | } 156 | 157 | schemaWatcher := &SchemaWatcher{ 158 | poller: config.SchemaPoller, 159 | schemaID: schemaID, 160 | includeSymbols: syms, 161 | lease: lease, 162 | cacheKey: cacheKey, 163 | callback: config.OnUpdate, 164 | errCallback: config.OnError, 165 | cache: config.Cache, 166 | stop: cancel, 167 | resolverReady: make(chan struct{}), 168 | resolveNow: make(chan struct{}, 1), 169 | } 170 | schemaWatcher.start(ctx, pollingPeriod, config.Jitter) 171 | return schemaWatcher, nil 172 | } 173 | 174 | func (s *SchemaWatcher) getResolver() *resolver { 175 | s.resolverMu.RLock() 176 | defer s.resolverMu.RUnlock() 177 | return s.resolver 178 | } 179 | 180 | func (s *SchemaWatcher) updateResolver(ctx context.Context) (err error) { 181 | var changed bool 182 | if s.callback != nil || s.errCallback != nil { 183 | // make sure to invoke callback at the end to notify application 184 | defer func() { 185 | if changed && s.callback != nil { 186 | go func() { 187 | // Lock forces sequential calls to callback and also 188 | // means callback does not need to be thread-safe. 189 | s.callbackMu.Lock() 190 | defer s.callbackMu.Unlock() 191 | s.callback(s) 192 | }() 193 | } else if err != nil && !errors.Is(err, ErrSchemaNotModified) && s.errCallback != nil { 194 | go func() { 195 | s.callbackMu.Lock() 196 | defer s.callbackMu.Unlock() 197 | s.errCallback(s, err) 198 | }() 199 | } 200 | }() 201 | } 202 | 203 | schema, schemaVersion, schemaTimestamp, err := s.getFileDescriptorSet(ctx) 204 | if err != nil { 205 | return fmt.Errorf("failed to fetch schema: %w", err) 206 | } 207 | 208 | s.resolverMu.RLock() 209 | prevSchema, prevTimestamp := s.resolvedSchema, s.resolveTime 210 | s.resolverMu.RUnlock() 211 | 212 | if prevSchema != nil { 213 | if schemaTimestamp.Before(prevTimestamp) { 214 | // Only possible if schemaTimestamp is loaded from cache entry that is 215 | // older than last successful load. If that happens, just leave 216 | // the existing resolver in place. 217 | return nil 218 | } 219 | if proto.Equal(prevSchema, schema) { 220 | // nothing changed 221 | return nil 222 | } 223 | } 224 | 225 | resolver, err := newResolver(schema) 226 | if err != nil { 227 | return fmt.Errorf("unable to create resolver from schema: %w", err) 228 | } 229 | 230 | if len(s.includeSymbols) > 0 { 231 | var missingSymbols []string 232 | for _, sym := range s.includeSymbols { 233 | _, err := resolver.FindDescriptorByName(protoreflect.FullName(sym)) 234 | if err != nil { 235 | missingSymbols = append(missingSymbols, sym) 236 | } 237 | } 238 | if len(missingSymbols) > 0 { 239 | sort.Strings(missingSymbols) 240 | for i, sym := range missingSymbols { 241 | missingSymbols[i] = strconv.Quote(sym) 242 | } 243 | return fmt.Errorf("schema poller returned incomplete schema: missing %v", strings.Join(missingSymbols, ", ")) 244 | } 245 | } 246 | 247 | s.resolverMu.Lock() 248 | defer s.resolverMu.Unlock() 249 | s.resolver = resolver 250 | s.resolveTime = schemaTimestamp 251 | s.resolvedSchema = schema 252 | s.resolvedVersion = schemaVersion 253 | s.resolverErr = nil 254 | changed = true 255 | return nil 256 | } 257 | 258 | func (s *SchemaWatcher) initialUpdateResolver(ctx context.Context, pollingPeriod time.Duration, jitter float64) (success bool) { 259 | defer func() { 260 | s.resolverMu.Lock() 261 | defer s.resolverMu.Unlock() 262 | close(s.resolverReady) 263 | s.resolverReady = nil 264 | if !success { 265 | s.stop = nil 266 | } 267 | }() 268 | 269 | var delay time.Duration 270 | for { 271 | err := s.updateResolver(ctx) 272 | if err == nil { 273 | // success! 274 | return true 275 | } 276 | s.resolverMu.Lock() 277 | s.resolverErr = err 278 | s.resolverMu.Unlock() 279 | if delay == 0 { 280 | // immediately retry, but delay 1s if it fails again 281 | delay = time.Second 282 | } else { 283 | timer := time.NewTimer(addJitter(delay, jitter)) 284 | select { 285 | case <-ctx.Done(): 286 | timer.Stop() 287 | return false 288 | case <-timer.C: 289 | } 290 | delay *= 2 // exponential backoff 291 | } 292 | 293 | // we never wait longer than configured polling period, so we only apply 294 | // exponential backoff up to this point 295 | if delay > pollingPeriod { 296 | delay = pollingPeriod 297 | } 298 | } 299 | } 300 | 301 | // AwaitReady returns a non-nil error when s has downloaded a schema and is 302 | // ready for use. If the given context is cancelled (or has a deadline that 303 | // elapses) before s is ready, a non-nil error is returned. If an error 304 | // occurred while trying to download a schema, that error will be returned 305 | // at that time. If no error has yet occurred (e.g. the context was cancelled 306 | // before a download attempt finished), this will return the context error. 307 | // 308 | // Even if an error is returned, the SchemaWatcher will still be trying to 309 | // download the schema. It will keep trying/polling until s.Stop is called or 310 | // until the context passed to [NewSchemaWatcher] is cancelled. 311 | func (s *SchemaWatcher) AwaitReady(ctx context.Context) error { 312 | s.resolverMu.RLock() 313 | ready, stop := s.resolverReady, s.stop 314 | s.resolverMu.RUnlock() 315 | if ready == nil { 316 | if stop == nil { 317 | return ErrSchemaWatcherStopped 318 | } 319 | return nil 320 | } 321 | select { 322 | case <-ready: 323 | s.resolverMu.RLock() 324 | stop = s.stop 325 | s.resolverMu.RUnlock() 326 | if stop == nil { 327 | return ErrSchemaWatcherStopped 328 | } 329 | return nil 330 | case <-ctx.Done(): 331 | s.resolverMu.RLock() 332 | err := s.resolverErr 333 | s.resolverMu.RUnlock() 334 | if err != nil { 335 | return err 336 | } 337 | return ctx.Err() 338 | } 339 | } 340 | 341 | // LastResolved returns the time that a schema was last successfully downloaded. 342 | // If the boolean value is false, the watcher is not yet ready and no schema has 343 | // yet been successfully downloaded. Otherwise, the returned time indicates when 344 | // the schema was downloaded. If the schema is loaded from a cache, the timestamp 345 | // will indicate when that cached schema was originally downloaded. 346 | // 347 | // This can be used for staleness heuristics if a partition occurs that makes 348 | // the remote registry unavailable. Under typical operations when no failures 349 | // are occurring, the maximum age will up to the configured polling period plus 350 | // the latency of the RPC to the remote registry. 351 | func (s *SchemaWatcher) LastResolved() (bool, time.Time) { 352 | s.resolverMu.RLock() 353 | defer s.resolverMu.RUnlock() 354 | if s.resolver == nil { 355 | return false, time.Time{} 356 | } 357 | return true, s.resolveTime 358 | } 359 | 360 | // ResolveNow tells the watcher to poll for a new schema immediately instead of 361 | // waiting until the next scheduled time per the configured polling period. 362 | func (s *SchemaWatcher) ResolveNow() { 363 | select { 364 | case s.resolveNow <- struct{}{}: 365 | default: 366 | // channel buffer is full, which means "resolve now" signal already pending 367 | } 368 | } 369 | 370 | // RangeFiles iterates over all registered files while f returns true. The 371 | // iteration order is undefined. 372 | // 373 | // This uses a snapshot of the most recently downloaded schema. So if the 374 | // schema is updated (via concurrent download) while iterating, f will only 375 | // see the contents of the older schema. 376 | // 377 | // If the s is not yet ready, this will not call f at all and instead immediately 378 | // return. This does not return an error so that the signature matches the method 379 | // of the same name of *protoregistry.Files, allowing *SchemaWatcher to provide 380 | // the same interface. 381 | func (s *SchemaWatcher) RangeFiles(f func(protoreflect.FileDescriptor) bool) { 382 | res := s.getResolver() 383 | if res == nil { 384 | return 385 | } 386 | res.RangeFiles(f) 387 | } 388 | 389 | // RangeFilesByPackage iterates over all registered files in a given proto package 390 | // while f returns true. The iteration order is undefined. 391 | // 392 | // This uses a snapshot of the most recently downloaded schema. So if the 393 | // schema is updated (via concurrent download) while iterating, f will only 394 | // see the contents of the older schema. 395 | // 396 | // If the s is not yet ready, this will not call f at all and instead immediately 397 | // return. This does not return an error so that the signature matches the method 398 | // of the same name of *protoregistry.Files, allowing *SchemaWatcher to provide 399 | // the same interface. 400 | func (s *SchemaWatcher) RangeFilesByPackage(name protoreflect.FullName, f func(protoreflect.FileDescriptor) bool) { 401 | res := s.getResolver() 402 | if res == nil { 403 | return 404 | } 405 | res.RangeFilesByPackage(name, f) 406 | } 407 | 408 | // FindFileByPath looks up a file by the path. 409 | // 410 | // This uses the most recently downloaded schema. 411 | func (s *SchemaWatcher) FindFileByPath(path string) (protoreflect.FileDescriptor, error) { 412 | res := s.getResolver() 413 | if res == nil { 414 | return nil, ErrSchemaWatcherNotReady 415 | } 416 | return res.FindFileByPath(path) 417 | } 418 | 419 | // FindDescriptorByName looks up a descriptor by the full name. 420 | // 421 | // This uses the most recently downloaded schema. 422 | func (s *SchemaWatcher) FindDescriptorByName(name protoreflect.FullName) (protoreflect.Descriptor, error) { 423 | res := s.getResolver() 424 | if res == nil { 425 | return nil, ErrSchemaWatcherNotReady 426 | } 427 | return res.FindDescriptorByName(name) 428 | } 429 | 430 | // FindExtensionByName looks up an extension field by the field's full name. 431 | // Note that this is the full name of the field as determined by 432 | // where the extension is declared and is unrelated to the full name of the 433 | // message being extended. 434 | // 435 | // Implements [Resolver] using the most recently downloaded schema. 436 | func (s *SchemaWatcher) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) { 437 | res := s.getResolver() 438 | if res == nil { 439 | return nil, ErrSchemaWatcherNotReady 440 | } 441 | return res.FindExtensionByName(field) 442 | } 443 | 444 | // FindExtensionByNumber looks up an extension field by the field number 445 | // within some parent message, identified by full name. 446 | // 447 | // Implements [Resolver] using the most recently downloaded schema. 448 | func (s *SchemaWatcher) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) { 449 | res := s.getResolver() 450 | if res == nil { 451 | return nil, ErrSchemaWatcherNotReady 452 | } 453 | return res.FindExtensionByNumber(message, field) 454 | } 455 | 456 | // FindMessageByName looks up a message by its full name. 457 | // E.g., "google.protobuf.Any" 458 | // 459 | // Implements [Resolver] using the most recently downloaded schema. 460 | func (s *SchemaWatcher) FindMessageByName(message protoreflect.FullName) (protoreflect.MessageType, error) { 461 | res := s.getResolver() 462 | if res == nil { 463 | return nil, ErrSchemaWatcherNotReady 464 | } 465 | return res.FindMessageByName(message) 466 | } 467 | 468 | // FindMessageByURL looks up a message by a URL identifier. 469 | // See documentation on google.protobuf.Any.type_url for the URL format. 470 | // 471 | // Implements [Resolver] using the most recently downloaded schema. 472 | func (s *SchemaWatcher) FindMessageByURL(url string) (protoreflect.MessageType, error) { 473 | res := s.getResolver() 474 | if res == nil { 475 | return nil, ErrSchemaWatcherNotReady 476 | } 477 | return res.FindMessageByURL(url) 478 | } 479 | 480 | // FindEnumByName looks up an enum by its full name. 481 | // E.g., "google.protobuf.Field.Kind". 482 | // 483 | // Implements [Resolver] using the most recently downloaded schema. 484 | func (s *SchemaWatcher) FindEnumByName(enum protoreflect.FullName) (protoreflect.EnumType, error) { 485 | res := s.getResolver() 486 | if res == nil { 487 | return nil, ErrSchemaWatcherNotReady 488 | } 489 | return res.FindEnumByName(enum) 490 | } 491 | 492 | // ResolvedSchema returns the resolved schema in the form of a 493 | // FileDescriptorSet. Until AwaitReady returns a non-nil status, the return 494 | // value of this function can be nil. 495 | // The caller must not mutate the returned file descriptor set. Clone the 496 | // returned file descriptor set using proto.Clone before performing mutations 497 | // on it. 498 | func (s *SchemaWatcher) ResolvedSchema() *descriptorpb.FileDescriptorSet { 499 | s.resolverMu.RLock() 500 | schema := s.resolvedSchema 501 | s.resolverMu.RUnlock() 502 | return schema 503 | } 504 | 505 | func (s *SchemaWatcher) start(ctx context.Context, pollingPeriod time.Duration, jitter float64) { 506 | go func() { 507 | if !s.initialUpdateResolver(ctx, pollingPeriod, jitter) { 508 | return 509 | } 510 | defer s.Stop() 511 | for { 512 | // consume any "resolve now" signal that arrived while we were concurrently resolving 513 | select { 514 | case <-s.resolveNow: 515 | default: 516 | } 517 | 518 | timer := time.NewTimer(addJitter(pollingPeriod, jitter)) 519 | select { 520 | case <-timer.C: 521 | if ctx.Err() != nil { 522 | // don't bother fetching a schema if context is done 523 | return 524 | } 525 | _ = s.updateResolver(ctx) 526 | case <-s.resolveNow: 527 | timer.Stop() 528 | if ctx.Err() != nil { 529 | // don't bother fetching a schema if context is done 530 | return 531 | } 532 | _ = s.updateResolver(ctx) 533 | case <-ctx.Done(): 534 | timer.Stop() 535 | return 536 | } 537 | } 538 | }() 539 | } 540 | 541 | // Stop the [SchemaWatcher] from polling the BSR for new schemas. Can be called 542 | // multiple times safely. 543 | func (s *SchemaWatcher) Stop() { 544 | s.resolverMu.Lock() 545 | defer s.resolverMu.Unlock() 546 | if s.stop != nil { 547 | s.stop() 548 | s.stop = nil 549 | } 550 | } 551 | 552 | func (s *SchemaWatcher) IsStopped() bool { 553 | s.resolverMu.RLock() 554 | defer s.resolverMu.RUnlock() 555 | return s.stop == nil 556 | } 557 | 558 | func (s *SchemaWatcher) getFileDescriptorSet(ctx context.Context) (*descriptorpb.FileDescriptorSet, string, time.Time, error) { 559 | s.resolverMu.RLock() 560 | currentVersion := s.resolvedVersion 561 | s.resolverMu.RUnlock() 562 | descriptors, version, err := s.poll(ctx, currentVersion) 563 | respTime := time.Now() 564 | if err != nil { //nolint:nestif 565 | if errors.Is(err, ErrSchemaNotModified) || s.cache == nil { 566 | return nil, "", time.Time{}, err 567 | } 568 | // try to fallback to cache 569 | data, cacheErr := s.cache.Load(ctx, s.cacheKey) 570 | if cacheErr != nil { 571 | return nil, "", time.Time{}, cacheMultiErr("failed to load from cache", err, cacheErr) 572 | } 573 | msg, cacheErr := decodeForCache(data) 574 | if cacheErr != nil { 575 | return nil, "", time.Time{}, cacheMultiErr("failed to decode cached value", err, cacheErr) 576 | } 577 | if !isCorrectCacheEntry(msg, s.schemaID, s.includeSymbols) { 578 | // Cache key collision! Do not use this result! 579 | isLeaseError := errors.As(err, new(leaseError)) 580 | if isLeaseError { 581 | return nil, "", time.Time{}, fmt.Errorf("%w (failed to load cached value: stored entry is for wrong schema)", err) 582 | } 583 | return nil, "", time.Time{}, err 584 | } 585 | return msg.GetSchema().GetDescriptors(), msg.GetSchema().GetVersion(), msg.GetSchemaTimestamp().AsTime(), nil 586 | } 587 | if s.cache != nil { 588 | go func() { 589 | data, err := encodeForCache(s.schemaID, s.includeSymbols, descriptors, version, respTime) 590 | if err != nil { 591 | // Since we got the data by unmarshalling it (either from RPC 592 | // response or cache), it must be marshallable. So this should 593 | // never actually happen. 594 | return 595 | } 596 | // though s.cache must be thread-safe, we use a mutex to 597 | // prevent racing, concurrent calls to Save, which could 598 | // potentially leave the cache in a bad/stale state if an 599 | // earlier call to Save actually succeeds last. 600 | s.cacheMu.Lock() 601 | defer s.cacheMu.Unlock() 602 | // We ignore the error since there's nothing we can do. 603 | // But keeping it in the interface signature means that 604 | // user code can wrap a cache implementation and observe 605 | // the error, in order to possibly take action (like write 606 | // a log message or update a counter metric, etc). 607 | _ = s.cache.Save(ctx, s.cacheKey, data) 608 | }() 609 | } 610 | return descriptors, version, respTime, nil 611 | } 612 | 613 | func (s *SchemaWatcher) poll(ctx context.Context, currentVersion string) (*descriptorpb.FileDescriptorSet, string, error) { 614 | if s.lease != nil { 615 | held, err := s.lease.IsHeld() 616 | if err != nil { 617 | return nil, "", leaseHolderUnknownError{err: err} 618 | } 619 | if !held { 620 | return nil, "", leaseNotHeldError{} 621 | } 622 | } 623 | return s.poller.GetSchema(ctx, s.includeSymbols, currentVersion) 624 | } 625 | 626 | type leaseError interface { 627 | leaseError() 628 | } 629 | 630 | var _ leaseError = leaseNotHeldError{} 631 | var _ leaseError = leaseHolderUnknownError{} 632 | 633 | type leaseNotHeldError struct{} 634 | 635 | func (e leaseNotHeldError) leaseError() {} 636 | 637 | func (e leaseNotHeldError) Error() string { 638 | return "cannot poll for schema because current process is not leaseholder" 639 | } 640 | 641 | type leaseHolderUnknownError struct { 642 | err error 643 | } 644 | 645 | func (e leaseHolderUnknownError) leaseError() {} 646 | 647 | func (e leaseHolderUnknownError) Error() string { 648 | return fmt.Sprintf("cannot poll for scheme because leaseholder is unknown: %v", e.err) 649 | } 650 | 651 | func (e leaseHolderUnknownError) Unwrap() error { 652 | return e.err 653 | } 654 | 655 | // cacheMultiErr wraps multiple errors with fmt.Errorf. 656 | func cacheMultiErr(msg string, err error, cacheErr error) error { 657 | return fmt.Errorf("%w (%s: %w)", err, msg, cacheErr) 658 | } 659 | --------------------------------------------------------------------------------