├── .github └── workflows │ ├── build.yaml │ └── publish.yaml ├── .gitignore ├── BUILD ├── Dockerfile ├── LICENSE ├── README.md ├── cmd ├── casload │ ├── .gitignore │ ├── BUILD │ ├── client.go │ └── main.go └── smoketest │ ├── .gitignore │ ├── BUILD │ └── main.go ├── go.mod ├── go.sum ├── pants.ci.toml ├── pants.toml ├── pants_from_sources ├── pkg ├── casutil │ ├── BUILD │ └── casutil.go ├── grpcutil │ ├── BUILD │ └── static_auth_token.go ├── load │ ├── BUILD │ ├── action.go │ ├── generate.go │ ├── load.go │ ├── read.go │ └── save.go ├── retry │ ├── BUILD │ └── retry.go └── stats │ ├── BUILD │ └── stats.go ├── protos ├── build │ └── bazel │ │ ├── remote │ │ └── execution │ │ │ └── v2 │ │ │ ├── BUILD │ │ │ ├── remote_execution.pb.go │ │ │ ├── remote_execution.proto │ │ │ └── remote_execution_grpc.pb.go │ │ └── semver │ │ ├── BUILD │ │ ├── semver.pb.go │ │ └── semver.proto └── google │ ├── api │ ├── annotations.proto │ ├── client.proto │ └── http.proto │ ├── longrunning │ └── operations.proto │ └── rpc │ └── status.proto └── scripts ├── build-docker.sh └── gen-protos.sh /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | # Trigger this workflow on any push (to main) or pull request. 3 | on: 4 | workflow_call: 5 | secrets: 6 | TOOLCHAIN_AUTH_TOKEN: 7 | required: true 8 | push: 9 | branches: 10 | - main 11 | pull_request: {} 12 | jobs: 13 | build: 14 | name: Build and test 15 | runs-on: ubuntu-20.04 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-go@v5 19 | with: 20 | go-version: '1.19' 21 | - uses: actions/setup-python@v5 22 | with: 23 | python-version: '3.11' 24 | - name: Initialize Pants 25 | uses: pantsbuild/actions/init-pants@v8 26 | with: 27 | # cache0 makes it easy to bust the cache if needed 28 | gha-cache-key: cache0- 29 | named-caches-hash: ${{ hashFiles('go.mod') }} 30 | - name: Set env vars 31 | run: | 32 | echo 'PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml"]' >> ${GITHUB_ENV} 33 | - name: Pants Bootstrap 34 | run: pants version 35 | - name: Lint & check 36 | run: | 37 | pants lint check :: 38 | - name: Tests 39 | run: | 40 | pants test :: 41 | - name: Build smoketest & casloader docker images 42 | run: | 43 | pants package cmd/:: 44 | # Keeping those here since the publish workflow wasn't ported to use pants. 45 | - name: Build smoketest docker image 46 | uses: docker/build-push-action@v3 47 | with: 48 | tags: smoketest:ci 49 | build-args: APP_NAME=smoketest 50 | - name: Build casload docker image 51 | uses: docker/build-push-action@v3 52 | with: 53 | tags: casload:ci 54 | build-args: APP_NAME=casload 55 | -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: Publish to DockerHub 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | publish-dockerhub: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | - name: Login to DockerHub 12 | uses: docker/login-action@v3 13 | with: 14 | username: ${{ secrets.DOCKER_USERNAME }} 15 | password: ${{ secrets.DOCKER_ACCESS_TOKEN }} 16 | - name: Build & Publish nightly smoketest 17 | id: docker_build_smoketest 18 | uses: docker/build-push-action@v5 19 | with: 20 | build-args: APP_NAME=smoketest 21 | tags: toolchainlabs/remote-api-tools-smoketest:nightly 22 | push: true 23 | - name: smoketest image digest 24 | run: echo ${{ steps.docker_build_smoketest.outputs.digest }} 25 | - name: Build & Publish nightly casload 26 | id: docker_build_casload 27 | uses: docker/build-push-action@v5 28 | with: 29 | build-args: APP_NAME=casload 30 | tags: toolchainlabs/remote-api-tools-casload:nightly 31 | push: true 32 | - name: casload image digest 33 | run: echo ${{ steps.docker_build_casload.outputs.digest }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore IntelliJ project files. 2 | .idea 3 | *.iml 4 | 5 | # Pants 6 | .pants.d 7 | .pids 8 | dist/ 9 | 10 | # TODO: Figure out why these are created in source root and not sandbox. 11 | /cache 12 | /gopath 13 | 14 | 15 | # MacOS from https://github.com/github/gitignore/blob/master/Global/OSX.gitignore 16 | .DS_Store 17 | .AppleDouble 18 | .LSOverride 19 | # Icon must end with two \r 20 | Icon 21 | # Thumbnails 22 | ._* 23 | # Files that might appear in the root of a volume 24 | .DocumentRevisions-V100 25 | .fseventsd 26 | .Spotlight-V100 27 | .TemporaryItems 28 | .Trashes 29 | .VolumeIcon.icns 30 | # Directories potentially created on remote AFP share 31 | .AppleDB 32 | .AppleDesktop 33 | Network Trash Folder 34 | Temporary Items 35 | .apdisk -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | go_mod(name="root") 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15.10-alpine3.13 AS builder 2 | ARG APP_NAME 3 | RUN apk add --update bash curl git && rm /var/cache/apk/* 4 | 5 | WORKDIR /build 6 | 7 | COPY go.mod . 8 | COPY go.sum . 9 | 10 | RUN go mod download 11 | 12 | COPY . . 13 | WORKDIR cmd/${APP_NAME} 14 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../../app 15 | 16 | FROM alpine:3.13.2 17 | WORKDIR /root/ 18 | COPY --from=builder /build/app /usr/local/bin/ 19 | 20 | ENTRYPOINT ["app"] 21 | CMD ["--help"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tooling for Remote Execution API 2 | 3 | This project contains miscellaneous tooling useful for dealing with 4 | [Remote Execution API](https://github.com/bazelbuild/remote-apis/blob/master/build/bazel/remote/execution/v2/remote_execution.proto) servers. 5 | Such servers include [BuildBarn](https://github.com/buildbarn/bb-remote-execution), 6 | [Buildfarm](https://github.com/bazelbuild/bazel-buildfarm/), and 7 | [BuildGrid](https://gitlab.com/BuildGrid/buildgrid). 8 | 9 | ## Tools 10 | 11 | ### REAPI Smoke Test 12 | 13 | The REAPAI "smoke test" is a simple operational check of an REAPI instance. It checks whether an REAPI server 14 | can handle basic CAS, Action Cache, and Execution service requests. The test creates a unique Command proto, 15 | requests its execution, and then checks the outputs and the Action Cache to ensure they are in the expected state. 16 | 17 | To build the `smoketest` binary: 18 | 19 | ``` 20 | $ cd cmd/smoketest 21 | $ go build 22 | ``` 23 | 24 | If you are running an REAPI instance at 127.0.0.1:8980, then you can run the smoketest against 25 | that instance with the `OSFamily` platform property set to `Linux` by running: 26 | 27 | ``` 28 | $ ./smoketest -r 127.0.0.1:8980 -p OSFamily=Linux 29 | ``` 30 | 31 | There are other options to enable TLS, send an authorization token, add additional platform properties, etc. 32 | Run `smoketest -h` to see the available options. 33 | 34 | ### Load Testing 35 | 36 | The `casload` tool generates CAS traffic according to one or more specifications of the form: 37 | NUM_REQUESTS:MIN_BLOB_SIZE:MAX_BLOB_SIZE[:CONCURRENCY]. 38 | 39 | To build the `casload` binary: 40 | 41 | ``` 42 | $ cd cmd/smoketest 43 | $ go build 44 | ``` 45 | 46 | If you are running a CAS instance at 127.0.0.1:8980, then you can generate 15,000 requests with blob sizes ranging 47 | from 5000 bytes to 10,000 bytes with 100 concurrent requests by running: 48 | 49 | ``` 50 | $ ./casload -r 127.0.0.1:8980 generate:15000:5000:10000:100 51 | ``` 52 | 53 | There are other options to enable TLS, send an authorization token, etc. 54 | Run `casload -h` to see the available options. 55 | 56 | `casload` takes "load actions" as parameters which drive the load test. Each load action has access to a set of 57 | "known digests". 58 | 59 | The load actions are: 60 | 61 | * `generate:NUM_BLOBS:MIN_BLOB_SIZE:MAX_BLOB_SIZE[:CONCURRENCY]` - This load action generates random blobs and 62 | writes them to the CAS. Each digest is added to the set of known digests. The blobs will be between 63 | `MIN_BLOB_SIZE` and `MAX_BLOB_SIZE` in size. The action will use CONCURRENCY workers to send requests. 64 | 65 | * `load-digests:FILENAME` - Load digests from a CSV file into the set of known digests. The CSV file consists of 66 | lines in the following format: `HASH,SIZE,true|false` where the third field is whether the digest is known to 67 | be present in the CAS or not. 68 | 69 | * `save-digests:FILENAME` - Save the set of known digests to a CSV file in the same format expected by the 70 | `load-digests` load action. 71 | 72 | * `read:NUM_DIGESTS_TO_READ:NUM_READS_PER_DIGEST[:CONCURRENCY]` - This load action randomly chooses 73 | `NUM_DIGESTS_TO_READ` from the set of known digests and issues NUM_READS_PER_DIGEST RPCs to read the blob from 74 | the CAS and verify its content matches the digest hash. The action will use CONCURRENCY workers to send requests. 75 | 76 | ## Development 77 | 78 | To rebuild the generated Go files, run: 79 | 80 | ``` 81 | $ ./scripts/gen-proto.sh 82 | ``` 83 | 84 | ## Contributions 85 | 86 | Contributions are welcome. Please open a pull request or file an issue. 87 | 88 | Please run `go fmt ./...` to format the code before submitting. 89 | 90 | ## Useful Links 91 | 92 | - [remote-apis-testing project](https://gitlab.com/remote-apis-testing/remote-apis-testing) 93 | -------------------------------------------------------------------------------- /cmd/casload/.gitignore: -------------------------------------------------------------------------------- 1 | casload 2 | -------------------------------------------------------------------------------- /cmd/casload/BUILD: -------------------------------------------------------------------------------- 1 | go_package() 2 | 3 | go_binary( 4 | name="bin", 5 | output_path="casload", 6 | ) 7 | 8 | docker_image( 9 | name="casload-docker", 10 | dependencies=[":bin"], 11 | repository="toolchainlabs/remote-api-tools-casload", 12 | instructions=[ 13 | "FROM alpine:3.13.2", 14 | "COPY casload /usr/local/bin/", 15 | 'ENTRYPOINT ["/usr/local/bin/casload"]', 16 | ], 17 | ) 18 | -------------------------------------------------------------------------------- /cmd/casload/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "crypto/tls" 20 | "fmt" 21 | log "github.com/sirupsen/logrus" 22 | "time" 23 | 24 | "github.com/toolchainlabs/remote-api-tools/pkg/grpcutil" 25 | remote_pb "github.com/toolchainlabs/remote-api-tools/protos/build/bazel/remote/execution/v2" 26 | bytestream_pb "google.golang.org/genproto/googleapis/bytestream" 27 | "google.golang.org/grpc" 28 | "google.golang.org/grpc/credentials" 29 | "google.golang.org/grpc/keepalive" 30 | ) 31 | 32 | type clientsStruct struct { 33 | conn *grpc.ClientConn 34 | instanceName string 35 | casClient remote_pb.ContentAddressableStorageClient 36 | bytestreamClient bytestream_pb.ByteStreamClient 37 | capabilitiesClient remote_pb.CapabilitiesClient 38 | maxBatchBlobSize int64 39 | } 40 | 41 | func (s *clientsStruct) Close() error { 42 | return s.conn.Close() 43 | } 44 | 45 | func setupClients( 46 | ctx context.Context, 47 | server, instanceName string, 48 | secure, allowInsecureAuth bool, 49 | authToken string, 50 | ) (*clientsStruct, error) { 51 | dialOptions := []grpc.DialOption{ 52 | grpc.WithBlock(), 53 | grpc.WithKeepaliveParams(keepalive.ClientParameters{ 54 | Time: time.Duration(1) * time.Second, 55 | Timeout: time.Duration(5) * time.Second, 56 | PermitWithoutStream: true, 57 | }), 58 | } 59 | 60 | if secure { 61 | tlsConfig := &tls.Config{} 62 | creds := credentials.NewTLS(tlsConfig) 63 | dialOptions = append(dialOptions, grpc.WithTransportCredentials(creds)) 64 | } else { 65 | dialOptions = append(dialOptions, grpc.WithInsecure()) 66 | } 67 | 68 | if authToken != "" { 69 | log.Info("enabling auth token") 70 | creds := grpcutil.NewStaticAuthToken(authToken, allowInsecureAuth) 71 | dialOptions = append(dialOptions, grpc.WithPerRPCCredentials(creds)) 72 | } 73 | 74 | conn, err := grpc.DialContext(ctx, server, dialOptions...) 75 | if err != nil { 76 | return nil, fmt.Errorf("dial error: %s", err) 77 | } 78 | 79 | casClient := remote_pb.NewContentAddressableStorageClient(conn) 80 | byteStreamClient := bytestream_pb.NewByteStreamClient(conn) 81 | capabilitiesClient := remote_pb.NewCapabilitiesClient(conn) 82 | 83 | capabilitiesRequest := remote_pb.GetCapabilitiesRequest{ 84 | InstanceName: instanceName, 85 | } 86 | capabilities, err := capabilitiesClient.GetCapabilities(ctx, &capabilitiesRequest) 87 | if err != nil { 88 | return nil, fmt.Errorf("failed to retrieve capabilities: %s", err) 89 | } 90 | 91 | cs := clientsStruct{ 92 | conn: conn, 93 | instanceName: instanceName, 94 | casClient: casClient, 95 | bytestreamClient: byteStreamClient, 96 | capabilitiesClient: capabilitiesClient, 97 | maxBatchBlobSize: capabilities.GetCacheCapabilities().MaxBatchTotalSizeBytes, 98 | } 99 | 100 | return &cs, nil 101 | } 102 | -------------------------------------------------------------------------------- /cmd/casload/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "github.com/toolchainlabs/remote-api-tools/pkg/load" 21 | "io/ioutil" 22 | "math/rand" 23 | "os" 24 | "strings" 25 | "time" 26 | 27 | log "github.com/sirupsen/logrus" 28 | "github.com/spf13/pflag" 29 | ) 30 | 31 | const version string = "0.1.0" 32 | 33 | func main() { 34 | rand.Seed(time.Now().Unix()) 35 | 36 | versionOpt := pflag.BoolP("version", "V", false, "display version and exit") 37 | remoteOpt := pflag.StringP("remote", "r", "", "remote server") 38 | authTokenFileOpt := pflag.StringP("auth-token-file", "a", "", "auth bearer token to use") 39 | authTokenEnvOpt := pflag.StringP("auth-token-env", "A", "", "name of environment variable with auth bearer token") 40 | secureOpt := pflag.BoolP("secure", "s", false, "enable secure mode (TLS)") 41 | instanceNameOpt := pflag.StringP("instance-name", "i", "", "instance name") 42 | instanceNameFromEnvOpt := pflag.StringP("instance-from-env", "I", "", "instance name from given environment variable") 43 | verboseOpt := pflag.CountP("verbose", "v", "increase logging verbosity") 44 | useJsonLoggingOpt := pflag.Bool("log-json", false, "log using JSON") 45 | allowInsecureAuthOpt := pflag.Bool("allow-insecure-auth", false, "allow credentials to be passed unencrypted (i.e., no TLS)") 46 | writeChunkSize := pflag.Int64("write-chunk-size", 512*1024, "number of bytes per bytestream chunk") 47 | maxBatchBlobSize := pflag.Int64("max-batch-blob-size", 512*1024, "maximum number of bytes per blob permitted to use BatchUpdateBlobs") 48 | 49 | pflag.Parse() 50 | 51 | if *versionOpt { 52 | fmt.Println(version) 53 | os.Exit(0) 54 | } 55 | 56 | if *useJsonLoggingOpt { 57 | log.SetFormatter(&log.JSONFormatter{}) 58 | } else { 59 | log.SetFormatter(&log.TextFormatter{ 60 | DisableColors: true, 61 | FullTimestamp: true, 62 | }) 63 | } 64 | 65 | if *remoteOpt == "" { 66 | log.Fatal("--remote option is required") 67 | } 68 | 69 | args := pflag.Args() 70 | if len(args) == 0 { 71 | log.Fatal("benchmark programs must be specified") 72 | } 73 | 74 | if *authTokenFileOpt != "" && *authTokenEnvOpt != "" { 75 | log.Fatalf("--auth-token-file and --auth-token-env are mutually exclusive") 76 | } 77 | 78 | authToken := "" 79 | if *authTokenFileOpt != "" { 80 | authTokenBytes, err := ioutil.ReadFile(*authTokenFileOpt) 81 | if err != nil { 82 | log.Fatalf("unable to read auth token from %s: %s", *authTokenFileOpt, err) 83 | } 84 | authToken = strings.TrimSpace(string(authTokenBytes)) 85 | } else if *authTokenEnvOpt != "" { 86 | envValue := os.Getenv(*authTokenEnvOpt) 87 | if envValue == "" { 88 | log.Fatalf("environment variable %s was either unset or empty", *authTokenEnvOpt) 89 | } 90 | authToken = strings.TrimSpace(envValue) 91 | } 92 | 93 | instanceName := "" 94 | if *instanceNameOpt != "" && *instanceNameFromEnvOpt != "" { 95 | log.Fatal("--instance and --instance-from-env are mutually exclusive") 96 | } else if *instanceNameOpt != "" { 97 | instanceName = *instanceNameOpt 98 | } else if *instanceNameFromEnvOpt != "" { 99 | instanceName = os.Getenv(*instanceNameFromEnvOpt) 100 | } 101 | 102 | if *verboseOpt > 1 { 103 | log.SetLevel(log.TraceLevel) 104 | } else if *verboseOpt == 1 { 105 | log.SetLevel(log.DebugLevel) 106 | } 107 | 108 | var actions []load.Action 109 | for _, arg := range args { 110 | action, err := load.ParseAction(arg) 111 | if err != nil { 112 | log.Fatalf("error parsing program: %s", err) 113 | } 114 | actions = append(actions, action) 115 | } 116 | 117 | ctx := context.Background() 118 | 119 | cs, err := setupClients(ctx, *remoteOpt, instanceName, *secureOpt, *allowInsecureAuthOpt, authToken) 120 | if err != nil { 121 | log.Fatalf("failed to setup connection: %s", err) 122 | } 123 | defer cs.Close() 124 | 125 | var finalMaxBatchBlobSize = *maxBatchBlobSize 126 | if cs.maxBatchBlobSize > 0 && cs.maxBatchBlobSize < finalMaxBatchBlobSize { 127 | finalMaxBatchBlobSize = cs.maxBatchBlobSize 128 | } 129 | 130 | actionContext := load.ActionContext{ 131 | InstanceName: instanceName, 132 | CasClient: cs.casClient, 133 | BytestreamClient: cs.bytestreamClient, 134 | Ctx: ctx, 135 | KnownDigests: make(map[string]bool), 136 | WriteChunkSize: *writeChunkSize, 137 | MaxBatchBlobSize: finalMaxBatchBlobSize, 138 | } 139 | 140 | for _, action := range actions { 141 | err := action.RunAction(&actionContext) 142 | if err != nil { 143 | log.Fatalf("error during load test: %s", err) 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /cmd/smoketest/.gitignore: -------------------------------------------------------------------------------- 1 | smoketest 2 | -------------------------------------------------------------------------------- /cmd/smoketest/BUILD: -------------------------------------------------------------------------------- 1 | go_package() 2 | 3 | go_binary( 4 | name="bin", 5 | output_path="smoketest", 6 | ) 7 | docker_image( 8 | name="smoketest-docker", 9 | dependencies=[":bin"], 10 | repository="toolchainlabs/remote-api-tools-smoketest", 11 | instructions=[ 12 | "FROM alpine:3.13.2", 13 | "COPY smoketest /usr/local/bin/", 14 | 'ENTRYPOINT ["/usr/local/bin/smoketest"]', 15 | ], 16 | ) 17 | -------------------------------------------------------------------------------- /cmd/smoketest/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "crypto/tls" 21 | "fmt" 22 | "io/ioutil" 23 | "os" 24 | "sort" 25 | "strconv" 26 | "strings" 27 | "sync" 28 | "time" 29 | 30 | "github.com/golang/protobuf/proto" 31 | log "github.com/sirupsen/logrus" 32 | "github.com/spf13/pflag" 33 | "github.com/toolchainlabs/remote-api-tools/pkg/casutil" 34 | "github.com/toolchainlabs/remote-api-tools/pkg/grpcutil" 35 | remote_pb "github.com/toolchainlabs/remote-api-tools/protos/build/bazel/remote/execution/v2" 36 | longrunning_pb "google.golang.org/genproto/googleapis/longrunning" 37 | "google.golang.org/grpc" 38 | "google.golang.org/grpc/codes" 39 | "google.golang.org/grpc/credentials" 40 | "google.golang.org/grpc/keepalive" 41 | "google.golang.org/grpc/status" 42 | ) 43 | 44 | const version string = "0.1.0" 45 | 46 | func makeCommand( 47 | now string, 48 | delaySecs uint, 49 | platformProperties []string, 50 | envVars map[string]string, 51 | ) *remote_pb.Command { 52 | delayCmd := "" 53 | if delaySecs > 0 { 54 | delayCmd = fmt.Sprintf("&& sleep %d", delaySecs) 55 | } 56 | 57 | var protoPlatformProperties []*remote_pb.Platform_Property 58 | for _, p := range platformProperties { 59 | parts := strings.SplitN(p, "=", 2) 60 | protoPlatformProperties = append(protoPlatformProperties, &remote_pb.Platform_Property{ 61 | Name: parts[0], 62 | Value: parts[1], 63 | }) 64 | } 65 | 66 | sort.Slice(protoPlatformProperties, func(i, j int) bool { 67 | left := protoPlatformProperties[i] 68 | right := protoPlatformProperties[j] 69 | 70 | cmp1 := strings.Compare(left.Name, right.Name) 71 | if cmp1 < 0 { 72 | return true 73 | } else if cmp1 > 0 { 74 | return false 75 | } 76 | 77 | cmp2 := strings.Compare(left.Value, right.Value) 78 | return cmp2 < 0 79 | }) 80 | 81 | envVars["NOW"] = string(now) 82 | var protoEnvironmentVariables []*remote_pb.Command_EnvironmentVariable 83 | for key, value := range envVars { 84 | protoEnvironmentVariables = append(protoEnvironmentVariables, &remote_pb.Command_EnvironmentVariable{ 85 | Name: key, 86 | Value: value, 87 | }) 88 | } 89 | 90 | sort.Slice(protoEnvironmentVariables, func(i, j int) bool { 91 | left := protoEnvironmentVariables[i] 92 | right := protoEnvironmentVariables[j] 93 | 94 | cmp1 := strings.Compare(left.Name, right.Name) 95 | if cmp1 < 0 { 96 | return true 97 | } else if cmp1 > 0 { 98 | return false 99 | } 100 | 101 | cmp2 := strings.Compare(left.Value, right.Value) 102 | return cmp2 < 0 103 | }) 104 | 105 | shellCommand := fmt.Sprintf("echo 'This is stdout.' && echo 'This is stderr.' 1>&2 && echo $NOW > foobar && mkdir -p xyzzy && echo $NOW > xyzzy/result %s", delayCmd) 106 | command := &remote_pb.Command{ 107 | Arguments: []string{"/bin/sh", "-c", shellCommand}, 108 | OutputFiles: []string{"foobar"}, 109 | OutputDirectories: []string{"xyzzy"}, 110 | EnvironmentVariables: protoEnvironmentVariables, 111 | Platform: &remote_pb.Platform{ 112 | Properties: protoPlatformProperties, 113 | }, 114 | } 115 | 116 | return command 117 | } 118 | 119 | func storeSimpleDirectory(ctx context.Context, casClient remote_pb.ContentAddressableStorageClient, instanceName string) (*remote_pb.Digest, error) { 120 | content := []byte("There is nothing to see here. Please move on.") 121 | contentDigest, err := casutil.PutBytes(ctx, casClient, content, instanceName) 122 | if err != nil { 123 | return nil, err 124 | } 125 | 126 | directory := &remote_pb.Directory{ 127 | Files: []*remote_pb.FileNode{ 128 | { 129 | Name: "just-a-file", 130 | Digest: contentDigest, 131 | }, 132 | }, 133 | } 134 | 135 | return casutil.PutProto(ctx, casClient, directory, instanceName) 136 | } 137 | 138 | func getActionResult(ctx context.Context, actionCacheClient remote_pb.ActionCacheClient, actionDigest *remote_pb.Digest, instanceName string) (*remote_pb.ActionResult, error) { 139 | req := remote_pb.GetActionResultRequest{ 140 | InstanceName: instanceName, 141 | ActionDigest: actionDigest, 142 | } 143 | 144 | actionResult, err := actionCacheClient.GetActionResult(ctx, &req) 145 | if err != nil { 146 | if s, ok := status.FromError(err); ok { 147 | if s.Code() == codes.NotFound { 148 | return nil, nil 149 | } else { 150 | return nil, err 151 | } 152 | } else { 153 | return nil, err 154 | } 155 | } 156 | 157 | return actionResult, nil 158 | } 159 | 160 | func verifyOutputFiles( 161 | ctx context.Context, 162 | actionResult *remote_pb.ActionResult, 163 | now string, 164 | casClient remote_pb.ContentAddressableStorageClient, 165 | instanceName string, 166 | ) error { 167 | if len(actionResult.OutputFiles) != 1 { 168 | return fmt.Errorf("action result should have one output file, got %d", len(actionResult.OutputFiles)) 169 | } 170 | 171 | outputFile := actionResult.OutputFiles[0] 172 | 173 | contentBytes, err := casutil.GetBytes(ctx, casClient, outputFile.Digest, instanceName) 174 | if err != nil { 175 | return err 176 | } 177 | 178 | if string(contentBytes) == now+"\n" { 179 | return nil 180 | } else { 181 | return fmt.Errorf("content from execution does not match, expected: %s, got: %s", now, string(contentBytes)) 182 | } 183 | } 184 | 185 | func verifyOutputDirectories( 186 | ctx context.Context, 187 | actionResult *remote_pb.ActionResult, 188 | now string, 189 | casClient remote_pb.ContentAddressableStorageClient, 190 | instanceName string, 191 | ) error { 192 | if len(actionResult.OutputDirectories) != 1 { 193 | return fmt.Errorf("action result should have one output directory, got %d", len(actionResult.OutputFiles)) 194 | } 195 | 196 | outputDirectory := actionResult.OutputDirectories[0] 197 | 198 | var tree remote_pb.Tree 199 | err := casutil.GetProto(ctx, casClient, outputDirectory.TreeDigest, instanceName, &tree) 200 | if err != nil { 201 | return err 202 | } 203 | 204 | if len(tree.Children) != 0 { 205 | return fmt.Errorf("output directory should not have subdirectories, got %d subdirs", len(tree.Children)) 206 | } 207 | 208 | if len(tree.Root.Directories) != 0 { 209 | return fmt.Errorf("output directory root should not have subdirectories, got %d subdirs", len(tree.Root.Directories)) 210 | } 211 | 212 | if len(tree.Root.Files) != 1 { 213 | return fmt.Errorf("output directory should have one file, got %d files", len(tree.Root.Files)) 214 | } 215 | 216 | contentBytes, err := casutil.GetBytes(ctx, casClient, tree.Root.Files[0].Digest, instanceName) 217 | if err != nil { 218 | return err 219 | } 220 | 221 | if string(contentBytes) == now+"\n" { 222 | return nil 223 | } else { 224 | return fmt.Errorf("content from execution does not match, expected: %s, got: %s", now, string(contentBytes)) 225 | } 226 | } 227 | 228 | func verifyActionResult( 229 | ctx context.Context, 230 | actionResult *remote_pb.ActionResult, 231 | now string, 232 | casClient remote_pb.ContentAddressableStorageClient, 233 | instanceName string, 234 | ) error { 235 | if actionResult.StdoutDigest != nil { 236 | stdoutBytes, err := casutil.GetBytes(ctx, casClient, actionResult.StdoutDigest, instanceName) 237 | if err != nil { 238 | return err 239 | } 240 | if log.IsLevelEnabled(log.DebugLevel) { 241 | log.WithField("content", string(stdoutBytes)).Debug("stdout") 242 | } 243 | if !bytes.Equal(stdoutBytes, []byte("This is stdout.\n")) { 244 | return fmt.Errorf("unexpected stdout: %s", string(stdoutBytes)) 245 | } 246 | } else { 247 | return fmt.Errorf("stdout was missing from action result") 248 | } 249 | 250 | if actionResult.StderrDigest != nil { 251 | stderrBytes, err := casutil.GetBytes(ctx, casClient, actionResult.StderrDigest, instanceName) 252 | if err != nil { 253 | return err 254 | } 255 | if log.IsLevelEnabled(log.DebugLevel) { 256 | log.WithField("content", string(stderrBytes)).Debug("stderr") 257 | } 258 | if !bytes.Equal(stderrBytes, []byte("This is stderr.\n")) { 259 | return fmt.Errorf("unexpected stderr: %s", string(stderrBytes)) 260 | } 261 | } else { 262 | return fmt.Errorf("stderr was missing from action result") 263 | } 264 | 265 | if actionResult.ExitCode != 0 { 266 | return fmt.Errorf("exit code was non-zero (code=%d)", actionResult.ExitCode) 267 | } 268 | 269 | err := verifyOutputFiles(ctx, actionResult, now, casClient, instanceName) 270 | if err != nil { 271 | return err 272 | } 273 | 274 | err = verifyOutputDirectories(ctx, actionResult, now, casClient, instanceName) 275 | if err != nil { 276 | return err 277 | } 278 | 279 | return nil 280 | } 281 | 282 | type clientsStruct struct { 283 | conn1 *grpc.ClientConn 284 | conn2 *grpc.ClientConn 285 | casClient remote_pb.ContentAddressableStorageClient 286 | reClient remote_pb.ExecutionClient 287 | actionCacheClient remote_pb.ActionCacheClient 288 | } 289 | 290 | func (s *clientsStruct) Close() error { 291 | err1 := s.conn1.Close() 292 | if s.conn2 != s.conn1 { 293 | err2 := s.conn2.Close() 294 | if err2 != nil { 295 | return err2 296 | } 297 | } 298 | return err1 299 | } 300 | 301 | func setupClients( 302 | ctx context.Context, 303 | casServer string, 304 | executionServer string, 305 | secure, allowInsecureAuth bool, 306 | authToken string, 307 | ) (*clientsStruct, error) { 308 | dialOptions := []grpc.DialOption{ 309 | grpc.WithBlock(), 310 | grpc.WithKeepaliveParams(keepalive.ClientParameters{ 311 | Time: time.Duration(1) * time.Second, 312 | Timeout: time.Duration(5) * time.Second, 313 | PermitWithoutStream: true, 314 | }), 315 | } 316 | 317 | if secure { 318 | tlsConfig := &tls.Config{} 319 | creds := credentials.NewTLS(tlsConfig) 320 | dialOptions = append(dialOptions, grpc.WithTransportCredentials(creds)) 321 | } else { 322 | dialOptions = append(dialOptions, grpc.WithInsecure()) 323 | } 324 | 325 | if authToken != "" { 326 | creds := grpcutil.NewStaticAuthToken(authToken, allowInsecureAuth) 327 | dialOptions = append(dialOptions, grpc.WithPerRPCCredentials(creds)) 328 | } 329 | 330 | conn1, err := grpc.DialContext(ctx, casServer, dialOptions...) 331 | if err != nil { 332 | return nil, fmt.Errorf("dial error: %s", err) 333 | } 334 | 335 | var conn2 *grpc.ClientConn 336 | if executionServer != casServer { 337 | conn2, err = grpc.DialContext(ctx, executionServer, dialOptions...) 338 | if err != nil { 339 | return nil, fmt.Errorf("dial error: %s", err) 340 | } 341 | } else { 342 | conn2 = conn1 343 | } 344 | 345 | casClient := remote_pb.NewContentAddressableStorageClient(conn1) 346 | actionCacheClient := remote_pb.NewActionCacheClient(conn1) 347 | reClient := remote_pb.NewExecutionClient(conn2) 348 | 349 | cs := clientsStruct{ 350 | conn1: conn1, 351 | conn2: conn2, 352 | casClient: casClient, 353 | reClient: reClient, 354 | actionCacheClient: actionCacheClient, 355 | } 356 | 357 | return &cs, nil 358 | } 359 | 360 | func digestToString(digest *remote_pb.Digest) string { 361 | return fmt.Sprintf("%s/%d", digest.GetHash(), digest.GetSizeBytes()) 362 | } 363 | 364 | func shouldRetryError(err error) bool { 365 | if s, ok := status.FromError(err); ok { 366 | switch s.Code() { 367 | case codes.Aborted, codes.Internal, codes.ResourceExhausted, codes.Unavailable, codes.Unknown: 368 | return true 369 | 370 | default: 371 | return false 372 | } 373 | } else { 374 | return false 375 | } 376 | } 377 | 378 | func smokeTest( 379 | ctx context.Context, 380 | cs *clientsStruct, 381 | instanceName string, 382 | platformProperties []string, 383 | delaySecs uint, 384 | extraEnvVars map[string]string, 385 | logFields log.Fields, 386 | ) error { 387 | nowBytes, _ := time.Now().MarshalText() 388 | now := string(nowBytes) 389 | command := makeCommand(now, delaySecs, platformProperties, extraEnvVars) 390 | log.WithFields(logFields).WithField("command", command.String()).Debug("constructed command") 391 | 392 | log.WithFields(logFields).Info("storing command in CAS") 393 | commandDigest, err := casutil.PutProto(ctx, cs.casClient, command, instanceName) 394 | if err != nil { 395 | return fmt.Errorf("failed to store command: %s", err) 396 | } 397 | log.WithFields(logFields).WithField("command_digest", digestToString(commandDigest)).Debug("command digest") 398 | 399 | missingDigests, err := casutil.FindMissingBlobs(ctx, cs.casClient, []*remote_pb.Digest{commandDigest}, instanceName) 400 | if err != nil { 401 | return fmt.Errorf("failed to verify command was stored: %s", err) 402 | } 403 | if len(missingDigests) > 0 { 404 | return fmt.Errorf("command was not stored in CAS") 405 | } 406 | 407 | log.WithFields(logFields).Info("storing input root in CAS") 408 | inputRootDigest, err := storeSimpleDirectory(ctx, cs.casClient, instanceName) 409 | if err != nil { 410 | return fmt.Errorf("failed to store input root: %s", err) 411 | } 412 | log.WithFields(logFields).WithField("input_root_digest", digestToString(inputRootDigest)).Debug("input root digest") 413 | 414 | log.WithFields(logFields).Info("storing action in CAS") 415 | action := &remote_pb.Action{ 416 | CommandDigest: commandDigest, 417 | InputRootDigest: inputRootDigest, 418 | } 419 | actionDigest, err := casutil.PutProto(ctx, cs.casClient, action, instanceName) 420 | if err != nil { 421 | return fmt.Errorf("failed to store action: %s", err) 422 | } 423 | log.WithFields(logFields).WithField("action_digest", digestToString(actionDigest)).Debug("action digest") 424 | 425 | missingDigests, err = casutil.FindMissingBlobs(ctx, cs.casClient, []*remote_pb.Digest{actionDigest}, instanceName) 426 | if err != nil { 427 | return fmt.Errorf("failed to verify action was stored: %s", err) 428 | } 429 | if len(missingDigests) > 0 { 430 | return fmt.Errorf("action was not stored in CAS") 431 | } 432 | 433 | // Check Action Cache to make sure the action was not run. It should not have been run since we insert 434 | // an environment variable with the current timestamp to make action unique. 435 | log.WithFields(logFields).Info("checking Action Cache to ensure action is not present (as expected)") 436 | actionResult, err := getActionResult(ctx, cs.actionCacheClient, actionDigest, instanceName) 437 | if err != nil { 438 | return fmt.Errorf("getActionResult: %s", err) 439 | } 440 | if actionResult != nil { 441 | return fmt.Errorf("illegal state: the action which is unique had an action result") 442 | } 443 | 444 | var operationName string 445 | 446 | OUTER: 447 | for { 448 | var recvOperation func() (*longrunning_pb.Operation, error) 449 | 450 | if operationName == "" { 451 | // No operation has been submitted. Submit the action for execution. 452 | log.WithFields(logFields).Info("submitting action for execution") 453 | executeRequest := &remote_pb.ExecuteRequest{ 454 | InstanceName: instanceName, 455 | ActionDigest: actionDigest, 456 | } 457 | client, err := cs.reClient.Execute(ctx, executeRequest) 458 | if err != nil { 459 | return fmt.Errorf("execute error: %s", err) 460 | } 461 | err = client.CloseSend() 462 | if err != nil { 463 | return fmt.Errorf("CloseSend error: %s", err) 464 | } 465 | recvOperation = func() (*longrunning_pb.Operation, error) { 466 | return client.Recv() 467 | } 468 | } else { 469 | // The operation has already been submitted. Reconnect to the operation stream. 470 | log.WithFields(logFields).Info("reconnecting to action's operation stream") 471 | waitExecutionRequest := &remote_pb.WaitExecutionRequest{ 472 | Name: operationName, 473 | } 474 | client, err := cs.reClient.WaitExecution(ctx, waitExecutionRequest) 475 | if err != nil { 476 | return fmt.Errorf("wait execution error: %s", err) 477 | } 478 | err = client.CloseSend() 479 | if err != nil { 480 | return fmt.Errorf("CloseSend error: %s", err) 481 | } 482 | recvOperation = func() (*longrunning_pb.Operation, error) { 483 | return client.Recv() 484 | } 485 | } 486 | 487 | // Loop over the operation status returned by the stream. 488 | for { 489 | operation, err := recvOperation() 490 | if err != nil { 491 | if shouldRetryError(err) { 492 | continue OUTER 493 | } 494 | return fmt.Errorf("stream recv: %s", err) 495 | } 496 | operationName = operation.Name 497 | 498 | if s := operation.GetError(); s != nil { 499 | s2 := status.FromProto(s) 500 | return fmt.Errorf("execution failed: %s", s2.Err()) 501 | } 502 | 503 | rawMetadata := operation.GetMetadata() 504 | if rawMetadata != nil { 505 | var metadata remote_pb.ExecuteOperationMetadata 506 | err = proto.Unmarshal(rawMetadata.Value, &metadata) 507 | if err != nil { 508 | return fmt.Errorf("unable to parse metadata: %s", err) 509 | } 510 | log.WithFields(logFields).WithFields(log.Fields{ 511 | "stage": metadata.Stage.String(), 512 | }).Debugf("metadata") 513 | } 514 | 515 | if operation.Done { 516 | if s := operation.GetError(); s != nil { 517 | s2 := status.FromProto(s) 518 | return fmt.Errorf("execution failed: %s", s2.Err()) 519 | } else if r := operation.GetResponse(); r != nil { 520 | if r.TypeUrl != "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteResponse" { 521 | return fmt.Errorf("illegal state: expected execute response to contain %s, got %s", 522 | "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteRespons", r.TypeUrl) 523 | } 524 | 525 | var executeResponse remote_pb.ExecuteResponse 526 | err = proto.Unmarshal(r.Value, &executeResponse) 527 | if err != nil { 528 | return fmt.Errorf("failed to decode execute response: %s", err) 529 | } 530 | log.WithFields(logFields).Infof("got action result: %s", executeResponse.Result) 531 | 532 | err = verifyActionResult(ctx, executeResponse.Result, now, cs.casClient, instanceName) 533 | if err != nil { 534 | return err 535 | } 536 | log.WithFields(logFields).Infof("verified output from execution is as expected") 537 | } else { 538 | return fmt.Errorf("unable to decode execution response") 539 | } 540 | 541 | // Exit the outer loop since execution was successful. 542 | break OUTER 543 | } 544 | 545 | log.WithFields(logFields).Debugf("operation: %s", operation) 546 | } 547 | } 548 | 549 | // Retrieve the action from the Action Cache. Given it just ran, this should succeed. 550 | log.WithFields(logFields).Info("checking Action Cache to ensure action is present") 551 | foundActionResult := false 552 | for retry := 1; retry <= 5; retry++ { 553 | actionResult, err = getActionResult(ctx, cs.actionCacheClient, actionDigest, instanceName) 554 | if err != nil { 555 | return fmt.Errorf("getActionResult: %s", err) 556 | } 557 | if actionResult == nil { 558 | log.WithFields(logFields).Warn("action result not in Action Cache yet, retrying...") 559 | time.Sleep(time.Duration(500) * time.Millisecond) 560 | continue 561 | } else { 562 | foundActionResult = true 563 | break 564 | } 565 | } 566 | 567 | if !foundActionResult { 568 | return fmt.Errorf("execution was successful but action cache does not have action result") 569 | } 570 | 571 | return nil 572 | } 573 | 574 | func parallelSmokeTest( 575 | ctx context.Context, 576 | cs *clientsStruct, 577 | instanceName string, 578 | platformProperties []string, 579 | count int, 580 | delaySecs uint, 581 | ) error { 582 | var wg sync.WaitGroup 583 | 584 | for i := 0; i < count; i++ { 585 | wg.Add(1) 586 | go func(i int, wg *sync.WaitGroup) { 587 | defer wg.Done() 588 | log.Infof("spawning request %d", i) 589 | 590 | err := smokeTest(ctx, cs, instanceName, platformProperties, delaySecs, map[string]string{ 591 | "COUNTER": strconv.Itoa(i), 592 | }, log.Fields{ 593 | "i": i, 594 | }) 595 | 596 | if err != nil { 597 | log.WithField("i", i).Errorf("request %d failed: %s", i, err) 598 | } else { 599 | log.WithField("i", i).Infof("request %d success", i) 600 | } 601 | }(i, &wg) 602 | } 603 | 604 | wg.Wait() 605 | 606 | return nil 607 | } 608 | 609 | func main() { 610 | versionOpt := pflag.BoolP("version", "V", false, "display version and exit") 611 | remoteOpt := pflag.StringP("remote", "r", "", "remote server") 612 | executionServerOpt := pflag.String("execution-server", "", "execution server") 613 | casServerOpt := pflag.String("cas-server", "", "CAS server") 614 | authTokenFileOpt := pflag.StringP("auth-token-file", "a", "", "auth bearer token to use") 615 | authTokenEnvOpt := pflag.StringP("auth-token-env", "A", "", "name of environment variable with auth bearer token") 616 | secureOpt := pflag.BoolP("secure", "s", false, "enable secure mode (TLS)") 617 | instanceNameOpt := pflag.StringP("instance-name", "i", "", "instance name") 618 | instanceNameFromEnvOpt := pflag.StringP("instance-from-env", "I", "", "instance name from given environment variable") 619 | timeoutSecsOpt := pflag.UintP("timeout-secs", "t", 0, "timeout in seconds") 620 | delaySecsOpt := pflag.UintP("delay", "d", 0, "delay on remote execution") 621 | verboseOpt := pflag.CountP("verbose", "v", "increase logging verbosity") 622 | useJsonLoggingOpt := pflag.Bool("log-json", false, "log using JSON") 623 | platformPropertiesOpt := pflag.StringSliceP("platform", "p", nil, "add platform property KEY=VALUE") 624 | allowInsecureAuthOpt := pflag.Bool("allow-insecure-auth", false, "allow credentials to be passed unencrypted (i.e., no TLS)") 625 | parallelModeOpt := pflag.Int("parallel", 1, "enable parallel mode") 626 | 627 | pflag.Parse() 628 | 629 | if *versionOpt { 630 | fmt.Println(version) 631 | os.Exit(0) 632 | } 633 | 634 | if *useJsonLoggingOpt { 635 | log.SetFormatter(&log.JSONFormatter{}) 636 | } else { 637 | log.SetFormatter(&log.TextFormatter{ 638 | DisableColors: true, 639 | FullTimestamp: true, 640 | }) 641 | } 642 | 643 | var executionServer string 644 | var casServer string 645 | 646 | if *remoteOpt != "" { 647 | if *executionServerOpt != "" || *casServerOpt != "" { 648 | log.Fatal("--remote is mutually exclusive with --execution-server / --cas-server") 649 | } 650 | executionServer = *remoteOpt 651 | casServer = *remoteOpt 652 | } else if *executionServerOpt != "" || *casServerOpt != "" { 653 | if *executionServerOpt == "" { 654 | log.Fatal("--execution-server must be specified when --cas-server is specified") 655 | } 656 | if *casServerOpt == "" { 657 | log.Fatal("--cas-server must be specified when --execution-server is specified") 658 | } 659 | 660 | executionServer = *executionServerOpt 661 | casServer = *casServerOpt 662 | } 663 | 664 | authToken := "" 665 | if *authTokenFileOpt != "" { 666 | authTokenBytes, err := ioutil.ReadFile(*authTokenFileOpt) 667 | if err != nil { 668 | log.Fatalf("unable to read auth token from %s: %s", *authTokenFileOpt, err) 669 | } 670 | authToken = strings.TrimSpace(string(authTokenBytes)) 671 | } else if *authTokenEnvOpt != "" { 672 | envValue := os.Getenv(*authTokenEnvOpt) 673 | if envValue == "" { 674 | log.Fatalf("environment variable %s was either unset or empty", *authTokenEnvOpt) 675 | } 676 | authToken = strings.TrimSpace(envValue) 677 | } 678 | 679 | instanceName := "" 680 | if *instanceNameOpt != "" && *instanceNameFromEnvOpt != "" { 681 | log.Fatal("--instance and --instance-from-env are mutually exclusive") 682 | } else if *instanceNameOpt != "" { 683 | instanceName = *instanceNameOpt 684 | } else if *instanceNameFromEnvOpt != "" { 685 | instanceName = os.Getenv(*instanceNameFromEnvOpt) 686 | } 687 | 688 | if *verboseOpt > 1 { 689 | log.SetLevel(log.TraceLevel) 690 | } else if *verboseOpt == 1 { 691 | log.SetLevel(log.DebugLevel) 692 | } 693 | 694 | ctx := context.Background() 695 | if *timeoutSecsOpt != 0 { 696 | duration := time.Duration(*timeoutSecsOpt) * time.Second 697 | ctx2, cancel := context.WithTimeout(ctx, duration) 698 | ctx = ctx2 699 | defer cancel() 700 | } 701 | 702 | cs, err := setupClients(ctx, casServer, executionServer, *secureOpt, *allowInsecureAuthOpt, authToken) 703 | if err != nil { 704 | log.Fatalf("failed to setup connection: %s", err) 705 | } 706 | defer cs.Close() 707 | 708 | if *parallelModeOpt < 2 { 709 | err = smokeTest(ctx, cs, instanceName, *platformPropertiesOpt, *delaySecsOpt, map[string]string{}, map[string]interface{}{}) 710 | if err != nil { 711 | log.Fatalf("smoke test failed: %s", err) 712 | } 713 | } else { 714 | err = parallelSmokeTest(ctx, cs, instanceName, *platformPropertiesOpt, *parallelModeOpt, *delaySecsOpt) 715 | if err != nil { 716 | log.Fatalf("parallel smoke test failed: %s", err) 717 | } 718 | } 719 | 720 | log.Info("smoke test succeeded") 721 | } 722 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/toolchainlabs/remote-api-tools 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/golang/protobuf v1.5.2 7 | github.com/google/uuid v1.3.0 8 | github.com/sirupsen/logrus v1.9.0 9 | github.com/spf13/pflag v1.0.5 10 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 11 | google.golang.org/grpc v1.33.2 12 | google.golang.org/protobuf v1.28.1 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 4 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 5 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 10 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 11 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 12 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 13 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 14 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 15 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 16 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 17 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 18 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 19 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 20 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 21 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 22 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 23 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 24 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 25 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 26 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 27 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 28 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 29 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 30 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 31 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 32 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 33 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 34 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 35 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 36 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 37 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 38 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 39 | github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= 40 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 41 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 42 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 43 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 44 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 45 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 46 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 47 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 48 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 49 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 50 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 51 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 52 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 53 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 54 | golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= 55 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 56 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 57 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 58 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 59 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 60 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 61 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 62 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= 63 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 64 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 65 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 66 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 67 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 68 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 69 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 70 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 71 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 72 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 73 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 74 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 75 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 76 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= 77 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 78 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 79 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 80 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 81 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 82 | google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= 83 | google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= 84 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 85 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 86 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 87 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 88 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 89 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 90 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 91 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 92 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 93 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 94 | google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= 95 | google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 96 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 97 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 98 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 99 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 100 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 101 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 102 | -------------------------------------------------------------------------------- /pants.ci.toml: -------------------------------------------------------------------------------- 1 | # See https://www.pantsbuild.org/docs/using-pants-in-ci. 2 | 3 | [GLOBAL] 4 | dynamic_ui = false 5 | colors = true 6 | -------------------------------------------------------------------------------- /pants.toml: -------------------------------------------------------------------------------- 1 | [GLOBAL] 2 | pants_version = "2.19.1" 3 | backend_packages.add = [ 4 | "pants.backend.build_files.fmt.black", 5 | "pants.backend.experimental.go", 6 | "pants.backend.docker", 7 | ] 8 | # Caching is disabled by default so that desktop builds are not cached. CI caching is enabled in pants.ci.toml. 9 | remote_cache_read = false 10 | remote_cache_write = false 11 | # We want to continue to get logs when remote caching errors. 12 | remote_cache_warnings = "backoff" 13 | 14 | [source] 15 | root_patterns = ["/"] 16 | 17 | [anonymous-telemetry] 18 | enabled = false # anonymous-telemetry is being deprecated from pants 19 | 20 | [golang] 21 | minimum_expected_version = "1.19" 22 | 23 | [python] 24 | interpreter_constraints = ['==3.11.*'] 25 | 26 | [docker.registries.dockerhub] 27 | address = "registry.hub.docker.com" 28 | default = true 29 | extra_image_tags = ["nightly"] 30 | -------------------------------------------------------------------------------- /pants_from_sources: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2020 Pants project contributors. 3 | # Licensed under the Apache License, Version 2.0 (see LICENSE). 4 | 5 | # Runs pants from sources. Useful for debugging. 6 | # Assumes you have the pantsbuild/pants repo checked out in a sibling dir of this dir, 7 | # named 'pants' but overridable using PANTS_SOURCE. 8 | 9 | set -euo pipefail 10 | 11 | cd "$(git rev-parse --show-toplevel)" 12 | 13 | PANTS_SOURCE="${PANTS_SOURCE:-../pants}" 14 | 15 | # When running pants from sources you are likely to be modifying those sources, so 16 | # you won't want pantsd running. You can override this by setting ENABLE_PANTSD=true. 17 | ENABLE_PANTSD="${ENABLE_PANTSD:-false}" 18 | 19 | backend_packages=( 20 | ) 21 | 22 | pythonpath=( 23 | ) 24 | 25 | plugins=( 26 | "toolchain.pants.plugin==0.22.0" 27 | ) 28 | 29 | function string_list() { 30 | eval local -r list_variable="\${$1[@]}" 31 | 32 | echo -n "[" 33 | for item in ${list_variable}; do 34 | echo -n "\"${item}\"," 35 | done 36 | echo -n "]" 37 | } 38 | 39 | export PANTS_VERSION="$(cat "${PANTS_SOURCE}/src/python/pants/VERSION")" 40 | export PANTS_PLUGINS="$(string_list plugins)" 41 | export PANTS_PYTHONPATH="+$(string_list pythonpath)" 42 | export PANTS_BACKEND_PACKAGES="+$(string_list backend_packages)" 43 | export PANTS_PANTSD="${ENABLE_PANTSD}" 44 | export no_proxy="*" 45 | 46 | exec "${PANTS_SOURCE}/pants" "--no-verify-config" "$@" 47 | -------------------------------------------------------------------------------- /pkg/casutil/BUILD: -------------------------------------------------------------------------------- 1 | go_package() 2 | -------------------------------------------------------------------------------- /pkg/casutil/casutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package casutil 16 | 17 | import ( 18 | "context" 19 | "crypto/sha256" 20 | "encoding/hex" 21 | "fmt" 22 | "io" 23 | 24 | "github.com/golang/protobuf/proto" 25 | "github.com/google/uuid" 26 | log "github.com/sirupsen/logrus" 27 | remote_pb "github.com/toolchainlabs/remote-api-tools/protos/build/bazel/remote/execution/v2" 28 | bytestream_pb "google.golang.org/genproto/googleapis/bytestream" 29 | "google.golang.org/grpc/codes" 30 | "google.golang.org/grpc/status" 31 | ) 32 | 33 | func ComputeDigest(data []byte) *remote_pb.Digest { 34 | h := sha256.Sum256(data) 35 | digest := hex.EncodeToString(h[:]) 36 | 37 | return &remote_pb.Digest{ 38 | Hash: digest, 39 | SizeBytes: int64(len(data)), 40 | } 41 | } 42 | 43 | func PutBytes( 44 | ctx context.Context, 45 | casClient remote_pb.ContentAddressableStorageClient, 46 | data []byte, 47 | instanceName string, 48 | ) (*remote_pb.Digest, error) { 49 | digest := ComputeDigest(data) 50 | 51 | req1 := remote_pb.FindMissingBlobsRequest{ 52 | BlobDigests: []*remote_pb.Digest{digest}, 53 | InstanceName: instanceName, 54 | } 55 | 56 | resp1, err := casClient.FindMissingBlobs(ctx, &req1) 57 | if err != nil { 58 | return nil, fmt.Errorf("FindMissingBlobs: %s", err) 59 | } 60 | 61 | if len(resp1.MissingBlobDigests) > 0 { 62 | req2 := remote_pb.BatchUpdateBlobsRequest{ 63 | InstanceName: instanceName, 64 | Requests: []*remote_pb.BatchUpdateBlobsRequest_Request{ 65 | { 66 | Digest: digest, 67 | Data: data, 68 | }, 69 | }, 70 | } 71 | 72 | resp2, err := casClient.BatchUpdateBlobs(context.Background(), &req2) 73 | if err != nil { 74 | return nil, fmt.Errorf("BatchUpdateBlobs: %s", err) 75 | } 76 | 77 | if len(resp2.Responses) != 1 { 78 | return nil, fmt.Errorf("illegal state: bad BatchUpdateBlobs response") 79 | } 80 | 81 | s := status.FromProto(resp2.Responses[0].Status) 82 | if s.Code() != codes.OK { 83 | return nil, fmt.Errorf("failed to store to CAS: %s", s.Err()) 84 | } 85 | } 86 | 87 | return digest, nil 88 | } 89 | 90 | func PutProto( 91 | ctx context.Context, 92 | casClient remote_pb.ContentAddressableStorageClient, 93 | m proto.Message, 94 | instanceName string, 95 | ) (*remote_pb.Digest, error) { 96 | data, err := proto.Marshal(m) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | return PutBytes(ctx, casClient, data, instanceName) 102 | } 103 | 104 | func GetBytes( 105 | ctx context.Context, 106 | casClient remote_pb.ContentAddressableStorageClient, 107 | digest *remote_pb.Digest, 108 | instanceName string, 109 | ) ([]byte, error) { 110 | req := remote_pb.BatchReadBlobsRequest{ 111 | InstanceName: instanceName, 112 | Digests: []*remote_pb.Digest{digest}, 113 | } 114 | 115 | resp, err := casClient.BatchReadBlobs(ctx, &req) 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | return resp.Responses[0].Data, nil 121 | } 122 | 123 | func GetProto( 124 | ctx context.Context, 125 | casClient remote_pb.ContentAddressableStorageClient, 126 | digest *remote_pb.Digest, 127 | instanceName string, 128 | m proto.Message, 129 | ) error { 130 | data, err := GetBytes(ctx, casClient, digest, instanceName) 131 | if err != nil { 132 | return err 133 | } 134 | 135 | err = proto.Unmarshal(data, m) 136 | if err != nil { 137 | return err 138 | } 139 | 140 | return nil 141 | } 142 | 143 | func FindMissingBlobs( 144 | ctx context.Context, 145 | casClient remote_pb.ContentAddressableStorageClient, 146 | digests []*remote_pb.Digest, 147 | instanceName string, 148 | ) ([]*remote_pb.Digest, error) { 149 | request := remote_pb.FindMissingBlobsRequest{ 150 | BlobDigests: digests, 151 | InstanceName: instanceName, 152 | } 153 | 154 | response, err := casClient.FindMissingBlobs(ctx, &request) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | return response.MissingBlobDigests, nil 160 | } 161 | 162 | func PutBytesStream( 163 | ctx context.Context, 164 | client bytestream_pb.ByteStreamClient, 165 | data []byte, 166 | chunkSize int64, 167 | instanceName string, 168 | ) (*remote_pb.Digest, error) { 169 | digest := ComputeDigest(data) 170 | 171 | writeClient, err := client.Write(ctx) 172 | if err != nil { 173 | return nil, err 174 | } 175 | 176 | var writeOffset int64 177 | 178 | id, err := uuid.NewRandom() 179 | if err != nil { 180 | return nil, err 181 | } 182 | 183 | resourceName := fmt.Sprintf( 184 | "%s/uploads/%s/blobs/%s/%d", 185 | instanceName, 186 | id.String(), 187 | digest.GetHash(), 188 | digest.GetSizeBytes()) 189 | includeResourceName := true 190 | 191 | for { 192 | request := bytestream_pb.WriteRequest{ 193 | WriteOffset: writeOffset, 194 | } 195 | 196 | if includeResourceName { 197 | request.ResourceName = resourceName 198 | includeResourceName = false 199 | } 200 | 201 | writeEndOffset := writeOffset + chunkSize 202 | if writeEndOffset > int64(len(data)) { 203 | writeEndOffset = int64(len(data)) 204 | request.FinishWrite = true 205 | } 206 | 207 | request.Data = data[writeOffset:writeEndOffset] 208 | 209 | log.WithFields(log.Fields{ 210 | "resource_name": request.ResourceName, 211 | "write_offset": request.WriteOffset, 212 | "finish_write": request.FinishWrite, 213 | "data_length": len(request.Data), 214 | }).Trace("bytestream write request") 215 | 216 | err = writeClient.Send(&request) 217 | if err != nil { 218 | return nil, fmt.Errorf("failed to write chunk: %s", err) 219 | } 220 | 221 | writeOffset += int64(len(request.Data)) 222 | 223 | if request.FinishWrite { 224 | break 225 | } 226 | } 227 | 228 | response, err := writeClient.CloseAndRecv() 229 | if err != nil && err != io.EOF { 230 | return nil, fmt.Errorf("failed to write digest: %s", err) 231 | } 232 | 233 | if response.CommittedSize != int64(len(data)) { 234 | return nil, fmt.Errorf("failed to write digest: %s", err) 235 | } 236 | 237 | return digest, nil 238 | } 239 | 240 | func GetBytesStream( 241 | ctx context.Context, 242 | client bytestream_pb.ByteStreamClient, 243 | digest *remote_pb.Digest, 244 | instanceName string, 245 | ) ([]byte, error) { 246 | resourceName := fmt.Sprintf( 247 | "%s/blobs/%s/%d", 248 | instanceName, 249 | digest.GetHash(), 250 | digest.GetSizeBytes()) 251 | 252 | request := bytestream_pb.ReadRequest{ 253 | ResourceName: resourceName, 254 | ReadOffset: 0, 255 | ReadLimit: 0, 256 | } 257 | 258 | readClient, err := client.Read(ctx, &request) 259 | if err != nil { 260 | return nil, fmt.Errorf("failed to read digest: %s", err) 261 | } 262 | 263 | data := make([]byte, 0, digest.SizeBytes) 264 | 265 | for { 266 | response, err := readClient.Recv() 267 | if err == io.EOF { 268 | break 269 | } 270 | if err != nil { 271 | return nil, fmt.Errorf("failed to read digest: %s", err) 272 | } 273 | 274 | data = append(data, response.Data...) 275 | } 276 | 277 | return data, nil 278 | } 279 | -------------------------------------------------------------------------------- /pkg/grpcutil/BUILD: -------------------------------------------------------------------------------- 1 | go_package() 2 | -------------------------------------------------------------------------------- /pkg/grpcutil/static_auth_token.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package grpcutil 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "google.golang.org/grpc/credentials" 22 | ) 23 | 24 | type staticAuthToken struct { 25 | token string 26 | allowInsecureAuth bool 27 | } 28 | 29 | func (s staticAuthToken) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { 30 | if !s.allowInsecureAuth { 31 | if err := credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil { 32 | return nil, fmt.Errorf("unable to transfer token PerRPCCredentials: %v", err) 33 | } 34 | } 35 | return map[string]string{ 36 | "authorization": fmt.Sprintf("Bearer %s", s.token), 37 | }, nil 38 | } 39 | 40 | func (s staticAuthToken) RequireTransportSecurity() bool { 41 | return !s.allowInsecureAuth 42 | } 43 | 44 | func NewStaticAuthToken(token string, allowInsecureAuth bool) credentials.PerRPCCredentials { 45 | return staticAuthToken{token: token, allowInsecureAuth: allowInsecureAuth} 46 | } 47 | -------------------------------------------------------------------------------- /pkg/load/BUILD: -------------------------------------------------------------------------------- 1 | go_package() 2 | -------------------------------------------------------------------------------- /pkg/load/action.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package load 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | remote_pb "github.com/toolchainlabs/remote-api-tools/protos/build/bazel/remote/execution/v2" 21 | bytestream_pb "google.golang.org/genproto/googleapis/bytestream" 22 | "strconv" 23 | "strings" 24 | "sync" 25 | ) 26 | 27 | type ActionContext struct { 28 | // Go context for the current operation. 29 | Ctx context.Context 30 | 31 | // Clients to use to access the CAS. 32 | CasClient remote_pb.ContentAddressableStorageClient 33 | BytestreamClient bytestream_pb.ByteStreamClient 34 | InstanceName string 35 | MaxBatchBlobSize int64 36 | WriteChunkSize int64 37 | 38 | // Map of a digest in string form to a bool representing whether it is known to be present or missing 39 | // in the CAS. 40 | KnownDigests map[string]bool 41 | mu sync.Mutex 42 | } 43 | 44 | func (ac *ActionContext) AddKnownDigest(digest *remote_pb.Digest, present bool) { 45 | ac.mu.Lock() 46 | defer ac.mu.Unlock() 47 | 48 | ac.KnownDigests[fmt.Sprintf("%s-%d", digest.Hash, digest.SizeBytes)] = present 49 | } 50 | 51 | func (ac *ActionContext) GetKnownDigests(presence bool) []*remote_pb.Digest { 52 | ac.mu.Lock() 53 | defer ac.mu.Unlock() 54 | 55 | result := []*remote_pb.Digest{} 56 | 57 | for digestKey, present := range ac.KnownDigests { 58 | if present != presence { 59 | continue 60 | } 61 | 62 | parts := strings.Split(digestKey, "-") 63 | if len(parts) != 2 { 64 | panic("illegal state: known digests key was not parsed correctly") 65 | } 66 | 67 | sizeBytes, err := strconv.Atoi(parts[1]) 68 | if err != nil { 69 | panic("illegal state: known digests key was not parsed correctly") 70 | } 71 | 72 | digest := remote_pb.Digest{ 73 | Hash: parts[0], 74 | SizeBytes: int64(sizeBytes), 75 | } 76 | 77 | result = append(result, &digest) 78 | } 79 | 80 | return result 81 | } 82 | 83 | type Action interface { 84 | RunAction(actionContext *ActionContext) error 85 | } 86 | 87 | func ParseAction(actionStr string) (Action, error) { 88 | parts := strings.Split(actionStr, ":") 89 | if len(parts) < 1 { 90 | return nil, fmt.Errorf("unable to parse load action: %s", actionStr) 91 | } 92 | 93 | switch parts[0] { 94 | case "generate": 95 | return ParseGenerateAction(parts[1:]) 96 | case "read": 97 | return ParseReadAction(parts[1:]) 98 | case "load-digests": 99 | return ParseLoadDigestsAction(parts[1:]) 100 | case "save-digests": 101 | return ParseSaveDigestsAction(parts[1:]) 102 | default: 103 | return nil, fmt.Errorf("unknown load action name: %s", parts[0]) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /pkg/load/generate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package load 16 | 17 | import ( 18 | "fmt" 19 | log "github.com/sirupsen/logrus" 20 | "math/rand" 21 | "strconv" 22 | "time" 23 | 24 | "github.com/toolchainlabs/remote-api-tools/pkg/casutil" 25 | "github.com/toolchainlabs/remote-api-tools/pkg/retry" 26 | "github.com/toolchainlabs/remote-api-tools/pkg/stats" 27 | remote_pb "github.com/toolchainlabs/remote-api-tools/protos/build/bazel/remote/execution/v2" 28 | ) 29 | 30 | type generateAction struct { 31 | numRequests int 32 | minBlobSize int 33 | maxBlobSize int 34 | concurrency int 35 | } 36 | 37 | func (g *generateAction) String() string { 38 | return fmt.Sprintf("%d:%d:%d", g.numRequests, g.minBlobSize, g.maxBlobSize) 39 | } 40 | 41 | type generateResult struct { 42 | startTime time.Time 43 | endTime time.Time 44 | success int 45 | errors int 46 | byteStreamWrites int 47 | blobWrites int 48 | } 49 | 50 | type generateWorkItem struct { 51 | actionContext *ActionContext 52 | blobSize int 53 | } 54 | 55 | type generateWorkResult struct { 56 | blobSize int 57 | err error 58 | elapsed time.Duration 59 | } 60 | 61 | func generateWorker(workChan <-chan *generateWorkItem, resultChan chan<- *generateWorkResult) { 62 | log.Debug("generateWorker started") 63 | for wi := range workChan { 64 | startTime := time.Now() 65 | result := processWorkItem(wi) 66 | result.elapsed = time.Now().Sub(startTime) 67 | resultChan <- result 68 | } 69 | log.Debug("generateWorker stopped") 70 | } 71 | 72 | func writeBlobBatch(actionContext *ActionContext, data []byte) (*remote_pb.Digest, error) { 73 | digest, err := casutil.PutBytes(actionContext.Ctx, actionContext.CasClient, data, actionContext.InstanceName) 74 | if err != nil { 75 | return nil, fmt.Errorf("failed to write blob: %s", err) 76 | } 77 | return digest, nil 78 | } 79 | 80 | func writeBlobStream(actionContext *ActionContext, data []byte) (*remote_pb.Digest, error) { 81 | digest, err := casutil.PutBytesStream(actionContext.Ctx, actionContext.BytestreamClient, data, actionContext.WriteChunkSize, 82 | actionContext.InstanceName) 83 | if err != nil { 84 | return nil, fmt.Errorf("failed to write blob (bytestream) %s", err) 85 | } 86 | return digest, nil 87 | } 88 | 89 | func processWorkItem(wi *generateWorkItem) *generateWorkResult { 90 | buf := make([]byte, wi.blobSize) 91 | n, err := rand.Read(buf) 92 | if err != nil { 93 | log.Errorf("rand failed: %s", err) 94 | return &generateWorkResult{ 95 | err: err, 96 | } 97 | } 98 | if n != wi.blobSize { 99 | log.Errorf("rand gave less than expected") 100 | return &generateWorkResult{ 101 | err: err, 102 | } 103 | } 104 | 105 | var digest *remote_pb.Digest 106 | if int64(wi.blobSize) < wi.actionContext.MaxBatchBlobSize { 107 | digest, err = writeBlobBatch(wi.actionContext, buf) 108 | } else { 109 | digest, err = writeBlobStream(wi.actionContext, buf) 110 | } 111 | if err != nil { 112 | return &generateWorkResult{ 113 | blobSize: wi.blobSize, 114 | err: err, 115 | } 116 | } 117 | 118 | _, err = retry.ExpBackoff(6, time.Duration(250)*time.Millisecond, time.Duration(5)*time.Second, func() (interface{}, error) { 119 | missingBlobs, err := casutil.FindMissingBlobs(wi.actionContext.Ctx, wi.actionContext.CasClient, []*remote_pb.Digest{digest}, 120 | wi.actionContext.InstanceName) 121 | if err != nil { 122 | return nil, err 123 | } 124 | if len(missingBlobs) > 0 { 125 | return nil, fmt.Errorf("just-written blob is reported as not present in the CAS") 126 | } 127 | return nil, nil 128 | }, func(err error) bool { 129 | return true 130 | }) 131 | if err != nil { 132 | return &generateWorkResult{ 133 | blobSize: wi.blobSize, 134 | err: fmt.Errorf("failed to verify existence of blob: %s", err), 135 | } 136 | } 137 | 138 | wi.actionContext.AddKnownDigest(digest, true) 139 | 140 | return &generateWorkResult{ 141 | blobSize: wi.blobSize, 142 | err: nil, 143 | } 144 | } 145 | 146 | func (g *generateAction) RunAction(actionContext *ActionContext) error { 147 | result := generateResult{ 148 | startTime: time.Now(), 149 | } 150 | 151 | workChan := make(chan *generateWorkItem) 152 | resultChan := make(chan *generateWorkResult) 153 | 154 | for c := 0; c < g.concurrency; c++ { 155 | go generateWorker(workChan, resultChan) 156 | } 157 | 158 | go func() { 159 | for i := 0; i < g.numRequests; i++ { 160 | wi := generateWorkItem{ 161 | actionContext: actionContext, 162 | blobSize: rand.Intn(g.maxBlobSize-g.minBlobSize) + g.minBlobSize, 163 | } 164 | workChan <- &wi 165 | } 166 | 167 | close(workChan) 168 | }() 169 | 170 | elapsedTimes := make([]time.Duration, g.numRequests) 171 | 172 | for i := 0; i < g.numRequests; i++ { 173 | r := <-resultChan 174 | if r.err == nil { 175 | result.success += 1 176 | } else { 177 | result.errors += 1 178 | log.WithFields(log.Fields{ 179 | "size": r.blobSize, 180 | "err": r.err.Error(), 181 | }).Error("request error") 182 | } 183 | elapsedTimes[i] = r.elapsed 184 | 185 | if int64(r.blobSize) < actionContext.MaxBatchBlobSize { 186 | result.blobWrites += 1 187 | } else { 188 | result.byteStreamWrites += 1 189 | } 190 | 191 | if i%100 == 0 { 192 | log.Debugf("progress: %d / %d", i, g.numRequests) 193 | } 194 | } 195 | 196 | result.endTime = time.Now() 197 | 198 | close(resultChan) 199 | 200 | fmt.Printf("program: %s\n startTime: %s\n endTime: %s\n success: %d\n errors: %d\n byteStreamWrites: %d\n blobWrites: %d\n", 201 | g.String(), 202 | result.startTime.String(), 203 | result.endTime.String(), 204 | result.success, 205 | result.errors, 206 | result.byteStreamWrites, 207 | result.blobWrites, 208 | ) 209 | 210 | stats.PrintTimingStats(elapsedTimes) 211 | 212 | return nil 213 | } 214 | 215 | func ParseGenerateAction(args []string) (Action, error) { 216 | if len(args) < 3 || len(args) > 4 { 217 | return nil, fmt.Errorf("unable to parse program: expected 3-4 fields, got %d fields", len(args)) 218 | } 219 | 220 | numRequests, err := strconv.Atoi(args[0]) 221 | if err != nil { 222 | return nil, fmt.Errorf("unable to parse number of requests: %s: %s", args[0], err) 223 | } 224 | 225 | minBlobSize, err := strconv.Atoi(args[1]) 226 | if err != nil { 227 | return nil, fmt.Errorf("unable to parse min blob size: %s: %s", args[1], err) 228 | } 229 | 230 | maxBlobSize, err := strconv.Atoi(args[2]) 231 | if err != nil { 232 | return nil, fmt.Errorf("unable to parse max blob size: %s: %s", args[2], err) 233 | } 234 | 235 | concurrency := 50 236 | if len(args) >= 4 { 237 | c, err := strconv.Atoi(args[3]) 238 | if err != nil { 239 | return nil, fmt.Errorf("unable to parse concurrency: %s: %s", args[3], err) 240 | } 241 | concurrency = c 242 | } 243 | 244 | action := generateAction{ 245 | numRequests: numRequests, 246 | minBlobSize: minBlobSize, 247 | maxBlobSize: maxBlobSize, 248 | concurrency: concurrency, 249 | } 250 | 251 | return &action, nil 252 | } 253 | -------------------------------------------------------------------------------- /pkg/load/load.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package load 16 | 17 | import ( 18 | "bufio" 19 | "fmt" 20 | "os" 21 | "strconv" 22 | "strings" 23 | 24 | remote_pb "github.com/toolchainlabs/remote-api-tools/protos/build/bazel/remote/execution/v2" 25 | ) 26 | 27 | // This action will load a set of digests from a file into the "known digests" of this run's action context. 28 | // This allows using a known set of digests in a load test (e.g., digests saved from a previous run via the 29 | // save digests action). 30 | 31 | type loadDigestsAction struct { 32 | filename string 33 | } 34 | 35 | func (self *loadDigestsAction) RunAction(actionContext *ActionContext) error { 36 | f, err := os.Open(self.filename) 37 | if err != nil { 38 | return fmt.Errorf("failed to load digests from file %s: %s", self.filename, err) 39 | } 40 | defer f.Close() 41 | 42 | lineNum := 0 43 | scanner := bufio.NewScanner(f) 44 | for scanner.Scan() { 45 | lineNum += 1 46 | line := scanner.Text() 47 | 48 | parts := strings.Split(line, ",") 49 | if len(parts) != 3 { 50 | return fmt.Errorf("invalid digest specifier (%s:%d)", self.filename, lineNum) 51 | } 52 | 53 | sizeBytes, err := strconv.Atoi(parts[1]) 54 | if err != nil { 55 | return fmt.Errorf("invalid digest size (%s:%d)", self.filename, lineNum) 56 | } 57 | 58 | present, err := strconv.ParseBool(parts[2]) 59 | if err != nil { 60 | return fmt.Errorf("invalid presennce specifier (%s:%d)", self.filename, lineNum) 61 | } 62 | 63 | digest := remote_pb.Digest{ 64 | Hash: parts[0], 65 | SizeBytes: int64(sizeBytes), 66 | } 67 | 68 | actionContext.AddKnownDigest(&digest, present) 69 | } 70 | 71 | return nil 72 | } 73 | 74 | func ParseLoadDigestsAction(args []string) (Action, error) { 75 | if len(args) != 1 { 76 | return nil, fmt.Errorf("filename to load must be specified") 77 | } 78 | 79 | action := loadDigestsAction{ 80 | filename: args[0], 81 | } 82 | 83 | return &action, nil 84 | } 85 | -------------------------------------------------------------------------------- /pkg/load/read.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package load 16 | 17 | import ( 18 | "fmt" 19 | log "github.com/sirupsen/logrus" 20 | "math/rand" 21 | "strconv" 22 | "time" 23 | 24 | "github.com/toolchainlabs/remote-api-tools/pkg/casutil" 25 | "github.com/toolchainlabs/remote-api-tools/pkg/stats" 26 | remote_pb "github.com/toolchainlabs/remote-api-tools/protos/build/bazel/remote/execution/v2" 27 | "google.golang.org/grpc/codes" 28 | "google.golang.org/grpc/status" 29 | ) 30 | 31 | // This action uses the known digests of the current run to schedule read requests to the CAS. 32 | // This is useful to verify CAS contents after generating known digests or loading a set of known digests. 33 | type readAction struct { 34 | numDigestsToRead int 35 | numReadsPerDigest int 36 | concurrency int 37 | } 38 | 39 | func (self *readAction) String() string { 40 | return fmt.Sprintf("read %d:%d:%d", self.numDigestsToRead, self.numReadsPerDigest, self.concurrency) 41 | } 42 | 43 | type readResult struct { 44 | startTime time.Time 45 | endTime time.Time 46 | success int 47 | errors int 48 | byteStreamReads int 49 | blobReads int 50 | } 51 | 52 | type readWorkItem struct { 53 | actionContext *ActionContext 54 | digest *remote_pb.Digest 55 | } 56 | 57 | type readWorkResult struct { 58 | digest *remote_pb.Digest 59 | err error 60 | elapsed time.Duration 61 | } 62 | 63 | func readWorker(workChan <-chan *readWorkItem, resultChan chan<- *readWorkResult) { 64 | log.Debug("readWorker started") 65 | for wi := range workChan { 66 | startTime := time.Now() 67 | result := processReadWorkItem(wi) 68 | result.elapsed = time.Now().Sub(startTime) 69 | resultChan <- result 70 | } 71 | log.Debug("readWorker stopped") 72 | } 73 | 74 | func readBlobBatch(actionContext *ActionContext, digest *remote_pb.Digest) ([]byte, error) { 75 | request := remote_pb.BatchReadBlobsRequest{ 76 | InstanceName: actionContext.InstanceName, 77 | Digests: []*remote_pb.Digest{ 78 | digest, 79 | }, 80 | } 81 | 82 | response, err := actionContext.CasClient.BatchReadBlobs(actionContext.Ctx, &request) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | s := status.FromProto(response.Responses[0].Status) 88 | if s.Code() != codes.OK { 89 | return nil, s.Err() 90 | } 91 | 92 | if len(response.Responses) != 1 { 93 | return nil, fmt.Errorf("expected 1 response in RPC call, got %d responses instead", len(response.Responses)) 94 | } 95 | 96 | digestResponse := response.Responses[0] 97 | 98 | if digestResponse.Digest.Hash != digest.Hash { 99 | return nil, fmt.Errorf("expected digest %s in RPC call, got digest %s instead", digest.Hash, digestResponse.Digest.Hash) 100 | } 101 | 102 | if digestResponse.Digest.SizeBytes != digest.SizeBytes { 103 | return nil, fmt.Errorf("expected size %d in RPC call, got size %d instead", digest.SizeBytes, digestResponse.Digest.SizeBytes) 104 | } 105 | 106 | return response.Responses[0].Data, nil 107 | } 108 | 109 | func readBlobStream(actionContext *ActionContext, digest *remote_pb.Digest) ([]byte, error) { 110 | data, err := casutil.GetBytesStream(actionContext.Ctx, actionContext.BytestreamClient, digest, actionContext.InstanceName) 111 | if err != nil { 112 | return nil, fmt.Errorf("failed to read blob (bytestream): %s", err) 113 | } 114 | 115 | return data, nil 116 | } 117 | 118 | func processReadWorkItem(wi *readWorkItem) *readWorkResult { 119 | var data []byte 120 | var err error 121 | 122 | if wi.digest.SizeBytes < wi.actionContext.MaxBatchBlobSize { 123 | data, err = readBlobBatch(wi.actionContext, wi.digest) 124 | } else { 125 | data, err = readBlobStream(wi.actionContext, wi.digest) 126 | } 127 | if err != nil { 128 | return &readWorkResult{ 129 | digest: wi.digest, 130 | err: err, 131 | } 132 | } 133 | 134 | if int64(len(data)) != wi.digest.SizeBytes { 135 | return &readWorkResult{ 136 | digest: wi.digest, 137 | err: fmt.Errorf("size mismatch, expected %d, actual %d", wi.digest.SizeBytes, len(data)), 138 | } 139 | } 140 | 141 | computedDigest := casutil.ComputeDigest(data) 142 | if computedDigest.Hash != wi.digest.Hash { 143 | return &readWorkResult{ 144 | digest: wi.digest, 145 | err: fmt.Errorf("digest hash mismatch, expected %s, actual %s", wi.digest, computedDigest.Hash), 146 | } 147 | } 148 | 149 | return &readWorkResult{ 150 | digest: wi.digest, 151 | err: nil, 152 | } 153 | } 154 | 155 | func (self *readAction) RunAction(actionContext *ActionContext) error { 156 | result := readResult{ 157 | startTime: time.Now(), 158 | } 159 | 160 | workChan := make(chan *readWorkItem) 161 | resultChan := make(chan *readWorkResult) 162 | 163 | for c := 0; c < self.concurrency; c++ { 164 | go readWorker(workChan, resultChan) 165 | } 166 | 167 | knownDigests := actionContext.GetKnownDigests(true) 168 | 169 | var workItems []*readWorkItem 170 | for i := 0; i < self.numDigestsToRead; i++ { 171 | index := rand.Int() % len(knownDigests) 172 | digest := knownDigests[index] 173 | for j := 0; j < self.numReadsPerDigest; j++ { 174 | workItem := &readWorkItem{ 175 | actionContext: actionContext, 176 | digest: digest, 177 | } 178 | workItems = append(workItems, workItem) 179 | } 180 | } 181 | 182 | // Shuffle the work items. 183 | rand.Shuffle(len(workItems), func(i, j int) { 184 | tmp := workItems[i] 185 | workItems[i] = workItems[j] 186 | workItems[j] = tmp 187 | }) 188 | 189 | // Inject the work items into the channel. 190 | go func() { 191 | for _, workItem := range workItems { 192 | workChan <- workItem 193 | } 194 | 195 | close(workChan) 196 | }() 197 | 198 | elapsedTimes := make([]time.Duration, len(workItems)) 199 | 200 | // Wait for results. 201 | for i := 0; i < len(workItems); i++ { 202 | r := <-resultChan 203 | if r.err == nil { 204 | result.success += 1 205 | } else { 206 | result.errors += 1 207 | log.WithFields(log.Fields{ 208 | "digest": r.digest, 209 | "err": r.err.Error(), 210 | }).Error("request error") 211 | } 212 | elapsedTimes[i] = r.elapsed 213 | if int64(r.digest.SizeBytes) < actionContext.MaxBatchBlobSize { 214 | result.blobReads += 1 215 | } else { 216 | result.byteStreamReads += 1 217 | } 218 | 219 | if i%100 == 0 { 220 | log.Debugf("progress: %d / %d", i, len(workItems)) 221 | } 222 | } 223 | 224 | result.endTime = time.Now() 225 | 226 | close(resultChan) 227 | 228 | fmt.Printf("program: %s\n startTime: %s\n endTime: %s\n success: %d\n errors: %d\n byteStreamReads: %d\n blobReads: %d\n", 229 | self.String(), 230 | result.startTime.String(), 231 | result.endTime.String(), 232 | result.success, 233 | result.errors, 234 | result.byteStreamReads, 235 | result.blobReads, 236 | ) 237 | 238 | stats.PrintTimingStats(elapsedTimes) 239 | 240 | return nil 241 | } 242 | 243 | func ParseReadAction(args []string) (Action, error) { 244 | if len(args) < 2 || len(args) > 3 { 245 | return nil, fmt.Errorf("unable to parse program: expected 2-3 fields, got %d fields", len(args)) 246 | } 247 | 248 | numDigestsToRead, err := strconv.Atoi(args[0]) 249 | if err != nil { 250 | return nil, fmt.Errorf("unable to parse number of digests to read from: %s: %s", args[0], err) 251 | } 252 | 253 | numReadsPerDigest, err := strconv.Atoi(args[1]) 254 | if err != nil { 255 | return nil, fmt.Errorf("unable to parse number of reads per digest: %s: %s", args[1], err) 256 | } 257 | 258 | concurrency := 50 259 | if len(args) >= 4 { 260 | c, err := strconv.Atoi(args[3]) 261 | if err != nil { 262 | return nil, fmt.Errorf("unable to parse concurrency: %s: %s", args[3], err) 263 | } 264 | concurrency = c 265 | } 266 | 267 | action := readAction{ 268 | numDigestsToRead: numDigestsToRead, 269 | numReadsPerDigest: numReadsPerDigest, 270 | concurrency: concurrency, 271 | } 272 | 273 | return &action, nil 274 | } 275 | -------------------------------------------------------------------------------- /pkg/load/save.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package load 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "strconv" 21 | "strings" 22 | ) 23 | 24 | // This action will save the "known digests" of a run to a file. This allows reuse of those digests in later 25 | // load tests.s action). 26 | 27 | type saveDigestsAction struct { 28 | filename string 29 | } 30 | 31 | func (self *saveDigestsAction) RunAction(actionContext *ActionContext) error { 32 | f, err := os.Create(self.filename) 33 | if err != nil { 34 | return fmt.Errorf("failed to load digests from file %s: %s", self.filename, err) 35 | } 36 | defer f.Close() 37 | 38 | for digest, present := range actionContext.KnownDigests { 39 | parts := strings.Split(digest, "-") 40 | if len(parts) != 2 { 41 | return fmt.Errorf("illegal state: known digests key was not parsed correctly") 42 | } 43 | 44 | line := fmt.Sprintf("%s,%s,%s\n", parts[0], parts[1], strconv.FormatBool(present)) 45 | _, err = f.WriteString(line) 46 | if err != nil { 47 | return fmt.Errorf("failed to write to file: %s", err) 48 | } 49 | } 50 | 51 | return nil 52 | } 53 | 54 | func ParseSaveDigestsAction(args []string) (Action, error) { 55 | if len(args) != 1 { 56 | return nil, fmt.Errorf("filename to save to must be specified") 57 | } 58 | 59 | action := saveDigestsAction{ 60 | filename: args[0], 61 | } 62 | 63 | return &action, nil 64 | } 65 | -------------------------------------------------------------------------------- /pkg/retry/BUILD: -------------------------------------------------------------------------------- 1 | go_package() 2 | -------------------------------------------------------------------------------- /pkg/retry/retry.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package retry 16 | 17 | import ( 18 | "errors" 19 | "math/rand" 20 | "time" 21 | ) 22 | 23 | var RetriesExceededError = errors.New("exceeded maximum number of retries") 24 | 25 | // Retry a function with exponential backoff. 26 | func ExpBackoff( 27 | n int, 28 | interval time.Duration, 29 | maxBackoffTime time.Duration, 30 | f func() (interface{}, error), 31 | isRetryable func(error) bool, 32 | ) (interface{}, error) { 33 | maxIntervals := 1 34 | attempt := 0 35 | 36 | for { 37 | result, err := f() 38 | if err == nil { 39 | return result, nil 40 | } 41 | 42 | if !isRetryable(err) { 43 | return nil, err 44 | } 45 | 46 | attempt += 1 47 | if attempt > n { 48 | return nil, RetriesExceededError 49 | } 50 | 51 | backoffTimeNanos := int64(1+rand.Int()%maxIntervals) * int64(interval) 52 | if backoffTimeNanos > int64(maxBackoffTime) { 53 | backoffTimeNanos = int64(maxBackoffTime) 54 | } 55 | time.Sleep(time.Duration(backoffTimeNanos)) 56 | 57 | maxIntervals *= 2 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/stats/BUILD: -------------------------------------------------------------------------------- 1 | go_package() 2 | -------------------------------------------------------------------------------- /pkg/stats/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Toolchain Labs, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package stats 16 | 17 | import ( 18 | "fmt" 19 | "sort" 20 | "time" 21 | ) 22 | 23 | func PrintTimingStats(times []time.Duration) { 24 | sort.Slice(times, func(i, j int) bool { 25 | return times[i] < times[j] 26 | }) 27 | 28 | total := time.Duration(0) 29 | for _, t := range times { 30 | total += t 31 | } 32 | 33 | average := time.Duration(float64(total) / float64(len(times))) 34 | 35 | fmt.Printf(" elapsed - total: %v\n elapsed - average: %v\n elapsed - min: %v\n elapsed - max: %v\n", 36 | total, 37 | average, 38 | times[0], 39 | times[len(times)-1]) 40 | } 41 | -------------------------------------------------------------------------------- /protos/build/bazel/remote/execution/v2/BUILD: -------------------------------------------------------------------------------- 1 | go_package(skip_gofmt=True) 2 | -------------------------------------------------------------------------------- /protos/build/bazel/remote/execution/v2/remote_execution_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.2.0 4 | // - protoc v3.21.6 5 | // source: build/bazel/remote/execution/v2/remote_execution.proto 6 | 7 | package v2 8 | 9 | import ( 10 | context "context" 11 | longrunning "google.golang.org/genproto/googleapis/longrunning" 12 | grpc "google.golang.org/grpc" 13 | codes "google.golang.org/grpc/codes" 14 | status "google.golang.org/grpc/status" 15 | ) 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the grpc package it is being compiled against. 19 | // Requires gRPC-Go v1.32.0 or later. 20 | const _ = grpc.SupportPackageIsVersion7 21 | 22 | // ExecutionClient is the client API for Execution service. 23 | // 24 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 25 | type ExecutionClient interface { 26 | // Execute an action remotely. 27 | // 28 | // In order to execute an action, the client must first upload all of the 29 | // inputs, the 30 | // [Command][build.bazel.remote.execution.v2.Command] to run, and the 31 | // [Action][build.bazel.remote.execution.v2.Action] into the 32 | // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. 33 | // It then calls `Execute` with an `action_digest` referring to them. The 34 | // server will run the action and eventually return the result. 35 | // 36 | // The input `Action`'s fields MUST meet the various canonicalization 37 | // requirements specified in the documentation for their types so that it has 38 | // the same digest as other logically equivalent `Action`s. The server MAY 39 | // enforce the requirements and return errors if a non-canonical input is 40 | // received. It MAY also proceed without verifying some or all of the 41 | // requirements, such as for performance reasons. If the server does not 42 | // verify the requirement, then it will treat the `Action` as distinct from 43 | // another logically equivalent action if they hash differently. 44 | // 45 | // Returns a stream of 46 | // [google.longrunning.Operation][google.longrunning.Operation] messages 47 | // describing the resulting execution, with eventual `response` 48 | // [ExecuteResponse][build.bazel.remote.execution.v2.ExecuteResponse]. The 49 | // `metadata` on the operation is of type 50 | // [ExecuteOperationMetadata][build.bazel.remote.execution.v2.ExecuteOperationMetadata]. 51 | // 52 | // If the client remains connected after the first response is returned after 53 | // the server, then updates are streamed as if the client had called 54 | // [WaitExecution][build.bazel.remote.execution.v2.Execution.WaitExecution] 55 | // until the execution completes or the request reaches an error. The 56 | // operation can also be queried using [Operations 57 | // API][google.longrunning.Operations.GetOperation]. 58 | // 59 | // The server NEED NOT implement other methods or functionality of the 60 | // Operations API. 61 | // 62 | // Errors discovered during creation of the `Operation` will be reported 63 | // as gRPC Status errors, while errors that occurred while running the 64 | // action will be reported in the `status` field of the `ExecuteResponse`. The 65 | // server MUST NOT set the `error` field of the `Operation` proto. 66 | // The possible errors include: 67 | // 68 | // * `INVALID_ARGUMENT`: One or more arguments are invalid. 69 | // * `FAILED_PRECONDITION`: One or more errors occurred in setting up the 70 | // action requested, such as a missing input or command or no worker being 71 | // available. The client may be able to fix the errors and retry. 72 | // * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to run 73 | // the action. 74 | // * `UNAVAILABLE`: Due to a transient condition, such as all workers being 75 | // occupied (and the server does not support a queue), the action could not 76 | // be started. The client should retry. 77 | // * `INTERNAL`: An internal error occurred in the execution engine or the 78 | // worker. 79 | // * `DEADLINE_EXCEEDED`: The execution timed out. 80 | // * `CANCELLED`: The operation was cancelled by the client. This status is 81 | // only possible if the server implements the Operations API CancelOperation 82 | // method, and it was called for the current execution. 83 | // 84 | // In the case of a missing input or command, the server SHOULD additionally 85 | // send a [PreconditionFailure][google.rpc.PreconditionFailure] error detail 86 | // where, for each requested blob not present in the CAS, there is a 87 | // `Violation` with a `type` of `MISSING` and a `subject` of 88 | // `"blobs/{hash}/{size}"` indicating the digest of the missing blob. 89 | // 90 | // The server does not need to guarantee that a call to this method leads to 91 | // at most one execution of the action. The server MAY execute the action 92 | // multiple times, potentially in parallel. These redundant executions MAY 93 | // continue to run, even if the operation is completed. 94 | Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (Execution_ExecuteClient, error) 95 | // Wait for an execution operation to complete. When the client initially 96 | // makes the request, the server immediately responds with the current status 97 | // of the execution. The server will leave the request stream open until the 98 | // operation completes, and then respond with the completed operation. The 99 | // server MAY choose to stream additional updates as execution progresses, 100 | // such as to provide an update as to the state of the execution. 101 | WaitExecution(ctx context.Context, in *WaitExecutionRequest, opts ...grpc.CallOption) (Execution_WaitExecutionClient, error) 102 | } 103 | 104 | type executionClient struct { 105 | cc grpc.ClientConnInterface 106 | } 107 | 108 | func NewExecutionClient(cc grpc.ClientConnInterface) ExecutionClient { 109 | return &executionClient{cc} 110 | } 111 | 112 | func (c *executionClient) Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (Execution_ExecuteClient, error) { 113 | stream, err := c.cc.NewStream(ctx, &Execution_ServiceDesc.Streams[0], "/build.bazel.remote.execution.v2.Execution/Execute", opts...) 114 | if err != nil { 115 | return nil, err 116 | } 117 | x := &executionExecuteClient{stream} 118 | if err := x.ClientStream.SendMsg(in); err != nil { 119 | return nil, err 120 | } 121 | if err := x.ClientStream.CloseSend(); err != nil { 122 | return nil, err 123 | } 124 | return x, nil 125 | } 126 | 127 | type Execution_ExecuteClient interface { 128 | Recv() (*longrunning.Operation, error) 129 | grpc.ClientStream 130 | } 131 | 132 | type executionExecuteClient struct { 133 | grpc.ClientStream 134 | } 135 | 136 | func (x *executionExecuteClient) Recv() (*longrunning.Operation, error) { 137 | m := new(longrunning.Operation) 138 | if err := x.ClientStream.RecvMsg(m); err != nil { 139 | return nil, err 140 | } 141 | return m, nil 142 | } 143 | 144 | func (c *executionClient) WaitExecution(ctx context.Context, in *WaitExecutionRequest, opts ...grpc.CallOption) (Execution_WaitExecutionClient, error) { 145 | stream, err := c.cc.NewStream(ctx, &Execution_ServiceDesc.Streams[1], "/build.bazel.remote.execution.v2.Execution/WaitExecution", opts...) 146 | if err != nil { 147 | return nil, err 148 | } 149 | x := &executionWaitExecutionClient{stream} 150 | if err := x.ClientStream.SendMsg(in); err != nil { 151 | return nil, err 152 | } 153 | if err := x.ClientStream.CloseSend(); err != nil { 154 | return nil, err 155 | } 156 | return x, nil 157 | } 158 | 159 | type Execution_WaitExecutionClient interface { 160 | Recv() (*longrunning.Operation, error) 161 | grpc.ClientStream 162 | } 163 | 164 | type executionWaitExecutionClient struct { 165 | grpc.ClientStream 166 | } 167 | 168 | func (x *executionWaitExecutionClient) Recv() (*longrunning.Operation, error) { 169 | m := new(longrunning.Operation) 170 | if err := x.ClientStream.RecvMsg(m); err != nil { 171 | return nil, err 172 | } 173 | return m, nil 174 | } 175 | 176 | // ExecutionServer is the server API for Execution service. 177 | // All implementations must embed UnimplementedExecutionServer 178 | // for forward compatibility 179 | type ExecutionServer interface { 180 | // Execute an action remotely. 181 | // 182 | // In order to execute an action, the client must first upload all of the 183 | // inputs, the 184 | // [Command][build.bazel.remote.execution.v2.Command] to run, and the 185 | // [Action][build.bazel.remote.execution.v2.Action] into the 186 | // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. 187 | // It then calls `Execute` with an `action_digest` referring to them. The 188 | // server will run the action and eventually return the result. 189 | // 190 | // The input `Action`'s fields MUST meet the various canonicalization 191 | // requirements specified in the documentation for their types so that it has 192 | // the same digest as other logically equivalent `Action`s. The server MAY 193 | // enforce the requirements and return errors if a non-canonical input is 194 | // received. It MAY also proceed without verifying some or all of the 195 | // requirements, such as for performance reasons. If the server does not 196 | // verify the requirement, then it will treat the `Action` as distinct from 197 | // another logically equivalent action if they hash differently. 198 | // 199 | // Returns a stream of 200 | // [google.longrunning.Operation][google.longrunning.Operation] messages 201 | // describing the resulting execution, with eventual `response` 202 | // [ExecuteResponse][build.bazel.remote.execution.v2.ExecuteResponse]. The 203 | // `metadata` on the operation is of type 204 | // [ExecuteOperationMetadata][build.bazel.remote.execution.v2.ExecuteOperationMetadata]. 205 | // 206 | // If the client remains connected after the first response is returned after 207 | // the server, then updates are streamed as if the client had called 208 | // [WaitExecution][build.bazel.remote.execution.v2.Execution.WaitExecution] 209 | // until the execution completes or the request reaches an error. The 210 | // operation can also be queried using [Operations 211 | // API][google.longrunning.Operations.GetOperation]. 212 | // 213 | // The server NEED NOT implement other methods or functionality of the 214 | // Operations API. 215 | // 216 | // Errors discovered during creation of the `Operation` will be reported 217 | // as gRPC Status errors, while errors that occurred while running the 218 | // action will be reported in the `status` field of the `ExecuteResponse`. The 219 | // server MUST NOT set the `error` field of the `Operation` proto. 220 | // The possible errors include: 221 | // 222 | // * `INVALID_ARGUMENT`: One or more arguments are invalid. 223 | // * `FAILED_PRECONDITION`: One or more errors occurred in setting up the 224 | // action requested, such as a missing input or command or no worker being 225 | // available. The client may be able to fix the errors and retry. 226 | // * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to run 227 | // the action. 228 | // * `UNAVAILABLE`: Due to a transient condition, such as all workers being 229 | // occupied (and the server does not support a queue), the action could not 230 | // be started. The client should retry. 231 | // * `INTERNAL`: An internal error occurred in the execution engine or the 232 | // worker. 233 | // * `DEADLINE_EXCEEDED`: The execution timed out. 234 | // * `CANCELLED`: The operation was cancelled by the client. This status is 235 | // only possible if the server implements the Operations API CancelOperation 236 | // method, and it was called for the current execution. 237 | // 238 | // In the case of a missing input or command, the server SHOULD additionally 239 | // send a [PreconditionFailure][google.rpc.PreconditionFailure] error detail 240 | // where, for each requested blob not present in the CAS, there is a 241 | // `Violation` with a `type` of `MISSING` and a `subject` of 242 | // `"blobs/{hash}/{size}"` indicating the digest of the missing blob. 243 | // 244 | // The server does not need to guarantee that a call to this method leads to 245 | // at most one execution of the action. The server MAY execute the action 246 | // multiple times, potentially in parallel. These redundant executions MAY 247 | // continue to run, even if the operation is completed. 248 | Execute(*ExecuteRequest, Execution_ExecuteServer) error 249 | // Wait for an execution operation to complete. When the client initially 250 | // makes the request, the server immediately responds with the current status 251 | // of the execution. The server will leave the request stream open until the 252 | // operation completes, and then respond with the completed operation. The 253 | // server MAY choose to stream additional updates as execution progresses, 254 | // such as to provide an update as to the state of the execution. 255 | WaitExecution(*WaitExecutionRequest, Execution_WaitExecutionServer) error 256 | mustEmbedUnimplementedExecutionServer() 257 | } 258 | 259 | // UnimplementedExecutionServer must be embedded to have forward compatible implementations. 260 | type UnimplementedExecutionServer struct { 261 | } 262 | 263 | func (UnimplementedExecutionServer) Execute(*ExecuteRequest, Execution_ExecuteServer) error { 264 | return status.Errorf(codes.Unimplemented, "method Execute not implemented") 265 | } 266 | func (UnimplementedExecutionServer) WaitExecution(*WaitExecutionRequest, Execution_WaitExecutionServer) error { 267 | return status.Errorf(codes.Unimplemented, "method WaitExecution not implemented") 268 | } 269 | func (UnimplementedExecutionServer) mustEmbedUnimplementedExecutionServer() {} 270 | 271 | // UnsafeExecutionServer may be embedded to opt out of forward compatibility for this service. 272 | // Use of this interface is not recommended, as added methods to ExecutionServer will 273 | // result in compilation errors. 274 | type UnsafeExecutionServer interface { 275 | mustEmbedUnimplementedExecutionServer() 276 | } 277 | 278 | func RegisterExecutionServer(s grpc.ServiceRegistrar, srv ExecutionServer) { 279 | s.RegisterService(&Execution_ServiceDesc, srv) 280 | } 281 | 282 | func _Execution_Execute_Handler(srv interface{}, stream grpc.ServerStream) error { 283 | m := new(ExecuteRequest) 284 | if err := stream.RecvMsg(m); err != nil { 285 | return err 286 | } 287 | return srv.(ExecutionServer).Execute(m, &executionExecuteServer{stream}) 288 | } 289 | 290 | type Execution_ExecuteServer interface { 291 | Send(*longrunning.Operation) error 292 | grpc.ServerStream 293 | } 294 | 295 | type executionExecuteServer struct { 296 | grpc.ServerStream 297 | } 298 | 299 | func (x *executionExecuteServer) Send(m *longrunning.Operation) error { 300 | return x.ServerStream.SendMsg(m) 301 | } 302 | 303 | func _Execution_WaitExecution_Handler(srv interface{}, stream grpc.ServerStream) error { 304 | m := new(WaitExecutionRequest) 305 | if err := stream.RecvMsg(m); err != nil { 306 | return err 307 | } 308 | return srv.(ExecutionServer).WaitExecution(m, &executionWaitExecutionServer{stream}) 309 | } 310 | 311 | type Execution_WaitExecutionServer interface { 312 | Send(*longrunning.Operation) error 313 | grpc.ServerStream 314 | } 315 | 316 | type executionWaitExecutionServer struct { 317 | grpc.ServerStream 318 | } 319 | 320 | func (x *executionWaitExecutionServer) Send(m *longrunning.Operation) error { 321 | return x.ServerStream.SendMsg(m) 322 | } 323 | 324 | // Execution_ServiceDesc is the grpc.ServiceDesc for Execution service. 325 | // It's only intended for direct use with grpc.RegisterService, 326 | // and not to be introspected or modified (even as a copy) 327 | var Execution_ServiceDesc = grpc.ServiceDesc{ 328 | ServiceName: "build.bazel.remote.execution.v2.Execution", 329 | HandlerType: (*ExecutionServer)(nil), 330 | Methods: []grpc.MethodDesc{}, 331 | Streams: []grpc.StreamDesc{ 332 | { 333 | StreamName: "Execute", 334 | Handler: _Execution_Execute_Handler, 335 | ServerStreams: true, 336 | }, 337 | { 338 | StreamName: "WaitExecution", 339 | Handler: _Execution_WaitExecution_Handler, 340 | ServerStreams: true, 341 | }, 342 | }, 343 | Metadata: "build/bazel/remote/execution/v2/remote_execution.proto", 344 | } 345 | 346 | // ActionCacheClient is the client API for ActionCache service. 347 | // 348 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 349 | type ActionCacheClient interface { 350 | // Retrieve a cached execution result. 351 | // 352 | // Implementations SHOULD ensure that any blobs referenced from the 353 | // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] 354 | // are available at the time of returning the 355 | // [ActionResult][build.bazel.remote.execution.v2.ActionResult] and will be 356 | // for some period of time afterwards. The lifetimes of the referenced blobs SHOULD be increased 357 | // if necessary and applicable. 358 | // 359 | // Errors: 360 | // 361 | // * `NOT_FOUND`: The requested `ActionResult` is not in the cache. 362 | GetActionResult(ctx context.Context, in *GetActionResultRequest, opts ...grpc.CallOption) (*ActionResult, error) 363 | // Upload a new execution result. 364 | // 365 | // In order to allow the server to perform access control based on the type of 366 | // action, and to assist with client debugging, the client MUST first upload 367 | // the [Action][build.bazel.remote.execution.v2.Execution] that produced the 368 | // result, along with its 369 | // [Command][build.bazel.remote.execution.v2.Command], into the 370 | // `ContentAddressableStorage`. 371 | // 372 | // Server implementations MAY modify the 373 | // `UpdateActionResultRequest.action_result` and return an equivalent value. 374 | // 375 | // Errors: 376 | // 377 | // * `INVALID_ARGUMENT`: One or more arguments are invalid. 378 | // * `FAILED_PRECONDITION`: One or more errors occurred in updating the 379 | // action result, such as a missing command or action. 380 | // * `RESOURCE_EXHAUSTED`: There is insufficient storage space to add the 381 | // entry to the cache. 382 | UpdateActionResult(ctx context.Context, in *UpdateActionResultRequest, opts ...grpc.CallOption) (*ActionResult, error) 383 | } 384 | 385 | type actionCacheClient struct { 386 | cc grpc.ClientConnInterface 387 | } 388 | 389 | func NewActionCacheClient(cc grpc.ClientConnInterface) ActionCacheClient { 390 | return &actionCacheClient{cc} 391 | } 392 | 393 | func (c *actionCacheClient) GetActionResult(ctx context.Context, in *GetActionResultRequest, opts ...grpc.CallOption) (*ActionResult, error) { 394 | out := new(ActionResult) 395 | err := c.cc.Invoke(ctx, "/build.bazel.remote.execution.v2.ActionCache/GetActionResult", in, out, opts...) 396 | if err != nil { 397 | return nil, err 398 | } 399 | return out, nil 400 | } 401 | 402 | func (c *actionCacheClient) UpdateActionResult(ctx context.Context, in *UpdateActionResultRequest, opts ...grpc.CallOption) (*ActionResult, error) { 403 | out := new(ActionResult) 404 | err := c.cc.Invoke(ctx, "/build.bazel.remote.execution.v2.ActionCache/UpdateActionResult", in, out, opts...) 405 | if err != nil { 406 | return nil, err 407 | } 408 | return out, nil 409 | } 410 | 411 | // ActionCacheServer is the server API for ActionCache service. 412 | // All implementations must embed UnimplementedActionCacheServer 413 | // for forward compatibility 414 | type ActionCacheServer interface { 415 | // Retrieve a cached execution result. 416 | // 417 | // Implementations SHOULD ensure that any blobs referenced from the 418 | // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] 419 | // are available at the time of returning the 420 | // [ActionResult][build.bazel.remote.execution.v2.ActionResult] and will be 421 | // for some period of time afterwards. The lifetimes of the referenced blobs SHOULD be increased 422 | // if necessary and applicable. 423 | // 424 | // Errors: 425 | // 426 | // * `NOT_FOUND`: The requested `ActionResult` is not in the cache. 427 | GetActionResult(context.Context, *GetActionResultRequest) (*ActionResult, error) 428 | // Upload a new execution result. 429 | // 430 | // In order to allow the server to perform access control based on the type of 431 | // action, and to assist with client debugging, the client MUST first upload 432 | // the [Action][build.bazel.remote.execution.v2.Execution] that produced the 433 | // result, along with its 434 | // [Command][build.bazel.remote.execution.v2.Command], into the 435 | // `ContentAddressableStorage`. 436 | // 437 | // Server implementations MAY modify the 438 | // `UpdateActionResultRequest.action_result` and return an equivalent value. 439 | // 440 | // Errors: 441 | // 442 | // * `INVALID_ARGUMENT`: One or more arguments are invalid. 443 | // * `FAILED_PRECONDITION`: One or more errors occurred in updating the 444 | // action result, such as a missing command or action. 445 | // * `RESOURCE_EXHAUSTED`: There is insufficient storage space to add the 446 | // entry to the cache. 447 | UpdateActionResult(context.Context, *UpdateActionResultRequest) (*ActionResult, error) 448 | mustEmbedUnimplementedActionCacheServer() 449 | } 450 | 451 | // UnimplementedActionCacheServer must be embedded to have forward compatible implementations. 452 | type UnimplementedActionCacheServer struct { 453 | } 454 | 455 | func (UnimplementedActionCacheServer) GetActionResult(context.Context, *GetActionResultRequest) (*ActionResult, error) { 456 | return nil, status.Errorf(codes.Unimplemented, "method GetActionResult not implemented") 457 | } 458 | func (UnimplementedActionCacheServer) UpdateActionResult(context.Context, *UpdateActionResultRequest) (*ActionResult, error) { 459 | return nil, status.Errorf(codes.Unimplemented, "method UpdateActionResult not implemented") 460 | } 461 | func (UnimplementedActionCacheServer) mustEmbedUnimplementedActionCacheServer() {} 462 | 463 | // UnsafeActionCacheServer may be embedded to opt out of forward compatibility for this service. 464 | // Use of this interface is not recommended, as added methods to ActionCacheServer will 465 | // result in compilation errors. 466 | type UnsafeActionCacheServer interface { 467 | mustEmbedUnimplementedActionCacheServer() 468 | } 469 | 470 | func RegisterActionCacheServer(s grpc.ServiceRegistrar, srv ActionCacheServer) { 471 | s.RegisterService(&ActionCache_ServiceDesc, srv) 472 | } 473 | 474 | func _ActionCache_GetActionResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 475 | in := new(GetActionResultRequest) 476 | if err := dec(in); err != nil { 477 | return nil, err 478 | } 479 | if interceptor == nil { 480 | return srv.(ActionCacheServer).GetActionResult(ctx, in) 481 | } 482 | info := &grpc.UnaryServerInfo{ 483 | Server: srv, 484 | FullMethod: "/build.bazel.remote.execution.v2.ActionCache/GetActionResult", 485 | } 486 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 487 | return srv.(ActionCacheServer).GetActionResult(ctx, req.(*GetActionResultRequest)) 488 | } 489 | return interceptor(ctx, in, info, handler) 490 | } 491 | 492 | func _ActionCache_UpdateActionResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 493 | in := new(UpdateActionResultRequest) 494 | if err := dec(in); err != nil { 495 | return nil, err 496 | } 497 | if interceptor == nil { 498 | return srv.(ActionCacheServer).UpdateActionResult(ctx, in) 499 | } 500 | info := &grpc.UnaryServerInfo{ 501 | Server: srv, 502 | FullMethod: "/build.bazel.remote.execution.v2.ActionCache/UpdateActionResult", 503 | } 504 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 505 | return srv.(ActionCacheServer).UpdateActionResult(ctx, req.(*UpdateActionResultRequest)) 506 | } 507 | return interceptor(ctx, in, info, handler) 508 | } 509 | 510 | // ActionCache_ServiceDesc is the grpc.ServiceDesc for ActionCache service. 511 | // It's only intended for direct use with grpc.RegisterService, 512 | // and not to be introspected or modified (even as a copy) 513 | var ActionCache_ServiceDesc = grpc.ServiceDesc{ 514 | ServiceName: "build.bazel.remote.execution.v2.ActionCache", 515 | HandlerType: (*ActionCacheServer)(nil), 516 | Methods: []grpc.MethodDesc{ 517 | { 518 | MethodName: "GetActionResult", 519 | Handler: _ActionCache_GetActionResult_Handler, 520 | }, 521 | { 522 | MethodName: "UpdateActionResult", 523 | Handler: _ActionCache_UpdateActionResult_Handler, 524 | }, 525 | }, 526 | Streams: []grpc.StreamDesc{}, 527 | Metadata: "build/bazel/remote/execution/v2/remote_execution.proto", 528 | } 529 | 530 | // ContentAddressableStorageClient is the client API for ContentAddressableStorage service. 531 | // 532 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 533 | type ContentAddressableStorageClient interface { 534 | // Determine if blobs are present in the CAS. 535 | // 536 | // Clients can use this API before uploading blobs to determine which ones are 537 | // already present in the CAS and do not need to be uploaded again. 538 | // 539 | // Servers SHOULD increase the lifetimes of the referenced blobs if necessary and 540 | // applicable. 541 | // 542 | // There are no method-specific errors. 543 | FindMissingBlobs(ctx context.Context, in *FindMissingBlobsRequest, opts ...grpc.CallOption) (*FindMissingBlobsResponse, error) 544 | // Upload many blobs at once. 545 | // 546 | // The server may enforce a limit of the combined total size of blobs 547 | // to be uploaded using this API. This limit may be obtained using the 548 | // [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. 549 | // Requests exceeding the limit should either be split into smaller 550 | // chunks or uploaded using the 551 | // [ByteStream API][google.bytestream.ByteStream], as appropriate. 552 | // 553 | // This request is equivalent to calling a Bytestream `Write` request 554 | // on each individual blob, in parallel. The requests may succeed or fail 555 | // independently. 556 | // 557 | // Errors: 558 | // 559 | // * `INVALID_ARGUMENT`: The client attempted to upload more than the 560 | // server supported limit. 561 | // 562 | // Individual requests may return the following errors, additionally: 563 | // 564 | // * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the blob. 565 | // * `INVALID_ARGUMENT`: The 566 | // [Digest][build.bazel.remote.execution.v2.Digest] does not match the 567 | // provided data. 568 | BatchUpdateBlobs(ctx context.Context, in *BatchUpdateBlobsRequest, opts ...grpc.CallOption) (*BatchUpdateBlobsResponse, error) 569 | // Download many blobs at once. 570 | // 571 | // The server may enforce a limit of the combined total size of blobs 572 | // to be downloaded using this API. This limit may be obtained using the 573 | // [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. 574 | // Requests exceeding the limit should either be split into smaller 575 | // chunks or downloaded using the 576 | // [ByteStream API][google.bytestream.ByteStream], as appropriate. 577 | // 578 | // This request is equivalent to calling a Bytestream `Read` request 579 | // on each individual blob, in parallel. The requests may succeed or fail 580 | // independently. 581 | // 582 | // Errors: 583 | // 584 | // * `INVALID_ARGUMENT`: The client attempted to read more than the 585 | // server supported limit. 586 | // 587 | // Every error on individual read will be returned in the corresponding digest 588 | // status. 589 | BatchReadBlobs(ctx context.Context, in *BatchReadBlobsRequest, opts ...grpc.CallOption) (*BatchReadBlobsResponse, error) 590 | // Fetch the entire directory tree rooted at a node. 591 | // 592 | // This request must be targeted at a 593 | // [Directory][build.bazel.remote.execution.v2.Directory] stored in the 594 | // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] 595 | // (CAS). The server will enumerate the `Directory` tree recursively and 596 | // return every node descended from the root. 597 | // 598 | // The GetTreeRequest.page_token parameter can be used to skip ahead in 599 | // the stream (e.g. when retrying a partially completed and aborted request), 600 | // by setting it to a value taken from GetTreeResponse.next_page_token of the 601 | // last successfully processed GetTreeResponse). 602 | // 603 | // The exact traversal order is unspecified and, unless retrieving subsequent 604 | // pages from an earlier request, is not guaranteed to be stable across 605 | // multiple invocations of `GetTree`. 606 | // 607 | // If part of the tree is missing from the CAS, the server will return the 608 | // portion present and omit the rest. 609 | // 610 | // Errors: 611 | // 612 | // * `NOT_FOUND`: The requested tree root is not present in the CAS. 613 | GetTree(ctx context.Context, in *GetTreeRequest, opts ...grpc.CallOption) (ContentAddressableStorage_GetTreeClient, error) 614 | } 615 | 616 | type contentAddressableStorageClient struct { 617 | cc grpc.ClientConnInterface 618 | } 619 | 620 | func NewContentAddressableStorageClient(cc grpc.ClientConnInterface) ContentAddressableStorageClient { 621 | return &contentAddressableStorageClient{cc} 622 | } 623 | 624 | func (c *contentAddressableStorageClient) FindMissingBlobs(ctx context.Context, in *FindMissingBlobsRequest, opts ...grpc.CallOption) (*FindMissingBlobsResponse, error) { 625 | out := new(FindMissingBlobsResponse) 626 | err := c.cc.Invoke(ctx, "/build.bazel.remote.execution.v2.ContentAddressableStorage/FindMissingBlobs", in, out, opts...) 627 | if err != nil { 628 | return nil, err 629 | } 630 | return out, nil 631 | } 632 | 633 | func (c *contentAddressableStorageClient) BatchUpdateBlobs(ctx context.Context, in *BatchUpdateBlobsRequest, opts ...grpc.CallOption) (*BatchUpdateBlobsResponse, error) { 634 | out := new(BatchUpdateBlobsResponse) 635 | err := c.cc.Invoke(ctx, "/build.bazel.remote.execution.v2.ContentAddressableStorage/BatchUpdateBlobs", in, out, opts...) 636 | if err != nil { 637 | return nil, err 638 | } 639 | return out, nil 640 | } 641 | 642 | func (c *contentAddressableStorageClient) BatchReadBlobs(ctx context.Context, in *BatchReadBlobsRequest, opts ...grpc.CallOption) (*BatchReadBlobsResponse, error) { 643 | out := new(BatchReadBlobsResponse) 644 | err := c.cc.Invoke(ctx, "/build.bazel.remote.execution.v2.ContentAddressableStorage/BatchReadBlobs", in, out, opts...) 645 | if err != nil { 646 | return nil, err 647 | } 648 | return out, nil 649 | } 650 | 651 | func (c *contentAddressableStorageClient) GetTree(ctx context.Context, in *GetTreeRequest, opts ...grpc.CallOption) (ContentAddressableStorage_GetTreeClient, error) { 652 | stream, err := c.cc.NewStream(ctx, &ContentAddressableStorage_ServiceDesc.Streams[0], "/build.bazel.remote.execution.v2.ContentAddressableStorage/GetTree", opts...) 653 | if err != nil { 654 | return nil, err 655 | } 656 | x := &contentAddressableStorageGetTreeClient{stream} 657 | if err := x.ClientStream.SendMsg(in); err != nil { 658 | return nil, err 659 | } 660 | if err := x.ClientStream.CloseSend(); err != nil { 661 | return nil, err 662 | } 663 | return x, nil 664 | } 665 | 666 | type ContentAddressableStorage_GetTreeClient interface { 667 | Recv() (*GetTreeResponse, error) 668 | grpc.ClientStream 669 | } 670 | 671 | type contentAddressableStorageGetTreeClient struct { 672 | grpc.ClientStream 673 | } 674 | 675 | func (x *contentAddressableStorageGetTreeClient) Recv() (*GetTreeResponse, error) { 676 | m := new(GetTreeResponse) 677 | if err := x.ClientStream.RecvMsg(m); err != nil { 678 | return nil, err 679 | } 680 | return m, nil 681 | } 682 | 683 | // ContentAddressableStorageServer is the server API for ContentAddressableStorage service. 684 | // All implementations must embed UnimplementedContentAddressableStorageServer 685 | // for forward compatibility 686 | type ContentAddressableStorageServer interface { 687 | // Determine if blobs are present in the CAS. 688 | // 689 | // Clients can use this API before uploading blobs to determine which ones are 690 | // already present in the CAS and do not need to be uploaded again. 691 | // 692 | // Servers SHOULD increase the lifetimes of the referenced blobs if necessary and 693 | // applicable. 694 | // 695 | // There are no method-specific errors. 696 | FindMissingBlobs(context.Context, *FindMissingBlobsRequest) (*FindMissingBlobsResponse, error) 697 | // Upload many blobs at once. 698 | // 699 | // The server may enforce a limit of the combined total size of blobs 700 | // to be uploaded using this API. This limit may be obtained using the 701 | // [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. 702 | // Requests exceeding the limit should either be split into smaller 703 | // chunks or uploaded using the 704 | // [ByteStream API][google.bytestream.ByteStream], as appropriate. 705 | // 706 | // This request is equivalent to calling a Bytestream `Write` request 707 | // on each individual blob, in parallel. The requests may succeed or fail 708 | // independently. 709 | // 710 | // Errors: 711 | // 712 | // * `INVALID_ARGUMENT`: The client attempted to upload more than the 713 | // server supported limit. 714 | // 715 | // Individual requests may return the following errors, additionally: 716 | // 717 | // * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the blob. 718 | // * `INVALID_ARGUMENT`: The 719 | // [Digest][build.bazel.remote.execution.v2.Digest] does not match the 720 | // provided data. 721 | BatchUpdateBlobs(context.Context, *BatchUpdateBlobsRequest) (*BatchUpdateBlobsResponse, error) 722 | // Download many blobs at once. 723 | // 724 | // The server may enforce a limit of the combined total size of blobs 725 | // to be downloaded using this API. This limit may be obtained using the 726 | // [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. 727 | // Requests exceeding the limit should either be split into smaller 728 | // chunks or downloaded using the 729 | // [ByteStream API][google.bytestream.ByteStream], as appropriate. 730 | // 731 | // This request is equivalent to calling a Bytestream `Read` request 732 | // on each individual blob, in parallel. The requests may succeed or fail 733 | // independently. 734 | // 735 | // Errors: 736 | // 737 | // * `INVALID_ARGUMENT`: The client attempted to read more than the 738 | // server supported limit. 739 | // 740 | // Every error on individual read will be returned in the corresponding digest 741 | // status. 742 | BatchReadBlobs(context.Context, *BatchReadBlobsRequest) (*BatchReadBlobsResponse, error) 743 | // Fetch the entire directory tree rooted at a node. 744 | // 745 | // This request must be targeted at a 746 | // [Directory][build.bazel.remote.execution.v2.Directory] stored in the 747 | // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] 748 | // (CAS). The server will enumerate the `Directory` tree recursively and 749 | // return every node descended from the root. 750 | // 751 | // The GetTreeRequest.page_token parameter can be used to skip ahead in 752 | // the stream (e.g. when retrying a partially completed and aborted request), 753 | // by setting it to a value taken from GetTreeResponse.next_page_token of the 754 | // last successfully processed GetTreeResponse). 755 | // 756 | // The exact traversal order is unspecified and, unless retrieving subsequent 757 | // pages from an earlier request, is not guaranteed to be stable across 758 | // multiple invocations of `GetTree`. 759 | // 760 | // If part of the tree is missing from the CAS, the server will return the 761 | // portion present and omit the rest. 762 | // 763 | // Errors: 764 | // 765 | // * `NOT_FOUND`: The requested tree root is not present in the CAS. 766 | GetTree(*GetTreeRequest, ContentAddressableStorage_GetTreeServer) error 767 | mustEmbedUnimplementedContentAddressableStorageServer() 768 | } 769 | 770 | // UnimplementedContentAddressableStorageServer must be embedded to have forward compatible implementations. 771 | type UnimplementedContentAddressableStorageServer struct { 772 | } 773 | 774 | func (UnimplementedContentAddressableStorageServer) FindMissingBlobs(context.Context, *FindMissingBlobsRequest) (*FindMissingBlobsResponse, error) { 775 | return nil, status.Errorf(codes.Unimplemented, "method FindMissingBlobs not implemented") 776 | } 777 | func (UnimplementedContentAddressableStorageServer) BatchUpdateBlobs(context.Context, *BatchUpdateBlobsRequest) (*BatchUpdateBlobsResponse, error) { 778 | return nil, status.Errorf(codes.Unimplemented, "method BatchUpdateBlobs not implemented") 779 | } 780 | func (UnimplementedContentAddressableStorageServer) BatchReadBlobs(context.Context, *BatchReadBlobsRequest) (*BatchReadBlobsResponse, error) { 781 | return nil, status.Errorf(codes.Unimplemented, "method BatchReadBlobs not implemented") 782 | } 783 | func (UnimplementedContentAddressableStorageServer) GetTree(*GetTreeRequest, ContentAddressableStorage_GetTreeServer) error { 784 | return status.Errorf(codes.Unimplemented, "method GetTree not implemented") 785 | } 786 | func (UnimplementedContentAddressableStorageServer) mustEmbedUnimplementedContentAddressableStorageServer() { 787 | } 788 | 789 | // UnsafeContentAddressableStorageServer may be embedded to opt out of forward compatibility for this service. 790 | // Use of this interface is not recommended, as added methods to ContentAddressableStorageServer will 791 | // result in compilation errors. 792 | type UnsafeContentAddressableStorageServer interface { 793 | mustEmbedUnimplementedContentAddressableStorageServer() 794 | } 795 | 796 | func RegisterContentAddressableStorageServer(s grpc.ServiceRegistrar, srv ContentAddressableStorageServer) { 797 | s.RegisterService(&ContentAddressableStorage_ServiceDesc, srv) 798 | } 799 | 800 | func _ContentAddressableStorage_FindMissingBlobs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 801 | in := new(FindMissingBlobsRequest) 802 | if err := dec(in); err != nil { 803 | return nil, err 804 | } 805 | if interceptor == nil { 806 | return srv.(ContentAddressableStorageServer).FindMissingBlobs(ctx, in) 807 | } 808 | info := &grpc.UnaryServerInfo{ 809 | Server: srv, 810 | FullMethod: "/build.bazel.remote.execution.v2.ContentAddressableStorage/FindMissingBlobs", 811 | } 812 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 813 | return srv.(ContentAddressableStorageServer).FindMissingBlobs(ctx, req.(*FindMissingBlobsRequest)) 814 | } 815 | return interceptor(ctx, in, info, handler) 816 | } 817 | 818 | func _ContentAddressableStorage_BatchUpdateBlobs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 819 | in := new(BatchUpdateBlobsRequest) 820 | if err := dec(in); err != nil { 821 | return nil, err 822 | } 823 | if interceptor == nil { 824 | return srv.(ContentAddressableStorageServer).BatchUpdateBlobs(ctx, in) 825 | } 826 | info := &grpc.UnaryServerInfo{ 827 | Server: srv, 828 | FullMethod: "/build.bazel.remote.execution.v2.ContentAddressableStorage/BatchUpdateBlobs", 829 | } 830 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 831 | return srv.(ContentAddressableStorageServer).BatchUpdateBlobs(ctx, req.(*BatchUpdateBlobsRequest)) 832 | } 833 | return interceptor(ctx, in, info, handler) 834 | } 835 | 836 | func _ContentAddressableStorage_BatchReadBlobs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 837 | in := new(BatchReadBlobsRequest) 838 | if err := dec(in); err != nil { 839 | return nil, err 840 | } 841 | if interceptor == nil { 842 | return srv.(ContentAddressableStorageServer).BatchReadBlobs(ctx, in) 843 | } 844 | info := &grpc.UnaryServerInfo{ 845 | Server: srv, 846 | FullMethod: "/build.bazel.remote.execution.v2.ContentAddressableStorage/BatchReadBlobs", 847 | } 848 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 849 | return srv.(ContentAddressableStorageServer).BatchReadBlobs(ctx, req.(*BatchReadBlobsRequest)) 850 | } 851 | return interceptor(ctx, in, info, handler) 852 | } 853 | 854 | func _ContentAddressableStorage_GetTree_Handler(srv interface{}, stream grpc.ServerStream) error { 855 | m := new(GetTreeRequest) 856 | if err := stream.RecvMsg(m); err != nil { 857 | return err 858 | } 859 | return srv.(ContentAddressableStorageServer).GetTree(m, &contentAddressableStorageGetTreeServer{stream}) 860 | } 861 | 862 | type ContentAddressableStorage_GetTreeServer interface { 863 | Send(*GetTreeResponse) error 864 | grpc.ServerStream 865 | } 866 | 867 | type contentAddressableStorageGetTreeServer struct { 868 | grpc.ServerStream 869 | } 870 | 871 | func (x *contentAddressableStorageGetTreeServer) Send(m *GetTreeResponse) error { 872 | return x.ServerStream.SendMsg(m) 873 | } 874 | 875 | // ContentAddressableStorage_ServiceDesc is the grpc.ServiceDesc for ContentAddressableStorage service. 876 | // It's only intended for direct use with grpc.RegisterService, 877 | // and not to be introspected or modified (even as a copy) 878 | var ContentAddressableStorage_ServiceDesc = grpc.ServiceDesc{ 879 | ServiceName: "build.bazel.remote.execution.v2.ContentAddressableStorage", 880 | HandlerType: (*ContentAddressableStorageServer)(nil), 881 | Methods: []grpc.MethodDesc{ 882 | { 883 | MethodName: "FindMissingBlobs", 884 | Handler: _ContentAddressableStorage_FindMissingBlobs_Handler, 885 | }, 886 | { 887 | MethodName: "BatchUpdateBlobs", 888 | Handler: _ContentAddressableStorage_BatchUpdateBlobs_Handler, 889 | }, 890 | { 891 | MethodName: "BatchReadBlobs", 892 | Handler: _ContentAddressableStorage_BatchReadBlobs_Handler, 893 | }, 894 | }, 895 | Streams: []grpc.StreamDesc{ 896 | { 897 | StreamName: "GetTree", 898 | Handler: _ContentAddressableStorage_GetTree_Handler, 899 | ServerStreams: true, 900 | }, 901 | }, 902 | Metadata: "build/bazel/remote/execution/v2/remote_execution.proto", 903 | } 904 | 905 | // CapabilitiesClient is the client API for Capabilities service. 906 | // 907 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 908 | type CapabilitiesClient interface { 909 | // GetCapabilities returns the server capabilities configuration of the 910 | // remote endpoint. 911 | // Only the capabilities of the services supported by the endpoint will 912 | // be returned: 913 | // * Execution + CAS + Action Cache endpoints should return both 914 | // CacheCapabilities and ExecutionCapabilities. 915 | // * Execution only endpoints should return ExecutionCapabilities. 916 | // * CAS + Action Cache only endpoints should return CacheCapabilities. 917 | GetCapabilities(ctx context.Context, in *GetCapabilitiesRequest, opts ...grpc.CallOption) (*ServerCapabilities, error) 918 | } 919 | 920 | type capabilitiesClient struct { 921 | cc grpc.ClientConnInterface 922 | } 923 | 924 | func NewCapabilitiesClient(cc grpc.ClientConnInterface) CapabilitiesClient { 925 | return &capabilitiesClient{cc} 926 | } 927 | 928 | func (c *capabilitiesClient) GetCapabilities(ctx context.Context, in *GetCapabilitiesRequest, opts ...grpc.CallOption) (*ServerCapabilities, error) { 929 | out := new(ServerCapabilities) 930 | err := c.cc.Invoke(ctx, "/build.bazel.remote.execution.v2.Capabilities/GetCapabilities", in, out, opts...) 931 | if err != nil { 932 | return nil, err 933 | } 934 | return out, nil 935 | } 936 | 937 | // CapabilitiesServer is the server API for Capabilities service. 938 | // All implementations must embed UnimplementedCapabilitiesServer 939 | // for forward compatibility 940 | type CapabilitiesServer interface { 941 | // GetCapabilities returns the server capabilities configuration of the 942 | // remote endpoint. 943 | // Only the capabilities of the services supported by the endpoint will 944 | // be returned: 945 | // * Execution + CAS + Action Cache endpoints should return both 946 | // CacheCapabilities and ExecutionCapabilities. 947 | // * Execution only endpoints should return ExecutionCapabilities. 948 | // * CAS + Action Cache only endpoints should return CacheCapabilities. 949 | GetCapabilities(context.Context, *GetCapabilitiesRequest) (*ServerCapabilities, error) 950 | mustEmbedUnimplementedCapabilitiesServer() 951 | } 952 | 953 | // UnimplementedCapabilitiesServer must be embedded to have forward compatible implementations. 954 | type UnimplementedCapabilitiesServer struct { 955 | } 956 | 957 | func (UnimplementedCapabilitiesServer) GetCapabilities(context.Context, *GetCapabilitiesRequest) (*ServerCapabilities, error) { 958 | return nil, status.Errorf(codes.Unimplemented, "method GetCapabilities not implemented") 959 | } 960 | func (UnimplementedCapabilitiesServer) mustEmbedUnimplementedCapabilitiesServer() {} 961 | 962 | // UnsafeCapabilitiesServer may be embedded to opt out of forward compatibility for this service. 963 | // Use of this interface is not recommended, as added methods to CapabilitiesServer will 964 | // result in compilation errors. 965 | type UnsafeCapabilitiesServer interface { 966 | mustEmbedUnimplementedCapabilitiesServer() 967 | } 968 | 969 | func RegisterCapabilitiesServer(s grpc.ServiceRegistrar, srv CapabilitiesServer) { 970 | s.RegisterService(&Capabilities_ServiceDesc, srv) 971 | } 972 | 973 | func _Capabilities_GetCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 974 | in := new(GetCapabilitiesRequest) 975 | if err := dec(in); err != nil { 976 | return nil, err 977 | } 978 | if interceptor == nil { 979 | return srv.(CapabilitiesServer).GetCapabilities(ctx, in) 980 | } 981 | info := &grpc.UnaryServerInfo{ 982 | Server: srv, 983 | FullMethod: "/build.bazel.remote.execution.v2.Capabilities/GetCapabilities", 984 | } 985 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 986 | return srv.(CapabilitiesServer).GetCapabilities(ctx, req.(*GetCapabilitiesRequest)) 987 | } 988 | return interceptor(ctx, in, info, handler) 989 | } 990 | 991 | // Capabilities_ServiceDesc is the grpc.ServiceDesc for Capabilities service. 992 | // It's only intended for direct use with grpc.RegisterService, 993 | // and not to be introspected or modified (even as a copy) 994 | var Capabilities_ServiceDesc = grpc.ServiceDesc{ 995 | ServiceName: "build.bazel.remote.execution.v2.Capabilities", 996 | HandlerType: (*CapabilitiesServer)(nil), 997 | Methods: []grpc.MethodDesc{ 998 | { 999 | MethodName: "GetCapabilities", 1000 | Handler: _Capabilities_GetCapabilities_Handler, 1001 | }, 1002 | }, 1003 | Streams: []grpc.StreamDesc{}, 1004 | Metadata: "build/bazel/remote/execution/v2/remote_execution.proto", 1005 | } 1006 | -------------------------------------------------------------------------------- /protos/build/bazel/semver/BUILD: -------------------------------------------------------------------------------- 1 | go_package() 2 | -------------------------------------------------------------------------------- /protos/build/bazel/semver/semver.pb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Bazel Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Code generated by protoc-gen-go. DO NOT EDIT. 16 | // versions: 17 | // protoc-gen-go v1.28.1 18 | // protoc v3.21.6 19 | // source: build/bazel/semver/semver.proto 20 | 21 | package semver 22 | 23 | import ( 24 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 25 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 26 | reflect "reflect" 27 | sync "sync" 28 | ) 29 | 30 | const ( 31 | // Verify that this generated code is sufficiently up-to-date. 32 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 33 | // Verify that runtime/protoimpl is sufficiently up-to-date. 34 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 35 | ) 36 | 37 | // The full version of a given tool. 38 | type SemVer struct { 39 | state protoimpl.MessageState 40 | sizeCache protoimpl.SizeCache 41 | unknownFields protoimpl.UnknownFields 42 | 43 | // The major version, e.g 10 for 10.2.3. 44 | Major int32 `protobuf:"varint,1,opt,name=major,proto3" json:"major,omitempty"` 45 | // The minor version, e.g. 2 for 10.2.3. 46 | Minor int32 `protobuf:"varint,2,opt,name=minor,proto3" json:"minor,omitempty"` 47 | // The patch version, e.g 3 for 10.2.3. 48 | Patch int32 `protobuf:"varint,3,opt,name=patch,proto3" json:"patch,omitempty"` 49 | // The pre-release version. Either this field or major/minor/patch fields 50 | // must be filled. They are mutually exclusive. Pre-release versions are 51 | // assumed to be earlier than any released versions. 52 | Prerelease string `protobuf:"bytes,4,opt,name=prerelease,proto3" json:"prerelease,omitempty"` 53 | } 54 | 55 | func (x *SemVer) Reset() { 56 | *x = SemVer{} 57 | if protoimpl.UnsafeEnabled { 58 | mi := &file_build_bazel_semver_semver_proto_msgTypes[0] 59 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 60 | ms.StoreMessageInfo(mi) 61 | } 62 | } 63 | 64 | func (x *SemVer) String() string { 65 | return protoimpl.X.MessageStringOf(x) 66 | } 67 | 68 | func (*SemVer) ProtoMessage() {} 69 | 70 | func (x *SemVer) ProtoReflect() protoreflect.Message { 71 | mi := &file_build_bazel_semver_semver_proto_msgTypes[0] 72 | if protoimpl.UnsafeEnabled && x != nil { 73 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 74 | if ms.LoadMessageInfo() == nil { 75 | ms.StoreMessageInfo(mi) 76 | } 77 | return ms 78 | } 79 | return mi.MessageOf(x) 80 | } 81 | 82 | // Deprecated: Use SemVer.ProtoReflect.Descriptor instead. 83 | func (*SemVer) Descriptor() ([]byte, []int) { 84 | return file_build_bazel_semver_semver_proto_rawDescGZIP(), []int{0} 85 | } 86 | 87 | func (x *SemVer) GetMajor() int32 { 88 | if x != nil { 89 | return x.Major 90 | } 91 | return 0 92 | } 93 | 94 | func (x *SemVer) GetMinor() int32 { 95 | if x != nil { 96 | return x.Minor 97 | } 98 | return 0 99 | } 100 | 101 | func (x *SemVer) GetPatch() int32 { 102 | if x != nil { 103 | return x.Patch 104 | } 105 | return 0 106 | } 107 | 108 | func (x *SemVer) GetPrerelease() string { 109 | if x != nil { 110 | return x.Prerelease 111 | } 112 | return "" 113 | } 114 | 115 | var File_build_bazel_semver_semver_proto protoreflect.FileDescriptor 116 | 117 | var file_build_bazel_semver_semver_proto_rawDesc = []byte{ 118 | 0x0a, 0x1f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2f, 0x62, 0x61, 0x7a, 0x65, 0x6c, 0x2f, 0x73, 0x65, 119 | 0x6d, 0x76, 0x65, 0x72, 0x2f, 0x73, 0x65, 0x6d, 0x76, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 120 | 0x6f, 0x12, 0x12, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x62, 0x61, 0x7a, 0x65, 0x6c, 0x2e, 0x73, 121 | 0x65, 0x6d, 0x76, 0x65, 0x72, 0x22, 0x6a, 0x0a, 0x06, 0x53, 0x65, 0x6d, 0x56, 0x65, 0x72, 0x12, 122 | 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 123 | 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x18, 0x02, 124 | 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 125 | 0x61, 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x70, 0x61, 0x74, 0x63, 126 | 0x68, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x72, 0x65, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x18, 127 | 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x65, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 128 | 0x65, 0x42, 0x83, 0x01, 0x0a, 0x12, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x62, 0x61, 0x7a, 0x65, 129 | 0x6c, 0x2e, 0x73, 0x65, 0x6d, 0x76, 0x65, 0x72, 0x42, 0x0b, 0x53, 0x65, 0x6d, 0x76, 0x65, 0x72, 130 | 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x43, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 131 | 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x6f, 0x6f, 0x6c, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x6c, 0x61, 0x62, 132 | 0x73, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2d, 0x61, 0x70, 0x69, 0x2d, 0x74, 0x6f, 0x6f, 133 | 0x6c, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2f, 134 | 0x62, 0x61, 0x7a, 0x65, 0x6c, 0x2f, 0x73, 0x65, 0x6d, 0x76, 0x65, 0x72, 0xa2, 0x02, 0x03, 0x53, 135 | 0x4d, 0x56, 0xaa, 0x02, 0x12, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x42, 0x61, 0x7a, 0x65, 0x6c, 136 | 0x2e, 0x53, 0x65, 0x6d, 0x76, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 137 | } 138 | 139 | var ( 140 | file_build_bazel_semver_semver_proto_rawDescOnce sync.Once 141 | file_build_bazel_semver_semver_proto_rawDescData = file_build_bazel_semver_semver_proto_rawDesc 142 | ) 143 | 144 | func file_build_bazel_semver_semver_proto_rawDescGZIP() []byte { 145 | file_build_bazel_semver_semver_proto_rawDescOnce.Do(func() { 146 | file_build_bazel_semver_semver_proto_rawDescData = protoimpl.X.CompressGZIP(file_build_bazel_semver_semver_proto_rawDescData) 147 | }) 148 | return file_build_bazel_semver_semver_proto_rawDescData 149 | } 150 | 151 | var file_build_bazel_semver_semver_proto_msgTypes = make([]protoimpl.MessageInfo, 1) 152 | var file_build_bazel_semver_semver_proto_goTypes = []interface{}{ 153 | (*SemVer)(nil), // 0: build.bazel.semver.SemVer 154 | } 155 | var file_build_bazel_semver_semver_proto_depIdxs = []int32{ 156 | 0, // [0:0] is the sub-list for method output_type 157 | 0, // [0:0] is the sub-list for method input_type 158 | 0, // [0:0] is the sub-list for extension type_name 159 | 0, // [0:0] is the sub-list for extension extendee 160 | 0, // [0:0] is the sub-list for field type_name 161 | } 162 | 163 | func init() { file_build_bazel_semver_semver_proto_init() } 164 | func file_build_bazel_semver_semver_proto_init() { 165 | if File_build_bazel_semver_semver_proto != nil { 166 | return 167 | } 168 | if !protoimpl.UnsafeEnabled { 169 | file_build_bazel_semver_semver_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 170 | switch v := v.(*SemVer); i { 171 | case 0: 172 | return &v.state 173 | case 1: 174 | return &v.sizeCache 175 | case 2: 176 | return &v.unknownFields 177 | default: 178 | return nil 179 | } 180 | } 181 | } 182 | type x struct{} 183 | out := protoimpl.TypeBuilder{ 184 | File: protoimpl.DescBuilder{ 185 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 186 | RawDescriptor: file_build_bazel_semver_semver_proto_rawDesc, 187 | NumEnums: 0, 188 | NumMessages: 1, 189 | NumExtensions: 0, 190 | NumServices: 0, 191 | }, 192 | GoTypes: file_build_bazel_semver_semver_proto_goTypes, 193 | DependencyIndexes: file_build_bazel_semver_semver_proto_depIdxs, 194 | MessageInfos: file_build_bazel_semver_semver_proto_msgTypes, 195 | }.Build() 196 | File_build_bazel_semver_semver_proto = out.File 197 | file_build_bazel_semver_semver_proto_rawDesc = nil 198 | file_build_bazel_semver_semver_proto_goTypes = nil 199 | file_build_bazel_semver_semver_proto_depIdxs = nil 200 | } 201 | -------------------------------------------------------------------------------- /protos/build/bazel/semver/semver.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Bazel Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package build.bazel.semver; 18 | 19 | option csharp_namespace = "Build.Bazel.Semver"; 20 | option go_package = "github.com/toolchainlabs/remote-api-tools/protos/build/bazel/semver"; 21 | option java_multiple_files = true; 22 | option java_outer_classname = "SemverProto"; 23 | option java_package = "build.bazel.semver"; 24 | option objc_class_prefix = "SMV"; 25 | 26 | // The full version of a given tool. 27 | message SemVer { 28 | // The major version, e.g 10 for 10.2.3. 29 | int32 major = 1; 30 | 31 | // The minor version, e.g. 2 for 10.2.3. 32 | int32 minor = 2; 33 | 34 | // The patch version, e.g 3 for 10.2.3. 35 | int32 patch = 3; 36 | 37 | // The pre-release version. Either this field or major/minor/patch fields 38 | // must be filled. They are mutually exclusive. Pre-release versions are 39 | // assumed to be earlier than any released versions. 40 | string prerelease = 4; 41 | } 42 | -------------------------------------------------------------------------------- /protos/google/api/annotations.proto: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/api/http.proto"; 20 | import "google/protobuf/descriptor.proto"; 21 | 22 | option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "AnnotationsProto"; 25 | option java_package = "com.google.api"; 26 | option objc_class_prefix = "GAPI"; 27 | 28 | extend google.protobuf.MethodOptions { 29 | // See `HttpRule`. 30 | HttpRule http = 72295728; 31 | } 32 | -------------------------------------------------------------------------------- /protos/google/api/client.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/protobuf/descriptor.proto"; 20 | 21 | option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; 22 | option java_multiple_files = true; 23 | option java_outer_classname = "ClientProto"; 24 | option java_package = "com.google.api"; 25 | option objc_class_prefix = "GAPI"; 26 | 27 | extend google.protobuf.MethodOptions { 28 | // A definition of a client library method signature. 29 | // 30 | // In client libraries, each proto RPC corresponds to one or more methods 31 | // which the end user is able to call, and calls the underlying RPC. 32 | // Normally, this method receives a single argument (a struct or instance 33 | // corresponding to the RPC request object). Defining this field will 34 | // add one or more overloads providing flattened or simpler method signatures 35 | // in some languages. 36 | // 37 | // The fields on the method signature are provided as a comma-separated 38 | // string. 39 | // 40 | // For example, the proto RPC and annotation: 41 | // 42 | // rpc CreateSubscription(CreateSubscriptionRequest) 43 | // returns (Subscription) { 44 | // option (google.api.method_signature) = "name,topic"; 45 | // } 46 | // 47 | // Would add the following Java overload (in addition to the method accepting 48 | // the request object): 49 | // 50 | // public final Subscription createSubscription(String name, String topic) 51 | // 52 | // The following backwards-compatibility guidelines apply: 53 | // 54 | // * Adding this annotation to an unannotated method is backwards 55 | // compatible. 56 | // * Adding this annotation to a method which already has existing 57 | // method signature annotations is backwards compatible if and only if 58 | // the new method signature annotation is last in the sequence. 59 | // * Modifying or removing an existing method signature annotation is 60 | // a breaking change. 61 | // * Re-ordering existing method signature annotations is a breaking 62 | // change. 63 | repeated string method_signature = 1051; 64 | } 65 | 66 | extend google.protobuf.ServiceOptions { 67 | // The hostname for this service. 68 | // This should be specified with no prefix or protocol. 69 | // 70 | // Example: 71 | // 72 | // service Foo { 73 | // option (google.api.default_host) = "foo.googleapi.com"; 74 | // ... 75 | // } 76 | string default_host = 1049; 77 | 78 | // OAuth scopes needed for the client. 79 | // 80 | // Example: 81 | // 82 | // service Foo { 83 | // option (google.api.oauth_scopes) = \ 84 | // "https://www.googleapis.com/auth/cloud-platform"; 85 | // ... 86 | // } 87 | // 88 | // If there is more than one scope, use a comma-separated string: 89 | // 90 | // Example: 91 | // 92 | // service Foo { 93 | // option (google.api.oauth_scopes) = \ 94 | // "https://www.googleapis.com/auth/cloud-platform," 95 | // "https://www.googleapis.com/auth/monitoring"; 96 | // ... 97 | // } 98 | string oauth_scopes = 1050; 99 | } 100 | -------------------------------------------------------------------------------- /protos/google/api/http.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | option cc_enable_arenas = true; 20 | option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; 21 | option java_multiple_files = true; 22 | option java_outer_classname = "HttpProto"; 23 | option java_package = "com.google.api"; 24 | option objc_class_prefix = "GAPI"; 25 | 26 | // Defines the HTTP configuration for an API service. It contains a list of 27 | // [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method 28 | // to one or more HTTP REST API methods. 29 | message Http { 30 | // A list of HTTP configuration rules that apply to individual API methods. 31 | // 32 | // **NOTE:** All service configuration rules follow "last one wins" order. 33 | repeated HttpRule rules = 1; 34 | 35 | // When set to true, URL path parameters will be fully URI-decoded except in 36 | // cases of single segment matches in reserved expansion, where "%2F" will be 37 | // left encoded. 38 | // 39 | // The default behavior is to not decode RFC 6570 reserved characters in multi 40 | // segment matches. 41 | bool fully_decode_reserved_expansion = 2; 42 | } 43 | 44 | // # gRPC Transcoding 45 | // 46 | // gRPC Transcoding is a feature for mapping between a gRPC method and one or 47 | // more HTTP REST endpoints. It allows developers to build a single API service 48 | // that supports both gRPC APIs and REST APIs. Many systems, including [Google 49 | // APIs](https://github.com/googleapis/googleapis), 50 | // [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC 51 | // Gateway](https://github.com/grpc-ecosystem/grpc-gateway), 52 | // and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature 53 | // and use it for large scale production services. 54 | // 55 | // `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies 56 | // how different portions of the gRPC request message are mapped to the URL 57 | // path, URL query parameters, and HTTP request body. It also controls how the 58 | // gRPC response message is mapped to the HTTP response body. `HttpRule` is 59 | // typically specified as an `google.api.http` annotation on the gRPC method. 60 | // 61 | // Each mapping specifies a URL path template and an HTTP method. The path 62 | // template may refer to one or more fields in the gRPC request message, as long 63 | // as each field is a non-repeated field with a primitive (non-message) type. 64 | // The path template controls how fields of the request message are mapped to 65 | // the URL path. 66 | // 67 | // Example: 68 | // 69 | // service Messaging { 70 | // rpc GetMessage(GetMessageRequest) returns (Message) { 71 | // option (google.api.http) = { 72 | // get: "/v1/{name=messages/*}" 73 | // }; 74 | // } 75 | // } 76 | // message GetMessageRequest { 77 | // string name = 1; // Mapped to URL path. 78 | // } 79 | // message Message { 80 | // string text = 1; // The resource content. 81 | // } 82 | // 83 | // This enables an HTTP REST to gRPC mapping as below: 84 | // 85 | // HTTP | gRPC 86 | // -----|----- 87 | // `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` 88 | // 89 | // Any fields in the request message which are not bound by the path template 90 | // automatically become HTTP query parameters if there is no HTTP request body. 91 | // For example: 92 | // 93 | // service Messaging { 94 | // rpc GetMessage(GetMessageRequest) returns (Message) { 95 | // option (google.api.http) = { 96 | // get:"/v1/messages/{message_id}" 97 | // }; 98 | // } 99 | // } 100 | // message GetMessageRequest { 101 | // message SubMessage { 102 | // string subfield = 1; 103 | // } 104 | // string message_id = 1; // Mapped to URL path. 105 | // int64 revision = 2; // Mapped to URL query parameter `revision`. 106 | // SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. 107 | // } 108 | // 109 | // This enables a HTTP JSON to RPC mapping as below: 110 | // 111 | // HTTP | gRPC 112 | // -----|----- 113 | // `GET /v1/messages/123456?revision=2&sub.subfield=foo` | 114 | // `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: 115 | // "foo"))` 116 | // 117 | // Note that fields which are mapped to URL query parameters must have a 118 | // primitive type or a repeated primitive type or a non-repeated message type. 119 | // In the case of a repeated type, the parameter can be repeated in the URL 120 | // as `...?param=A¶m=B`. In the case of a message type, each field of the 121 | // message is mapped to a separate parameter, such as 122 | // `...?foo.a=A&foo.b=B&foo.c=C`. 123 | // 124 | // For HTTP methods that allow a request body, the `body` field 125 | // specifies the mapping. Consider a REST update method on the 126 | // message resource collection: 127 | // 128 | // service Messaging { 129 | // rpc UpdateMessage(UpdateMessageRequest) returns (Message) { 130 | // option (google.api.http) = { 131 | // patch: "/v1/messages/{message_id}" 132 | // body: "message" 133 | // }; 134 | // } 135 | // } 136 | // message UpdateMessageRequest { 137 | // string message_id = 1; // mapped to the URL 138 | // Message message = 2; // mapped to the body 139 | // } 140 | // 141 | // The following HTTP JSON to RPC mapping is enabled, where the 142 | // representation of the JSON in the request body is determined by 143 | // protos JSON encoding: 144 | // 145 | // HTTP | gRPC 146 | // -----|----- 147 | // `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: 148 | // "123456" message { text: "Hi!" })` 149 | // 150 | // The special name `*` can be used in the body mapping to define that 151 | // every field not bound by the path template should be mapped to the 152 | // request body. This enables the following alternative definition of 153 | // the update method: 154 | // 155 | // service Messaging { 156 | // rpc UpdateMessage(Message) returns (Message) { 157 | // option (google.api.http) = { 158 | // patch: "/v1/messages/{message_id}" 159 | // body: "*" 160 | // }; 161 | // } 162 | // } 163 | // message Message { 164 | // string message_id = 1; 165 | // string text = 2; 166 | // } 167 | // 168 | // 169 | // The following HTTP JSON to RPC mapping is enabled: 170 | // 171 | // HTTP | gRPC 172 | // -----|----- 173 | // `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: 174 | // "123456" text: "Hi!")` 175 | // 176 | // Note that when using `*` in the body mapping, it is not possible to 177 | // have HTTP parameters, as all fields not bound by the path end in 178 | // the body. This makes this option more rarely used in practice when 179 | // defining REST APIs. The common usage of `*` is in custom methods 180 | // which don't use the URL at all for transferring data. 181 | // 182 | // It is possible to define multiple HTTP methods for one RPC by using 183 | // the `additional_bindings` option. Example: 184 | // 185 | // service Messaging { 186 | // rpc GetMessage(GetMessageRequest) returns (Message) { 187 | // option (google.api.http) = { 188 | // get: "/v1/messages/{message_id}" 189 | // additional_bindings { 190 | // get: "/v1/users/{user_id}/messages/{message_id}" 191 | // } 192 | // }; 193 | // } 194 | // } 195 | // message GetMessageRequest { 196 | // string message_id = 1; 197 | // string user_id = 2; 198 | // } 199 | // 200 | // This enables the following two alternative HTTP JSON to RPC mappings: 201 | // 202 | // HTTP | gRPC 203 | // -----|----- 204 | // `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` 205 | // `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: 206 | // "123456")` 207 | // 208 | // ## Rules for HTTP mapping 209 | // 210 | // 1. Leaf request fields (recursive expansion nested messages in the request 211 | // message) are classified into three categories: 212 | // - Fields referred by the path template. They are passed via the URL path. 213 | // - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP 214 | // request body. 215 | // - All other fields are passed via the URL query parameters, and the 216 | // parameter name is the field path in the request message. A repeated 217 | // field can be represented as multiple query parameters under the same 218 | // name. 219 | // 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields 220 | // are passed via URL path and HTTP request body. 221 | // 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all 222 | // fields are passed via URL path and URL query parameters. 223 | // 224 | // ### Path template syntax 225 | // 226 | // Template = "/" Segments [ Verb ] ; 227 | // Segments = Segment { "/" Segment } ; 228 | // Segment = "*" | "**" | LITERAL | Variable ; 229 | // Variable = "{" FieldPath [ "=" Segments ] "}" ; 230 | // FieldPath = IDENT { "." IDENT } ; 231 | // Verb = ":" LITERAL ; 232 | // 233 | // The syntax `*` matches a single URL path segment. The syntax `**` matches 234 | // zero or more URL path segments, which must be the last part of the URL path 235 | // except the `Verb`. 236 | // 237 | // The syntax `Variable` matches part of the URL path as specified by its 238 | // template. A variable template must not contain other variables. If a variable 239 | // matches a single path segment, its template may be omitted, e.g. `{var}` 240 | // is equivalent to `{var=*}`. 241 | // 242 | // The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` 243 | // contains any reserved character, such characters should be percent-encoded 244 | // before the matching. 245 | // 246 | // If a variable contains exactly one path segment, such as `"{var}"` or 247 | // `"{var=*}"`, when such a variable is expanded into a URL path on the client 248 | // side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The 249 | // server side does the reverse decoding. Such variables show up in the 250 | // [Discovery 251 | // Document](https://developers.google.com/discovery/v1/reference/apis) as 252 | // `{var}`. 253 | // 254 | // If a variable contains multiple path segments, such as `"{var=foo/*}"` 255 | // or `"{var=**}"`, when such a variable is expanded into a URL path on the 256 | // client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. 257 | // The server side does the reverse decoding, except "%2F" and "%2f" are left 258 | // unchanged. Such variables show up in the 259 | // [Discovery 260 | // Document](https://developers.google.com/discovery/v1/reference/apis) as 261 | // `{+var}`. 262 | // 263 | // ## Using gRPC API Service Configuration 264 | // 265 | // gRPC API Service Configuration (service config) is a configuration language 266 | // for configuring a gRPC service to become a user-facing product. The 267 | // service config is simply the YAML representation of the `google.api.Service` 268 | // proto message. 269 | // 270 | // As an alternative to annotating your proto file, you can configure gRPC 271 | // transcoding in your service config YAML files. You do this by specifying a 272 | // `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same 273 | // effect as the proto annotation. This can be particularly useful if you 274 | // have a proto that is reused in multiple services. Note that any transcoding 275 | // specified in the service config will override any matching transcoding 276 | // configuration in the proto. 277 | // 278 | // Example: 279 | // 280 | // http: 281 | // rules: 282 | // # Selects a gRPC method and applies HttpRule to it. 283 | // - selector: example.v1.Messaging.GetMessage 284 | // get: /v1/messages/{message_id}/{sub.subfield} 285 | // 286 | // ## Special notes 287 | // 288 | // When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the 289 | // proto to JSON conversion must follow the [proto3 290 | // specification](https://developers.google.com/protocol-buffers/docs/proto3#json). 291 | // 292 | // While the single segment variable follows the semantics of 293 | // [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String 294 | // Expansion, the multi segment variable **does not** follow RFC 6570 Section 295 | // 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion 296 | // does not expand special characters like `?` and `#`, which would lead 297 | // to invalid URLs. As the result, gRPC Transcoding uses a custom encoding 298 | // for multi segment variables. 299 | // 300 | // The path variables **must not** refer to any repeated or mapped field, 301 | // because client libraries are not capable of handling such variable expansion. 302 | // 303 | // The path variables **must not** capture the leading "/" character. The reason 304 | // is that the most common use case "{var}" does not capture the leading "/" 305 | // character. For consistency, all path variables must share the same behavior. 306 | // 307 | // Repeated message fields must not be mapped to URL query parameters, because 308 | // no client library can support such complicated mapping. 309 | // 310 | // If an API needs to use a JSON array for request or response body, it can map 311 | // the request or response body to a repeated field. However, some gRPC 312 | // Transcoding implementations may not support this feature. 313 | message HttpRule { 314 | // Selects a method to which this rule applies. 315 | // 316 | // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. 317 | string selector = 1; 318 | 319 | // Determines the URL pattern is matched by this rules. This pattern can be 320 | // used with any of the {get|put|post|delete|patch} methods. A custom method 321 | // can be defined using the 'custom' field. 322 | oneof pattern { 323 | // Maps to HTTP GET. Used for listing and getting information about 324 | // resources. 325 | string get = 2; 326 | 327 | // Maps to HTTP PUT. Used for replacing a resource. 328 | string put = 3; 329 | 330 | // Maps to HTTP POST. Used for creating a resource or performing an action. 331 | string post = 4; 332 | 333 | // Maps to HTTP DELETE. Used for deleting a resource. 334 | string delete = 5; 335 | 336 | // Maps to HTTP PATCH. Used for updating a resource. 337 | string patch = 6; 338 | 339 | // The custom pattern is used for specifying an HTTP method that is not 340 | // included in the `pattern` field, such as HEAD, or "*" to leave the 341 | // HTTP method unspecified for this rule. The wild-card rule is useful 342 | // for services that provide content to Web (HTML) clients. 343 | CustomHttpPattern custom = 8; 344 | } 345 | 346 | // The name of the request field whose value is mapped to the HTTP request 347 | // body, or `*` for mapping all request fields not captured by the path 348 | // pattern to the HTTP body, or omitted for not having any HTTP request body. 349 | // 350 | // NOTE: the referred field must be present at the top-level of the request 351 | // message type. 352 | string body = 7; 353 | 354 | // Optional. The name of the response field whose value is mapped to the HTTP 355 | // response body. When omitted, the entire response message will be used 356 | // as the HTTP response body. 357 | // 358 | // NOTE: The referred field must be present at the top-level of the response 359 | // message type. 360 | string response_body = 12; 361 | 362 | // Additional HTTP bindings for the selector. Nested bindings must 363 | // not contain an `additional_bindings` field themselves (that is, 364 | // the nesting may only be one level deep). 365 | repeated HttpRule additional_bindings = 11; 366 | } 367 | 368 | // A custom pattern is used for defining custom HTTP verb. 369 | message CustomHttpPattern { 370 | // The name of this custom HTTP verb. 371 | string kind = 1; 372 | 373 | // The path matched by this custom verb. 374 | string path = 2; 375 | } 376 | -------------------------------------------------------------------------------- /protos/google/longrunning/operations.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.longrunning; 18 | 19 | import "google/api/annotations.proto"; 20 | import "google/api/client.proto"; 21 | import "google/protobuf/any.proto"; 22 | import "google/protobuf/duration.proto"; 23 | import "google/protobuf/empty.proto"; 24 | import "google/rpc/status.proto"; 25 | import "google/protobuf/descriptor.proto"; 26 | 27 | option cc_enable_arenas = true; 28 | option csharp_namespace = "Google.LongRunning"; 29 | option go_package = "google.golang.org/genproto/googleapis/longrunning;longrunning"; 30 | option java_multiple_files = true; 31 | option java_outer_classname = "OperationsProto"; 32 | option java_package = "com.google.longrunning"; 33 | option php_namespace = "Google\\LongRunning"; 34 | 35 | extend google.protobuf.MethodOptions { 36 | // Additional information regarding long-running operations. 37 | // In particular, this specifies the types that are returned from 38 | // long-running operations. 39 | // 40 | // Required for methods that return `google.longrunning.Operation`; invalid 41 | // otherwise. 42 | google.longrunning.OperationInfo operation_info = 1049; 43 | } 44 | 45 | // Manages long-running operations with an API service. 46 | // 47 | // When an API method normally takes long time to complete, it can be designed 48 | // to return [Operation][google.longrunning.Operation] to the client, and the client can use this 49 | // interface to receive the real response asynchronously by polling the 50 | // operation resource, or pass the operation resource to another API (such as 51 | // Google Cloud Pub/Sub API) to receive the response. Any API service that 52 | // returns long-running operations should implement the `Operations` interface 53 | // so developers can have a consistent client experience. 54 | service Operations { 55 | option (google.api.default_host) = "longrunning.googleapis.com"; 56 | 57 | // Lists operations that match the specified filter in the request. If the 58 | // server doesn't support this method, it returns `UNIMPLEMENTED`. 59 | // 60 | // NOTE: the `name` binding allows API services to override the binding 61 | // to use different resource name schemes, such as `users/*/operations`. To 62 | // override the binding, API services can add a binding such as 63 | // `"/v1/{name=users/*}/operations"` to their service configuration. 64 | // For backwards compatibility, the default name includes the operations 65 | // collection id, however overriding users must ensure the name binding 66 | // is the parent resource, without the operations collection id. 67 | rpc ListOperations(ListOperationsRequest) returns (ListOperationsResponse) { 68 | option (google.api.http) = { 69 | get: "/v1/{name=operations}" 70 | }; 71 | option (google.api.method_signature) = "name,filter"; 72 | } 73 | 74 | // Gets the latest state of a long-running operation. Clients can use this 75 | // method to poll the operation result at intervals as recommended by the API 76 | // service. 77 | rpc GetOperation(GetOperationRequest) returns (Operation) { 78 | option (google.api.http) = { 79 | get: "/v1/{name=operations/**}" 80 | }; 81 | option (google.api.method_signature) = "name"; 82 | } 83 | 84 | // Deletes a long-running operation. This method indicates that the client is 85 | // no longer interested in the operation result. It does not cancel the 86 | // operation. If the server doesn't support this method, it returns 87 | // `google.rpc.Code.UNIMPLEMENTED`. 88 | rpc DeleteOperation(DeleteOperationRequest) returns (google.protobuf.Empty) { 89 | option (google.api.http) = { 90 | delete: "/v1/{name=operations/**}" 91 | }; 92 | option (google.api.method_signature) = "name"; 93 | } 94 | 95 | // Starts asynchronous cancellation on a long-running operation. The server 96 | // makes a best effort to cancel the operation, but success is not 97 | // guaranteed. If the server doesn't support this method, it returns 98 | // `google.rpc.Code.UNIMPLEMENTED`. Clients can use 99 | // [Operations.GetOperation][google.longrunning.Operations.GetOperation] or 100 | // other methods to check whether the cancellation succeeded or whether the 101 | // operation completed despite cancellation. On successful cancellation, 102 | // the operation is not deleted; instead, it becomes an operation with 103 | // an [Operation.error][google.longrunning.Operation.error] value with a [google.rpc.Status.code][google.rpc.Status.code] of 1, 104 | // corresponding to `Code.CANCELLED`. 105 | rpc CancelOperation(CancelOperationRequest) returns (google.protobuf.Empty) { 106 | option (google.api.http) = { 107 | post: "/v1/{name=operations/**}:cancel" 108 | body: "*" 109 | }; 110 | option (google.api.method_signature) = "name"; 111 | } 112 | 113 | // Waits for the specified long-running operation until it is done or reaches 114 | // at most a specified timeout, returning the latest state. If the operation 115 | // is already done, the latest state is immediately returned. If the timeout 116 | // specified is greater than the default HTTP/RPC timeout, the HTTP/RPC 117 | // timeout is used. If the server does not support this method, it returns 118 | // `google.rpc.Code.UNIMPLEMENTED`. 119 | // Note that this method is on a best-effort basis. It may return the latest 120 | // state before the specified timeout (including immediately), meaning even an 121 | // immediate response is no guarantee that the operation is done. 122 | rpc WaitOperation(WaitOperationRequest) returns (Operation) { 123 | } 124 | } 125 | 126 | // This resource represents a long-running operation that is the result of a 127 | // network API call. 128 | message Operation { 129 | // The server-assigned name, which is only unique within the same service that 130 | // originally returns it. If you use the default HTTP mapping, the 131 | // `name` should be a resource name ending with `operations/{unique_id}`. 132 | string name = 1; 133 | 134 | // Service-specific metadata associated with the operation. It typically 135 | // contains progress information and common metadata such as create time. 136 | // Some services might not provide such metadata. Any method that returns a 137 | // long-running operation should document the metadata type, if any. 138 | google.protobuf.Any metadata = 2; 139 | 140 | // If the value is `false`, it means the operation is still in progress. 141 | // If `true`, the operation is completed, and either `error` or `response` is 142 | // available. 143 | bool done = 3; 144 | 145 | // The operation result, which can be either an `error` or a valid `response`. 146 | // If `done` == `false`, neither `error` nor `response` is set. 147 | // If `done` == `true`, exactly one of `error` or `response` is set. 148 | oneof result { 149 | // The error result of the operation in case of failure or cancellation. 150 | google.rpc.Status error = 4; 151 | 152 | // The normal response of the operation in case of success. If the original 153 | // method returns no data on success, such as `Delete`, the response is 154 | // `google.protobuf.Empty`. If the original method is standard 155 | // `Get`/`Create`/`Update`, the response should be the resource. For other 156 | // methods, the response should have the type `XxxResponse`, where `Xxx` 157 | // is the original method name. For example, if the original method name 158 | // is `TakeSnapshot()`, the inferred response type is 159 | // `TakeSnapshotResponse`. 160 | google.protobuf.Any response = 5; 161 | } 162 | } 163 | 164 | // The request message for [Operations.GetOperation][google.longrunning.Operations.GetOperation]. 165 | message GetOperationRequest { 166 | // The name of the operation resource. 167 | string name = 1; 168 | } 169 | 170 | // The request message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. 171 | message ListOperationsRequest { 172 | // The name of the operation's parent resource. 173 | string name = 4; 174 | 175 | // The standard list filter. 176 | string filter = 1; 177 | 178 | // The standard list page size. 179 | int32 page_size = 2; 180 | 181 | // The standard list page token. 182 | string page_token = 3; 183 | } 184 | 185 | // The response message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. 186 | message ListOperationsResponse { 187 | // A list of operations that matches the specified filter in the request. 188 | repeated Operation operations = 1; 189 | 190 | // The standard List next-page token. 191 | string next_page_token = 2; 192 | } 193 | 194 | // The request message for [Operations.CancelOperation][google.longrunning.Operations.CancelOperation]. 195 | message CancelOperationRequest { 196 | // The name of the operation resource to be cancelled. 197 | string name = 1; 198 | } 199 | 200 | // The request message for [Operations.DeleteOperation][google.longrunning.Operations.DeleteOperation]. 201 | message DeleteOperationRequest { 202 | // The name of the operation resource to be deleted. 203 | string name = 1; 204 | } 205 | 206 | // The request message for [Operations.WaitOperation][google.longrunning.Operations.WaitOperation]. 207 | message WaitOperationRequest { 208 | // The name of the operation resource to wait on. 209 | string name = 1; 210 | 211 | // The maximum duration to wait before timing out. If left blank, the wait 212 | // will be at most the time permitted by the underlying HTTP/RPC protocol. 213 | // If RPC context deadline is also specified, the shorter one will be used. 214 | google.protobuf.Duration timeout = 2; 215 | } 216 | 217 | // A message representing the message types used by a long-running operation. 218 | // 219 | // Example: 220 | // 221 | // rpc LongRunningRecognize(LongRunningRecognizeRequest) 222 | // returns (google.longrunning.Operation) { 223 | // option (google.longrunning.operation_info) = { 224 | // response_type: "LongRunningRecognizeResponse" 225 | // metadata_type: "LongRunningRecognizeMetadata" 226 | // }; 227 | // } 228 | message OperationInfo { 229 | // Required. The message name of the primary return type for this 230 | // long-running operation. 231 | // This type will be used to deserialize the LRO's response. 232 | // 233 | // If the response is in a different package from the rpc, a fully-qualified 234 | // message name must be used (e.g. `google.protobuf.Struct`). 235 | // 236 | // Note: Altering this value constitutes a breaking change. 237 | string response_type = 1; 238 | 239 | // Required. The message name of the metadata type for this long-running 240 | // operation. 241 | // 242 | // If the response is in a different package from the rpc, a fully-qualified 243 | // message name must be used (e.g. `google.protobuf.Struct`). 244 | // 245 | // Note: Altering this value constitutes a breaking change. 246 | string metadata_type = 2; 247 | } 248 | -------------------------------------------------------------------------------- /protos/google/rpc/status.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.rpc; 18 | 19 | import "google/protobuf/any.proto"; 20 | 21 | option cc_enable_arenas = true; 22 | option go_package = "google.golang.org/genproto/googleapis/rpc/status;status"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "StatusProto"; 25 | option java_package = "com.google.rpc"; 26 | option objc_class_prefix = "RPC"; 27 | 28 | // The `Status` type defines a logical error model that is suitable for 29 | // different programming environments, including REST APIs and RPC APIs. It is 30 | // used by [gRPC](https://github.com/grpc). Each `Status` message contains 31 | // three pieces of data: error code, error message, and error details. 32 | // 33 | // You can find out more about this error model and how to work with it in the 34 | // [API Design Guide](https://cloud.google.com/apis/design/errors). 35 | message Status { 36 | // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. 37 | int32 code = 1; 38 | 39 | // A developer-facing error message, which should be in English. Any 40 | // user-facing error message should be localized and sent in the 41 | // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. 42 | string message = 2; 43 | 44 | // A list of messages that carry the error details. There is a common set of 45 | // message types for APIs to use. 46 | repeated google.protobuf.Any details = 3; 47 | } 48 | -------------------------------------------------------------------------------- /scripts/build-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd "$(git rev-parse --show-toplevel)" 4 | 5 | docker build . --build-arg APP_NAME=smoketest --tag smoketest 6 | docker build . --build-arg APP_NAME=casload --tag casload 7 | 8 | # sanity check, run apps in containers (they will show help) 9 | docker run -it --rm smoketest 10 | docker run -it --rm casload 11 | -------------------------------------------------------------------------------- /scripts/gen-protos.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ ! -d protos ]; then 4 | echo "ERROR: This script must be run from the root of the repository." 1>&2 5 | exit 1 6 | fi 7 | 8 | protoc \ 9 | -Iprotos/ \ 10 | --go_out=protos/ \ 11 | --go-grpc_out=protos/ \ 12 | --go_opt=Mbuild/bazel/semver/semver.proto=github.com/toolchainlabs/remote-api-tools/protos/build/bazel/semver \ 13 | protos/build/bazel/semver/*.proto \ 14 | protos/build/bazel/remote/execution/v2/*.proto 15 | --------------------------------------------------------------------------------