├── .bazelrc ├── .bazelversion ├── .clang-format ├── .gitignore ├── BUILD ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── SECURITY.md ├── WORKSPACE ├── perf_benchmark ├── BUILD ├── README.md ├── benchmark.proto ├── benchmark_input_stream.cc ├── benchmark_input_stream.h ├── benchmark_input_stream_test.cc ├── benchmark_input_stream_translator_integration_test.cc ├── benchmark_main.cc ├── benchmark_service.textproto ├── image │ ├── array_length.jpg │ ├── body_length.jpg │ ├── nested_layers.jpg │ ├── num_message_segment.jpg │ ├── num_variable_bindings.jpg │ ├── value_data_type.png │ └── variable_binding_depth.jpg ├── utils.cc ├── utils.h └── utils_test.cc ├── repositories.bzl ├── script └── ci.sh ├── src ├── BUILD ├── http_template.cc ├── include │ └── grpc_transcoding │ │ ├── http_template.h │ │ ├── json_request_translator.h │ │ ├── message_reader.h │ │ ├── message_stream.h │ │ ├── path_matcher.h │ │ ├── path_matcher_node.h │ │ ├── path_matcher_utility.h │ │ ├── percent_encoding.h │ │ ├── prefix_writer.h │ │ ├── request_message_translator.h │ │ ├── request_stream_translator.h │ │ ├── request_weaver.h │ │ ├── response_to_json_translator.h │ │ ├── status_error_listener.h │ │ ├── transcoder.h │ │ ├── transcoder_input_stream.h │ │ └── type_helper.h ├── json_request_translator.cc ├── message_reader.cc ├── message_stream.cc ├── path_matcher_node.cc ├── prefix_writer.cc ├── request_message_translator.cc ├── request_stream_translator.cc ├── request_weaver.cc ├── response_to_json_translator.cc ├── status_error_listener.cc └── type_helper.cc └── test ├── BUILD ├── bookstore.proto ├── http_template_fuzz_test.cc ├── http_template_fuzz_test_corpus ├── literal ├── path-double-wildcard ├── path-double-wildcard-capture ├── path-wildcard └── path-wildcard-capture ├── http_template_test.cc ├── json_request_translator_test.cc ├── message_reader_fuzz_test.cc ├── message_reader_fuzz_test_corpus └── grpc-delimiter.txt ├── message_reader_test.cc ├── message_stream_test.cc ├── path_matcher_test.cc ├── path_matcher_utility_test.cc ├── prefix_writer_test.cc ├── proto_stream_tester.cc ├── proto_stream_tester.h ├── request_message_translator_test.cc ├── request_stream_translator_test.cc ├── request_translator_test_base.cc ├── request_translator_test_base.h ├── request_weaver_test.cc ├── response_to_json_translator_test.cc ├── status_error_listener_test.cc ├── test_common.cc ├── test_common.h ├── testdata └── bookstore_service.pb.txt └── type_helper_test.cc /.bazelrc: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Copied from bazel rules_fuzzing. 16 | # https://github.com/bazelbuild/rules_fuzzing/blob/f6062a88d83463e2900e47bc218547ba046dad44/.bazelrc 17 | 18 | # Force the use of Clang for all builds. 19 | build --action_env=CC=clang-14 20 | build --action_env=CXX=clang++-14 21 | 22 | build --cxxopt=-std=c++20 23 | build --host_cxxopt=-std=c++20 24 | 25 | # The ASAN configuration suitable for C++ unit tests. 26 | build:asan --copt=-fsanitize=address 27 | build:asan --linkopt=-fsanitize=address 28 | 29 | # LibFuzzer + ASAN 30 | build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer 31 | build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer 32 | build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=asan 33 | 34 | # LibFuzzer + MSAN 35 | build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer 36 | build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer 37 | build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=msan 38 | 39 | # LibFuzzer + MSAN (reproduction mode) 40 | build:msan-libfuzzer-repro --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer 41 | build:msan-libfuzzer-repro --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer 42 | build:msan-libfuzzer-repro --@rules_fuzzing//fuzzing:cc_engine_sanitizer=msan-origin-tracking 43 | 44 | # Honggfuzz + ASAN 45 | build:asan-honggfuzz --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:honggfuzz 46 | build:asan-honggfuzz --@rules_fuzzing//fuzzing:cc_engine_instrumentation=honggfuzz 47 | build:asan-honggfuzz --@rules_fuzzing//fuzzing:cc_engine_sanitizer=asan 48 | 49 | # Honggfuzz + MSAN 50 | build:msan-honggfuzz --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:honggfuzz 51 | build:msan-honggfuzz --@rules_fuzzing//fuzzing:cc_engine_instrumentation=honggfuzz 52 | build:msan-honggfuzz --@rules_fuzzing//fuzzing:cc_engine_sanitizer=msan 53 | 54 | # Replay + ASAN 55 | build:asan-replay --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:replay 56 | build:asan-replay --@rules_fuzzing//fuzzing:cc_engine_instrumentation=none 57 | build:asan-replay --@rules_fuzzing//fuzzing:cc_engine_sanitizer=asan 58 | 59 | build:oss-fuzz --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine 60 | build:oss-fuzz --@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz 61 | build:oss-fuzz --@rules_fuzzing//fuzzing:cc_engine_sanitizer=none 62 | 63 | # Default build parameters so that `bazel build //...` can always find them. 64 | build --define=PUSH_REGISTRY=gcr.io --define=PUSH_PROJECT=project-id --define=PUSH_TAG=latest 65 | -------------------------------------------------------------------------------- /.bazelversion: -------------------------------------------------------------------------------- 1 | 6.1.0 2 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | BasedOnStyle: Google 4 | --- 5 | Language: Proto 6 | BasedOnStyle: Google 7 | --- 8 | Language: TextProto 9 | BasedOnStyle: Google 10 | ... 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # bazel build folders 2 | /bazel-bin 3 | /bazel-grpc-httpjson-transcoding 4 | /bazel-out 5 | /bazel-testlogs 6 | 7 | # IDE 8 | .clwb/ 9 | .ijwb/ 10 | .idea/ 11 | -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | ################################################################################ 16 | # 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Want to contribute? Great! First, read this page (including the small print at the end). 2 | 3 | ### Before you contribute 4 | Before we can use your code, you must sign the 5 | [Google Individual Contributor License Agreement] 6 | (https://cla.developers.google.com/about/google-individual) 7 | (CLA), which you can do online. The CLA is necessary mainly because you own the 8 | copyright to your changes, even after your contribution becomes part of our 9 | codebase, so we need your permission to use and distribute your code. We also 10 | need to be sure of various other things—for instance that you'll tell us if you 11 | know that your code infringes on other people's patents. You don't have to sign 12 | the CLA until after you've submitted your code for review and a member has 13 | approved it, but you must do it before we can put your code into our codebase. 14 | Before you start working on a larger contribution, you should get in touch with 15 | us first through the issue tracker with your idea so that we can help out and 16 | possibly guide you. Coordinating up front makes it much easier to avoid 17 | frustration later on. 18 | 19 | ### Code reviews 20 | All submissions, including submissions by project members, require review. We 21 | use Github pull requests for this purpose. 22 | 23 | ### The small print 24 | Contributions made by corporations are covered by a different agreement than 25 | the one above, the 26 | [Software Grant and Corporate Contributor License Agreement] 27 | (https://cla.developers.google.com/about/google-corporate). 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CPP_PROTO_FILES = $(shell find . -type f \ 2 | -regex "./\(src\|api\)/.*[.]\(h\|cc\|proto\)" \ 3 | -not -path "./vendor/*") 4 | 5 | .PHONY: build 6 | build: clang-format 7 | bazelisk build //... 8 | 9 | .PHONY: test 10 | test: clang-format 11 | bazelisk test //... 12 | 13 | .PHONY: clang-format 14 | clang-format: 15 | @echo "--> formatting code with 'clang-format' tool" 16 | @echo $(CPP_PROTO_FILES) | xargs clang-format-14 -i 17 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - paulhong01 3 | - gopkarthik 4 | - dchakarwarti 5 | - numanelahi 6 | - Elliot-xq 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # grpc-httpjson-transcoding 2 | 3 | grpc-httpjson-transcoding is a library that supports 4 | [transcoding](https://github.com/googleapis/googleapis/blob/master/google/api/http.proto) 5 | so that HTTP/JSON can be converted to gRPC. 6 | 7 | It helps you to provide your APIs in both gRPC and RESTful style at the same 8 | time. The code is used in istio [proxy](https://github.com/istio/proxy) and 9 | cloud [endpoints](https://cloud.google.com/endpoints/) to provide HTTP+JSON 10 | interface to gRPC service. 11 | 12 | [![CI Status](https://oss.gprow.dev/badge.svg?jobs=grpc-transcoder-periodic)](https://testgrid.k8s.io/googleoss-grpc-transcoder#Summary) 13 | 14 | [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/grpc-httpjson-transcoding.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:grpc-httpjson-transcoding) 15 | 16 | ## Develop 17 | 18 | [Bazel](https://bazel.build/) is used for build and dependency management. The 19 | following commands build and test sources: 20 | 21 | ```bash 22 | $ bazel build //... 23 | $ bazel test //... 24 | ``` 25 | 26 | Use the following script to check and fix code format: 27 | 28 | ```bash 29 | $ script/check-style 30 | ``` 31 | 32 | ## Toolchain 33 | 34 | The Bazel build system defaults to using clang 14 to enable reproducible builds. 35 | 36 | ## Continuous Integration 37 | 38 | This repository is integrated with [OSS Prow](https://github.com/kubernetes/test-infra/tree/master/prow). Prow will run the [presubmit script](https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/blob/master/script/ci.sh) on each Pull Request to verify tests pass. Note: 39 | 40 | - PR submission is only allowed if the job passes. 41 | - If you are an outside contributor, Prow may not run until a Googler LGTMs. 42 | 43 | # Contribution 44 | See [CONTRIBUTING.md](CONTRIBUTING.md). 45 | 46 | # License 47 | grpc-httpjson-transcoding is licensed under the Apache 2.0 license. See 48 | [LICENSE](LICENSE) for more details. 49 | 50 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Follow envoy security policy 4 | 5 | This code is mainly used by [envoy](https://github.com/envoyproxy/envoy). Please follow envoy security [policy](https://github.com/envoyproxy/envoy/security/policy). 6 | 7 | 8 | ## Supported Versions 9 | 10 | This repo is small, there is not versioned releases. Please always use top of the tree. If there is any bugs or security fixes, they will be fixed in the master branch. 11 | 12 | 13 | ## Reporting a Vulnerability 14 | 15 | Please report any vulnerability to envoy according to its security [policy](https://github.com/envoyproxy/envoy/security/policy). 16 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | ################################################################################ 16 | # 17 | 18 | workspace(name = "grpc-httpjson-transcoding") 19 | 20 | load( 21 | "//:repositories.bzl", 22 | "absl_repositories", 23 | "googleapis_repositories", 24 | "googlebenchmark_repositories", 25 | "googletest_repositories", 26 | "io_bazel_rules_docker", 27 | "nlohmannjson_repositories", 28 | "protobuf_repositories", 29 | "protoconverter_repositories", 30 | ) 31 | 32 | # See 33 | # https://github.com/bazelbuild/rules_fuzzing/blob/master/README.md#configuring-the-workspace. 34 | # The fuzzing rules must be first because if they are not, bazel will 35 | # pull in incompatible versions of absl and rules_python. 36 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 37 | 38 | http_archive( 39 | name = "rules_fuzzing", 40 | sha256 = "d9002dd3cd6437017f08593124fdd1b13b3473c7b929ceb0e60d317cb9346118", 41 | strip_prefix = "rules_fuzzing-0.3.2", 42 | urls = ["https://github.com/bazelbuild/rules_fuzzing/archive/v0.3.2.zip"], 43 | ) 44 | 45 | absl_repositories() 46 | 47 | load("@rules_fuzzing//fuzzing:repositories.bzl", "rules_fuzzing_dependencies") 48 | 49 | rules_fuzzing_dependencies() 50 | 51 | load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init") 52 | 53 | rules_fuzzing_init() 54 | 55 | protobuf_repositories() 56 | 57 | load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") 58 | 59 | protobuf_deps() 60 | 61 | protoconverter_repositories() 62 | 63 | googletest_repositories() 64 | 65 | googleapis_repositories() 66 | 67 | googlebenchmark_repositories() 68 | 69 | nlohmannjson_repositories() 70 | 71 | # Followed https://github.com/bazelbuild/rules_docker#setup and 72 | # https://github.com/bazelbuild/rules_docker#cc_image. 73 | # BEGIN io_bazel_rules_docker 74 | io_bazel_rules_docker() 75 | 76 | load( 77 | "@io_bazel_rules_docker//repositories:repositories.bzl", 78 | container_repositories = "repositories", 79 | ) 80 | 81 | container_repositories() 82 | 83 | load( 84 | "@io_bazel_rules_docker//cc:image.bzl", 85 | _cc_image_repos = "repositories", 86 | ) 87 | 88 | _cc_image_repos() 89 | # END io_bazel_rules_docker 90 | 91 | load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") 92 | 93 | rules_proto_toolchains() 94 | 95 | load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") 96 | 97 | switched_rules_by_language( 98 | name = "com_google_googleapis_imports", 99 | cc = True, 100 | ) 101 | -------------------------------------------------------------------------------- /perf_benchmark/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | ################################################################################ 16 | # 17 | load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") 18 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 19 | load("@io_bazel_rules_docker//cc:image.bzl", "cc_image") 20 | load("@io_bazel_rules_docker//container:container.bzl", "container_push") 21 | 22 | cc_proto_library( 23 | name = "benchmark_cc_proto", 24 | testonly = 1, 25 | deps = [":benchmark_proto"], 26 | ) 27 | 28 | cc_proto_library( 29 | name = "com_google_protobuf_struct_cc_proto", 30 | testonly = 1, 31 | deps = ["@com_google_protobuf//:struct_proto"], 32 | ) 33 | 34 | proto_library( 35 | name = "benchmark_proto", 36 | testonly = 1, 37 | srcs = ["benchmark.proto"], 38 | deps = [ 39 | "@com_google_googleapis//google/api:annotations_proto", 40 | "@com_google_protobuf//:struct_proto", 41 | ], 42 | ) 43 | 44 | cc_binary( 45 | name = "benchmark_main", 46 | testonly = 1, 47 | srcs = ["benchmark_main.cc"], 48 | data = [ 49 | "benchmark_service.textproto", 50 | ], 51 | deps = [ 52 | "benchmark_cc_proto", 53 | ":benchmark_input_stream", 54 | ":utils", 55 | "//src:json_request_translator", 56 | "//src:response_to_json_translator", 57 | "//src:type_helper", 58 | "@com_google_absl//absl/memory", 59 | "@com_google_absl//absl/random", 60 | "@com_google_absl//absl/status", 61 | "@com_google_absl//absl/status:statusor", 62 | "@com_google_absl//absl/strings", 63 | "@com_google_benchmark//:benchmark", 64 | "@com_google_benchmark//:benchmark_main", 65 | "@com_google_googleapis//google/api:service_cc_proto", 66 | ], 67 | ) 68 | 69 | # Note: cc_image rule won't build the binary inside the container. It builds the 70 | # binary locally and copies the binary into the container. 71 | # This cc_image rule with v.0.25.0 rules_docker release uses Debian 11 as base. 72 | # Please make sure if you're building from Debian 11 system as well. 73 | cc_image( 74 | name = "benchmark_main_image", 75 | testonly = 1, 76 | binary = ":benchmark_main", 77 | ) 78 | 79 | # Example run command: 80 | # bazel run //perf_benchmark:benchmark_main_image_push \ 81 | # --define=PUSH_REGISTRY=gcr.io \ 82 | # --define=PUSH_PROJECT=project-id \ 83 | # --define=PUSH_TAG=latest 84 | container_push( 85 | name = "benchmark_main_image_push", 86 | testonly = 1, 87 | format = "Docker", 88 | image = ":benchmark_main_image", 89 | registry = "$(PUSH_REGISTRY)", 90 | repository = "$(PUSH_PROJECT)/grpc-httpjson-transcoding-benchmark", 91 | tag = "$(PUSH_TAG)", 92 | ) 93 | 94 | cc_library( 95 | name = "utils", 96 | testonly = 1, 97 | srcs = ["utils.cc"], 98 | hdrs = ["utils.h"], 99 | deps = [ 100 | "benchmark_cc_proto", 101 | "//src:type_helper", 102 | "//test:test_common", 103 | "@com_github_nlohmann_json//:json", 104 | "@com_google_absl//absl/random", 105 | "@com_google_absl//absl/status", 106 | "@com_google_absl//absl/status:statusor", 107 | "@com_google_absl//absl/strings", 108 | "@com_google_googleapis//google/api:service_cc_proto", 109 | "@com_google_protobuf//:protobuf", 110 | ], 111 | ) 112 | 113 | cc_library( 114 | name = "benchmark_input_stream", 115 | testonly = 1, 116 | srcs = ["benchmark_input_stream.cc"], 117 | hdrs = ["benchmark_input_stream.h"], 118 | deps = [ 119 | "//src:transcoder_input_stream", 120 | "@com_google_absl//absl/log:absl_check", 121 | "@com_google_absl//absl/log:absl_log", 122 | "@com_google_absl//absl/strings", 123 | "@com_google_protobuf//:protobuf", 124 | ], 125 | ) 126 | 127 | cc_test( 128 | name = "utils_test", 129 | srcs = [ 130 | "utils_test.cc", 131 | ], 132 | deps = [ 133 | ":utils", 134 | "@com_google_absl//absl/strings", 135 | "@com_google_googletest//:gtest_main", 136 | ], 137 | ) 138 | 139 | cc_test( 140 | name = "benchmark_input_stream_test", 141 | srcs = [ 142 | "benchmark_input_stream_test.cc", 143 | ], 144 | data = [ 145 | "benchmark_service.textproto", 146 | ], 147 | deps = [ 148 | ":benchmark_input_stream", 149 | "@com_google_absl//absl/strings", 150 | "@com_google_googletest//:gtest_main", 151 | ], 152 | ) 153 | 154 | cc_test( 155 | name = "benchmark_input_stream_translator_integration_test", 156 | srcs = [ 157 | "benchmark_input_stream_translator_integration_test.cc", 158 | ], 159 | data = [ 160 | "benchmark_service.textproto", 161 | ], 162 | deps = [ 163 | "com_google_protobuf_struct_cc_proto", 164 | ":benchmark_cc_proto", 165 | ":benchmark_input_stream", 166 | ":utils", 167 | "//src:json_request_translator", 168 | "//src:response_to_json_translator", 169 | "//src:type_helper", 170 | "@com_github_nlohmann_json//:json", 171 | "@com_google_absl//absl/log:absl_check", 172 | "@com_google_googleapis//google/api:service_cc_proto", 173 | "@com_google_googletest//:gtest_main", 174 | ], 175 | ) 176 | -------------------------------------------------------------------------------- /perf_benchmark/benchmark.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | // Test proto for transcoding 18 | syntax = "proto3"; 19 | package google.grpc.transcoding.perf_benchmark; 20 | 21 | import "google/api/annotations.proto"; 22 | import "google/protobuf/struct.proto"; 23 | 24 | service Benchmark { 25 | rpc BytesPayloadBM(BytesPayload) returns (BytesPayload) { 26 | option (google.api.http) = { 27 | post: "/payload/bytes", 28 | body: "*" 29 | }; 30 | } 31 | rpc StringPayloadBM(StringPayload) returns (StringPayload) { 32 | option (google.api.http) = { 33 | post: "/payload/string", 34 | body: "*" 35 | }; 36 | } 37 | rpc Int32ArrayPayloadBM(Int32ArrayPayload) returns (Int32ArrayPayload) { 38 | option (google.api.http) = { 39 | post: "/payload/int32_array", 40 | body: "*" 41 | }; 42 | } 43 | rpc DoubleArrayPayloadBM(DoubleArrayPayload) returns (DoubleArrayPayload) { 44 | option (google.api.http) = { 45 | post: "/payload/double_array", 46 | body: "*" 47 | }; 48 | } 49 | rpc StringArrayPayloadBM(StringArrayPayload) returns (StringArrayPayload) { 50 | option (google.api.http) = { 51 | post: "/payload/string_array", 52 | body: "*" 53 | }; 54 | } 55 | rpc NestedPayloadBM(NestedPayload) returns (NestedPayload) { 56 | option (google.api.http) = { 57 | post: "/payload/nested", 58 | body: "*" 59 | }; 60 | } 61 | rpc StructPayloadBM(google.protobuf.Struct) returns (google.protobuf.Struct) { 62 | option (google.api.http) = { 63 | post: "/payload/struct", 64 | body: "*" 65 | }; 66 | } 67 | rpc MultiStringFieldPayloadBM(MultiStringFieldPayload) 68 | returns (MultiStringFieldPayload) { 69 | option (google.api.http) = { 70 | post: "/payload/multi_string_field", 71 | body: "*" 72 | }; 73 | } 74 | } 75 | 76 | message BytesPayload { 77 | optional bytes payload = 1; 78 | } 79 | 80 | message StringPayload { 81 | optional string payload = 1; 82 | } 83 | 84 | message Int32ArrayPayload { 85 | repeated int32 payload = 1; 86 | } 87 | 88 | message DoubleArrayPayload { 89 | repeated double payload = 1; 90 | } 91 | 92 | message StringArrayPayload { 93 | repeated string payload = 1; 94 | } 95 | 96 | message NestedPayload { 97 | optional NestedPayload nested = 1; 98 | optional string payload = 2; 99 | } 100 | 101 | message MultiStringFieldPayload { 102 | optional string f1 = 1; 103 | optional string f2 = 2; 104 | optional string f3 = 3; 105 | optional string f4 = 4; 106 | optional string f5 = 5; 107 | optional string f6 = 6; 108 | optional string f7 = 7; 109 | optional string f8 = 8; 110 | } 111 | -------------------------------------------------------------------------------- /perf_benchmark/benchmark_input_stream.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | 18 | #include "perf_benchmark/benchmark_input_stream.h" 19 | #include "absl/log/absl_check.h" 20 | 21 | namespace google { 22 | namespace grpc { 23 | namespace transcoding { 24 | 25 | namespace perf_benchmark { 26 | BenchmarkZeroCopyInputStream::BenchmarkZeroCopyInputStream( 27 | std::string json_data, uint64_t num_checks) 28 | : finished_(false), 29 | msg_(std::move(json_data)), 30 | chunk_size_(msg_.size() / num_checks), 31 | pos_(0) { 32 | ABSL_CHECK(num_checks <= msg_.size()); 33 | } 34 | 35 | int64_t BenchmarkZeroCopyInputStream::BytesAvailable() const { 36 | if (finished_) { 37 | return 0; 38 | } 39 | // check if we are at the last chunk 40 | if (pos_ + chunk_size_ >= msg_.size()) { 41 | return msg_.size() - pos_; 42 | } 43 | return chunk_size_; 44 | } 45 | 46 | bool BenchmarkZeroCopyInputStream::Next(const void** data, int* size) { 47 | if (finished_) { 48 | *size = 0; 49 | return false; 50 | } 51 | 52 | *data = msg_.data() + pos_; 53 | if (pos_ + chunk_size_ >= msg_.size()) { // last message to be sent 54 | *size = msg_.size() - pos_; 55 | } else { 56 | *size = chunk_size_; 57 | } 58 | pos_ += *size; 59 | 60 | // Check if we have reached the end. 61 | if (pos_ >= msg_.size()) { 62 | finished_ = true; 63 | } 64 | return true; 65 | } 66 | 67 | void BenchmarkZeroCopyInputStream::Reset() { 68 | finished_ = false; 69 | pos_ = 0; 70 | } 71 | 72 | uint64_t BenchmarkZeroCopyInputStream::TotalBytes() const { 73 | return msg_.size(); 74 | } 75 | 76 | void BenchmarkZeroCopyInputStream::BackUp(int count) { 77 | ABSL_CHECK(count <= pos_); 78 | pos_ -= count; 79 | finished_ = false; 80 | } 81 | 82 | int64_t BenchmarkZeroCopyInputStream::ByteCount() const { return pos_; } 83 | } // namespace perf_benchmark 84 | 85 | } // namespace transcoding 86 | } // namespace grpc 87 | } // namespace google 88 | -------------------------------------------------------------------------------- /perf_benchmark/benchmark_input_stream.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #ifndef PERF_BENCHMARK_BENCHMARK_INPUT_STREAM_H_ 18 | #define PERF_BENCHMARK_BENCHMARK_INPUT_STREAM_H_ 19 | 20 | #include "absl/strings/string_view.h" 21 | #include "absl/log/absl_log.h" 22 | #include "grpc_transcoding/transcoder_input_stream.h" 23 | 24 | namespace google { 25 | namespace grpc { 26 | namespace transcoding { 27 | 28 | namespace perf_benchmark { 29 | // An implementation of ZeroCopyInputStream for benchmarking. 30 | // Subclasses of this should store the entire input message and return pointer 31 | // to the stored message for each round of Next(). This is useful during 32 | // benchmark since the same input message will be read multiple times without 33 | // introducing a large runtime overhead. 34 | // 35 | // For streaming JSON messages, pass in an array of JSON objects as the msg. 36 | // 37 | // After each benchmark iteration, Reset() needs to be called. 38 | class BenchmarkZeroCopyInputStream : public TranscoderInputStream { 39 | public: 40 | // Pre-Conditions: 41 | // - num_checks <= json_data.size() 42 | // 43 | // json_data - a std::string containing the JSON data to be read. 44 | // num_chunks - controls the number of calls to Next() that would yield the 45 | // full JSON message. 46 | // Note: the actual number of checks could be off by a few chunks due to int 47 | // rounding. 48 | explicit BenchmarkZeroCopyInputStream(std::string json_data, 49 | uint64_t num_checks); 50 | ~BenchmarkZeroCopyInputStream() override = default; 51 | 52 | int64_t BytesAvailable() const override; 53 | bool Finished() const override { return finished_; }; 54 | 55 | bool Next(const void** data, int* size) override; 56 | void BackUp(int count) override; 57 | bool Skip(int count) override { 58 | ABSL_LOG(FATAL) << "Not implemented"; 59 | return false; 60 | }; 61 | int64_t ByteCount() const override; 62 | 63 | // Reset the input stream back to the original start state. 64 | // This should be called after one iteration of benchmark. 65 | virtual void Reset(); 66 | 67 | // Return the total number of bytes of the entire JSON message. 68 | virtual uint64_t TotalBytes() const; 69 | 70 | private: 71 | bool finished_; 72 | const std::string msg_; 73 | const uint64_t chunk_size_; 74 | uint64_t pos_; 75 | }; 76 | 77 | } // namespace perf_benchmark 78 | 79 | } // namespace transcoding 80 | } // namespace grpc 81 | } // namespace google 82 | 83 | #endif // PERF_BENCHMARK_BENCHMARK_INPUT_STREAM_H_ 84 | -------------------------------------------------------------------------------- /perf_benchmark/benchmark_input_stream_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "perf_benchmark/benchmark_input_stream.h" 18 | #include "absl/strings/escaping.h" 19 | #include "gtest/gtest.h" 20 | 21 | namespace google { 22 | namespace grpc { 23 | namespace transcoding { 24 | 25 | namespace perf_benchmark { 26 | TEST(BenchmarkInputStreamTest, BenchmarkZeroCopyInputStreamSimple) { 27 | absl::string_view json_msg_input[] = { 28 | R"({"Hello":"World!"})", 29 | R"([{"Hello":"World!"}])", 30 | R"([{"Hello":"World!"},{"Hello":"World, Again!"}])"}; 31 | 32 | for (auto& json_msg : json_msg_input) { 33 | BenchmarkZeroCopyInputStream is(std::string(json_msg), 1); 34 | 35 | // TotalBytes and BytesAvailable should equal to json_msg. 36 | EXPECT_EQ(is.TotalBytes(), json_msg.size()); 37 | EXPECT_EQ(is.BytesAvailable(), json_msg.size()); 38 | // Stream should not be finished. 39 | EXPECT_FALSE(is.Finished()); 40 | 41 | // Reading data. 42 | const void* data = nullptr; 43 | int size; 44 | is.Next(&data, &size); 45 | EXPECT_EQ(size, json_msg.size()); 46 | EXPECT_EQ(std::string(static_cast(data)), json_msg); 47 | 48 | // Stream should be finished 49 | EXPECT_TRUE(is.Finished()); 50 | 51 | // Reset should reset everything as if Next() is not called. 52 | is.Reset(); 53 | EXPECT_EQ(is.TotalBytes(), json_msg.size()); 54 | EXPECT_EQ(is.BytesAvailable(), json_msg.size()); 55 | EXPECT_FALSE(is.Finished()); 56 | } 57 | } 58 | 59 | TEST(BenchmarkInputStreamTest, BenchmarkZeroCopyInputStreamChunk) { 60 | absl::string_view json_msg = R"({"Hello":"World!"})"; 61 | const uint64_t num_checks_input[] = {1, 2, 4, json_msg.size() - 1, 62 | json_msg.size()}; 63 | 64 | for (uint64_t num_checks : num_checks_input) { 65 | BenchmarkZeroCopyInputStream is(std::string(json_msg), num_checks); 66 | uint64_t expected_chunk_size = json_msg.size() / num_checks; 67 | 68 | // Reading data. 69 | const void* data = nullptr; 70 | int size; 71 | uint64_t total_bytes_read = 0; 72 | std::string str_read; 73 | while (!is.Finished()) { 74 | // TotalBytes should equal to json_msg. 75 | EXPECT_EQ(is.TotalBytes(), json_msg.size()); 76 | if (json_msg.size() - total_bytes_read >= expected_chunk_size) { 77 | // BytesAvailable should equal to the chunk size unless we are reading 78 | // the last message. 79 | EXPECT_EQ(is.BytesAvailable(), expected_chunk_size); 80 | } 81 | 82 | is.Next(&data, &size); 83 | total_bytes_read += size; 84 | str_read += std::string(static_cast(data), size); 85 | 86 | if (json_msg.size() - total_bytes_read >= expected_chunk_size) { 87 | // size should equal to the expected_chunk_size unless it's the last 88 | // message. 89 | EXPECT_EQ(size, expected_chunk_size); 90 | } 91 | if (total_bytes_read == json_msg.size()) { 92 | // Stream should be finished 93 | EXPECT_TRUE(is.Finished()); 94 | } 95 | } 96 | EXPECT_EQ(total_bytes_read, json_msg.size()); 97 | EXPECT_EQ(str_read, json_msg); 98 | 99 | // Reset should reset everything as if Next() is not called. 100 | is.Reset(); 101 | EXPECT_EQ(is.TotalBytes(), json_msg.size()); 102 | EXPECT_EQ(is.BytesAvailable(), expected_chunk_size); 103 | EXPECT_FALSE(is.Finished()); 104 | } 105 | } 106 | 107 | } // namespace perf_benchmark 108 | 109 | } // namespace transcoding 110 | } // namespace grpc 111 | } // namespace google 112 | -------------------------------------------------------------------------------- /perf_benchmark/benchmark_service.textproto: -------------------------------------------------------------------------------- 1 | name: "grpc-httpjson-transcoding-benchmark.googleapis.com" 2 | apis { 3 | methods { 4 | name: "BytesPayloadBM" 5 | request_type_url: "type.googleapis.com/BytesPayload" 6 | response_type_url: "type.googleapis.com/BytesPayload" 7 | } 8 | methods { 9 | name: "StringPayloadBM" 10 | request_type_url: "type.googleapis.com/StringPayload" 11 | response_type_url: "type.googleapis.com/StringPayload" 12 | } 13 | methods { 14 | name: "Int32ArrayPayloadBM" 15 | request_type_url: "type.googleapis.com/Int32ArrayPayload" 16 | response_type_url: "type.googleapis.com/Int32ArrayPayload" 17 | } 18 | methods { 19 | name: "DoubleArrayPayloadBM" 20 | request_type_url: "type.googleapis.com/DoubleArrayPayload" 21 | response_type_url: "type.googleapis.com/DoubleArrayPayload" 22 | } 23 | methods { 24 | name: "StringArrayPayloadBM" 25 | request_type_url: "type.googleapis.com/StringArrayPayload" 26 | response_type_url: "type.googleapis.com/StringArrayPayload" 27 | } 28 | methods { 29 | name: "NestedPayloadBM" 30 | request_type_url: "type.googleapis.com/NestedPayload" 31 | response_type_url: "type.googleapis.com/NestedPayload" 32 | } 33 | methods { 34 | name: "StructPayloadBM" 35 | request_type_url: "type.googleapis.com/google.protobuf.Struct" 36 | response_type_url: "type.googleapis.com/google.protobuf.Struct" 37 | } 38 | methods { 39 | name: "MultiStringFieldPayloadBM" 40 | request_type_url: "type.googleapis.com/MultiStringFieldPayload" 41 | response_type_url: "type.googleapis.com/MultiStringFieldPayload" 42 | } 43 | } 44 | types { 45 | name: "BytesPayload" 46 | fields { 47 | kind: TYPE_BYTES 48 | cardinality: CARDINALITY_OPTIONAL 49 | number: 1 50 | name: "payload" 51 | json_name: "payload" 52 | } 53 | source_context { 54 | } 55 | } 56 | types { 57 | name: "StringPayload" 58 | fields { 59 | kind: TYPE_STRING 60 | cardinality: CARDINALITY_OPTIONAL 61 | number: 1 62 | name: "payload" 63 | json_name: "payload" 64 | } 65 | source_context { 66 | } 67 | } 68 | types { 69 | name: "Int32ArrayPayload" 70 | fields { 71 | kind: TYPE_INT32 72 | cardinality: CARDINALITY_REPEATED 73 | number: 1 74 | name: "payload" 75 | json_name: "payload" 76 | } 77 | source_context { 78 | } 79 | } 80 | types { 81 | name: "DoubleArrayPayload" 82 | fields { 83 | kind: TYPE_DOUBLE 84 | cardinality: CARDINALITY_REPEATED 85 | number: 1 86 | name: "payload" 87 | json_name: "payload" 88 | } 89 | source_context { 90 | } 91 | } 92 | types { 93 | name: "StringArrayPayload" 94 | fields { 95 | kind: TYPE_STRING 96 | cardinality: CARDINALITY_REPEATED 97 | number: 1 98 | name: "payload" 99 | json_name: "payload" 100 | } 101 | source_context { 102 | } 103 | } 104 | types { 105 | name: "NestedPayload" 106 | fields { 107 | kind: TYPE_MESSAGE 108 | cardinality: CARDINALITY_OPTIONAL 109 | number: 1 110 | name: "nested" 111 | type_url: "type.googleapis.com/NestedPayload" 112 | json_name: "nested" 113 | } 114 | fields { 115 | kind: TYPE_STRING 116 | cardinality: CARDINALITY_OPTIONAL 117 | number: 2 118 | name: "payload" 119 | json_name: "payload" 120 | } 121 | source_context { 122 | } 123 | } 124 | types { 125 | name: "MultiStringFieldPayload" 126 | fields { 127 | kind: TYPE_STRING 128 | cardinality: CARDINALITY_OPTIONAL 129 | number: 1 130 | name: "f1" 131 | json_name: "f1" 132 | } 133 | fields { 134 | kind: TYPE_STRING 135 | cardinality: CARDINALITY_OPTIONAL 136 | number: 2 137 | name: "f2" 138 | json_name: "f2" 139 | } 140 | fields { 141 | kind: TYPE_STRING 142 | cardinality: CARDINALITY_OPTIONAL 143 | number: 3 144 | name: "f3" 145 | json_name: "f3" 146 | } 147 | fields { 148 | kind: TYPE_STRING 149 | cardinality: CARDINALITY_OPTIONAL 150 | number: 4 151 | name: "f4" 152 | json_name: "f4" 153 | } 154 | fields { 155 | kind: TYPE_STRING 156 | cardinality: CARDINALITY_OPTIONAL 157 | number: 5 158 | name: "f5" 159 | json_name: "f5" 160 | } 161 | fields { 162 | kind: TYPE_STRING 163 | cardinality: CARDINALITY_OPTIONAL 164 | number: 6 165 | name: "f6" 166 | json_name: "f6" 167 | } 168 | fields { 169 | kind: TYPE_STRING 170 | cardinality: CARDINALITY_OPTIONAL 171 | number: 7 172 | name: "f7" 173 | json_name: "f7" 174 | } 175 | fields { 176 | kind: TYPE_STRING 177 | cardinality: CARDINALITY_OPTIONAL 178 | number: 8 179 | name: "f8" 180 | json_name: "f8" 181 | } 182 | source_context { 183 | } 184 | } 185 | types { 186 | name: "google.protobuf.ListValue" 187 | fields { 188 | kind: TYPE_MESSAGE 189 | cardinality: CARDINALITY_REPEATED 190 | number: 1 191 | name: "values" 192 | type_url: "type.googleapis.com/google.protobuf.Value" 193 | json_name: "values" 194 | } 195 | source_context { 196 | file_name: "struct.proto" 197 | } 198 | } 199 | types { 200 | name: "google.protobuf.Struct" 201 | fields { 202 | kind: TYPE_MESSAGE 203 | cardinality: CARDINALITY_REPEATED 204 | number: 1 205 | name: "fields" 206 | type_url: "type.googleapis.com/google.protobuf.Struct.FieldsEntry" 207 | json_name: "fields" 208 | } 209 | source_context { 210 | file_name: "struct.proto" 211 | } 212 | } 213 | types { 214 | name: "google.protobuf.Struct.FieldsEntry" 215 | fields { 216 | kind: TYPE_STRING 217 | cardinality: CARDINALITY_OPTIONAL 218 | number: 1 219 | name: "key" 220 | json_name: "key" 221 | } 222 | fields { 223 | kind: TYPE_MESSAGE 224 | cardinality: CARDINALITY_OPTIONAL 225 | number: 2 226 | name: "value" 227 | type_url: "type.googleapis.com/google.protobuf.Value" 228 | json_name: "value" 229 | } 230 | options { 231 | name: "map_entry" 232 | value { 233 | type_url: "type.googleapis.com/google.protobuf.BoolValue" 234 | value: "" 235 | } 236 | } 237 | source_context { 238 | file_name: "struct.proto" 239 | } 240 | } 241 | types { 242 | name: "google.protobuf.Empty" 243 | source_context { 244 | file_name: "struct.proto" 245 | } 246 | } 247 | types { 248 | name: "google.protobuf.Value" 249 | fields { 250 | kind: TYPE_ENUM 251 | cardinality: CARDINALITY_OPTIONAL 252 | number: 1 253 | name: "null_value" 254 | type_url: "type.googleapis.com/google.protobuf.NullValue" 255 | json_name: "nullValue" 256 | } 257 | fields { 258 | kind: TYPE_DOUBLE 259 | cardinality: CARDINALITY_OPTIONAL 260 | number: 2 261 | name: "number_value" 262 | json_name: "numberValue" 263 | } 264 | fields { 265 | kind: TYPE_STRING 266 | cardinality: CARDINALITY_OPTIONAL 267 | number: 3 268 | name: "string_value" 269 | json_name: "stringValue" 270 | } 271 | fields { 272 | kind: TYPE_BOOL 273 | cardinality: CARDINALITY_OPTIONAL 274 | number: 4 275 | name: "bool_value" 276 | json_name: "boolValue" 277 | } 278 | fields { 279 | kind: TYPE_MESSAGE 280 | cardinality: CARDINALITY_OPTIONAL 281 | number: 5 282 | name: "struct_value" 283 | type_url: "type.googleapis.com/google.protobuf.Struct" 284 | json_name: "structValue" 285 | } 286 | fields { 287 | kind: TYPE_MESSAGE 288 | cardinality: CARDINALITY_OPTIONAL 289 | number: 6 290 | name: "list_value" 291 | type_url: "type.googleapis.com/google.protobuf.ListValue" 292 | json_name: "listValue" 293 | } 294 | source_context { 295 | file_name: "struct.proto" 296 | } 297 | } 298 | types { 299 | name: "google.protobuf.BoolValue" 300 | fields { 301 | kind: TYPE_BOOL 302 | cardinality: CARDINALITY_REQUIRED 303 | number: 1 304 | name: "value" 305 | json_name: "value" 306 | } 307 | source_context { 308 | file_name: "wrappers.proto" 309 | } 310 | } 311 | enums { 312 | name: "google.protobuf.NullValue" 313 | enumvalue { 314 | name: "NULL_VALUE" 315 | } 316 | source_context { 317 | file_name: "struct.proto" 318 | } 319 | } 320 | http { 321 | rules { 322 | selector: "BytesPayload" 323 | post: "/payload/bytes" 324 | body: "*" 325 | } 326 | rules { 327 | selector: "StringPayload" 328 | post: "/payload/string" 329 | body: "*" 330 | } 331 | rules { 332 | selector: "Int32ArrayPayload" 333 | post: "/payload/int32_array" 334 | body: "*" 335 | } 336 | rules { 337 | selector: "DoubleArrayPayload" 338 | post: "/payload/double_array" 339 | body: "*" 340 | } 341 | rules { 342 | selector: "StringArrayPayloadBM" 343 | post: "/payload/string_array" 344 | body: "*" 345 | } 346 | rules { 347 | selector: "NestedPayloadBM" 348 | post: "/payload/nested" 349 | body: "*" 350 | } 351 | rules { 352 | selector: "StructPayloadBM" 353 | post: "/payload/struct" 354 | body: "*" 355 | } 356 | } -------------------------------------------------------------------------------- /perf_benchmark/image/array_length.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grpc-ecosystem/grpc-httpjson-transcoding/a6e226f9a2e656a973df3ad48f0ee5efacce1a28/perf_benchmark/image/array_length.jpg -------------------------------------------------------------------------------- /perf_benchmark/image/body_length.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grpc-ecosystem/grpc-httpjson-transcoding/a6e226f9a2e656a973df3ad48f0ee5efacce1a28/perf_benchmark/image/body_length.jpg -------------------------------------------------------------------------------- /perf_benchmark/image/nested_layers.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grpc-ecosystem/grpc-httpjson-transcoding/a6e226f9a2e656a973df3ad48f0ee5efacce1a28/perf_benchmark/image/nested_layers.jpg -------------------------------------------------------------------------------- /perf_benchmark/image/num_message_segment.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grpc-ecosystem/grpc-httpjson-transcoding/a6e226f9a2e656a973df3ad48f0ee5efacce1a28/perf_benchmark/image/num_message_segment.jpg -------------------------------------------------------------------------------- /perf_benchmark/image/num_variable_bindings.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grpc-ecosystem/grpc-httpjson-transcoding/a6e226f9a2e656a973df3ad48f0ee5efacce1a28/perf_benchmark/image/num_variable_bindings.jpg -------------------------------------------------------------------------------- /perf_benchmark/image/value_data_type.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grpc-ecosystem/grpc-httpjson-transcoding/a6e226f9a2e656a973df3ad48f0ee5efacce1a28/perf_benchmark/image/value_data_type.png -------------------------------------------------------------------------------- /perf_benchmark/image/variable_binding_depth.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grpc-ecosystem/grpc-httpjson-transcoding/a6e226f9a2e656a973df3ad48f0ee5efacce1a28/perf_benchmark/image/variable_binding_depth.jpg -------------------------------------------------------------------------------- /perf_benchmark/utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #ifndef PERF_BENCHMARK_UTILS_H_ 18 | #define PERF_BENCHMARK_UTILS_H_ 19 | 20 | #include 21 | 22 | #include "absl/status/status.h" 23 | #include "absl/strings/string_view.h" 24 | #include "google/api/service.pb.h" 25 | #include "google/protobuf/util/converter/type_info.h" 26 | #include "perf_benchmark/benchmark.pb.h" 27 | #include "src/include/grpc_transcoding/type_helper.h" 28 | 29 | namespace google { 30 | namespace grpc { 31 | namespace transcoding { 32 | 33 | namespace perf_benchmark { 34 | 35 | // Load service from a proto text file. Returns true if loading succeeds; 36 | // otherwise returns false. 37 | absl::Status LoadService(absl::string_view config_pb_txt_file, 38 | absl::string_view benchmark_path, 39 | ::google::api::Service* service); 40 | absl::Status LoadService(absl::string_view config_pb_txt_file, 41 | ::google::api::Service* service); 42 | 43 | // Return the given percentile of the vector v. 44 | double GetPercentile(const std::vector& v, double perc); 45 | 46 | // This method is not thread-safe since it uses a shared absl::BitGen. 47 | // Return a random string of the given length. 48 | // length - Length of the returned string. If base64 == true, the actual 49 | // returned string length is 33–37% larger due to the encoding. 50 | // base64 - True if the returned string should be base64 encoded. This is 51 | // required for bytes proto message. 52 | std::string GetRandomBytesString(uint64_t length, bool base64); 53 | 54 | // This method is not thread-safe since it uses a shared absl::BitGen. 55 | // Return a random alphanumeric string of the given length. 56 | // length - Length of the returned string. 57 | std::string GetRandomAlphanumericString(uint64_t length); 58 | 59 | // This method is not thread-safe since it uses a shared absl::BitGen. 60 | // Return a random string representing an array of int32, e.g. "[1,2,3]" 61 | // length - Length of the integer array. 62 | std::string GetRandomInt32ArrayString(uint64_t length); 63 | 64 | // Return an array string of the given length with repeated values, 65 | // e.g. "[0, 0, 0]" for GetRepeatedValueArrayString("0", 3). 66 | // val - Unescaped string value to be put in the array. 67 | // length - Length of the integer array. 68 | std::string GetRepeatedValueArrayString(absl::string_view val, uint64_t length); 69 | 70 | // Return a nested JSON string with the innermost value being a payload string, 71 | // e.g. "{"nested": {"nested": {"inner_key": "inner_val"}}}" 72 | // layers - Number of nested layer. The value needs >= 0. 0 is a flat JSON. 73 | // nested_field_name - JSON key name for the nested field. 74 | // inner_key - Field name for the innermost json field. 75 | // payload_msg - String value for the innermost json field. 76 | std::string GetNestedJsonString(uint64_t layers, 77 | absl::string_view nested_field_name, 78 | absl::string_view inner_key, 79 | absl::string_view inner_val); 80 | 81 | // Return an HTTP/JSON string that corresponds to gRPC streaming message. 82 | // This is essentially wrapping the json_msg repetitively around a JSON array. 83 | // for stream_size == 1 -> "[json_msg]" 84 | // for stream_size > 1 -> "[json_msg,...,json_msg]" 85 | std::string GetStreamedJson(absl::string_view json_msg, uint64_t stream_size); 86 | 87 | // Prefix the binary with a size to delimiter data segment and return. 88 | std::string WrapGrpcMessageWithDelimiter(absl::string_view proto_binary); 89 | 90 | // Return a unique_ptr to a NestedPayload object having the given `layers`. 91 | std::unique_ptr GetNestedPayload(uint64_t layers, 92 | absl::string_view inner_val); 93 | 94 | // Return a unique_ptr to a ::google::protobuf::Struct object. 95 | std::unique_ptr<::google::protobuf::Struct> GetNestedStructPayload( 96 | uint64_t layers, absl::string_view nested_field_name, 97 | absl::string_view inner_key, absl::string_view inner_val); 98 | 99 | // Parse a dot delimited field path string into a vector of actual field 100 | // pointers. 101 | std::vector ParseFieldPath( 102 | const TypeHelper& type_helper, absl::string_view msg_type, 103 | const std::string& field_path_str); 104 | 105 | // Generate a JSON string corresponds to MultiStringFieldMessage. 106 | // For the 8 fields in the message, we will fill in the first 107 | // `num_fields_exist` 108 | // number of fields with the given `val`. 109 | std::string GenerateMultiStringFieldPayloadJsonStr( 110 | uint64_t num_fields_exist, absl::string_view field_prefix, 111 | absl::string_view val); 112 | 113 | } // namespace perf_benchmark 114 | 115 | } // namespace transcoding 116 | } // namespace grpc 117 | } // namespace google 118 | 119 | // Macros 120 | 121 | // Macro for running a benchmark with p25, p75, p90, p99, p999 percentiles. 122 | // Other statistics - mean, median, standard deviation, coefficient of variation 123 | // are automatically captured. 124 | // Note that running with 1000 iterations only gives 1 data point. Therefore, 125 | // it is recommended to run with --benchmark_repetitions=1000 CLI argument to 126 | // get comparable results. 127 | // Use this marco the same way as BENCHMARK macro. 128 | #define BENCHMARK_WITH_PERCENTILE(func) \ 129 | BENCHMARK(func) \ 130 | ->ComputeStatistics("p25", \ 131 | [](const std::vector& v) -> double { \ 132 | return GetPercentile(v, 25); \ 133 | }) \ 134 | ->ComputeStatistics("p75", \ 135 | [](const std::vector& v) -> double { \ 136 | return GetPercentile(v, 75); \ 137 | }) \ 138 | ->ComputeStatistics("p90", \ 139 | [](const std::vector& v) -> double { \ 140 | return GetPercentile(v, 90); \ 141 | }) \ 142 | ->ComputeStatistics("p99", \ 143 | [](const std::vector& v) -> double { \ 144 | return GetPercentile(v, 99); \ 145 | }) \ 146 | ->ComputeStatistics("p999", [](const std::vector& v) -> double { \ 147 | return GetPercentile(v, 99.9); \ 148 | }) 149 | 150 | #define BENCHMARK_STREAMING_WITH_PERCENTILE(func) \ 151 | BENCHMARK_WITH_PERCENTILE(func)->Arg(1)->Arg(1 << 2)->Arg(1 << 4)->Arg(1 << 6) 152 | 153 | #endif // PERF_BENCHMARK_UTILS_H_ 154 | -------------------------------------------------------------------------------- /perf_benchmark/utils_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "perf_benchmark/utils.h" 18 | #include 19 | #include "absl/strings/ascii.h" 20 | #include "absl/strings/escaping.h" 21 | #include "absl/strings/str_split.h" 22 | #include "gtest/gtest.h" 23 | 24 | namespace google { 25 | namespace grpc { 26 | namespace transcoding { 27 | 28 | namespace perf_benchmark { 29 | 30 | TEST(UtilsTest, GetRandomBytesStringLength) { 31 | const int test_length_input[] = {0, 1, 10, 100}; 32 | for (auto length : test_length_input) { 33 | // Regular random string 34 | EXPECT_EQ(GetRandomBytesString(length, false).length(), length); 35 | 36 | // Base64 encoded random string should have the given length after decoding 37 | std::string decoded; 38 | absl::Base64Unescape(GetRandomBytesString(length, true), &decoded); 39 | EXPECT_EQ(decoded.length(), length); 40 | } 41 | } 42 | 43 | TEST(UtilsTest, GetPercentile) { 44 | // Fill in an array of 0 to 99 45 | std::vector arr; 46 | int arr_length = 100; 47 | for (int i = 0; i < arr_length; ++i) { 48 | arr.push_back(double(i)); 49 | } 50 | 51 | // i^th percentile should equal to i 52 | for (int i = 0; i < arr.size(); ++i) { 53 | EXPECT_EQ(GetPercentile(arr, double(i)), double(i)); 54 | } 55 | 56 | // p999 should get the largest value 57 | EXPECT_EQ(GetPercentile(arr, 99.9), 99.0); 58 | } 59 | 60 | TEST(UtilsTest, GetRandomAlphanumericString) { 61 | for (auto ch : GetRandomAlphanumericString(100)) { 62 | std::cout << ch << std::endl; 63 | EXPECT_TRUE(absl::ascii_isalnum(ch)); 64 | } 65 | } 66 | 67 | TEST(UtilsTest, GetRandomAlphanumericStringLength) { 68 | const int test_length_input[] = {0, 1, 10, 100}; 69 | for (auto length : test_length_input) { 70 | EXPECT_EQ(GetRandomAlphanumericString(length).length(), length); 71 | } 72 | } 73 | 74 | TEST(UtilsTest, GetRandomInt32ArrayString) { 75 | const int test_length_input[] = {0, 1, 10, 100}; 76 | for (auto length : test_length_input) { 77 | std::string res = GetRandomInt32ArrayString(length); 78 | EXPECT_EQ(res.front(), '['); 79 | EXPECT_EQ(res.back(), ']'); 80 | 81 | // Verify length 82 | std::vector split = 83 | absl::StrSplit(res.substr(1, res.size() - 2), ','); 84 | if (!split.empty() && split.at(0) != "") { // if a delimiter is found 85 | EXPECT_EQ(split.size(), length); 86 | } 87 | } 88 | } 89 | 90 | TEST(UtilsTest, GetRepeatedValueArrayString) { 91 | const int test_length_input[] = {0, 1, 10, 100}; 92 | absl::string_view test_val = "TEST"; 93 | absl::string_view expected_json_val = R"("TEST")"; 94 | for (auto length : test_length_input) { 95 | std::string res = GetRepeatedValueArrayString(test_val, length); 96 | EXPECT_EQ(res.front(), '['); 97 | EXPECT_EQ(res.back(), ']'); 98 | 99 | // Verify length 100 | std::vector split = 101 | absl::StrSplit(res.substr(1, res.size() - 2), ','); 102 | if (split.at(0) != "") { // if a delimiter is found 103 | EXPECT_EQ(split.size(), length); 104 | for (const auto& s : split) { 105 | EXPECT_EQ(expected_json_val, s); 106 | } 107 | } 108 | } 109 | } 110 | 111 | TEST(UtilsTest, GetNestedJsonStringZeroLayer) { 112 | EXPECT_EQ( 113 | R"({"inner_val":"inner_key"})", 114 | GetNestedJsonString(0, "doesnt_matter", "inner_val", "inner_key")); 115 | } 116 | 117 | TEST(UtilsTest, GetNestedJsonStringMultiLayers) { 118 | EXPECT_EQ( 119 | R"({"nested_field_name":{"inner_val":"inner_key"}})", 120 | GetNestedJsonString(1, "nested_field_name", "inner_val", "inner_key")); 121 | EXPECT_EQ( 122 | R"({"nested_field_name":{"nested_field_name":{"inner_val":"inner_key"}}})", 123 | GetNestedJsonString(2, "nested_field_name", "inner_val", "inner_key")); 124 | } 125 | 126 | TEST(UtilsTest, GetNestedPayload) { 127 | std::string payload = "Hello World!"; 128 | for (uint64_t num_layers : {0, 5, 50, 100}) { 129 | std::unique_ptr proto = 130 | GetNestedPayload(num_layers, payload); 131 | uint64_t counter = 0; 132 | const NestedPayload* it = proto.get(); 133 | while (it->has_nested()) { 134 | ++counter; 135 | it = &it->nested(); 136 | } 137 | EXPECT_EQ(it->payload(), payload); 138 | EXPECT_EQ(counter, num_layers); 139 | } 140 | } 141 | 142 | TEST(UtilsTest, GetNestedStructPayload) { 143 | std::string inner_val = "Hello World!"; 144 | for (uint64_t num_layers : {0, 5, 50, 100}) { 145 | std::unique_ptr<::google::protobuf::Struct> proto = 146 | GetNestedStructPayload(num_layers, "nested", "payload", inner_val); 147 | uint64_t counter = 0; 148 | const ::google::protobuf::Struct* it = proto.get(); 149 | while (it->fields().contains("nested")) { 150 | ++counter; 151 | it = &it->fields().at("nested").struct_value(); 152 | } 153 | EXPECT_EQ(it->fields().at("payload").string_value(), inner_val); 154 | EXPECT_EQ(counter, num_layers); 155 | } 156 | } 157 | 158 | } // namespace perf_benchmark 159 | 160 | } // namespace transcoding 161 | } // namespace grpc 162 | } // namespace google 163 | -------------------------------------------------------------------------------- /repositories.bzl: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | ################################################################################ 16 | # 17 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 18 | load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") 19 | 20 | def absl_repositories(bind = True): 21 | http_archive( 22 | name = "com_google_absl", 23 | sha256 = "ea1d31db00eb37e607bfda17ffac09064670ddf05da067944c4766f517876390", 24 | strip_prefix = "abseil-cpp-c2435f8342c2d0ed8101cb43adfd605fdc52dca2", # May 04, 2023. 25 | urls = ["https://github.com/abseil/abseil-cpp/archive/c2435f8342c2d0ed8101cb43adfd605fdc52dca2.zip"], 26 | ) 27 | 28 | PROTOBUF_COMMIT = "315ffb5be89460f2857387d20aefc59b76b8bdc3" # May 31, 2023 29 | PROTOBUF_SHA256 = "aa61db6ff113a1c76eac9408144c6e996c5e2d6b2410818fd7f1b0d222a50bf8" 30 | 31 | def protobuf_repositories(bind = True): 32 | http_archive( 33 | name = "com_google_protobuf", 34 | strip_prefix = "protobuf-" + PROTOBUF_COMMIT, 35 | urls = [ 36 | "https://github.com/google/protobuf/archive/" + PROTOBUF_COMMIT + ".tar.gz", 37 | ], 38 | sha256 = PROTOBUF_SHA256, 39 | ) 40 | 41 | GOOGLETEST_COMMIT = "f8d7d77c06936315286eb55f8de22cd23c188571" # v1.14.0: Aug 2, 2023 42 | GOOGLETEST_SHA256 = "7ff5db23de232a39cbb5c9f5143c355885e30ac596161a6b9fc50c4538bfbf01" 43 | 44 | def googletest_repositories(bind = True): 45 | http_archive( 46 | name = "com_google_googletest", 47 | strip_prefix = "googletest-" + GOOGLETEST_COMMIT, 48 | url = "https://github.com/google/googletest/archive/" + GOOGLETEST_COMMIT + ".tar.gz", 49 | sha256 = GOOGLETEST_SHA256, 50 | ) 51 | 52 | GOOGLEAPIS_COMMIT = "1d5522ad1056f16a6d593b8f3038d831e64daeea" # Sept 03, 2020 53 | GOOGLEAPIS_SHA256 = "cd13e547cffaad217c942084fd5ae0985a293d0cce3e788c20796e5e2ea54758" 54 | 55 | def googleapis_repositories(bind = True): 56 | http_archive( 57 | name = "com_google_googleapis", 58 | strip_prefix = "googleapis-" + GOOGLEAPIS_COMMIT, 59 | url = "https://github.com/googleapis/googleapis/archive/" + GOOGLEAPIS_COMMIT + ".tar.gz", 60 | sha256 = GOOGLEAPIS_SHA256, 61 | ) 62 | 63 | GOOGLEBENCHMARK_COMMIT = "1.7.0" # Jul 25, 2022 64 | GOOGLEBENCHMARK_SHA256 = "3aff99169fa8bdee356eaa1f691e835a6e57b1efeadb8a0f9f228531158246ac" 65 | 66 | def googlebenchmark_repositories(bind = True): 67 | http_archive( 68 | name = "com_google_benchmark", 69 | strip_prefix = "benchmark-" + GOOGLEBENCHMARK_COMMIT, 70 | url = "https://github.com/google/benchmark/archive/v" + GOOGLEBENCHMARK_COMMIT + ".tar.gz", 71 | sha256 = GOOGLEBENCHMARK_SHA256, 72 | ) 73 | 74 | def nlohmannjson_repositories(bind = True): 75 | http_archive( 76 | name = "com_github_nlohmann_json", 77 | strip_prefix = "json-3.11.3", 78 | urls = ["https://github.com/nlohmann/json/archive/v3.11.3.tar.gz"], 79 | sha256 = "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406", 80 | ) 81 | 82 | RULES_DOCKER_COMMIT = "0.25.0" # Jun 22, 2022 83 | RULES_DOCKER_SHA256 = "b1e80761a8a8243d03ebca8845e9cc1ba6c82ce7c5179ce2b295cd36f7e394bf" 84 | 85 | def io_bazel_rules_docker(bind = True): 86 | http_archive( 87 | name = "io_bazel_rules_docker", 88 | sha256 = RULES_DOCKER_SHA256, 89 | urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v" + RULES_DOCKER_COMMIT + "/rules_docker-v" + RULES_DOCKER_COMMIT + ".tar.gz"], 90 | ) 91 | 92 | def protoconverter_repositories(bind = True): 93 | http_archive( 94 | name = "com_google_protoconverter", 95 | sha256 = "6081836fa3838ebb1aa15089a5c3e20f877a0244c7a39b92a2000efb40408dcb", 96 | strip_prefix = "proto-converter-d77ff301f48bf2e7a0f8935315e847c1a8e00017", 97 | urls = ["https://github.com/grpc-ecosystem/proto-converter/archive/d77ff301f48bf2e7a0f8935315e847c1a8e00017.zip"], 98 | ) 99 | -------------------------------------------------------------------------------- /script/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2021 Google LLC. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ################################################################################ 18 | 19 | # TODO: Remove these two lines after getting rid of python2 dependencies. 20 | apt-get upgrade 21 | apt-get -y install python 22 | bazelisk build //... 23 | bazelisk test //... --test_output=errors 24 | -------------------------------------------------------------------------------- /src/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | ################################################################################ 16 | # 17 | package(default_visibility = ["//visibility:public"]) 18 | 19 | cc_library( 20 | name = "prefix_writer", 21 | srcs = [ 22 | "prefix_writer.cc", 23 | ], 24 | hdrs = [ 25 | "include/grpc_transcoding/prefix_writer.h", 26 | ], 27 | includes = [ 28 | "include/", 29 | ], 30 | deps = [ 31 | "@com_google_absl//absl/strings", 32 | "@com_google_protobuf//:protobuf", 33 | "@com_google_protoconverter//:all", 34 | ], 35 | ) 36 | 37 | cc_library( 38 | name = "request_weaver", 39 | srcs = [ 40 | "request_weaver.cc", 41 | ], 42 | hdrs = [ 43 | "include/grpc_transcoding/request_weaver.h", 44 | ], 45 | includes = [ 46 | "include/", 47 | ], 48 | deps = [ 49 | ":status_error_listener", 50 | "@com_google_absl//absl/strings", 51 | "@com_google_absl//absl/strings:str_format", 52 | "@com_google_protobuf//:protobuf", 53 | "@com_google_protoconverter//:all", 54 | ], 55 | ) 56 | 57 | cc_library( 58 | name = "message_stream", 59 | srcs = [ 60 | "message_stream.cc", 61 | ], 62 | hdrs = [ 63 | "include/grpc_transcoding/message_stream.h", 64 | ], 65 | includes = [ 66 | "include/", 67 | ], 68 | deps = [ 69 | ":transcoder_input_stream", 70 | "@com_google_absl//absl/status", 71 | "@com_google_protobuf//:protobuf", 72 | ], 73 | ) 74 | 75 | cc_library( 76 | name = "request_message_translator", 77 | srcs = [ 78 | "request_message_translator.cc", 79 | ], 80 | hdrs = [ 81 | "include/grpc_transcoding/request_message_translator.h", 82 | ], 83 | includes = [ 84 | "include/", 85 | ], 86 | deps = [ 87 | ":message_stream", 88 | ":prefix_writer", 89 | ":request_weaver", 90 | "@com_google_absl//absl/strings", 91 | "@com_google_protobuf//:protobuf", 92 | "@com_google_protoconverter//:all", 93 | ], 94 | ) 95 | 96 | cc_library( 97 | name = "request_stream_translator", 98 | srcs = [ 99 | "request_stream_translator.cc", 100 | ], 101 | hdrs = [ 102 | "include/grpc_transcoding/request_stream_translator.h", 103 | ], 104 | includes = [ 105 | "include/", 106 | ], 107 | deps = [ 108 | ":request_message_translator", 109 | "@com_google_protobuf//:protobuf", 110 | ], 111 | ) 112 | 113 | cc_library( 114 | name = "percent_encoding_lib", 115 | hdrs = [ 116 | "include/grpc_transcoding/percent_encoding.h", 117 | ], 118 | includes = [ 119 | "include/", 120 | ], 121 | deps = [ 122 | "@com_google_absl//absl/strings", 123 | ], 124 | ) 125 | 126 | cc_library( 127 | name = "path_matcher", 128 | srcs = [ 129 | "include/grpc_transcoding/path_matcher_node.h", 130 | "path_matcher_node.cc", 131 | ], 132 | hdrs = [ 133 | "include/grpc_transcoding/path_matcher.h", 134 | ], 135 | includes = [ 136 | "include/", 137 | ], 138 | deps = [ 139 | ":http_template", 140 | ":percent_encoding_lib", 141 | ], 142 | ) 143 | 144 | cc_library( 145 | name = "http_template", 146 | srcs = [ 147 | "http_template.cc", 148 | ], 149 | hdrs = [ 150 | "include/grpc_transcoding/http_template.h", 151 | ], 152 | includes = [ 153 | "include/", 154 | ], 155 | ) 156 | 157 | cc_library( 158 | name = "path_matcher_utility", 159 | hdrs = [ 160 | "include/grpc_transcoding/path_matcher_utility.h", 161 | ], 162 | includes = [ 163 | "include/", 164 | ], 165 | deps = [ 166 | ":path_matcher", 167 | "@com_google_googleapis//google/api:http_cc_proto", 168 | ], 169 | ) 170 | 171 | cc_library( 172 | name = "json_request_translator", 173 | srcs = [ 174 | "json_request_translator.cc", 175 | ], 176 | hdrs = [ 177 | "include/grpc_transcoding/json_request_translator.h", 178 | ], 179 | includes = [ 180 | "include/", 181 | ], 182 | deps = [ 183 | ":request_message_translator", 184 | ":request_stream_translator", 185 | "@com_google_absl//absl/strings", 186 | "@com_google_protobuf//:protobuf", 187 | ], 188 | ) 189 | 190 | cc_library( 191 | name = "message_reader", 192 | srcs = [ 193 | "message_reader.cc", 194 | ], 195 | hdrs = [ 196 | "include/grpc_transcoding/message_reader.h", 197 | ], 198 | includes = [ 199 | "include/", 200 | ], 201 | deps = [ 202 | ":transcoder_input_stream", 203 | "@com_google_protobuf//:protobuf", 204 | ], 205 | ) 206 | 207 | cc_library( 208 | name = "response_to_json_translator", 209 | srcs = [ 210 | "response_to_json_translator.cc", 211 | ], 212 | hdrs = [ 213 | "include/grpc_transcoding/response_to_json_translator.h", 214 | ], 215 | includes = [ 216 | "include/", 217 | ], 218 | deps = [ 219 | ":message_reader", 220 | ":message_stream", 221 | "@com_google_protobuf//:protobuf", 222 | ], 223 | ) 224 | 225 | cc_library( 226 | name = "transcoder_input_stream", 227 | hdrs = [ 228 | "include/grpc_transcoding/transcoder_input_stream.h", 229 | ], 230 | includes = [ 231 | "include/", 232 | ], 233 | deps = [ 234 | "@com_google_protobuf//:protobuf", 235 | ], 236 | ) 237 | 238 | cc_library( 239 | name = "transcoding", 240 | hdrs = [ 241 | "include/grpc_transcoding/transcoder.h", 242 | ], 243 | includes = [ 244 | "include/", 245 | ], 246 | deps = [ 247 | ":json_request_translator", 248 | ":message_stream", 249 | ":path_matcher_utility", 250 | ":response_to_json_translator", 251 | ":type_helper", 252 | "@com_google_protobuf//:protobuf", 253 | ], 254 | ) 255 | 256 | cc_library( 257 | name = "type_helper", 258 | srcs = [ 259 | "type_helper.cc", 260 | ], 261 | hdrs = [ 262 | "include/grpc_transcoding/type_helper.h", 263 | ], 264 | includes = [ 265 | "include/", 266 | ], 267 | deps = [ 268 | ":percent_encoding_lib", 269 | "@com_google_absl//absl/strings", 270 | "@com_google_absl//absl/synchronization", 271 | "@com_google_protobuf//:protobuf", 272 | "@com_google_protoconverter//:all", 273 | ], 274 | ) 275 | 276 | cc_library( 277 | name = "status_error_listener", 278 | srcs = [ 279 | "status_error_listener.cc", 280 | ], 281 | hdrs = [ 282 | "include/grpc_transcoding/status_error_listener.h", 283 | ], 284 | includes = [ 285 | "include/", 286 | ], 287 | deps = [ 288 | "@com_google_absl//absl/status", 289 | "@com_google_absl//absl/strings", 290 | "@com_google_absl//absl/strings:str_format", 291 | "@com_google_protobuf//:protobuf", 292 | "@com_google_protoconverter//:all", 293 | ], 294 | ) 295 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/http_template.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_HTTP_TEMPLATE_H_ 16 | #define GRPC_TRANSCODING_HTTP_TEMPLATE_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | namespace google { 23 | namespace grpc { 24 | namespace transcoding { 25 | 26 | class HttpTemplate { 27 | public: 28 | static std::unique_ptr Parse(const std::string &ht); 29 | const std::vector &segments() const { return segments_; } 30 | const std::string &verb() const { return verb_; } 31 | 32 | // The info about a variable binding {variable=subpath} in the template. 33 | struct Variable { 34 | // Specifies the range of segments [start_segment, end_segment) the 35 | // variable binds to. Both start_segment and end_segment are 0 based. 36 | // end_segment can also be negative, which means that the position is 37 | // specified relative to the end such that -1 corresponds to the end 38 | // of the path. 39 | int start_segment; 40 | int end_segment; 41 | 42 | // The path of the protobuf field the variable binds to. 43 | std::vector field_path; 44 | 45 | // Do we have a ** in the variable template? 46 | bool has_wildcard_path; 47 | }; 48 | 49 | std::vector &Variables() { return variables_; } 50 | 51 | // '/.': match any single path segment. 52 | static const char kSingleParameterKey[]; 53 | // '*': Wildcard match for one path segment. 54 | static const char kWildCardPathPartKey[]; 55 | // '**': Wildcard match the remaining path. 56 | static const char kWildCardPathKey[]; 57 | 58 | private: 59 | HttpTemplate(std::vector &&segments, std::string &&verb, 60 | std::vector &&variables) 61 | : segments_(std::move(segments)), 62 | verb_(std::move(verb)), 63 | variables_(std::move(variables)) {} 64 | const std::vector segments_; 65 | std::string verb_; 66 | std::vector variables_; 67 | }; 68 | 69 | /** 70 | * VariableBinding specifies a value for a single field in the request message. 71 | * When transcoding HTTP/REST/JSON to gRPC/proto the request message is 72 | * constructed using the HTTP body and the variable bindings (specified through 73 | * request url). 74 | * See 75 | * https://github.com/googleapis/googleapis/blob/master/google/api/http.proto 76 | * for details of variable binding. 77 | */ 78 | struct VariableBinding { 79 | // The location of the field in the protobuf message, where the value 80 | // needs to be inserted, e.g. "shelf.theme" would mean the "theme" field 81 | // of the nested "shelf" message of the request protobuf message. 82 | std::vector field_path; 83 | // The value to be inserted. 84 | std::string value; 85 | }; 86 | 87 | } // namespace transcoding 88 | } // namespace grpc 89 | } // namespace google 90 | 91 | #endif // GRPC_TRANSCODING_HTTP_TEMPLATE_H_ 92 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/json_request_translator.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_JSON_REQUEST_TRANSLATOR_H_ 16 | #define GRPC_TRANSCODING_JSON_REQUEST_TRANSLATOR_H_ 17 | 18 | #include 19 | 20 | #include "google/protobuf/io/zero_copy_stream.h" 21 | #include "google/protobuf/util/converter/json_stream_parser.h" 22 | #include "google/protobuf/util/type_resolver.h" 23 | #include "message_stream.h" 24 | #include "request_message_translator.h" 25 | #include "request_stream_translator.h" 26 | 27 | namespace google { 28 | namespace grpc { 29 | 30 | namespace transcoding { 31 | 32 | // JsonRequestTranslator translates HTTP JSON request into gRPC message(s) 33 | // according to the http rules defined in the service config (see 34 | // third_party/config/google/api/http.proto). 35 | // 36 | // It supports streaming, weaving variable bindings and prefixing the messages 37 | // with GRPC message delimiters (see http://www.grpc.io/docs/guides/wire.html). 38 | // Also to support flow-control JsonRequestTranslator does the translation in a 39 | // lazy fashion, i.e. reads the input stream only as needed. 40 | // 41 | // Example: 42 | // JsonRequestTranslator translator(type_resolver, input_json, request_info, 43 | // /*streaming*/true, 44 | // /*output_delimiters*/true); 45 | // 46 | // MessageStream& out = translator.Output(); 47 | // 48 | // if (!out.Status().ok()) { 49 | // printf("Error: %s\n", out.Status().ErrorMessage().as_string().c_str()); 50 | // return; 51 | // } 52 | // 53 | // std::string message; 54 | // while (out.NextMessage(&message)) { 55 | // printf("Message=%s\n", message.c_str()); 56 | // } 57 | // 58 | // The implementation uses JsonStreamParser to parse the incoming JSON and 59 | // RequestMessageTranslator or RequestStreamTranslator to translate it into 60 | // protobuf message(s). 61 | // - JsonStreamParser converts the incoming JSON into ObjectWriter events, 62 | // - in a non-streaming case RequestMessageTranslator translates these 63 | // events into a protobuf message, 64 | // - in a streaming case RequestStreamTranslator translates these events 65 | // into a stream of protobuf messages. 66 | class JsonRequestTranslator { 67 | public: 68 | // type_resolver - provides type information necessary for translation ( 69 | // passed to the underlying RequestMessageTranslator or 70 | // RequestStreamTranslator). Note that JsonRequestTranslator 71 | // doesn't maintain the ownership of type_resolver. 72 | // json_input - the input JSON stream representated through 73 | // a ZeroCopyInputStream. Note that JsonRequestTranslator does 74 | // not maintain the ownership of json_input. 75 | // request_info - information about the request being translated (passed to 76 | // the underlying RequestMessageTranslator or 77 | // RequestStreamTranslator). 78 | // streaming - whether this is a streaming call or not 79 | // output_delimiters - whether to ouptut gRPC message delimiters or not 80 | JsonRequestTranslator(::google::protobuf::util::TypeResolver* type_resolver, 81 | ::google::protobuf::io::ZeroCopyInputStream* json_input, 82 | RequestInfo request_info, bool streaming, 83 | bool output_delimiters); 84 | 85 | // The translated output stream 86 | MessageStream& Output() { return *output_; } 87 | 88 | private: 89 | // The JSON parser 90 | std::unique_ptr<::google::protobuf::util::converter::JsonStreamParser> 91 | parser_; 92 | 93 | // The output stream 94 | std::unique_ptr output_; 95 | 96 | // A single message translator (empty unique_ptr if this is a streaming call) 97 | std::unique_ptr message_translator_; 98 | 99 | // A message stream translator (empty unique_ptr if this is a non-streaming 100 | // call) 101 | std::unique_ptr stream_translator_; 102 | 103 | JsonRequestTranslator(const JsonRequestTranslator&) = delete; 104 | JsonRequestTranslator& operator=(const JsonRequestTranslator&) = delete; 105 | }; 106 | 107 | } // namespace transcoding 108 | 109 | } // namespace grpc 110 | } // namespace google 111 | 112 | #endif // GRPC_TRANSCODING_REQUEST_TRANSLATOR_H 113 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/message_reader.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_MESSAGE_READER_H_ 16 | #define GRPC_TRANSCODING_MESSAGE_READER_H_ 17 | 18 | #include 19 | 20 | #include "absl/status/status.h" 21 | #include "transcoder_input_stream.h" 22 | 23 | namespace google { 24 | namespace grpc { 25 | 26 | namespace transcoding { 27 | 28 | // The number of bytes in the delimiter for gRPC wire format's 29 | // `Length-Prefixed-Message`. 30 | constexpr size_t kGrpcDelimiterByteSize = 5; 31 | 32 | // Return type that contains both the proto message and the preceding gRPC data 33 | // frame. 34 | struct MessageAndGrpcFrame { 35 | std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> message; 36 | unsigned char grpc_frame[kGrpcDelimiterByteSize]; 37 | // The size (in bytes) of the full gRPC message, excluding the frame header. 38 | uint32_t message_size; 39 | }; 40 | 41 | // MessageReader helps extract full messages from a ZeroCopyInputStream of 42 | // messages in gRPC wire format (http://www.grpc.io/docs/guides/wire.html). Each 43 | // message is returned in a ZeroCopyInputStream. MessageReader doesn't advance 44 | // the underlying ZeroCopyInputStream unless there is a full message available. 45 | // This is done to avoid copying while buffering. 46 | // 47 | // Example: 48 | // MessageReader reader(&input); 49 | // 50 | // while (!reader.Finished()) { 51 | // auto message = reader.NextMessage(); 52 | // if (!message) { 53 | // // No message is available at this moment. 54 | // break; 55 | // } 56 | // 57 | // const void* buffer = nullptr; 58 | // int size = 0; 59 | // while (message.Next(&buffer, &size)) { 60 | // // Process the message data. 61 | // ... 62 | // } 63 | // } 64 | // 65 | // NOTE: MessageReader is unable to recognize the case when there is an 66 | // incomplete message at the end of the input. The callers will need to 67 | // detect it and act appropriately. 68 | // This is because the MessageReader doesn't call Next() on the input 69 | // stream until there is a full message available. So, if there is an 70 | // incomplete message at the end of the input, MessageReader won't call 71 | // Next() and won't know that the stream has finished. 72 | // 73 | class MessageReader { 74 | public: 75 | MessageReader(TranscoderInputStream* in); 76 | 77 | // If a full message is available, NextMessage() returns a ZeroCopyInputStream 78 | // over the message. Otherwise returns nullptr - this might be temporary, the 79 | // caller can call NextMessage() again later to check. 80 | // NOTE: the caller must consume the entire message before calling 81 | // NextMessage() again. 82 | // That's because the returned ZeroCopyInputStream is a wrapper on top 83 | // of the original ZeroCopyInputStream and the MessageReader relies on 84 | // the caller to advance the stream to the next message before calling 85 | // NextMessage() again. 86 | // NOTE: the caller should check `Status()` is OK after calling this method. 87 | std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> NextMessage(); 88 | 89 | // An overload that also outputs the gRPC message delimiter for the parsed 90 | // message. The caller is free to take ownership of contents in `grpc_frame`. 91 | // NOTE: the caller must check the `message` is NOT nullptr and the `Status()` 92 | // is OK before consuming the `grpc_frame`. 93 | MessageAndGrpcFrame NextMessageAndGrpcFrame(); 94 | 95 | absl::Status Status() const { return status_; } 96 | 97 | // Returns true if the stream has ended (this is permanent); otherwise returns 98 | // false. 99 | bool Finished() const { return finished_ || !status_.ok(); } 100 | 101 | private: 102 | TranscoderInputStream* in_; 103 | // The size of the current message. 104 | uint32_t current_message_size_; 105 | // Whether we have read the current message size or not 106 | bool have_current_message_size_; 107 | // Are we all done? 108 | bool finished_; 109 | // Status 110 | absl::Status status_; 111 | // Buffer to store the current delimiter value. 112 | unsigned char delimiter_[kGrpcDelimiterByteSize]; 113 | 114 | MessageReader(const MessageReader&) = delete; 115 | MessageReader& operator=(const MessageReader&) = delete; 116 | }; 117 | 118 | } // namespace transcoding 119 | 120 | } // namespace grpc 121 | } // namespace google 122 | 123 | #endif // GRPC_TRANSCODING_MESSAGE_READER_H_ 124 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/message_stream.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_MESSAGE_STREAM_H_ 16 | #define GRPC_TRANSCODING_MESSAGE_STREAM_H_ 17 | 18 | #include 19 | #include 20 | 21 | #include "absl/status/status.h" 22 | #include "google/protobuf/io/zero_copy_stream.h" 23 | #include "transcoder_input_stream.h" 24 | 25 | namespace google { 26 | namespace grpc { 27 | 28 | namespace transcoding { 29 | 30 | // MessageStream abstracts a stream of std::string represented messages. Among 31 | // other things MessageStream helps us to reuse some code for streaming and 32 | // non-streaming implementations of request translation. 33 | // We'll use this simple interface internally in transcoding (ESP) and will 34 | // implement ZeroCopyInputStream based on this for exposing it to Nginx 35 | // integration code (or potentially other consumers of transcoding). 36 | // 37 | // Example: 38 | // MessageStream& stream = GetTranslatorStream(); 39 | // 40 | // if (!stream.Status().ok()) { 41 | // printf("Error - %s", stream.Status().error_message().c_str()); 42 | // return; 43 | // } 44 | // 45 | // std::string message; 46 | // while (stream.Message(&message)) { 47 | // printf("Message=%s\n", message.c_str()); 48 | // } 49 | // 50 | // if (stream.Finished()) { 51 | // printf("Finished\n"); 52 | // } else { 53 | // printf("No messages at this time. Try again later \n"); 54 | // } 55 | // 56 | // Unlike ZeroCopyInputStream this interface doesn't support features like 57 | // BackUp(), Skip() and is easier to implement. At the same time the 58 | // implementations can achieve "zero copy" by moving the std::string messages. 59 | // However, this assumes a particular implementation (std::string based), so 60 | // we use it only internally. 61 | // 62 | class MessageStream { 63 | public: 64 | // Retrieves the next message from the stream if there is one available. 65 | // The implementation can use move assignment operator to avoid a copy. 66 | // If no message is available at this time, returns false (this might be 67 | // temporary); otherwise returns true. 68 | virtual bool NextMessage(std::string* message) = 0; 69 | // Returns true if no messages are left (this is permanent); otherwise return 70 | // false. 71 | virtual bool Finished() const = 0; 72 | // Stream status to report errors 73 | virtual absl::Status Status() const = 0; 74 | // Virtual destructor 75 | virtual ~MessageStream() {} 76 | // Creates ZeroCopyInputStream implementation based on this stream 77 | std::unique_ptr CreateInputStream(); 78 | }; 79 | 80 | } // namespace transcoding 81 | 82 | } // namespace grpc 83 | } // namespace google 84 | 85 | #endif // GRPC_TRANSCODING_MESSAGE_STREAM_H_ 86 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/path_matcher_node.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_PATH_MATCHER_NODE_H_ 16 | #define GRPC_TRANSCODING_PATH_MATCHER_NODE_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | namespace google { 25 | namespace grpc { 26 | namespace transcoding { 27 | 28 | typedef std::string HttpMethod; 29 | 30 | struct PathMatcherLookupResult { 31 | PathMatcherLookupResult() : data(nullptr), is_multiple(false) {} 32 | 33 | PathMatcherLookupResult(void* data, bool is_multiple) 34 | : data(data), is_multiple(is_multiple) {} 35 | 36 | // The WrapperGraph that is registered to a method (or HTTP path). 37 | void* data; 38 | // Whether the method (or path) has been registered for more than once. 39 | bool is_multiple; 40 | }; 41 | 42 | // PathMatcherNodes represents a path part in a PathMatcher trie. Children nodes 43 | // represent adjacent path parts. A node can have many literal children, one 44 | // single-parameter child, and one repeated-parameter child. 45 | // 46 | // Thread Compatible. 47 | class PathMatcherNode { 48 | public: 49 | // Provides information for inserting templates into the trie. Clients can 50 | // instantiate PathInfo with the provided Builder. 51 | class PathInfo { 52 | public: 53 | class Builder { 54 | public: 55 | friend class PathInfo; 56 | Builder() : path_() {} 57 | ~Builder() {} 58 | 59 | PathMatcherNode::PathInfo Build() const; 60 | 61 | // Appends a node that must match the string value of a request part. The 62 | // strings "/." and "/.." are disallowed. 63 | // 64 | // Example: 65 | // 66 | // builder.AppendLiteralNode("a") 67 | // .AppendLiteralNode("b") 68 | // .AppendLiteralNode("c"); 69 | // 70 | // Matches the request path: a/b/c 71 | Builder& AppendLiteralNode(std::string name); 72 | 73 | // Appends a node that ignores the string value and matches any single 74 | // request part. 75 | // 76 | // Example: 77 | // 78 | // builder.AppendLiteralNode("a") 79 | // .AppendSingleParameterNode() 80 | // .AppendLiteralNode("c"); 81 | // 82 | // Matching request paths: a/foo/c, a/bar/c, a/1/c 83 | Builder& AppendSingleParameterNode(); 84 | 85 | // TODO: Appends a node that ignores string values and matches any 86 | // number of consecutive request parts. 87 | // 88 | // Example: 89 | // 90 | // builder.AppendLiteralNode("a") 91 | // .AppendLiteralNode("b") 92 | // .AppendRepeatedParameterNode(); 93 | // 94 | // Matching request paths: a/b/1/2/3/4/5, a/b/c 95 | // Builder& AppendRepeatedParameterNode(); 96 | 97 | private: 98 | std::vector path_; 99 | }; // class Builder 100 | 101 | ~PathInfo() {} 102 | 103 | // Returns path information used to insert a new path into a PathMatcherNode 104 | // trie. 105 | const std::vector& path_info() const { return path_; } 106 | 107 | private: 108 | explicit PathInfo(const Builder& builder) : path_(builder.path_) {} 109 | std::vector path_; 110 | }; // class PathInfo 111 | 112 | typedef std::vector RequestPathParts; 113 | 114 | // Creates a Root node with an empty WrapperGraph map. 115 | PathMatcherNode() : result_map_(), children_(), wildcard_(false) {} 116 | 117 | ~PathMatcherNode(); 118 | 119 | // Creates a clone of this node and its subtrie 120 | std::unique_ptr Clone() const; 121 | 122 | // Searches subtrie by finding a matching child for the current path part. If 123 | // a matching child exists, this function recurses on current + 1 with that 124 | // child as the receiver. If a matching descendant is found for the last part 125 | // in then this method copies the matching descendant's WrapperGraph, 126 | // VariableBindingInfoMap to the result pointers. 127 | void LookupPath(const RequestPathParts::const_iterator current, 128 | const RequestPathParts::const_iterator end, 129 | HttpMethod http_method, 130 | PathMatcherLookupResult* result) const; 131 | 132 | // This method inserts a path of nodes into this subtrie. The WrapperGraph, 133 | // VariableBindingInfoMap are inserted at the terminal descendant node. 134 | // Returns true if the template didn't previously exist. Returns false 135 | // otherwise and depends on if mark_duplicates is true, the template will be 136 | // marked as having been registered for more than once and the lookup of the 137 | // template will yield a special error reporting WrapperGraph. 138 | bool InsertPath(const PathInfo& node_path_info, std::string http_method, 139 | void* method_data, bool mark_duplicates); 140 | 141 | void set_wildcard(bool wildcard) { wildcard_ = wildcard; } 142 | 143 | private: 144 | // This method inserts a path of nodes into this subtrie (described by the 145 | // vector, starting from the |current| position in the iterator of path 146 | // parts, and if necessary, creating intermediate nodes along the way. The 147 | // WrapperGraph, VariableBindingInfoMap are inserted at the terminal 148 | // descendant node (which corresponds to the string part in the iterator). 149 | // Returns true if the template didn't previously exist. Returns false 150 | // otherwise and depends on if mark_duplicates is true, the template will be 151 | // marked as having been registered for more than once and the lookup of the 152 | // template will yield a special error reporting WrapperGraph. 153 | bool InsertTemplate(const std::vector::const_iterator current, 154 | const std::vector::const_iterator end, 155 | HttpMethod http_method, void* method_data, 156 | bool mark_duplicates); 157 | 158 | // Helper method for LookupPath. If the given child key exists, search 159 | // continues on the child node pointed by the child key with the next part 160 | // in the path. Returns true if found a match for the path eventually. 161 | bool LookupPathFromChild(const std::string child_key, 162 | const RequestPathParts::const_iterator current, 163 | const RequestPathParts::const_iterator end, 164 | HttpMethod http_method, 165 | PathMatcherLookupResult* result) const; 166 | 167 | // If a WrapperGraph is found for the provided key, then this method returns 168 | // true and copies the WrapperGraph to the provided result pointer. If no 169 | // match is found, this method returns false and leaves the result unmodified. 170 | // 171 | // NB: If result == nullptr, method will return bool value without modifying 172 | // result. 173 | bool GetResultForHttpMethod(HttpMethod key, 174 | PathMatcherLookupResult* result) const; 175 | 176 | std::map result_map_; 177 | 178 | // Lookup must be FAST 179 | // 180 | // n: the number of paths registered per client varies, but we can expect the 181 | // size of |children_| to range from ~5 to ~100 entries. 182 | // 183 | // To ensure fast lookups when n grows large, it is prudent to consider an 184 | // alternative to binary search on a sorted vector. 185 | std::unordered_map> children_; 186 | 187 | // True if this node represents a wildcard path '**'. 188 | bool wildcard_; 189 | }; 190 | 191 | } // namespace transcoding 192 | } // namespace grpc 193 | } // namespace google 194 | 195 | #endif // GRPC_TRANSCODING_PATH_MATCHER_NODE_H_ 196 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/path_matcher_utility.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef GRPC_HTTPJSON_TRANSCODING_PATH_MATCHER_UTILITY_H 17 | #define GRPC_HTTPJSON_TRANSCODING_PATH_MATCHER_UTILITY_H 18 | 19 | #include "google/api/http.pb.h" 20 | #include "path_matcher.h" 21 | 22 | namespace google { 23 | namespace grpc { 24 | namespace transcoding { 25 | 26 | class PathMatcherUtility { 27 | public: 28 | template 29 | static bool RegisterByHttpRule( 30 | PathMatcherBuilder &pmb, const google::api::HttpRule &http_rule, 31 | const std::unordered_set &system_query_parameter_names, 32 | const Method &method); 33 | 34 | template 35 | static bool RegisterByHttpRule(PathMatcherBuilder &pmb, 36 | const google::api::HttpRule &http_rule, 37 | const Method &method) { 38 | return RegisterByHttpRule(pmb, http_rule, std::unordered_set(), 39 | method); 40 | } 41 | }; 42 | 43 | template 44 | bool PathMatcherUtility::RegisterByHttpRule( 45 | PathMatcherBuilder &pmb, const google::api::HttpRule &http_rule, 46 | const std::unordered_set &system_query_parameter_names, 47 | const Method &method) { 48 | bool ok = true; 49 | switch (http_rule.pattern_case()) { 50 | case ::google::api::HttpRule::kGet: 51 | ok = pmb.Register("GET", http_rule.get(), http_rule.body(), 52 | system_query_parameter_names, method); 53 | break; 54 | case ::google::api::HttpRule::kPut: 55 | ok = pmb.Register("PUT", http_rule.put(), http_rule.body(), 56 | system_query_parameter_names, method); 57 | break; 58 | case ::google::api::HttpRule::kPost: 59 | ok = pmb.Register("POST", http_rule.post(), http_rule.body(), 60 | system_query_parameter_names, method); 61 | break; 62 | case ::google::api::HttpRule::kDelete: 63 | ok = pmb.Register("DELETE", http_rule.delete_(), http_rule.body(), 64 | system_query_parameter_names, method); 65 | break; 66 | case ::google::api::HttpRule::kPatch: 67 | ok = pmb.Register("PATCH", http_rule.patch(), http_rule.body(), 68 | system_query_parameter_names, method); 69 | break; 70 | case ::google::api::HttpRule::kCustom: 71 | ok = pmb.Register(http_rule.custom().kind(), http_rule.custom().path(), 72 | http_rule.body(), system_query_parameter_names, method); 73 | break; 74 | default: // ::google::api::HttpRule::PATTEN_NOT_SET 75 | break; 76 | } 77 | 78 | for (const auto &additional_binding : http_rule.additional_bindings()) { 79 | if (!ok) { 80 | return ok; 81 | } 82 | ok = RegisterByHttpRule(pmb, additional_binding, 83 | system_query_parameter_names, method); 84 | } 85 | 86 | return ok; 87 | } 88 | 89 | } // namespace transcoding 90 | } // namespace grpc 91 | } // namespace google 92 | 93 | #endif // GRPC_HTTPJSON_TRANSCODING_PATH_MATCHER_UTILITY_H 94 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/percent_encoding.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2022 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_PERCENT_ENCODING_H_ 16 | #define GRPC_TRANSCODING_PERCENT_ENCODING_H_ 17 | 18 | #include 19 | 20 | #include "absl/strings/string_view.h" 21 | 22 | namespace google { 23 | namespace grpc { 24 | namespace transcoding { 25 | 26 | enum class UrlUnescapeSpec { 27 | // URL path parameters will not decode RFC 6570 reserved characters. 28 | // This is the default behavior. 29 | kAllCharactersExceptReserved = 0, 30 | // URL path parameters will be fully URI-decoded except in 31 | // cases of single segment matches in reserved expansion, where "%2F" will be 32 | // left encoded. 33 | kAllCharactersExceptSlash, 34 | // URL path parameters will be fully URI-decoded. 35 | kAllCharacters, 36 | }; 37 | 38 | inline bool IsReservedChar(char c) { 39 | // Reserved characters according to RFC 6570 40 | switch (c) { 41 | case '!': 42 | case '#': 43 | case '$': 44 | case '&': 45 | case '\'': 46 | case '(': 47 | case ')': 48 | case '*': 49 | case '+': 50 | case ',': 51 | case '/': 52 | case ':': 53 | case ';': 54 | case '=': 55 | case '?': 56 | case '@': 57 | case '[': 58 | case ']': 59 | return true; 60 | default: 61 | return false; 62 | } 63 | } 64 | 65 | // Check if an ASCII character is a hex digit. We can't use ctype's 66 | // isxdigit() because it is affected by locale. This function is applied 67 | // to the escaped characters in a url, not to natural-language 68 | // strings, so locale should not be taken into account. 69 | inline bool ascii_isxdigit(char c) { 70 | return ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F') || 71 | ('0' <= c && c <= '9'); 72 | } 73 | 74 | inline int hex_digit_to_int(char c) { 75 | /* Assume ASCII. */ 76 | int x = static_cast(c); 77 | if (x > '9') { 78 | x += 9; 79 | } 80 | return x & 0xf; 81 | } 82 | 83 | // This is a helper function for UrlUnescapeString. It takes a string and 84 | // the index of where we are within that string. 85 | // 86 | // The function returns true if the next three characters are of the format: 87 | // "%[0-9A-Fa-f]{2}". 88 | // 89 | // If the next three characters are an escaped character then this function will 90 | // also return what character is escaped. 91 | // 92 | // If unescape_plus is true, unescape '+' to space. 93 | // 94 | // return value: 0: not unescaped, >0: unescaped, number of used original 95 | // characters. 96 | // 97 | inline int GetEscapedChar(absl::string_view src, size_t i, 98 | UrlUnescapeSpec unescape_spec, bool unescape_plus, 99 | char* out) { 100 | if (unescape_plus && src[i] == '+') { 101 | *out = ' '; 102 | return 1; 103 | } 104 | if (i + 2 < src.size() && src[i] == '%') { 105 | if (ascii_isxdigit(src[i + 1]) && ascii_isxdigit(src[i + 2])) { 106 | char c = 107 | (hex_digit_to_int(src[i + 1]) << 4) | hex_digit_to_int(src[i + 2]); 108 | switch (unescape_spec) { 109 | case UrlUnescapeSpec::kAllCharactersExceptReserved: 110 | if (IsReservedChar(c)) { 111 | return 0; 112 | } 113 | break; 114 | case UrlUnescapeSpec::kAllCharactersExceptSlash: 115 | if (c == '/') { 116 | return 0; 117 | } 118 | break; 119 | case UrlUnescapeSpec::kAllCharacters: 120 | break; 121 | } 122 | *out = c; 123 | return 3; 124 | } 125 | } 126 | return 0; 127 | } 128 | 129 | inline bool IsUrlEscapedString(absl::string_view part, 130 | UrlUnescapeSpec unescape_spec, 131 | bool unescape_plus) { 132 | char ch = '\0'; 133 | for (size_t i = 0; i < part.size(); ++i) { 134 | if (GetEscapedChar(part, i, unescape_spec, unescape_plus, &ch) > 0) { 135 | return true; 136 | } 137 | } 138 | return false; 139 | } 140 | 141 | inline bool IsUrlEscapedString(absl::string_view part) { 142 | return IsUrlEscapedString(part, UrlUnescapeSpec::kAllCharacters, false); 143 | } 144 | 145 | // Unescapes string 'part' and returns the unescaped string. Reserved characters 146 | // (as specified in RFC 6570) are not escaped if unescape_reserved_chars is 147 | // false. 148 | inline std::string UrlUnescapeString(absl::string_view part, 149 | UrlUnescapeSpec unescape_spec, 150 | bool unescape_plus) { 151 | // Check whether we need to escape at all. 152 | if (!IsUrlEscapedString(part, unescape_spec, unescape_plus)) { 153 | return std::string(part); 154 | } 155 | 156 | std::string unescaped; 157 | char ch = '\0'; 158 | unescaped.resize(part.size()); 159 | 160 | char* begin = &(unescaped)[0]; 161 | char* p = begin; 162 | 163 | for (size_t i = 0; i < part.size();) { 164 | int skip = GetEscapedChar(part, i, unescape_spec, unescape_plus, &ch); 165 | if (skip > 0) { 166 | *p++ = ch; 167 | i += skip; 168 | } else { 169 | *p++ = part[i]; 170 | i += 1; 171 | } 172 | } 173 | 174 | unescaped.resize(p - begin); 175 | return unescaped; 176 | } 177 | 178 | inline std::string UrlUnescapeString(absl::string_view part) { 179 | return UrlUnescapeString(part, UrlUnescapeSpec::kAllCharacters, false); 180 | } 181 | 182 | } // namespace transcoding 183 | } // namespace grpc 184 | } // namespace google 185 | 186 | #endif // GRPC_TRANSCODING_PERCENT_ENCODING_H_ 187 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/prefix_writer.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_PREFIX_WRITER_H_ 16 | #define GRPC_TRANSCODING_PREFIX_WRITER_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "absl/strings/string_view.h" 23 | #include "google/protobuf/util/converter/object_writer.h" 24 | 25 | namespace google { 26 | namespace grpc { 27 | 28 | namespace transcoding { 29 | 30 | // PrefixWriter is helper ObjectWriter implementation that for each incoming 31 | // object 32 | // 1) writes the given prefix path by starting objects to the output 33 | // ObjectWriter, 34 | // 2) forwards the writer events for a single object, 35 | // 3) unwinds the prefix, by closing objects in the reverse order. 36 | // 37 | // E.g. 38 | // 39 | // PrefixWriter pw("A.B.C", out); 40 | // pw.StartObject("Root"); 41 | // ... 42 | // pw.RenderString("x", "value"); 43 | // ... 44 | // pw.EndObject("Root"); 45 | // 46 | // is equivalent to 47 | // 48 | // out.StartObject("Root"); 49 | // out.StartObject("A"); 50 | // out.StartObject("B"); 51 | // out.StartObject("C"); 52 | // ... 53 | // pw.RenderString("x", "value"); 54 | // ... 55 | // out.EndObject("C"); 56 | // out.EndObject("B"); 57 | // out.EndObject("A"); 58 | // out.EndObject("Root"); 59 | // 60 | class PrefixWriter : public google::protobuf::util::converter::ObjectWriter { 61 | public: 62 | // prefix is a '.' delimited prefix path to be added 63 | PrefixWriter(const std::string& prefix, 64 | google::protobuf::util::converter::ObjectWriter* ow); 65 | 66 | // ObjectWriter methods. 67 | PrefixWriter* StartObject(absl::string_view name); 68 | PrefixWriter* EndObject(); 69 | PrefixWriter* StartList(absl::string_view name); 70 | PrefixWriter* EndList(); 71 | PrefixWriter* RenderBool(absl::string_view name, bool value); 72 | PrefixWriter* RenderInt32(absl::string_view name, int32_t value); 73 | PrefixWriter* RenderUint32(absl::string_view name, uint32_t value); 74 | PrefixWriter* RenderInt64(absl::string_view name, int64_t value); 75 | PrefixWriter* RenderUint64(absl::string_view name, uint64_t value); 76 | PrefixWriter* RenderDouble(absl::string_view name, double value); 77 | PrefixWriter* RenderFloat(absl::string_view name, float value); 78 | PrefixWriter* RenderString(absl::string_view name, absl::string_view value); 79 | PrefixWriter* RenderBytes(absl::string_view name, absl::string_view value); 80 | PrefixWriter* RenderNull(absl::string_view name); 81 | 82 | private: 83 | // Helper method to start the prefix and return the name to use for the value. 84 | absl::string_view StartPrefix(absl::string_view name); 85 | 86 | // Helper method to end the prefix. 87 | void EndPrefix(); 88 | 89 | // The path prefix if the HTTP body maps to a nested message in the proto. 90 | std::vector prefix_; 91 | 92 | // Tracks the depth within the output, so we know when to write the prefix 93 | // and when to close it off. 94 | int non_actionable_depth_; 95 | 96 | // The output object writer to forward the writer events. 97 | google::protobuf::util::converter::ObjectWriter* writer_; 98 | 99 | PrefixWriter(const PrefixWriter&) = delete; 100 | PrefixWriter& operator=(const PrefixWriter&) = delete; 101 | }; 102 | 103 | } // namespace transcoding 104 | 105 | } // namespace grpc 106 | } // namespace google 107 | 108 | #endif // GRPC_TRANSCODING_PREFIX_WRITER_H_ 109 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/request_message_translator.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_REQUEST_MESSAGE_TRANSLATOR_H_ 16 | #define GRPC_TRANSCODING_REQUEST_MESSAGE_TRANSLATOR_H_ 17 | 18 | #include 19 | #include 20 | 21 | #include "absl/strings/string_view.h" 22 | #include "google/protobuf/stubs/bytestream.h" 23 | #include "google/protobuf/type.pb.h" 24 | #include "google/protobuf/util/converter/error_listener.h" 25 | #include "google/protobuf/util/converter/protostream_objectwriter.h" 26 | #include "google/protobuf/util/type_resolver.h" 27 | #include "message_stream.h" 28 | #include "prefix_writer.h" 29 | #include "request_weaver.h" 30 | 31 | namespace google { 32 | namespace grpc { 33 | 34 | namespace transcoding { 35 | 36 | // RequestInfo contains the information needed for request translation. 37 | struct RequestInfo { 38 | // The protobuf type that we are translating to. 39 | const google::protobuf::Type* message_type; 40 | 41 | // body_field_path is a dot-delimited chain of protobuf field names that 42 | // defines the (potentially nested) location in the message, where the 43 | // translated HTTP body must be inserted. E.g. "shelf.theme" means that the 44 | // translated HTTP body must be inserted into the "theme" field of the "shelf" 45 | // field of the request message. 46 | std::string body_field_path; 47 | 48 | // A collection of variable bindings extracted from the HTTP url or other 49 | // sources that must be injected into certain fields of the translated 50 | // message. 51 | std::vector variable_bindings; 52 | 53 | // Whether to reject the request if the binding value and the body value 54 | // are conflicted. 55 | bool reject_binding_body_field_collisions = false; 56 | 57 | // Proto enum values are supposed to be all in upper cases. 58 | // If true, enum values can be in lower cases. 59 | bool case_insensitive_enum_parsing = false; 60 | }; 61 | 62 | // RequestMessageTranslator translates ObjectWriter events into a single 63 | // protobuf message. The protobuf message is built based on the input 64 | // ObjectWriter events and a RequestInfo. 65 | // If output_delimiter is true, RequestMessageTranslator will prepend the output 66 | // message with a GRPC message delimiter - a 1-byte compression flag and a 67 | // 4-byte message length (see http://www.grpc.io/docs/guides/wire.html). 68 | // The translated message is exposed through MessageStream interface. 69 | // 70 | // The implementation uses a pipeline of ObjectWriters to do the job: 71 | // PrefixWriter -> RequestWeaver -> ProtoStreamObjectWriter 72 | // 73 | // - PrefixWriter writes the body prefix making sure that the body goes to the 74 | // right place and forwards the writer events to the RequestWeaver. This link 75 | // will be absent if the prefix is empty. 76 | // - RequestWeaver injects the variable bindings and forwards the writer events 77 | // to the ProtoStreamObjectWriter. This link will be absent if there are no 78 | // variable bindings to weave. 79 | // - ProtoStreamObjectWriter does the actual proto writing. 80 | // 81 | // Example: 82 | // RequestMessageTranslator t(type_resolver, true, std::move(request_info)); 83 | // 84 | // ObjectWriter& input = t.Input(); 85 | // 86 | // input.StartObject(""); 87 | // ... 88 | // write the request body using input ObjectWriter 89 | // ... 90 | // input.EndObject(); 91 | // 92 | // if (!t.Status().ok()) { 93 | // printf("Error: %s\n", t->Status().ErrorMessage().as_string().c_str()); 94 | // return; 95 | // } 96 | // 97 | // std::string message; 98 | // if (t.NextMessage(&message)) { 99 | // printf("Message=%s\n", message.c_str()); 100 | // } 101 | // 102 | class RequestMessageTranslator : public MessageStream { 103 | public: 104 | // type_resolver is forwarded to the ProtoStreamObjectWriter that does the 105 | // actual proto writing. 106 | // output_delimiter specifies whether to output the GRPC 5 byte message 107 | // delimiter before the message or not. 108 | RequestMessageTranslator(google::protobuf::util::TypeResolver& type_resolver, 109 | bool output_delimiter, RequestInfo request_info); 110 | 111 | ~RequestMessageTranslator(); 112 | 113 | // An ObjectWriter that takes the input object to translate 114 | google::protobuf::util::converter::ObjectWriter& Input() { 115 | return *writer_pipeline_; 116 | } 117 | 118 | // MessageStream methods 119 | bool NextMessage(std::string* message); 120 | bool Finished() const; 121 | absl::Status Status() const { return error_listener_.status(); } 122 | 123 | private: 124 | // Reserves space (5 bytes) for the GRPC delimiter to be written later. As it 125 | // requires the length of the message, we can't write it before the message 126 | // itself. 127 | void ReserveDelimiterSpace(); 128 | 129 | // Writes the wire delimiter into the reserved delimiter space at the begining 130 | // of this->message_. 131 | void WriteDelimiter(); 132 | 133 | // The message being written 134 | std::string message_; 135 | 136 | // StringByteSink instance that appends the bytes to this->message_. We pass 137 | // this to the ProtoStreamObjectWriter for writing the translated message. 138 | google::protobuf::strings::StringByteSink sink_; 139 | 140 | // ErrorListener implementation that converts the error events into 141 | // a status. 142 | StatusErrorListener error_listener_; 143 | 144 | // The proto writer for writing the actual proto bytes 145 | google::protobuf::util::converter::ProtoStreamObjectWriter proto_writer_; 146 | 147 | // A RequestWeaver for writing the variable bindings 148 | std::unique_ptr request_weaver_; 149 | 150 | // A PrefixWriter for writing the body prefix 151 | std::unique_ptr prefix_writer_; 152 | 153 | // The ObjectWriter that will receive the events 154 | // This is either &proto_writer_, request_weaver_.get() or 155 | // prefix_writer_.get() 156 | google::protobuf::util::converter::ObjectWriter* writer_pipeline_; 157 | 158 | // Whether to ouput a delimiter before the message or not 159 | bool output_delimiter_; 160 | 161 | // A flag that indicates whether the message has been already read or not 162 | // This helps with the MessageStream implementation. 163 | bool finished_; 164 | 165 | // GRPC delimiter size = 1 + 4 - 1-byte compression flag and 4-byte message 166 | // length. 167 | static const int kDelimiterSize = 5; 168 | 169 | RequestMessageTranslator(const RequestMessageTranslator&) = delete; 170 | RequestMessageTranslator& operator=(const RequestMessageTranslator&) = delete; 171 | }; 172 | 173 | } // namespace transcoding 174 | 175 | } // namespace grpc 176 | } // namespace google 177 | 178 | #endif // GRPC_TRANSCODING_REQUEST_MESSAGE_TRANSLATOR_H_ 179 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/request_stream_translator.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_REQUEST_STREAM_TRANSLATOR_H_ 16 | #define GRPC_TRANSCODING_REQUEST_STREAM_TRANSLATOR_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "absl/strings/string_view.h" 24 | #include "google/protobuf/util/converter/object_writer.h" 25 | #include "google/protobuf/util/type_resolver.h" 26 | #include "message_stream.h" 27 | #include "request_message_translator.h" 28 | 29 | namespace google { 30 | namespace grpc { 31 | 32 | namespace transcoding { 33 | 34 | // Translates ObjectWriter events into protobuf messages for streaming requests. 35 | // RequestStreamTranslator handles the outermost array and for each element uses 36 | // a RequestMessageTranslator to translate it to a proto message. Collects the 37 | // translated messages into a deque and exposes those through MessageStream 38 | // interface. 39 | // Example: 40 | // RequestMessageTranslator t(type_resolver, true, std::move(request_info)); 41 | // 42 | // t.StartList(""); 43 | // ... 44 | // t.StartObject(""); 45 | // write object 1 46 | // t.EndObject(); 47 | // ... 48 | // t.StartObject(""); 49 | // write object 2 50 | // t.EndObject(); 51 | // ... 52 | // t.EndList(); 53 | // 54 | // if (!t.Status().ok()) { 55 | // printf("Error: %s\n", t->Status().ErrorMessage().as_string().c_str()); 56 | // return; 57 | // } 58 | // 59 | // std::string message; 60 | // while (t.NextMessage(&message)) { 61 | // printf("Message=%s\n", message.c_str()); 62 | // } 63 | // 64 | class RequestStreamTranslator 65 | : public google::protobuf::util::converter::ObjectWriter, 66 | public MessageStream { 67 | public: 68 | RequestStreamTranslator(google::protobuf::util::TypeResolver& type_resolver, 69 | bool output_delimiters, RequestInfo request_info); 70 | ~RequestStreamTranslator(); 71 | 72 | // MessageStream methods 73 | bool NextMessage(std::string* message); 74 | bool Finished() const; 75 | absl::Status Status() const { return status_; } 76 | 77 | private: 78 | // ObjectWriter methods. 79 | RequestStreamTranslator* StartObject(absl::string_view name); 80 | RequestStreamTranslator* EndObject(); 81 | RequestStreamTranslator* StartList(absl::string_view name); 82 | RequestStreamTranslator* EndList(); 83 | RequestStreamTranslator* RenderBool(absl::string_view name, bool value); 84 | RequestStreamTranslator* RenderInt32(absl::string_view name, int32_t value); 85 | RequestStreamTranslator* RenderUint32(absl::string_view name, uint32_t value); 86 | RequestStreamTranslator* RenderInt64(absl::string_view name, int64_t value); 87 | RequestStreamTranslator* RenderUint64(absl::string_view name, uint64_t value); 88 | RequestStreamTranslator* RenderDouble(absl::string_view name, double value); 89 | RequestStreamTranslator* RenderFloat(absl::string_view name, float value); 90 | RequestStreamTranslator* RenderString(absl::string_view name, 91 | absl::string_view value); 92 | RequestStreamTranslator* RenderBytes(absl::string_view name, 93 | absl::string_view value); 94 | RequestStreamTranslator* RenderNull(absl::string_view name); 95 | 96 | // Sets up the ProtoMessageHelper to handle writing data. 97 | void StartMessageTranslator(); 98 | 99 | // Closes down the ProtoMessageHelper and stores its message. 100 | void EndMessageTranslator(); 101 | 102 | // Helper method to render a single piece of data, to reuse code. 103 | void RenderData(absl::string_view name, std::function renderer); 104 | 105 | // TypeResolver to be passed to the RequestMessageTranslator 106 | google::protobuf::util::TypeResolver& type_resolver_; 107 | 108 | // The status of the translation 109 | absl::Status status_; 110 | 111 | // The request info 112 | RequestInfo request_info_; 113 | 114 | // Whether to prefix each message with a delimiter or not 115 | bool output_delimiters_; 116 | 117 | // The ProtoMessageWriter that is currently writing a message, or null if we 118 | // are at the root or have invalid input. 119 | std::unique_ptr translator_; 120 | 121 | // Holds the messages we've translated so far. 122 | std::deque messages_; 123 | 124 | // Depth within the object tree. We special case the root level. 125 | int depth_; 126 | 127 | // Done with the translation (i.e., have seen the last EndList()) 128 | bool done_; 129 | 130 | RequestStreamTranslator(const RequestStreamTranslator&) = delete; 131 | RequestStreamTranslator& operator=(const RequestStreamTranslator&) = delete; 132 | }; 133 | 134 | } // namespace transcoding 135 | 136 | } // namespace grpc 137 | } // namespace google 138 | #endif // GRPC_TRANSCODING_REQUEST_STREAM_TRANSLATOR_H_ 139 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/request_weaver.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_REQUEST_WEAVER_H_ 16 | #define GRPC_TRANSCODING_REQUEST_WEAVER_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "absl/strings/string_view.h" 25 | #include "google/protobuf/type.pb.h" 26 | #include "google/protobuf/util/converter/object_writer.h" 27 | #include "grpc_transcoding/status_error_listener.h" 28 | 29 | namespace google { 30 | namespace grpc { 31 | 32 | namespace transcoding { 33 | 34 | // RequestWeaver is an ObjectWriter implementation that weaves-in given variable 35 | // bindings together with the input ObjectWriter events and forwards it to the 36 | // output ObjectWriter specified in the constructor. 37 | // 38 | // E.g., assume we have the {"shelf.theme" -> "Russian Classics"} binding and 39 | // the caller is "writing" an object calling the weaver methods as follows: 40 | // 41 | // weaver.StartObject(""); 42 | // ... 43 | // weaver.StartObject("shelf"); 44 | // weaver.RenderString("name", "1"); 45 | // weaver.EndObject(); 46 | // ... 47 | // weaver.EndObject(); 48 | // 49 | // The request weaver will forward all these events to the output ObjectWriter 50 | // and will also inject the "shelf.theme" value: 51 | // 52 | // out.StartObject(""); 53 | // ... 54 | // out.StartObject("shelf"); 55 | // out.RenderString("name", "1"); 56 | // out.RenderString("theme", "Russian Classics"); <-- weaved value 57 | // out.EndObject(); 58 | // ... 59 | // out.EndObject(); 60 | // 61 | class RequestWeaver : public google::protobuf::util::converter::ObjectWriter { 62 | public: 63 | // a single binding to be weaved-in into the message 64 | struct BindingInfo { 65 | // field_path is a chain of protobuf fields that defines the (potentially 66 | // nested) location in the message, where the value should be weaved-in. 67 | // E.g. {"shelf", "theme"} field_path means that the value should be 68 | // inserted into the "theme" field of the "shelf" field of the request 69 | // message. 70 | std::vector field_path; 71 | std::string value; 72 | }; 73 | 74 | // We accept 'bindings' by value to enable moving if the caller doesn't need 75 | // the passed object anymore. 76 | // RequestWeaver does not take the ownership of 'ow'. The caller must make 77 | // sure that it exists throughout the lifetime of the RequestWeaver. 78 | RequestWeaver(std::vector bindings, 79 | google::protobuf::util::converter::ObjectWriter* ow, 80 | StatusErrorListener* el, bool report_collisions); 81 | 82 | absl::Status Status() { return error_listener_->status(); } 83 | 84 | // ObjectWriter methods 85 | RequestWeaver* StartObject(absl::string_view name); 86 | RequestWeaver* EndObject(); 87 | RequestWeaver* StartList(absl::string_view name); 88 | RequestWeaver* EndList(); 89 | RequestWeaver* RenderBool(absl::string_view name, bool value); 90 | RequestWeaver* RenderInt32(absl::string_view name, int32_t value); 91 | RequestWeaver* RenderUint32(absl::string_view name, uint32_t value); 92 | RequestWeaver* RenderInt64(absl::string_view name, int64_t value); 93 | RequestWeaver* RenderUint64(absl::string_view name, uint64_t value); 94 | RequestWeaver* RenderDouble(absl::string_view name, double value); 95 | RequestWeaver* RenderFloat(absl::string_view name, float value); 96 | RequestWeaver* RenderString(absl::string_view name, absl::string_view value); 97 | RequestWeaver* RenderNull(absl::string_view name); 98 | RequestWeaver* RenderBytes(absl::string_view name, absl::string_view value); 99 | 100 | private: 101 | // Container for information to be weaved. 102 | // WeaveInfo represents an internal node in the weave tree. 103 | // messages: list of non-leaf children nodes. 104 | // bindings: list of binding values (leaf nodes) in this node. 105 | struct WeaveInfo { 106 | // NOTE: using list instead of map/unordered_map as the number of keys is 107 | // going to be small. 108 | std::list> messages; 109 | std::list> bindings; 110 | 111 | // Find the entry for the speciied field in messages list . 112 | WeaveInfo* FindWeaveMsg(absl::string_view field_name); 113 | 114 | // Create an entry in messages for the given field. The caller must make 115 | // sure that there is no existing entry for the same field before calling. 116 | WeaveInfo* CreateWeaveMsg(const google::protobuf::Field* field); 117 | 118 | // Ensure that there is an entry for the given field and return it. 119 | WeaveInfo* FindOrCreateWeaveMsg(const google::protobuf::Field* field); 120 | }; 121 | 122 | // Bind value to location indicated by fields. 123 | void Bind(std::vector field_path, 124 | std::string value); 125 | 126 | // Write out the whole subtree rooted at info to the ProtoStreamObjectWriter. 127 | void WeaveTree(WeaveInfo* info); 128 | 129 | // Checks if any repeated fields with the same field name are in the current 130 | // node of the weave tree. Output them if there are any. 131 | void CollisionCheck( 132 | absl::string_view name, 133 | const ::google::protobuf::util::converter::DataPiece& value); 134 | 135 | // All the headers, variable bindings and parameter bindings to be weaved in. 136 | // root_ : root of the tree to be weaved in. 137 | // current_: stack of nodes in the current visit path from the root. 138 | // NOTE: current_ points to the nodes owned by root_. It doesn't maintain the 139 | // ownership itself. 140 | WeaveInfo root_; 141 | std::stack current_; 142 | 143 | // Destination ObjectWriter for final output. 144 | google::protobuf::util::converter::ObjectWriter* ow_; 145 | 146 | // Counter for number of uninteresting nested messages. 147 | int non_actionable_depth_; 148 | 149 | // Error listener to report errors. 150 | StatusErrorListener* error_listener_; 151 | 152 | // Whether to report binding and body value collisions in the error listener. 153 | bool report_collisions_; 154 | 155 | RequestWeaver(const RequestWeaver&) = delete; 156 | RequestWeaver& operator=(const RequestWeaver&) = delete; 157 | }; 158 | 159 | } // namespace transcoding 160 | 161 | } // namespace grpc 162 | } // namespace google 163 | 164 | #endif // GRPC_TRANSCODING_REQUEST_WEAVER_H_ 165 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/response_to_json_translator.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_RESPONSE_TO_JSON_TRANSLATOR_H_ 16 | #define GRPC_TRANSCODING_RESPONSE_TO_JSON_TRANSLATOR_H_ 17 | 18 | #include 19 | 20 | #include "google/protobuf/io/zero_copy_stream.h" 21 | #include "google/protobuf/util/json_util.h" 22 | #include "google/protobuf/util/type_resolver.h" 23 | #include "message_reader.h" 24 | #include "message_stream.h" 25 | 26 | namespace google { 27 | namespace grpc { 28 | 29 | namespace transcoding { 30 | 31 | // ResponseToJsonTranslator translates gRPC response message(s) into JSON. It 32 | // accepts the input from a ZeroCopyInputStream and exposes the output through a 33 | // MessageStream implementation. Supports streaming calls. 34 | // 35 | // The implementation uses a MessageReader to extract complete messages from the 36 | // input stream and ::google::protobuf::util::BinaryToJsonStream() to do the 37 | // actual translation. For streaming calls emits '[', ',' and ']' in appropriate 38 | // locations to construct a JSON array. 39 | // 40 | // Example: 41 | // ResponseToJsonTranslator translator(type_resolver, 42 | // "type.googleapis.com/Shelf", 43 | // true, input_stream); 44 | // 45 | // std::string message; 46 | // while (translator.NextMessage(&message)) { 47 | // printf("Message=%s\n", message.c_str()); 48 | // } 49 | // 50 | // if (!translator.Status().ok()) { 51 | // printf("Error: %s\n", 52 | // translator.Status().error_message().as_string().c_str()); 53 | // return; 54 | // } 55 | // 56 | // NOTE: ResponseToJsonTranslator is unable to recognize the case when there is 57 | // an incomplete message at the end of the input. The callers will need to 58 | // detect it and act appropriately. 59 | // 60 | 61 | // Control various aspects of the generated JSON during response translation 62 | struct JsonResponseTranslateOptions { 63 | // JsonPrintOptions 64 | // (https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.util.json_util#JsonPrintOptions) 65 | // to configures the printing of individual messages as JSON 66 | ::google::protobuf::util::JsonPrintOptions json_print_options; 67 | 68 | // Whether the stream emits messages with newline-delimiters or not. 69 | // If set to true, newline "\n" is used to separate streaming messages. 70 | // If set to false, all streaming messages are treated as a JSON array and 71 | // separated by comma. 72 | bool stream_newline_delimited = false; 73 | 74 | // If true, enforces Server-Sent Events (SSE) message framing (`data: \n\n`) 75 | // and, `stream_newline_delimited` is ignored. 76 | // If false, message framing is determined by `stream_newline_delimited`. 77 | bool stream_sse_style_delimited = false; 78 | }; 79 | 80 | class ResponseToJsonTranslator : public MessageStream { 81 | public: 82 | // type_resolver - passed to BinaryToJsonStream() to do the translation 83 | // type_url - the type of input proto message(s) 84 | // streaming - whether this is a streaming call or not 85 | // in - the input stream of delimited proto message(s) as in the gRPC wire 86 | // format (http://www.grpc.io/docs/guides/wire.html) 87 | // options - control various aspects for the generated JSON 88 | ResponseToJsonTranslator( 89 | ::google::protobuf::util::TypeResolver* type_resolver, 90 | std::string type_url, bool streaming, TranscoderInputStream* in, 91 | const JsonResponseTranslateOptions& options = { 92 | ::google::protobuf::util::JsonPrintOptions(), false, false}); 93 | 94 | // MessageStream implementation 95 | bool NextMessage(std::string* message); 96 | bool Finished() const { return finished_ || !status_.ok(); } 97 | absl::Status Status() const { return status_; } 98 | 99 | private: 100 | // Translates a single message 101 | bool TranslateMessage(::google::protobuf::io::ZeroCopyInputStream* proto_in, 102 | std::string* json_out); 103 | 104 | ::google::protobuf::util::TypeResolver* type_resolver_; 105 | std::string type_url_; 106 | const JsonResponseTranslateOptions options_; 107 | bool streaming_; 108 | 109 | // A MessageReader to extract full messages 110 | MessageReader reader_; 111 | 112 | // Whether this is the first message of a streaming call or not. Used to emit 113 | // the opening '['. 114 | bool first_; 115 | 116 | bool finished_; 117 | absl::Status status_; 118 | }; 119 | 120 | } // namespace transcoding 121 | 122 | } // namespace grpc 123 | } // namespace google 124 | 125 | #endif // GRPC_TRANSCODING_RESPONSE_TO_JSON_TRANSLATOR_H_ 126 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/status_error_listener.h: -------------------------------------------------------------------------------- 1 | #ifndef GRPC_TRANSCODING_STATUS_ERROR_LISTENER_H_ 2 | #define GRPC_TRANSCODING_STATUS_ERROR_LISTENER_H_ 3 | 4 | #include "absl/status/status.h" 5 | #include "absl/strings/string_view.h" 6 | #include "google/protobuf/util/converter/error_listener.h" 7 | 8 | namespace google { 9 | namespace grpc { 10 | 11 | namespace transcoding { 12 | 13 | // StatusErrorListener converts the error events into a Status 14 | class StatusErrorListener 15 | : public ::google::protobuf::util::converter::ErrorListener { 16 | public: 17 | StatusErrorListener() {} 18 | virtual ~StatusErrorListener() {} 19 | 20 | absl::Status status() const { return status_; } 21 | 22 | // ErrorListener implementation 23 | void InvalidName( 24 | const ::google::protobuf::util::converter::LocationTrackerInterface& loc, 25 | absl::string_view unknown_name, absl::string_view message); 26 | void InvalidValue( 27 | const ::google::protobuf::util::converter::LocationTrackerInterface& loc, 28 | absl::string_view type_name, absl::string_view value); 29 | void MissingField( 30 | const ::google::protobuf::util::converter::LocationTrackerInterface& loc, 31 | absl::string_view missing_name); 32 | 33 | void set_status(absl::Status status) { status_ = status; } 34 | 35 | private: 36 | absl::Status status_; 37 | 38 | StatusErrorListener(const StatusErrorListener&) = delete; 39 | StatusErrorListener& operator=(const StatusErrorListener&) = delete; 40 | }; 41 | 42 | } // namespace transcoding 43 | 44 | } // namespace grpc 45 | } // namespace google 46 | #endif // GRPC_TRANSCODING_STATUS_ERROR_LISTENER_H_ 47 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/transcoder.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_TRANSCODER_H_ 16 | #define GRPC_TRANSCODING_TRANSCODER_H_ 17 | 18 | #include "transcoder_input_stream.h" 19 | 20 | namespace google { 21 | namespace grpc { 22 | namespace transcoding { 23 | 24 | // Transcoder interface that transcodes a single request. It holds 25 | // - translated request stream, 26 | // - status of request translation, 27 | // - translated response stream, 28 | // - status of response translation. 29 | // 30 | // NOTE: Transcoder uses TranscoderInputStream for carrying the payloads 31 | // both for input and output. It assumes the following interpretation 32 | // of the TranscoderInputStream interface: 33 | // 34 | // bool TranscoderInputStream::Next(const void** data, int* size); 35 | // 36 | // Obtains a chunk of data from the stream. 37 | // 38 | // Preconditions: 39 | // * "size" and "data" are not NULL. 40 | // 41 | // Postconditions: 42 | // * If the returned value is false, there is no more data to return or an error 43 | // occurred. This is permanent. 44 | // * Otherwise, "size" points to the actual number of bytes read and "data" 45 | // points to a pointer to a buffer containing these bytes. 46 | // * Ownership of this buffer remains with the stream, and the buffer remains 47 | // valid only until some other method of the stream is called or the stream is 48 | // destroyed. 49 | // * It is legal for the returned buffer to have zero size. That means there is 50 | // no data available at this point. This is temporary. The caller needs to try 51 | // again later. 52 | // 53 | // 54 | // void TranscoderInputStream::BackUp(int count); 55 | // 56 | // Backs up a number of bytes, so that the next call to Next() returns 57 | // data again that was already returned by the last call to Next(). This 58 | // is useful when writing procedures that are only supposed to read up 59 | // to a certain point in the input, then return. If Next() returns a 60 | // buffer that goes beyond what you wanted to read, you can use BackUp() 61 | // to return to the point where you intended to finish. 62 | // 63 | // Preconditions: 64 | // * The last method called must have been Next(). 65 | // * count must be less than or equal to the size of the last buffer 66 | // returned by Next(). 67 | // 68 | // Postconditions: 69 | // * The last "count" bytes of the last buffer returned by Next() will be 70 | // pushed back into the stream. Subsequent calls to Next() will return 71 | // the same data again before producing new data. 72 | // 73 | // 74 | // bool TranscoderInputStream::Skip(int count); 75 | // 76 | // Not used and not implemented by the Transcoder. 77 | // 78 | // 79 | // int64_t TranscoderInputStream::BytesAvailable() const; 80 | // 81 | // Returns the number of bytes available for reading at this moment 82 | // 83 | // 84 | // NOTE: To support flow-control the translation & reading the input stream 85 | // happens on-demand in both directions. I.e. Transcoder doesn't call 86 | // Next() on the input stream unless Next() is called on the output stream 87 | // and it ran out of input to translate. 88 | // 89 | // EXAMPLE: 90 | // Transcoder* t = transcoder_factory->Create(...); 91 | // 92 | // const void* buffer = nullptr; 93 | // int size = 0; 94 | // while (backend can accept request) { 95 | // if (!t->RequestOutput()->Next(&buffer, &size)) { 96 | // // end of input or error 97 | // if (t->RequestStatus().ok()) { 98 | // // half-close the request 99 | // } else { 100 | // // error 101 | // } 102 | // } else if (size == 0) { 103 | // // no transcoded request data available at this point; wait for more 104 | // // request data to arrive and run this loop again later. 105 | // break; 106 | // } else { 107 | // // send the buffer to the backend 108 | // ... 109 | // } 110 | // } 111 | // 112 | // const void* buffer = nullptr; 113 | // int size = 0; 114 | // while (client can accept response) { 115 | // if (!t->ResponseOutput()->Next(&buffer, &size)) { 116 | // // end of input or error 117 | // if (t->ResponseStatus().ok()) { 118 | // // close the request 119 | // } else { 120 | // // error 121 | // } 122 | // } else if (size == 0) { 123 | // // no transcoded response data available at this point; wait for more 124 | // // response data to arrive and run this loop again later. 125 | // break; 126 | // } else { 127 | // // send the buffer to the client 128 | // ... 129 | // } 130 | // } 131 | // 132 | class Transcoder { 133 | public: 134 | // ZeroCopyInputStream to read the transcoded request. 135 | virtual TranscoderInputStream* RequestOutput() = 0; 136 | 137 | // The status of request transcoding 138 | virtual absl::Status RequestStatus() = 0; 139 | 140 | // ZeroCopyInputStream to read the transcoded response. 141 | virtual ::google::protobuf::io::ZeroCopyInputStream* ResponseOutput() = 0; 142 | 143 | // The status of response transcoding 144 | virtual absl::Status ResponseStatus() = 0; 145 | 146 | // Virtual destructor 147 | virtual ~Transcoder() {} 148 | }; 149 | 150 | } // namespace transcoding 151 | } // namespace grpc 152 | } // namespace google 153 | 154 | #endif // GRPC_TRANSCODER_H_ 155 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/transcoder_input_stream.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_TRANSCODER_INPUT_STREAM_H_ 16 | #define GRPC_TRANSCODING_TRANSCODER_INPUT_STREAM_H_ 17 | 18 | #include "google/protobuf/io/zero_copy_stream.h" 19 | 20 | namespace google { 21 | namespace grpc { 22 | namespace transcoding { 23 | 24 | class TranscoderInputStream 25 | : public virtual google::protobuf::io::ZeroCopyInputStream { 26 | public: 27 | // returns the number of bytes available to read at the moment. 28 | virtual int64_t BytesAvailable() const = 0; 29 | 30 | // returns if the stream is finished. i.e. No more data will be added. 31 | virtual bool Finished() const = 0; 32 | }; 33 | 34 | } // namespace transcoding 35 | } // namespace grpc 36 | } // namespace google 37 | 38 | #endif // GRPC_TRANSCODER_H_ 39 | -------------------------------------------------------------------------------- /src/include/grpc_transcoding/type_helper.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_TYPE_HELPER_H_ 16 | #define GRPC_TRANSCODING_TYPE_HELPER_H_ 17 | 18 | #include 19 | #include 20 | 21 | #include "google/protobuf/type.pb.h" 22 | #include "google/protobuf/util/converter/type_info.h" 23 | #include "google/protobuf/util/type_resolver.h" 24 | 25 | namespace google { 26 | namespace grpc { 27 | 28 | namespace transcoding { 29 | 30 | // Provides ::google::protobuf::util::TypeResolver and 31 | // ::google::protobuf::util::converter::TypeInfo implementations based on a 32 | // collection of types and a collection of enums. 33 | // This object is thread-safe 34 | class TypeHelper { 35 | public: 36 | template 37 | TypeHelper(const Types& types, const Enums& enums); 38 | 39 | TypeHelper(::google::protobuf::util::TypeResolver* type_resolver); 40 | 41 | ~TypeHelper(); 42 | 43 | ::google::protobuf::util::TypeResolver* Resolver() const; 44 | ::google::protobuf::util::converter::TypeInfo* Info() const; 45 | 46 | // Takes a string representation of a field path & resolves it into actual 47 | // protobuf Field pointers. 48 | // 49 | // A field path is a sequence of fields that identifies a potentially nested 50 | // field in the message. It can be empty as well, which identifies the entire 51 | // message. 52 | // E.g. "shelf.theme" field path would correspond to the "theme" field of the 53 | // "shelf" field of the top-level message. The type of the top-level message 54 | // is passed to ResolveFieldPath(). 55 | // 56 | // The string representation of the field path is just the dot-delimited 57 | // list of the field names or empty: 58 | // FieldPath = "" | Field {"." Field}; 59 | // Field = ; 60 | absl::Status ResolveFieldPath( 61 | const ::google::protobuf::Type& type, const std::string& field_path_str, 62 | std::vector* field_path) const; 63 | 64 | // Resolve a field path specified through a vector of field names into a 65 | // vector of actual protobuf Field pointers. 66 | // Similiar to the above method but accepts the field path as a vector of 67 | // names instead of one dot-delimited string. 68 | absl::Status ResolveFieldPath( 69 | const ::google::protobuf::Type& type, 70 | const std::vector& field_path_unresolved, 71 | std::vector* field_path_resolved) const; 72 | 73 | private: 74 | void Initialize(); 75 | void AddType(const ::google::protobuf::Type& t); 76 | void AddEnum(const ::google::protobuf::Enum& e); 77 | 78 | const google::protobuf::Field* FindField(const google::protobuf::Type* type, 79 | absl::string_view name) const; 80 | 81 | ::google::protobuf::util::TypeResolver* type_resolver_; 82 | std::unique_ptr<::google::protobuf::util::converter::TypeInfo> type_info_; 83 | 84 | TypeHelper() = delete; 85 | TypeHelper(const TypeHelper&) = delete; 86 | TypeHelper& operator=(const TypeHelper&) = delete; 87 | }; 88 | 89 | template 90 | TypeHelper::TypeHelper(const Types& types, const Enums& enums) 91 | : type_resolver_(nullptr), type_info_() { 92 | Initialize(); 93 | for (const auto& t : types) { 94 | AddType(t); 95 | } 96 | for (const auto& e : enums) { 97 | AddEnum(e); 98 | } 99 | } 100 | 101 | } // namespace transcoding 102 | 103 | } // namespace grpc 104 | } // namespace google 105 | 106 | #endif // GRPC_TRANSCODING_TYPE_HELPER_H_ 107 | -------------------------------------------------------------------------------- /src/json_request_translator.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "grpc_transcoding/json_request_translator.h" 18 | 19 | #include 20 | 21 | #include "absl/strings/string_view.h" 22 | #include "google/protobuf/io/zero_copy_stream.h" 23 | #include "google/protobuf/util/converter/json_stream_parser.h" 24 | #include "google/protobuf/util/converter/object_writer.h" 25 | #include "grpc_transcoding/message_stream.h" 26 | #include "grpc_transcoding/request_message_translator.h" 27 | #include "grpc_transcoding/request_stream_translator.h" 28 | 29 | namespace google { 30 | namespace grpc { 31 | 32 | namespace transcoding { 33 | namespace { 34 | 35 | namespace pb = ::google::protobuf; 36 | namespace pbio = ::google::protobuf::io; 37 | namespace pbutil = ::google::protobuf::util; 38 | namespace pbconv = ::google::protobuf::util::converter; 39 | 40 | // An on-demand request translation implementation where the reading of the 41 | // input and translation happen only as needed when the caller asks for an 42 | // output message. 43 | // 44 | // LazyRequestTranslator is given 45 | // - a ZeroCopyInputStream (json_input) to read the input JSON from, 46 | // - a JsonStreamParser (parser) - the input end of the translation 47 | // pipeline, i.e. that takes the input JSON, 48 | // - a MessageStream (translated), the output end of the translation 49 | // pipeline, i.e. where the output proto messages appear. 50 | // When asked for a message it reads chunks from the input stream and passes 51 | // to the json parser until a message appears in the output (translated) 52 | // stream, or until the input JSON stream runs out of data (in this case, caller 53 | // will call NextMessage again in the future when more data is available). 54 | class LazyRequestTranslator : public MessageStream { 55 | public: 56 | LazyRequestTranslator(pbio::ZeroCopyInputStream* json_input, 57 | pbconv::JsonStreamParser* json_parser, 58 | MessageStream* translated) 59 | : input_json_(json_input), 60 | json_parser_(json_parser), 61 | translated_(translated), 62 | seen_input_(false) {} 63 | 64 | // MessageStream implementation 65 | bool NextMessage(std::string* message) { 66 | // Keep translating chunks until a message appears in the translated stream. 67 | while (!translated_->NextMessage(message)) { 68 | if (!TranslateChunk()) { 69 | // Error or no more input to translate. 70 | return false; 71 | } 72 | } 73 | return true; 74 | } 75 | bool Finished() const { return translated_->Finished() || !status_.ok(); } 76 | absl::Status Status() const { return status_; } 77 | 78 | private: 79 | // Translates one chunk of data. Returns true, if there was input to 80 | // translate; otherwise or in case of an error returns false. 81 | bool TranslateChunk() { 82 | if (Finished()) { 83 | return false; 84 | } 85 | // Read the next chunk of data from input_json_ 86 | const void* data = nullptr; 87 | int size = 0; 88 | if (!input_json_->Next(&data, &size)) { 89 | // End of input 90 | if (!seen_input_) { 91 | // If there was no input at all translate an empty JSON object ("{}"). 92 | return CheckParsingStatus(json_parser_->Parse("{}")); 93 | } 94 | // No more data to translate, finish the parser and return false. 95 | CheckParsingStatus(json_parser_->FinishParse()); 96 | return false; 97 | } else if (0 == size) { 98 | // No data at this point, but there might be more input later. 99 | return false; 100 | } 101 | seen_input_ = true; 102 | 103 | // Feed the chunk to the parser & check the status. 104 | return CheckParsingStatus(json_parser_->Parse( 105 | absl::string_view(reinterpret_cast(data), size))); 106 | } 107 | 108 | // If parsing status fails, return false. 109 | // check translated status, if fails, return false. 110 | // save failed status. 111 | bool CheckParsingStatus(absl::Status parsing_status) { 112 | status_ = parsing_status; 113 | if (!status_.ok()) { 114 | return false; 115 | } 116 | // Check the translation status 117 | status_ = translated_->Status(); 118 | if (!status_.ok()) { 119 | return false; 120 | } 121 | return true; 122 | } 123 | 124 | // The input JSON stream 125 | pbio::ZeroCopyInputStream* input_json_; 126 | 127 | // The JSON parser that is the starting point of the translation pipeline 128 | pbconv::JsonStreamParser* json_parser_; 129 | 130 | // The stream where the translated messages appear 131 | MessageStream* translated_; 132 | 133 | // Whether we have seen any input or not 134 | bool seen_input_; 135 | 136 | // Translation status 137 | absl::Status status_; 138 | }; 139 | 140 | } // namespace 141 | 142 | JsonRequestTranslator::JsonRequestTranslator( 143 | pbutil::TypeResolver* type_resolver, pbio::ZeroCopyInputStream* json_input, 144 | RequestInfo request_info, bool streaming, bool output_delimiters) { 145 | // A writer that accepts input ObjectWriter events for translation 146 | pbconv::ObjectWriter* writer = nullptr; 147 | // The stream where translated messages appear 148 | MessageStream* translated = nullptr; 149 | if (streaming) { 150 | // Streaming - we'll need a RequestStreamTranslator 151 | stream_translator_.reset(new RequestStreamTranslator( 152 | *type_resolver, output_delimiters, std::move(request_info))); 153 | writer = stream_translator_.get(); 154 | translated = stream_translator_.get(); 155 | } else { 156 | // No streaming - use a RequestMessageTranslator 157 | message_translator_.reset(new RequestMessageTranslator( 158 | *type_resolver, output_delimiters, std::move(request_info))); 159 | writer = &message_translator_->Input(); 160 | translated = message_translator_.get(); 161 | } 162 | parser_.reset(new pbconv::JsonStreamParser(writer)); 163 | output_.reset( 164 | new LazyRequestTranslator(json_input, parser_.get(), translated)); 165 | } 166 | 167 | } // namespace transcoding 168 | 169 | } // namespace grpc 170 | } // namespace google 171 | -------------------------------------------------------------------------------- /src/message_reader.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "grpc_transcoding/message_reader.h" 18 | 19 | #include 20 | 21 | #include "google/protobuf/io/zero_copy_stream_impl.h" 22 | 23 | namespace google { 24 | namespace grpc { 25 | 26 | namespace transcoding { 27 | 28 | namespace pb = ::google::protobuf; 29 | namespace pbio = ::google::protobuf::io; 30 | 31 | MessageReader::MessageReader(TranscoderInputStream* in) 32 | : in_(in), 33 | current_message_size_(0), 34 | have_current_message_size_(false), 35 | finished_(false) {} 36 | 37 | namespace { 38 | 39 | // A helper function that reads the given number of bytes from a 40 | // ZeroCopyInputStream and copies it to the given buffer 41 | bool ReadStream(pbio::ZeroCopyInputStream* stream, unsigned char* buffer, 42 | int size) { 43 | int size_in = 0; 44 | const void* data_in = nullptr; 45 | // While we have bytes to read 46 | while (size > 0) { 47 | if (!stream->Next(&data_in, &size_in)) { 48 | return false; 49 | } 50 | int to_copy = std::min(size, size_in); 51 | memcpy(buffer, data_in, to_copy); 52 | // Advance buffer and subtract the size to reflect the number of bytes left 53 | buffer += to_copy; 54 | size -= to_copy; 55 | // Keep track of uncopied bytes 56 | size_in -= to_copy; 57 | } 58 | // Return the uncopied bytes 59 | stream->BackUp(size_in); 60 | return true; 61 | } 62 | 63 | // A helper function to extract the size from a gRPC wire format message 64 | // delimiter - see http://www.grpc.io/docs/guides/wire.html. 65 | uint32_t DelimiterToSize(const unsigned char* delimiter) { 66 | unsigned size = 0; 67 | // Bytes 1-4 are big-endian 32-bit message size 68 | size = size | static_cast(delimiter[1]); 69 | size <<= 8; 70 | size = size | static_cast(delimiter[2]); 71 | size <<= 8; 72 | size = size | static_cast(delimiter[3]); 73 | size <<= 8; 74 | size = size | static_cast(delimiter[4]); 75 | return size; 76 | } 77 | 78 | } // namespace 79 | 80 | std::unique_ptr MessageReader::NextMessage() { 81 | if (Finished()) { 82 | // The stream has ended 83 | return nullptr; 84 | } 85 | 86 | // Check if we have the current message size. If not try to read it. 87 | if (!have_current_message_size_) { 88 | if (in_->BytesAvailable() < 89 | static_cast(kGrpcDelimiterByteSize)) { 90 | // We don't have 5 bytes available to read the length of the message. 91 | // Find out whether the stream is finished and return false. 92 | finished_ = in_->Finished(); 93 | if (finished_ && in_->BytesAvailable() != 0) { 94 | status_ = absl::Status(absl::StatusCode::kInternal, 95 | "Incomplete gRPC frame header received"); 96 | } 97 | return nullptr; 98 | } 99 | 100 | // Try to read the delimiter. 101 | memset(delimiter_, 0, kGrpcDelimiterByteSize); 102 | if (!ReadStream(in_, delimiter_, kGrpcDelimiterByteSize)) { 103 | finished_ = true; 104 | return nullptr; 105 | } 106 | 107 | if (delimiter_[0] != 0) { 108 | status_ = absl::Status( 109 | absl::StatusCode::kInternal, 110 | "Unsupported gRPC frame flag: " + std::to_string(delimiter_[0])); 111 | return nullptr; 112 | } 113 | 114 | current_message_size_ = DelimiterToSize(delimiter_); 115 | have_current_message_size_ = true; 116 | } 117 | 118 | if (in_->BytesAvailable() < static_cast(current_message_size_)) { 119 | if (in_->Finished()) { 120 | status_ = absl::Status( 121 | absl::StatusCode::kInternal, 122 | "Incomplete gRPC frame expected size: " + 123 | std::to_string(current_message_size_) + 124 | " actual size: " + std::to_string(in_->BytesAvailable())); 125 | } 126 | // We don't have a full message 127 | return nullptr; 128 | } 129 | 130 | // Reset the have_current_message_size_ for the next message 131 | have_current_message_size_ = false; 132 | 133 | // We have a message! Use LimitingInputStream to wrap the input stream and 134 | // limit it to current_message_size_ bytes to cover only the current message. 135 | return std::unique_ptr( 136 | new pbio::LimitingInputStream(in_, current_message_size_)); 137 | } 138 | 139 | MessageAndGrpcFrame MessageReader::NextMessageAndGrpcFrame() { 140 | MessageAndGrpcFrame out; 141 | out.message = NextMessage(); 142 | memcpy(out.grpc_frame, delimiter_, kGrpcDelimiterByteSize); 143 | out.message_size = current_message_size_; 144 | return out; 145 | } 146 | 147 | } // namespace transcoding 148 | 149 | } // namespace grpc 150 | } // namespace google 151 | -------------------------------------------------------------------------------- /src/message_stream.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "grpc_transcoding/message_stream.h" 18 | 19 | #include 20 | #include 21 | 22 | #include "google/protobuf/io/zero_copy_stream_impl_lite.h" 23 | 24 | namespace google { 25 | namespace grpc { 26 | 27 | namespace transcoding { 28 | 29 | namespace pbio = ::google::protobuf::io; 30 | 31 | namespace { 32 | 33 | // a ZeroCopyInputStream implementation over a MessageStream implementation 34 | class InputStreamOverMessageStream : public TranscoderInputStream { 35 | public: 36 | // src - the underlying MessageStream. InputStreamOverMessageStream doesn't 37 | // maintain the ownership of src, the caller must make sure it exists 38 | // throughtout the lifetime of InputStreamOverMessageStream. 39 | InputStreamOverMessageStream(MessageStream* src) 40 | : src_(src), message_(), position_(0) {} 41 | 42 | // ZeroCopyInputStream implementation 43 | bool Next(const void** data, int* size) { 44 | // Done with the current message, try to get another one. 45 | if (position_ >= message_.size()) { 46 | ReadNextMessage(); 47 | } 48 | 49 | if (position_ < message_.size()) { 50 | *data = static_cast(&message_[position_]); 51 | // Assuming message_.size() - position_ < INT_MAX 52 | *size = static_cast(message_.size() - position_); 53 | // Advance the position 54 | position_ = message_.size(); 55 | return true; 56 | } else { 57 | // No data at this point. 58 | *size = 0; 59 | // Return false if the source stream has finished as this is the end 60 | // of the data; otherwise return true. 61 | return !src_->Finished(); 62 | } 63 | } 64 | 65 | void BackUp(int count) { 66 | if (count > 0 && static_cast(count) <= position_) { 67 | position_ -= static_cast(count); 68 | } 69 | // Otherwise, BackUp has been called illegaly, so we ignore it. 70 | } 71 | 72 | bool Skip(int) { return false; } // Not implemented (no need) 73 | 74 | int64_t ByteCount() const { return 0; } // Not implemented 75 | 76 | int64_t BytesAvailable() const { 77 | if (position_ >= message_.size()) { 78 | // If the current message is all done, try to read the next message 79 | // to make sure we return the correct byte count. 80 | const_cast(this)->ReadNextMessage(); 81 | } 82 | return static_cast(message_.size() - position_); 83 | } 84 | 85 | bool Finished() const { return src_->Finished(); } 86 | 87 | private: 88 | // Updates the current message and creates an ArrayInputStream over it. 89 | void ReadNextMessage() { 90 | message_.clear(); 91 | position_ = 0; 92 | // Try to find the next non-empty message in the stream 93 | while (message_.empty() && src_->NextMessage(&message_)) { 94 | } 95 | } 96 | 97 | // The source MessageStream 98 | MessageStream* src_; 99 | 100 | // The current message being read 101 | std::string message_; 102 | 103 | // The current position in the current message 104 | size_t position_; 105 | }; 106 | 107 | } // namespace 108 | 109 | std::unique_ptr MessageStream::CreateInputStream() { 110 | return std::unique_ptr( 111 | new InputStreamOverMessageStream(this)); 112 | } 113 | 114 | } // namespace transcoding 115 | 116 | } // namespace grpc 117 | } // namespace google 118 | -------------------------------------------------------------------------------- /src/prefix_writer.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "grpc_transcoding/prefix_writer.h" 18 | 19 | #include 20 | #include 21 | 22 | #include "absl/strings/str_split.h" 23 | #include "google/protobuf/util/converter/object_writer.h" 24 | 25 | namespace google { 26 | namespace grpc { 27 | 28 | namespace transcoding { 29 | 30 | PrefixWriter::PrefixWriter(const std::string& prefix, 31 | google::protobuf::util::converter::ObjectWriter* ow) 32 | : prefix_(absl::StrSplit(prefix, ".", absl::SkipEmpty())), 33 | non_actionable_depth_(0), 34 | writer_(ow) {} 35 | 36 | PrefixWriter* PrefixWriter::StartObject(absl::string_view name) { 37 | if (++non_actionable_depth_ == 1) { 38 | name = StartPrefix(name); 39 | } 40 | writer_->StartObject(name); 41 | return this; 42 | } 43 | 44 | PrefixWriter* PrefixWriter::EndObject() { 45 | writer_->EndObject(); 46 | if (--non_actionable_depth_ == 0) { 47 | EndPrefix(); 48 | } 49 | return this; 50 | } 51 | 52 | PrefixWriter* PrefixWriter::StartList(absl::string_view name) { 53 | if (++non_actionable_depth_ == 1) { 54 | name = StartPrefix(name); 55 | } 56 | writer_->StartList(name); 57 | return this; 58 | } 59 | 60 | PrefixWriter* PrefixWriter::EndList() { 61 | writer_->EndList(); 62 | if (--non_actionable_depth_ == 0) { 63 | EndPrefix(); 64 | } 65 | return this; 66 | } 67 | 68 | PrefixWriter* PrefixWriter::RenderBool(absl::string_view name, bool value) { 69 | bool root = non_actionable_depth_ == 0; 70 | if (root) { 71 | name = StartPrefix(name); 72 | } 73 | writer_->RenderBool(name, value); 74 | if (root) { 75 | EndPrefix(); 76 | } 77 | return this; 78 | } 79 | 80 | PrefixWriter* PrefixWriter::RenderInt32(absl::string_view name, int32_t value) { 81 | bool root = non_actionable_depth_ == 0; 82 | if (root) { 83 | name = StartPrefix(name); 84 | } 85 | writer_->RenderInt32(name, value); 86 | if (root) { 87 | EndPrefix(); 88 | } 89 | return this; 90 | } 91 | 92 | PrefixWriter* PrefixWriter::RenderUint32(absl::string_view name, 93 | uint32_t value) { 94 | bool root = non_actionable_depth_ == 0; 95 | if (root) { 96 | name = StartPrefix(name); 97 | } 98 | writer_->RenderUint32(name, value); 99 | if (root) { 100 | EndPrefix(); 101 | } 102 | return this; 103 | } 104 | 105 | PrefixWriter* PrefixWriter::RenderInt64(absl::string_view name, int64_t value) { 106 | bool root = non_actionable_depth_ == 0; 107 | if (root) { 108 | name = StartPrefix(name); 109 | } 110 | writer_->RenderInt64(name, value); 111 | if (root) { 112 | EndPrefix(); 113 | } 114 | return this; 115 | } 116 | 117 | PrefixWriter* PrefixWriter::RenderUint64(absl::string_view name, 118 | uint64_t value) { 119 | bool root = non_actionable_depth_ == 0; 120 | if (root) { 121 | name = StartPrefix(name); 122 | } 123 | writer_->RenderUint64(name, value); 124 | if (root) { 125 | EndPrefix(); 126 | } 127 | return this; 128 | } 129 | 130 | PrefixWriter* PrefixWriter::RenderDouble(absl::string_view name, double value) { 131 | bool root = non_actionable_depth_ == 0; 132 | if (root) { 133 | name = StartPrefix(name); 134 | } 135 | writer_->RenderDouble(name, value); 136 | if (root) { 137 | EndPrefix(); 138 | } 139 | return this; 140 | } 141 | 142 | PrefixWriter* PrefixWriter::RenderFloat(absl::string_view name, float value) { 143 | bool root = non_actionable_depth_ == 0; 144 | if (root) { 145 | name = StartPrefix(name); 146 | } 147 | writer_->RenderFloat(name, value); 148 | if (root) { 149 | EndPrefix(); 150 | } 151 | return this; 152 | } 153 | 154 | PrefixWriter* PrefixWriter::RenderString(absl::string_view name, 155 | absl::string_view value) { 156 | bool root = non_actionable_depth_ == 0; 157 | if (root) { 158 | name = StartPrefix(name); 159 | } 160 | writer_->RenderString(name, value); 161 | if (root) { 162 | EndPrefix(); 163 | } 164 | return this; 165 | } 166 | 167 | PrefixWriter* PrefixWriter::RenderBytes(absl::string_view name, 168 | absl::string_view value) { 169 | bool root = non_actionable_depth_ == 0; 170 | if (root) { 171 | name = StartPrefix(name); 172 | } 173 | writer_->RenderBytes(name, value); 174 | if (root) { 175 | EndPrefix(); 176 | } 177 | return this; 178 | } 179 | 180 | PrefixWriter* PrefixWriter::RenderNull(absl::string_view name) { 181 | bool root = non_actionable_depth_ == 0; 182 | if (root) { 183 | name = StartPrefix(name); 184 | } 185 | 186 | writer_->RenderNull(name); 187 | if (root) { 188 | EndPrefix(); 189 | } 190 | return this; 191 | } 192 | 193 | absl::string_view PrefixWriter::StartPrefix(absl::string_view name) { 194 | for (const auto& prefix : prefix_) { 195 | writer_->StartObject(name); 196 | name = prefix; 197 | } 198 | return name; 199 | } 200 | 201 | void PrefixWriter::EndPrefix() { 202 | for (size_t i = 0; i < prefix_.size(); ++i) { 203 | writer_->EndObject(); 204 | } 205 | } 206 | 207 | } // namespace transcoding 208 | 209 | } // namespace grpc 210 | } // namespace google 211 | -------------------------------------------------------------------------------- /src/request_message_translator.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "grpc_transcoding/request_message_translator.h" 18 | 19 | #include 20 | 21 | #include "google/protobuf/stubs/bytestream.h" 22 | #include "google/protobuf/util/converter/error_listener.h" 23 | #include "google/protobuf/util/converter/protostream_objectwriter.h" 24 | #include "grpc_transcoding/prefix_writer.h" 25 | #include "grpc_transcoding/request_weaver.h" 26 | 27 | namespace pbconv = ::google::protobuf::util::converter; 28 | 29 | namespace google { 30 | namespace grpc { 31 | 32 | namespace transcoding { 33 | 34 | namespace { 35 | 36 | pbconv::ProtoStreamObjectWriter::Options GetProtoWriterOptions( 37 | bool case_insensitive_enum_parsing) { 38 | auto options = pbconv::ProtoStreamObjectWriter::Options::Defaults(); 39 | // Don't fail the translation if there are unknown fields in JSON. 40 | // This will make sure that we allow backward and forward compatible APIs. 41 | options.ignore_unknown_fields = true; 42 | options.case_insensitive_enum_parsing = case_insensitive_enum_parsing; 43 | return options; 44 | } 45 | 46 | } // namespace 47 | 48 | RequestMessageTranslator::RequestMessageTranslator( 49 | google::protobuf::util::TypeResolver& type_resolver, bool output_delimiter, 50 | RequestInfo request_info) 51 | : message_(), 52 | sink_(&message_), 53 | error_listener_(), 54 | proto_writer_( 55 | &type_resolver, *request_info.message_type, &sink_, &error_listener_, 56 | GetProtoWriterOptions(request_info.case_insensitive_enum_parsing)), 57 | request_weaver_(), 58 | prefix_writer_(), 59 | writer_pipeline_(&proto_writer_), 60 | output_delimiter_(output_delimiter), 61 | finished_(false) { 62 | // Relax Base64 decoding to support RFC 2045 Base64 63 | proto_writer_.set_use_strict_base64_decoding(false); 64 | 65 | // Create a RequestWeaver if we have variable bindings to weave 66 | if (!request_info.variable_bindings.empty()) { 67 | request_weaver_.reset(new RequestWeaver( 68 | std::move(request_info.variable_bindings), writer_pipeline_, 69 | &error_listener_, request_info.reject_binding_body_field_collisions)); 70 | writer_pipeline_ = request_weaver_.get(); 71 | } 72 | 73 | // Create a PrefixWriter if there is a prefix to write 74 | if (!request_info.body_field_path.empty() && 75 | "*" != request_info.body_field_path) { 76 | prefix_writer_.reset( 77 | new PrefixWriter(request_info.body_field_path, writer_pipeline_)); 78 | writer_pipeline_ = prefix_writer_.get(); 79 | } 80 | 81 | if (output_delimiter_) { 82 | // Reserve space for the delimiter at the begining of the message_ 83 | ReserveDelimiterSpace(); 84 | } 85 | } 86 | 87 | RequestMessageTranslator::~RequestMessageTranslator() {} 88 | 89 | bool RequestMessageTranslator::Finished() const { return finished_; } 90 | 91 | bool RequestMessageTranslator::NextMessage(std::string* message) { 92 | if (Finished()) { 93 | // Finished reading 94 | return false; 95 | } 96 | if (!proto_writer_.done()) { 97 | // No full message yet 98 | return false; 99 | } 100 | if (output_delimiter_) { 101 | WriteDelimiter(); 102 | } 103 | *message = std::move(message_); 104 | finished_ = true; 105 | return true; 106 | } 107 | 108 | void RequestMessageTranslator::ReserveDelimiterSpace() { 109 | static char reserved[kDelimiterSize] = {0}; 110 | sink_.Append(reserved, sizeof(reserved)); 111 | } 112 | 113 | namespace { 114 | 115 | void SizeToDelimiter(unsigned size, unsigned char* delimiter) { 116 | delimiter[0] = 0; // compression bit 117 | 118 | // big-endian 32-bit length 119 | delimiter[4] = 0xFF & size; 120 | size >>= 8; 121 | delimiter[3] = 0xFF & size; 122 | size >>= 8; 123 | delimiter[2] = 0xFF & size; 124 | size >>= 8; 125 | delimiter[1] = 0xFF & size; 126 | } 127 | 128 | } // namespace 129 | 130 | void RequestMessageTranslator::WriteDelimiter() { 131 | // Asumming that the message_.size() - kDelimiterSize is less than UINT_MAX 132 | SizeToDelimiter(static_cast(message_.size() - kDelimiterSize), 133 | reinterpret_cast(&message_[0])); 134 | } 135 | 136 | } // namespace transcoding 137 | 138 | } // namespace grpc 139 | } // namespace google 140 | -------------------------------------------------------------------------------- /src/response_to_json_translator.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "grpc_transcoding/response_to_json_translator.h" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "google/protobuf/io/zero_copy_stream_impl_lite.h" 24 | #include "google/protobuf/util/type_resolver.h" 25 | 26 | namespace google { 27 | namespace grpc { 28 | 29 | namespace transcoding { 30 | ResponseToJsonTranslator::ResponseToJsonTranslator( 31 | ::google::protobuf::util::TypeResolver* type_resolver, std::string type_url, 32 | bool streaming, TranscoderInputStream* in, 33 | const JsonResponseTranslateOptions& options) 34 | : type_resolver_(type_resolver), 35 | type_url_(std::move(type_url)), 36 | options_(options), 37 | streaming_(streaming), 38 | reader_(in), 39 | first_(true), 40 | finished_(false) {} 41 | 42 | bool ResponseToJsonTranslator::NextMessage(std::string* message) { 43 | if (Finished()) { 44 | // All done 45 | return false; 46 | } 47 | 48 | // Try to read a message 49 | auto proto_in = reader_.NextMessage(); 50 | status_ = reader_.Status(); 51 | if (!status_.ok()) { 52 | return false; 53 | } 54 | 55 | if (proto_in) { 56 | std::string json_out; 57 | if (TranslateMessage(proto_in.get(), &json_out)) { 58 | *message = std::move(json_out); 59 | if (!streaming_) { 60 | // This is a non-streaming call, so we don't expect more messages. 61 | finished_ = true; 62 | } 63 | return true; 64 | } else { 65 | // TranslateMessage() failed - return false. The error details are stored 66 | // in status_. 67 | return false; 68 | } 69 | } else if (streaming_ && reader_.Finished()) { 70 | if (!options_.stream_newline_delimited && 71 | !options_.stream_sse_style_delimited) { 72 | // This is a non-newline-delimited and non-SSE-style-delimited streaming 73 | // call and the input is finished. Return the final ']' or "[]" in case 74 | // this was an empty stream. 75 | *message = first_ ? "[]" : "]"; 76 | } 77 | finished_ = true; 78 | return true; 79 | } else { 80 | // Don't have an input message 81 | return false; 82 | } 83 | } 84 | 85 | namespace { 86 | 87 | // A helper to write a single char to a ZeroCopyOutputStream 88 | bool WriteChar(::google::protobuf::io::ZeroCopyOutputStream* stream, char c) { 89 | int size = 0; 90 | void* data = 0; 91 | if (!stream->Next(&data, &size) || 0 == size) { 92 | return false; 93 | } 94 | // Write the char to the first byte of the buffer and return the rest size-1 95 | // bytes to the stream. 96 | *reinterpret_cast(data) = c; 97 | stream->BackUp(size - 1); 98 | return true; 99 | } 100 | 101 | // A helper to write a string to a ZeroCopyOutputStream. 102 | bool WriteString(::google::protobuf::io::ZeroCopyOutputStream* stream, 103 | const std::string& str) { 104 | int bytes_to_write = str.size(); 105 | int bytes_written = 0; 106 | while (bytes_written < bytes_to_write) { 107 | int size = 0; 108 | void* data; 109 | if (!stream->Next(&data, &size) || size == 0) { 110 | return false; 111 | } 112 | int bytes_to_write_this_iteration = 113 | std::min(bytes_to_write - bytes_written, size); 114 | memcpy(data, str.data() + bytes_written, bytes_to_write_this_iteration); 115 | bytes_written += bytes_to_write_this_iteration; 116 | if (bytes_to_write_this_iteration < size) { 117 | stream->BackUp(size - bytes_to_write_this_iteration); 118 | } 119 | } 120 | return true; 121 | } 122 | 123 | } // namespace 124 | 125 | bool ResponseToJsonTranslator::TranslateMessage( 126 | ::google::protobuf::io::ZeroCopyInputStream* proto_in, 127 | std::string* json_out) { 128 | ::google::protobuf::io::StringOutputStream json_stream(json_out); 129 | 130 | if (streaming_ && options_.stream_sse_style_delimited) { 131 | if (!WriteString(&json_stream, "data: ")) { 132 | status_ = absl::Status(absl::StatusCode::kInternal, 133 | "Failed to build the response message."); 134 | return false; 135 | } 136 | } else if (streaming_ && !options_.stream_newline_delimited) { 137 | if (first_) { 138 | // This is a non-newline-delimited streaming call and this is the first 139 | // message, so prepend the 140 | // output JSON with a '['. 141 | if (!WriteChar(&json_stream, '[')) { 142 | status_ = absl::Status(absl::StatusCode::kInternal, 143 | "Failed to build the response message."); 144 | return false; 145 | } 146 | first_ = false; 147 | } else { 148 | // For non-newline-delimited streaming calls add a ',' before each message 149 | // except the first. 150 | if (!WriteChar(&json_stream, ',')) { 151 | status_ = absl::Status(absl::StatusCode::kInternal, 152 | "Failed to build the response message."); 153 | return false; 154 | } 155 | } 156 | } 157 | 158 | // Do the actual translation. 159 | status_ = ::google::protobuf::util::BinaryToJsonStream( 160 | type_resolver_, type_url_, proto_in, &json_stream, 161 | options_.json_print_options); 162 | 163 | if (!status_.ok()) { 164 | return false; 165 | } 166 | 167 | // Append a newline delimiter after the message if needed. 168 | if (streaming_ && options_.stream_sse_style_delimited) { 169 | if (!WriteString(&json_stream, "\n\n")) { 170 | status_ = absl::Status(absl::StatusCode::kInternal, 171 | "Failed to build the response message."); 172 | return false; 173 | } 174 | } else if (streaming_ && options_.stream_newline_delimited) { 175 | if (!WriteChar(&json_stream, '\n')) { 176 | status_ = absl::Status(absl::StatusCode::kInternal, 177 | "Failed to build the response message."); 178 | return false; 179 | } 180 | } 181 | 182 | return true; 183 | } 184 | 185 | } // namespace transcoding 186 | 187 | } // namespace grpc 188 | } // namespace google 189 | -------------------------------------------------------------------------------- /src/status_error_listener.cc: -------------------------------------------------------------------------------- 1 | #include "grpc_transcoding/status_error_listener.h" 2 | 3 | #include 4 | 5 | namespace google { 6 | namespace grpc { 7 | 8 | namespace transcoding { 9 | 10 | void StatusErrorListener::InvalidName( 11 | const ::google::protobuf::util::converter::LocationTrackerInterface& loc, 12 | absl::string_view unknown_name, absl::string_view message) { 13 | status_ = absl::Status(absl::StatusCode::kInvalidArgument, 14 | loc.ToString() + ": " + std::string(message)); 15 | } 16 | 17 | void StatusErrorListener::InvalidValue( 18 | const ::google::protobuf::util::converter::LocationTrackerInterface& loc, 19 | absl::string_view type_name, absl::string_view value) { 20 | status_ = 21 | absl::Status(absl::StatusCode::kInvalidArgument, 22 | loc.ToString() + ": invalid value " + std::string(value) + 23 | " for type " + std::string(type_name)); 24 | } 25 | 26 | void StatusErrorListener::MissingField( 27 | const ::google::protobuf::util::converter::LocationTrackerInterface& loc, 28 | absl::string_view missing_name) { 29 | status_ = absl::Status( 30 | absl::StatusCode::kInvalidArgument, 31 | loc.ToString() + ": missing field " + std::string(missing_name)); 32 | } 33 | 34 | } // namespace transcoding 35 | 36 | } // namespace grpc 37 | } // namespace google -------------------------------------------------------------------------------- /test/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | ################################################################################ 16 | # 17 | load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") 18 | load("@rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test") 19 | 20 | package(default_visibility = ["//visibility:public"]) 21 | 22 | cc_test( 23 | name = "http_template_test", 24 | size = "small", 25 | srcs = [ 26 | "http_template_test.cc", 27 | ], 28 | linkstatic = 1, 29 | deps = [ 30 | "//src:http_template", 31 | "@com_google_googletest//:gtest_main", 32 | ], 33 | ) 34 | 35 | cc_fuzz_test( 36 | name = "http_template_fuzz_test", 37 | srcs = [ 38 | "http_template_fuzz_test.cc", 39 | ], 40 | corpus = glob(["http_template_fuzz_test_corpus/**"]), 41 | deps = [ 42 | "//src:http_template", 43 | ], 44 | ) 45 | 46 | cc_test( 47 | name = "path_matcher_test", 48 | size = "small", 49 | srcs = [ 50 | "path_matcher_test.cc", 51 | ], 52 | linkstatic = 1, 53 | deps = [ 54 | "//src:path_matcher", 55 | "@com_google_googletest//:gtest_main", 56 | ], 57 | ) 58 | 59 | cc_test( 60 | name = "path_matcher_utility_test", 61 | size = "small", 62 | srcs = [ 63 | "path_matcher_utility_test.cc", 64 | ], 65 | linkstatic = 1, 66 | deps = [ 67 | "//src:path_matcher_utility", 68 | "@com_google_googletest//:gtest_main", 69 | ], 70 | ) 71 | 72 | cc_test( 73 | name = "prefix_writer_test", 74 | size = "small", 75 | srcs = [ 76 | "prefix_writer_test.cc", 77 | ], 78 | deps = [ 79 | "//src:prefix_writer", 80 | "@com_google_googletest//:gtest_main", 81 | "@com_google_protobuf//:protobuf", 82 | "@com_google_protoconverter//:testing", 83 | ], 84 | ) 85 | 86 | cc_test( 87 | name = "request_weaver_test", 88 | size = "small", 89 | srcs = [ 90 | "request_weaver_test.cc", 91 | ], 92 | deps = [ 93 | "//src:request_weaver", 94 | "@com_google_absl//absl/strings", 95 | "@com_google_googletest//:gtest_main", 96 | "@com_google_protoconverter//:testing", 97 | ], 98 | ) 99 | 100 | cc_test( 101 | name = "status_error_listener_test", 102 | size = "small", 103 | srcs = [ 104 | "status_error_listener_test.cc", 105 | ], 106 | deps = [ 107 | "//src:status_error_listener", 108 | "@com_google_absl//absl/strings", 109 | "@com_google_googletest//:gtest_main", 110 | ], 111 | ) 112 | 113 | proto_library( 114 | name = "bookstore_proto", 115 | testonly = 1, 116 | srcs = ["bookstore.proto"], 117 | ) 118 | 119 | cc_proto_library( 120 | name = "bookstore_cc_proto", 121 | testonly = 1, 122 | deps = [":bookstore_proto"], 123 | ) 124 | 125 | cc_test( 126 | name = "type_helper_test", 127 | size = "small", 128 | srcs = [ 129 | "type_helper_test.cc", 130 | ], 131 | data = [ 132 | "testdata/bookstore_service.pb.txt", 133 | ], 134 | deps = [ 135 | ":test_common", 136 | "//src:type_helper", 137 | "@com_google_googleapis//google/api:service_cc_proto", 138 | "@com_google_googletest//:gtest_main", 139 | ], 140 | ) 141 | 142 | cc_library( 143 | name = "test_common", 144 | testonly = 1, 145 | srcs = ["test_common.cc"], 146 | hdrs = ["test_common.h"], 147 | deps = [ 148 | "//src:transcoder_input_stream", 149 | "@com_google_googleapis//google/api:service_cc_proto", 150 | "@com_google_googletest//:gtest", 151 | "@com_google_protobuf//:protobuf", 152 | ], 153 | ) 154 | 155 | cc_library( 156 | name = "request_translator_test_base", 157 | testonly = 1, 158 | srcs = [ 159 | "proto_stream_tester.cc", 160 | "proto_stream_tester.h", 161 | "request_translator_test_base.cc", 162 | ], 163 | hdrs = [ 164 | "request_translator_test_base.h", 165 | ], 166 | deps = [ 167 | ":bookstore_cc_proto", 168 | ":test_common", 169 | "//src:request_message_translator", 170 | "//src:type_helper", 171 | "@com_google_googleapis//google/api:service_cc_proto", 172 | "@com_google_googletest//:gtest", 173 | "@com_google_protobuf//:protobuf", 174 | ], 175 | ) 176 | 177 | cc_test( 178 | name = "request_message_translator_test", 179 | size = "small", 180 | srcs = [ 181 | "request_message_translator_test.cc", 182 | ], 183 | data = [ 184 | "testdata/bookstore_service.pb.txt", 185 | ], 186 | deps = [ 187 | ":bookstore_cc_proto", 188 | ":request_translator_test_base", 189 | ":test_common", 190 | "//src:request_message_translator", 191 | "@com_google_googletest//:gtest_main", 192 | ], 193 | ) 194 | 195 | cc_test( 196 | name = "request_stream_translator_test", 197 | size = "small", 198 | srcs = [ 199 | "request_stream_translator_test.cc", 200 | ], 201 | data = [ 202 | "testdata/bookstore_service.pb.txt", 203 | ], 204 | deps = [ 205 | ":bookstore_cc_proto", 206 | ":request_translator_test_base", 207 | "//src:request_stream_translator", 208 | "@com_google_googletest//:gtest_main", 209 | ], 210 | ) 211 | 212 | cc_test( 213 | name = "json_request_translator_test", 214 | size = "small", 215 | srcs = [ 216 | "json_request_translator_test.cc", 217 | ], 218 | data = [ 219 | "testdata/bookstore_service.pb.txt", 220 | ], 221 | deps = [ 222 | ":bookstore_cc_proto", 223 | ":request_translator_test_base", 224 | ":test_common", 225 | "//src:json_request_translator", 226 | "@com_google_googletest//:gtest_main", 227 | ], 228 | ) 229 | 230 | cc_test( 231 | name = "message_reader_test", 232 | size = "small", 233 | srcs = [ 234 | "message_reader_test.cc", 235 | ], 236 | deps = [ 237 | ":test_common", 238 | "//src:message_reader", 239 | "@com_google_googletest//:gtest_main", 240 | ], 241 | ) 242 | 243 | cc_fuzz_test( 244 | name = "message_reader_fuzz_test", 245 | testonly = 1, 246 | srcs = [ 247 | "message_reader_fuzz_test.cc", 248 | ], 249 | corpus = glob(["message_reader_fuzz_test_corpus/**"]), 250 | deps = [ 251 | ":test_common", 252 | "//src:message_reader", 253 | ], 254 | ) 255 | 256 | cc_test( 257 | name = "response_to_json_translator_test", 258 | size = "small", 259 | srcs = [ 260 | "response_to_json_translator_test.cc", 261 | ], 262 | data = [ 263 | "testdata/bookstore_service.pb.txt", 264 | ], 265 | deps = [ 266 | ":bookstore_cc_proto", 267 | ":test_common", 268 | "//src:message_reader", 269 | "//src:response_to_json_translator", 270 | "//src:type_helper", 271 | "@com_google_googletest//:gtest_main", 272 | ], 273 | ) 274 | 275 | cc_test( 276 | name = "message_stream_test", 277 | size = "small", 278 | srcs = [ 279 | "message_stream_test.cc", 280 | ], 281 | deps = [ 282 | ":test_common", 283 | "//src:message_stream", 284 | "@com_google_googletest//:gtest_main", 285 | ], 286 | ) 287 | -------------------------------------------------------------------------------- /test/bookstore.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | // bookstore.proto 18 | // Test proto for transcoding 19 | syntax = "proto3"; 20 | package google.grpc.transcoding; 21 | message Biography { 22 | int64 year_born = 1; 23 | int64 year_died = 2; 24 | string text = 3; 25 | } 26 | message AuthorInfo { 27 | string first_name = 1; 28 | string last_name = 2; 29 | Biography bio = 3; 30 | } 31 | message Book { 32 | string author = 1; 33 | string name = 2; 34 | string title = 3; 35 | AuthorInfo author_info = 4; 36 | } 37 | message Shelf { 38 | string name = 1; 39 | string theme = 2; 40 | // All the types. 41 | enum TYPE { 42 | CLASSIC = 0; 43 | COMIC = 1; 44 | HORROR = 2; 45 | OTHER = 3; 46 | } 47 | TYPE type = 3; 48 | } 49 | message ListShelvesResponse { 50 | repeated Shelf shelves = 1; 51 | } 52 | message CreateShelfRequest { 53 | Shelf shelf = 1; 54 | } 55 | message GetShelfRequest { 56 | int64 shelf = 1; 57 | } 58 | message DeleteShelfRequest { 59 | int64 shelf = 1; 60 | } 61 | message ListBooksRequest { 62 | int64 shelf = 1; 63 | } 64 | message CreateBookRequest { 65 | int64 shelf = 1; 66 | Book book = 2; 67 | } 68 | message GetBookRequest { 69 | int64 shelf = 1; 70 | int64 book = 2; 71 | } 72 | message DeleteBookRequest { 73 | int64 shelf = 1; 74 | int64 book = 2; 75 | } 76 | -------------------------------------------------------------------------------- /test/http_template_fuzz_test.cc: -------------------------------------------------------------------------------- 1 | #include "grpc_transcoding/http_template.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { 8 | std::string path((const char *)data, size); 9 | google::grpc::transcoding::HttpTemplate::Parse(path); 10 | return 0; 11 | } 12 | -------------------------------------------------------------------------------- /test/http_template_fuzz_test_corpus/literal: -------------------------------------------------------------------------------- 1 | /foo 2 | -------------------------------------------------------------------------------- /test/http_template_fuzz_test_corpus/path-double-wildcard: -------------------------------------------------------------------------------- 1 | /foo/** 2 | -------------------------------------------------------------------------------- /test/http_template_fuzz_test_corpus/path-double-wildcard-capture: -------------------------------------------------------------------------------- 1 | /{root=**} 2 | -------------------------------------------------------------------------------- /test/http_template_fuzz_test_corpus/path-wildcard: -------------------------------------------------------------------------------- 1 | /foo/* 2 | -------------------------------------------------------------------------------- /test/http_template_fuzz_test_corpus/path-wildcard-capture: -------------------------------------------------------------------------------- 1 | /{root=*} 2 | -------------------------------------------------------------------------------- /test/message_reader_fuzz_test.cc: -------------------------------------------------------------------------------- 1 | #include "grpc_transcoding/message_reader.h" 2 | 3 | #include "test_common.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace google { 11 | namespace grpc { 12 | namespace transcoding { 13 | namespace testing { 14 | namespace { 15 | 16 | extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { 17 | FuzzedDataProvider provider(data, size); 18 | 19 | TestZeroCopyInputStream input_stream; 20 | MessageReader reader(&input_stream); 21 | 22 | while (provider.remaining_bytes() > 0) { 23 | // Add a few chucks of data to the input stream. 24 | for (int i = 0; i < provider.ConsumeIntegralInRange(0, 5); i++) { 25 | input_stream.AddChunk(provider.ConsumeRandomLengthString(100)); 26 | } 27 | 28 | // Run the message reader to get the next message. 29 | (void)reader.NextMessageAndGrpcFrame(); 30 | 31 | // Handle end of input or error due to malformed bytes. 32 | if (reader.Finished()) { 33 | return 0; 34 | } 35 | } 36 | 37 | return 0; 38 | } 39 | 40 | } // namespace 41 | } // namespace testing 42 | } // namespace transcoding 43 | } // namespace grpc 44 | } // namespace google 45 | -------------------------------------------------------------------------------- /test/message_reader_fuzz_test_corpus/grpc-delimiter.txt: -------------------------------------------------------------------------------- 1 | 1\0\0\0\0a2830232 -------------------------------------------------------------------------------- /test/prefix_writer_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "grpc_transcoding/prefix_writer.h" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "google/protobuf/util/converter/expecting_objectwriter.h" 24 | #include "gtest/gtest.h" 25 | 26 | namespace google { 27 | namespace grpc { 28 | 29 | namespace transcoding { 30 | namespace testing { 31 | namespace { 32 | 33 | using ::testing::InSequence; 34 | 35 | class PrefixWriterTest : public ::testing::Test { 36 | protected: 37 | PrefixWriterTest() : mock_(), expect_(&mock_) {} 38 | 39 | std::unique_ptr Create(const std::string& prefix) { 40 | return std::unique_ptr(new PrefixWriter(prefix, &mock_)); 41 | } 42 | 43 | google::protobuf::util::converter::MockObjectWriter mock_; 44 | google::protobuf::util::converter::ExpectingObjectWriter expect_; 45 | InSequence seq_; // all our expectations must be ordered 46 | }; 47 | 48 | TEST_F(PrefixWriterTest, EmptyPrefix) { 49 | expect_.StartObject(""); 50 | expect_.StartObject("A"); 51 | expect_.RenderString("x", "a"); 52 | expect_.RenderBytes("by", "b"); 53 | expect_.RenderInt32("i", int32_t(1)); 54 | expect_.RenderUint32("ui", uint32_t(2)); 55 | expect_.RenderInt64("i64", int64_t(3)); 56 | expect_.RenderUint64("ui64", uint64_t(4)); 57 | expect_.RenderBool("b", true); 58 | expect_.RenderNull("null"); 59 | expect_.StartObject("B"); 60 | expect_.RenderString("y", "b"); 61 | expect_.EndObject(); // B 62 | expect_.EndObject(); // A 63 | expect_.EndObject(); // "" 64 | 65 | auto w = Create(""); 66 | 67 | w->StartObject(""); 68 | w->StartObject("A"); 69 | w->RenderString("x", "a"); 70 | w->RenderBytes("by", "b"); 71 | w->RenderInt32("i", int32_t(1)); 72 | w->RenderUint32("ui", uint32_t(2)); 73 | w->RenderInt64("i64", int64_t(3)); 74 | w->RenderUint64("ui64", uint64_t(4)); 75 | w->RenderBool("b", true); 76 | w->RenderNull("null"); 77 | w->StartObject("B"); 78 | w->RenderString("y", "b"); 79 | w->EndObject(); // B 80 | w->EndObject(); // A 81 | w->EndObject(); // "" 82 | } 83 | 84 | TEST_F(PrefixWriterTest, OneLevelPrefix1) { 85 | expect_.StartObject(""); 86 | expect_.StartObject("A"); 87 | expect_.RenderString("x", "a"); 88 | expect_.StartObject("B"); 89 | expect_.RenderString("y", "b"); 90 | expect_.EndObject(); // B 91 | expect_.EndObject(); // A 92 | expect_.EndObject(); // "" 93 | 94 | expect_.StartObject("C"); 95 | expect_.StartObject("A"); 96 | expect_.RenderString("z", "c"); 97 | expect_.EndObject(); // C 98 | expect_.EndObject(); // A 99 | 100 | auto w = Create("A"); 101 | 102 | w->StartObject(""); 103 | w->RenderString("x", "a"); 104 | w->StartObject("B"); 105 | w->RenderString("y", "b"); 106 | w->EndObject(); // B 107 | w->EndObject(); // A, "" 108 | 109 | w->StartObject("C"); 110 | w->RenderString("z", "c"); 111 | w->EndObject(); // C, A 112 | } 113 | 114 | TEST_F(PrefixWriterTest, OneLevelPrefix2) { 115 | expect_.StartObject("x"); 116 | expect_.RenderString("A", "a"); 117 | expect_.EndObject(); // "A" 118 | 119 | expect_.StartObject("by"); 120 | expect_.RenderBytes("A", "b"); 121 | expect_.EndObject(); // "A" 122 | 123 | expect_.StartObject("i32"); 124 | expect_.RenderInt32("A", int32_t(-32)); 125 | expect_.EndObject(); // "A" 126 | 127 | expect_.StartObject("ui32"); 128 | expect_.RenderUint32("A", uint32_t(32)); 129 | expect_.EndObject(); // "A" 130 | 131 | expect_.StartObject("i64"); 132 | expect_.RenderInt64("A", int64_t(-64)); 133 | expect_.EndObject(); // "A" 134 | 135 | expect_.StartObject("ui64"); 136 | expect_.RenderUint64("A", uint64_t(64)); 137 | expect_.EndObject(); // "A" 138 | 139 | expect_.StartObject("b"); 140 | expect_.RenderBool("A", false); 141 | expect_.EndObject(); // "A" 142 | 143 | expect_.StartObject("nil"); 144 | expect_.RenderNull("A"); 145 | expect_.EndObject(); // "A" 146 | 147 | auto w = Create("A"); 148 | 149 | w->RenderString("x", "a"); 150 | w->RenderBytes("by", "b"); 151 | w->RenderInt32("i32", int32_t(-32)); 152 | w->RenderUint32("ui32", uint32_t(32)); 153 | w->RenderInt64("i64", int64_t(-64)); 154 | w->RenderUint64("ui64", uint64_t(64)); 155 | w->RenderBool("b", false); 156 | w->RenderNull("nil"); 157 | } 158 | 159 | TEST_F(PrefixWriterTest, TwoLevelPrefix) { 160 | expect_.StartObject(""); 161 | expect_.StartObject("A"); 162 | expect_.StartObject("B"); 163 | expect_.RenderString("x", "a"); 164 | expect_.EndObject(); // B 165 | expect_.EndObject(); // A 166 | expect_.EndObject(); // "" 167 | 168 | expect_.StartObject("C"); 169 | expect_.StartObject("A"); 170 | expect_.StartObject("B"); 171 | expect_.RenderString("y", "b"); 172 | expect_.EndObject(); // B 173 | expect_.EndObject(); // A 174 | expect_.EndObject(); // C 175 | 176 | auto w = Create("A.B"); 177 | 178 | w->StartObject(""); 179 | w->RenderString("x", "a"); 180 | w->EndObject(); // B, A, "" 181 | 182 | w->StartObject("C"); 183 | w->RenderString("y", "b"); 184 | w->EndObject(); // B, A, C 185 | } 186 | 187 | TEST_F(PrefixWriterTest, ThreeLevelPrefix) { 188 | expect_.StartObject(""); 189 | expect_.StartObject("A"); 190 | expect_.StartObject("B"); 191 | expect_.StartObject("C"); 192 | expect_.RenderString("x", "a"); 193 | expect_.EndObject(); // C 194 | expect_.EndObject(); // B 195 | expect_.EndObject(); // A 196 | expect_.EndObject(); // "" 197 | 198 | auto w = Create("A.B.C"); 199 | 200 | w->StartObject(""); 201 | w->RenderString("x", "a"); 202 | w->EndObject(); // C, B, A, "" 203 | } 204 | 205 | } // namespace 206 | } // namespace testing 207 | } // namespace transcoding 208 | 209 | } // namespace grpc 210 | } // namespace google 211 | -------------------------------------------------------------------------------- /test/proto_stream_tester.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "proto_stream_tester.h" 18 | 19 | #include 20 | #include "gtest/gtest.h" 21 | 22 | namespace google { 23 | namespace grpc { 24 | 25 | namespace transcoding { 26 | namespace testing { 27 | 28 | ProtoStreamTester::ProtoStreamTester(MessageStream& stream, bool delimiters) 29 | : stream_(stream), delimiters_(delimiters) {} 30 | 31 | bool ProtoStreamTester::ExpectNone() { 32 | // First check the status of the stream 33 | if (!ExpectStatusEq(absl::StatusCode::kOk)) { 34 | return false; 35 | } 36 | std::string message; 37 | if (stream_.NextMessage(&message)) { 38 | ADD_FAILURE() << "ProtoStreamTester::ValidateNone: NextMessage() returned " 39 | "true; expected false.\n"; 40 | return false; 41 | } 42 | return true; 43 | } 44 | 45 | bool ProtoStreamTester::ExpectFinishedEq(bool expected) { 46 | // First check the status of the stream 47 | if (!ExpectStatusEq(absl::StatusCode::kOk)) { 48 | return false; 49 | } 50 | if (expected != stream_.Finished()) { 51 | ADD_FAILURE() 52 | << (expected 53 | ? "The stream was expected to be finished, but it's not.\n" 54 | : "The stream was not expected to be finished, but it is.\n"); 55 | EXPECT_EQ(expected, stream_.Finished()); 56 | return false; 57 | } 58 | return true; 59 | } 60 | 61 | namespace { 62 | 63 | unsigned DelimiterToSize(const unsigned char* delimiter) { 64 | unsigned size = 0; 65 | size = size | static_cast(delimiter[1]); 66 | size <<= 8; 67 | size = size | static_cast(delimiter[2]); 68 | size <<= 8; 69 | size = size | static_cast(delimiter[3]); 70 | size <<= 8; 71 | size = size | static_cast(delimiter[4]); 72 | return size; 73 | } 74 | 75 | } // namespace 76 | 77 | bool ProtoStreamTester::ValidateDelimiter(const std::string& message) { 78 | // First check the status of the stream 79 | if (!ExpectStatusEq(absl::StatusCode::kOk)) { 80 | return false; 81 | } 82 | if (message.size() < kDelimiterSize) { 83 | ADD_FAILURE() << "ProtoStreamTester::ValidateSizeDelimiter: message size " 84 | "is less than a delimiter size (5).\n"; 85 | } 86 | int size_actual = 87 | DelimiterToSize(reinterpret_cast(&message[0])); 88 | int size_expected = static_cast(message.size() - kDelimiterSize); 89 | if (size_expected != size_actual) { 90 | ADD_FAILURE() << "The size extracted from the message delimiter: " 91 | << size_actual 92 | << " doesn't match the size of the message: " << size_expected 93 | << std::endl; 94 | return false; 95 | } 96 | return true; 97 | } 98 | 99 | bool ProtoStreamTester::ExpectStatusEq( 100 | absl::StatusCode error_code) { 101 | if (error_code != stream_.Status().code()) { 102 | ADD_FAILURE() 103 | << "ObjectTranslatorTest::ValidateStatus: Status doesn't match " 104 | "expected: " 105 | << static_cast(error_code) 106 | << " actual: " << static_cast(stream_.Status().code()) << " - " 107 | << stream_.Status().message() << std::endl; 108 | return false; 109 | } 110 | return true; 111 | } 112 | 113 | } // namespace testing 114 | } // namespace transcoding 115 | 116 | } // namespace grpc 117 | } // namespace google 118 | -------------------------------------------------------------------------------- /test/proto_stream_tester.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_PROTO_STREAM_TESTER_H_ 16 | #define GRPC_TRANSCODING_PROTO_STREAM_TESTER_H_ 17 | 18 | #include 19 | 20 | #include "google/protobuf/text_format.h" 21 | #include "google/protobuf/util/message_differencer.h" 22 | #include "grpc_transcoding/message_stream.h" 23 | #include "gtest/gtest.h" 24 | 25 | namespace google { 26 | namespace grpc { 27 | 28 | namespace transcoding { 29 | namespace testing { 30 | 31 | // A helper that makes it easy to test a stream of protobuf messages 32 | // represented through a MessageStream interface. It handles matching 33 | // proto messages, validating the GRPC message delimiter (see 34 | // http://www.grpc.io/docs/guides/wire.html) and automatically checking the 35 | // stream status. 36 | class ProtoStreamTester { 37 | public: 38 | // stream - the stream to be tested 39 | // delimiters - whether the messages have delimiters or not 40 | ProtoStreamTester(MessageStream& stream, bool delimiters); 41 | 42 | // Validation methods 43 | template 44 | bool ExpectNextEq(const std::string& expected_proto_text); 45 | bool ExpectNone(); 46 | bool ExpectFinishedEq(bool expected); 47 | bool ExpectStatusEq(absl::StatusCode error_code); 48 | 49 | private: 50 | // Validates the GRPC message delimiter at the beginning 51 | // of the message. 52 | bool ValidateDelimiter(const std::string& message); 53 | 54 | MessageStream& stream_; 55 | bool delimiters_; 56 | 57 | static const int kDelimiterSize = 5; 58 | }; 59 | 60 | template 61 | bool ProtoStreamTester::ExpectNextEq(const std::string& expected_proto_text) { 62 | // First check the status of the stream 63 | if (!ExpectStatusEq(absl::StatusCode::kOk)) { 64 | return false; 65 | } 66 | // Try to get a message 67 | std::string message; 68 | if (!stream_.NextMessage(&message)) { 69 | ADD_FAILURE() << "ProtoStreamTester::ValidateNext: NextMessage() " 70 | "returned false\n"; 71 | // Use ExpectStatusEq() to output the status if it's not OK. 72 | ExpectStatusEq(absl::StatusCode::kOk); 73 | return false; 74 | } 75 | // Validate the delimiter if it's expected 76 | if (delimiters_) { 77 | if (!ValidateDelimiter(message)) { 78 | return false; 79 | } else { 80 | // Strip the delimiter 81 | message = message.substr(kDelimiterSize); 82 | } 83 | } 84 | // Parse the actual message 85 | MessageType actual; 86 | if (!actual.ParseFromString(message)) { 87 | ADD_FAILURE() << "ProtoStreamTester::ValidateNext: couldn't parse " 88 | "the actual message:\n" 89 | << message << std::endl; 90 | return false; 91 | } 92 | // Parse the expected message 93 | MessageType expected; 94 | if (!google::protobuf::TextFormat::ParseFromString(expected_proto_text, 95 | &expected)) { 96 | ADD_FAILURE() << "ProtoStreamTester::ValidateNext: couldn't parse " 97 | "the expected message:\n" 98 | << expected_proto_text << std::endl; 99 | return false; 100 | } 101 | // Now try matching the protos 102 | if (!google::protobuf::util::MessageDifferencer::Equivalent(expected, 103 | actual)) { 104 | // Use EXPECT_EQ on debug strings to output the diff 105 | EXPECT_EQ(expected.DebugString(), actual.DebugString()); 106 | return false; 107 | } 108 | return true; 109 | } 110 | 111 | } // namespace testing 112 | } // namespace transcoding 113 | 114 | } // namespace grpc 115 | } // namespace google 116 | 117 | #endif // GRPC_TRANSCODING_PROTO_STREAM_TESTER_H_ 118 | -------------------------------------------------------------------------------- /test/request_translator_test_base.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | // 17 | #include "request_translator_test_base.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "absl/strings/str_split.h" 26 | 27 | #include "google/api/service.pb.h" 28 | #include "google/protobuf/text_format.h" 29 | #include "google/protobuf/type.pb.h" 30 | #include "google/protobuf/util/converter/type_info.h" 31 | #include "grpc_transcoding/message_stream.h" 32 | #include "gtest/gtest.h" 33 | #include "test_common.h" 34 | 35 | namespace google { 36 | namespace grpc { 37 | 38 | namespace transcoding { 39 | namespace testing { 40 | namespace { 41 | 42 | // Parse a dot delimited field path string into a vector of actual field 43 | // pointers 44 | std::vector ParseFieldPath( 45 | const google::protobuf::Type& type, 46 | google::protobuf::util::converter::TypeInfo& type_info, 47 | const std::string& field_path_str) { 48 | // First, split the field names 49 | std::vector field_names = 50 | absl::StrSplit(field_path_str, ".", absl::SkipEmpty()); 51 | 52 | auto current_type = &type; 53 | std::vector field_path; 54 | for (size_t i = 0; i < field_names.size(); ++i) { 55 | // Find the field by name 56 | auto field = type_info.FindField(current_type, field_names[i]); 57 | EXPECT_NE(nullptr, field) << "Could not find field " << field_names[i] 58 | << " in type " << current_type->name() 59 | << " while parsing field path " << field_path_str 60 | << std::endl; 61 | field_path.push_back(field); 62 | 63 | if (i < field_names.size() - 1) { 64 | // If it's not the last one in the path, it must be a message 65 | EXPECT_EQ(google::protobuf::Field::TYPE_MESSAGE, field->kind()) 66 | << "Encountered a non-leaf field " << field->name() 67 | << " that is not a message while parsing field path" << field_path_str 68 | << std::endl; 69 | 70 | // Update the type of the current field 71 | current_type = type_info.GetTypeByTypeUrl(field->type_url()); 72 | EXPECT_NE(nullptr, current_type) 73 | << "Could not resolve type url " << field->type_url() 74 | << " of the field " << field_names[i] << " while parsing field path " 75 | << field_path_str << std::endl; 76 | } 77 | } 78 | return field_path; 79 | } 80 | 81 | } // namespace 82 | 83 | RequestTranslatorTestBase::RequestTranslatorTestBase() 84 | : type_(), 85 | body_prefix_(), 86 | bindings_(), 87 | output_delimiters_(false), 88 | tester_() {} 89 | 90 | RequestTranslatorTestBase::~RequestTranslatorTestBase() {} 91 | 92 | void RequestTranslatorTestBase::LoadService( 93 | const std::string& config_pb_txt_file) { 94 | EXPECT_TRUE(transcoding::testing::LoadService(config_pb_txt_file, &service_)); 95 | type_helper_.reset(new TypeHelper(service_.types(), service_.enums())); 96 | } 97 | 98 | void RequestTranslatorTestBase::SetMessageType(const std::string& type_name) { 99 | type_ = type_helper_->Info()->GetTypeByTypeUrl("type.googleapis.com/" + 100 | type_name); 101 | EXPECT_NE(nullptr, type_) << "Could not resolve the message type " 102 | << type_name << std::endl; 103 | } 104 | 105 | void RequestTranslatorTestBase::AddVariableBinding( 106 | const std::string& field_path_str, std::string value) { 107 | auto field_path = 108 | ParseFieldPath(*type_, *type_helper_->Info(), field_path_str); 109 | bindings_.emplace_back( 110 | RequestWeaver::BindingInfo{field_path, std::move(value)}); 111 | } 112 | 113 | void RequestTranslatorTestBase::Build() { 114 | RequestInfo request_info; 115 | request_info.message_type = type_; 116 | request_info.body_field_path = body_prefix_; 117 | request_info.variable_bindings = bindings_; 118 | 119 | auto output_stream = Create(*type_helper_->Resolver(), output_delimiters_, 120 | std::move(request_info)); 121 | 122 | tester_.reset(new ProtoStreamTester(*output_stream, output_delimiters_)); 123 | } 124 | 125 | } // namespace testing 126 | } // namespace transcoding 127 | 128 | } // namespace grpc 129 | } // namespace google 130 | -------------------------------------------------------------------------------- /test/request_translator_test_base.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_REQUEST_TRANSLATOR_TEST_BASE_H_ 16 | #define GRPC_TRANSCODING_REQUEST_TRANSLATOR_TEST_BASE_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "google/api/service.pb.h" 23 | #include "google/protobuf/type.pb.h" 24 | #include "google/protobuf/util/type_resolver.h" 25 | #include "grpc_transcoding/message_stream.h" 26 | #include "grpc_transcoding/request_message_translator.h" 27 | #include "grpc_transcoding/type_helper.h" 28 | #include "gtest/gtest.h" 29 | #include "proto_stream_tester.h" 30 | 31 | namespace google { 32 | namespace grpc { 33 | 34 | namespace transcoding { 35 | namespace testing { 36 | 37 | // A base that provides common functionality for streaming and non-streaming 38 | // translator tests. 39 | class RequestTranslatorTestBase : public ::testing::Test { 40 | protected: 41 | RequestTranslatorTestBase(); 42 | ~RequestTranslatorTestBase(); 43 | 44 | // Loads the test service from a config file. Must be the first call of each 45 | // test. 46 | void LoadService(const std::string& config_pb_txt_file); 47 | 48 | // Methods that tests can use to build the translator 49 | void SetMessageType(const std::string& type_name); 50 | void SetBodyPrefix(const std::string& body_prefix) { 51 | body_prefix_ = body_prefix; 52 | } 53 | void AddVariableBinding(const std::string& field_path_str, std::string value); 54 | void SetOutputDelimiters(bool output_delimiters) { 55 | output_delimiters_ = output_delimiters; 56 | } 57 | void Build(); 58 | 59 | // ProtoStreamTester that the tests can use to validate the output 60 | ProtoStreamTester& Tester() { return *tester_; } 61 | 62 | private: 63 | // Virtual Create() function that each test class must override to create the 64 | // translator and return the output MessageStream. 65 | virtual MessageStream* Create( 66 | google::protobuf::util::TypeResolver& type_resolver, 67 | bool output_delimiters, RequestInfo info) = 0; 68 | 69 | // The test service config 70 | google::api::Service service_; 71 | 72 | // TypeHelper for the service types (helps with resolving/navigating service 73 | // type information) 74 | std::unique_ptr type_helper_; 75 | 76 | // Input for building the translator 77 | const google::protobuf::Type* type_; 78 | std::string body_prefix_; 79 | std::vector bindings_; 80 | bool output_delimiters_; 81 | 82 | std::unique_ptr tester_; 83 | }; 84 | 85 | } // namespace testing 86 | } // namespace transcoding 87 | 88 | } // namespace grpc 89 | } // namespace google 90 | 91 | #endif // GRPC_TRANSCODING_REQUEST_TRANSLATOR_TEST_BASE_H_ 92 | -------------------------------------------------------------------------------- /test/status_error_listener_test.cc: -------------------------------------------------------------------------------- 1 | #include "grpc_transcoding/status_error_listener.h" 2 | 3 | #include "gmock/gmock.h" 4 | #include "google/protobuf/util/converter/object_location_tracker.h" 5 | #include "gtest/gtest.h" 6 | 7 | namespace google { 8 | namespace grpc { 9 | 10 | namespace transcoding { 11 | namespace testing { 12 | namespace { 13 | 14 | using ::testing::HasSubstr; 15 | using absl::Status; 16 | using absl::StatusCode; 17 | namespace pbconv = google::protobuf::util::converter; 18 | 19 | class StatusErrorListenerTest : public ::testing::Test { 20 | protected: 21 | StatusErrorListenerTest() : listener_() {} 22 | 23 | StatusErrorListener listener_; 24 | }; 25 | 26 | TEST_F(StatusErrorListenerTest, ReportFailures) { 27 | listener_.set_status(Status(StatusCode::kInvalidArgument, "invalid args")); 28 | EXPECT_EQ(listener_.status().code(), StatusCode::kInvalidArgument); 29 | EXPECT_THAT(listener_.status().ToString(), HasSubstr("invalid args")); 30 | 31 | listener_.InvalidName(pbconv::ObjectLocationTracker{}, "invalid name", 32 | "invalid_name_foo"); 33 | EXPECT_EQ(listener_.status().code(), StatusCode::kInvalidArgument); 34 | EXPECT_THAT(listener_.status().ToString(), HasSubstr("invalid_name_foo")); 35 | listener_.InvalidValue(pbconv::ObjectLocationTracker{}, "invalid value", 36 | "invalid_value_foo"); 37 | EXPECT_EQ(listener_.status().code(), StatusCode::kInvalidArgument); 38 | EXPECT_THAT(listener_.status().ToString(), HasSubstr("invalid_value_foo")); 39 | listener_.MissingField(pbconv::ObjectLocationTracker{}, "missing value"); 40 | EXPECT_EQ(listener_.status().code(), StatusCode::kInvalidArgument); 41 | EXPECT_THAT(listener_.status().ToString(), 42 | HasSubstr("missing field missing value")); 43 | } 44 | 45 | } // namespace 46 | } // namespace testing 47 | } // namespace transcoding 48 | 49 | } // namespace grpc 50 | } // namespace google -------------------------------------------------------------------------------- /test/test_common.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | #ifndef GRPC_TRANSCODING_TEST_COMMON_H_ 16 | #define GRPC_TRANSCODING_TEST_COMMON_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "google/api/service.pb.h" 24 | #include "google/protobuf/io/zero_copy_stream.h" 25 | #include "google/protobuf/text_format.h" 26 | #include "grpc_transcoding/transcoder_input_stream.h" 27 | #include "gtest/gtest.h" 28 | 29 | namespace google { 30 | namespace grpc { 31 | 32 | namespace transcoding { 33 | namespace testing { 34 | 35 | // An implementation of ZeroCopyInputStream for testing. 36 | // The tests define the chunks that TestZeroCopyInputStream produces. 37 | class TestZeroCopyInputStream : public TranscoderInputStream { 38 | public: 39 | TestZeroCopyInputStream(); 40 | 41 | // Add an input chunk 42 | void AddChunk(std::string chunk); 43 | 44 | // Ends the stream 45 | void Finish() { finished_ = true; } 46 | 47 | // Returns whether Finish() has been called on this stream or not. 48 | bool Finished() const { return finished_; } 49 | 50 | // ZeroCopyInputStream methods 51 | bool Next(const void** data, int* size); 52 | void BackUp(int count); 53 | int64_t BytesAvailable() const; 54 | int64_t ByteCount() const { return 0; } // Not implemented 55 | bool Skip(int) { return false; } // Not implemented 56 | 57 | private: 58 | std::deque chunks_; 59 | bool finished_; 60 | 61 | int position_; 62 | std::string current_; 63 | }; 64 | 65 | // Test the translation test case with different partitions of the input. The 66 | // test will generate combinations of partitioning input into specified number 67 | // of chunks (chunk_count). For each of the input partition, test assertion is 68 | // verified. The partition is passed to the test assertion as a std::vector of 69 | // partitioning points in the input. 70 | // Because the number of partitionings is O(N^chunk_count) we use a coefficient 71 | // which controls which fraction of partitionings is generated and tested. 72 | // The process of generating partitionings is deterministic. 73 | // 74 | // chunk_count - the number of parts (chunks) in each partition 75 | // partitioning_coefficient - a real number in (0, 1] interval that defines how 76 | // exhaustive the test should be, i.e. what part of 77 | // all partitions of the input string should be 78 | // tested (1.0 means all partitions). 79 | // input - the input string 80 | // test - the test to run 81 | bool RunTestForInputPartitions( 82 | size_t chunk_count, double partitioning_coefficient, 83 | const std::string& input, 84 | std::function& t)> test); 85 | 86 | // Generate an input string of the specified size using the specified seed. 87 | std::string GenerateInput(const std::string& seed, size_t size); 88 | 89 | // Load service from a proto text file. Returns true if loading succeeds; 90 | // otherwise returns false. 91 | bool LoadService(const std::string& config_pb_txt_file, 92 | const std::string& testdata_path, 93 | ::google::api::Service* service); 94 | bool LoadService(const std::string& config_pb_txt_file, 95 | ::google::api::Service* service); 96 | 97 | // Parses the gRPC message delimiter and returns the size of the message. 98 | unsigned DelimiterToSize(const unsigned char* delimiter); 99 | 100 | // Generates a gRPC message delimiter with the given message size. 101 | std::string SizeToDelimiter(unsigned size); 102 | 103 | // Genereate a proto message with the gRPC delimiter from proto text 104 | template 105 | std::string GenerateGrpcMessage(const std::string& proto_text) { 106 | // Parse the message from text & serialize to binary 107 | MessageType message; 108 | EXPECT_TRUE( 109 | ::google::protobuf::TextFormat::ParseFromString(proto_text, &message)); 110 | std::string binary; 111 | EXPECT_TRUE(message.SerializeToString(&binary)); 112 | 113 | // Now prefix the binary with a delimiter and return 114 | return SizeToDelimiter(binary.size()) + binary; 115 | } 116 | 117 | // Compares JSON objects 118 | bool ExpectJsonObjectEq(const std::string& expected, const std::string& actual); 119 | 120 | // Compares JSON arrays 121 | bool ExpectJsonArrayEq(const std::string& expected, const std::string& actual); 122 | 123 | // JSON array tester that supports matching partial arrays. 124 | class JsonArrayTester { 125 | public: 126 | // Tests a new element of the array. 127 | // expected - the expected new element of the array to match 128 | // actual - the actual JSON chunk (which will include "[", "]" or "," if 129 | // needed) 130 | bool TestElement(const std::string& expected, const std::string& actual); 131 | 132 | // Tests a new chunk of the array (potentially multiple elements). 133 | // expected - the expected new chunk of the array to match (including "[", "]" 134 | // or "," if needed) 135 | // actual - the actual JSON chunk (including "[", "]" or "," if needed) 136 | // closes - indicates whether the chunk closes the array or not. 137 | bool TestChunk(const std::string& expected, const std::string& actual, 138 | bool closes); 139 | 140 | // Test that the array is closed after adding the given JSON chunk (i.e. must 141 | // be "]" modulo whitespace) 142 | bool TestClosed(const std::string& actual); 143 | 144 | private: 145 | std::string expected_so_far_; 146 | std::string actual_so_far_; 147 | }; 148 | 149 | } // namespace testing 150 | } // namespace transcoding 151 | 152 | } // namespace grpc 153 | } // namespace google 154 | 155 | #endif // GRPC_TRANSCODING_MESSAGE_READER_H_ 156 | --------------------------------------------------------------------------------