├── test ├── test │ ├── resources │ │ ├── testdata.txt │ │ ├── https │ │ │ └── keystore.jks │ │ ├── empty.proto │ │ ├── addressbook.proto │ │ └── grpctest.proto │ ├── protojure │ │ ├── test │ │ │ ├── utils.clj │ │ │ └── grpc │ │ │ │ └── TestService │ │ │ │ ├── server.cljc │ │ │ │ └── client.cljc │ │ ├── iostream_test.clj │ │ ├── grpc_web_test.clj │ │ ├── pedestal_test.clj │ │ └── protobuf_test.clj │ ├── example │ │ ├── hello │ │ │ └── Greeter.clj │ │ ├── hello.clj │ │ └── types.clj │ └── com │ │ └── example │ │ ├── empty.cljc │ │ └── addressbook.cljc ├── dev-resources │ ├── log4j2.xml │ └── user.clj └── project.clj ├── .gitignore ├── modules ├── grpc-client │ ├── src │ │ └── protojure │ │ │ ├── internal │ │ │ ├── README.md │ │ │ └── grpc │ │ │ │ └── client │ │ │ │ └── providers │ │ │ │ └── http2 │ │ │ │ ├── core.clj │ │ │ │ └── jetty.clj │ │ │ └── grpc │ │ │ └── client │ │ │ ├── utils.clj │ │ │ ├── providers │ │ │ └── http2.clj │ │ │ └── api.clj │ └── project.clj ├── io │ ├── src │ │ └── protojure │ │ │ └── internal │ │ │ ├── io │ │ │ ├── AsyncInputStream.java │ │ │ ├── AsyncOutputStream.java │ │ │ ├── ProxyInputStream.java │ │ │ └── ProxyOutputStream.java │ │ │ └── io.clj │ └── project.clj ├── core │ ├── src │ │ └── protojure │ │ │ ├── protobuf │ │ │ ├── protocol.cljc │ │ │ ├── serdes │ │ │ │ ├── stream.clj │ │ │ │ ├── complex.cljc │ │ │ │ ├── utils.cljc │ │ │ │ └── core.clj │ │ │ └── any.cljc │ │ │ ├── protobuf.clj │ │ │ └── grpc │ │ │ ├── status.clj │ │ │ └── codec │ │ │ ├── compression.clj │ │ │ └── lpm.clj │ └── project.clj └── grpc-server │ ├── project.clj │ └── src │ └── protojure │ └── pedestal │ ├── ssl.clj │ ├── interceptors │ ├── grpc_web.clj │ ├── authz.clj │ └── grpc.clj │ └── routes.clj ├── .circleci └── config.yml ├── CONTRIBUTORS.md ├── README.md ├── Makefile ├── project.clj ├── LICENSE └── lein /test/test/resources/testdata.txt: -------------------------------------------------------------------------------- 1 | testdata! 2 | -------------------------------------------------------------------------------- /test/test/resources/https/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/protojure/lib/HEAD/test/test/resources/https/keystore.jks -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | *.iml 13 | .idea/ 14 | *.DS_Store 15 | -------------------------------------------------------------------------------- /modules/grpc-client/src/protojure/internal/README.md: -------------------------------------------------------------------------------- 1 | ## protojure.internal 2 | 3 | This namespace is used for functions that are internal to Protojure and are not part of its public API. The only thing special about this namespace is that it is excluded from showing up in codox documentation -------------------------------------------------------------------------------- /test/test/resources/empty.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package com.example.empty; 4 | 5 | message Empty {} 6 | 7 | message NonEmpty { 8 | int32 i = 1; 9 | } 10 | 11 | message Selection { 12 | oneof opt { 13 | Empty e = 1; 14 | NonEmpty ne = 2; 15 | } 16 | } 17 | 18 | message Container { 19 | Empty e = 1; 20 | NonEmpty ne = 2; 21 | } 22 | -------------------------------------------------------------------------------- /modules/io/src/protojure/internal/io/AsyncInputStream.java: -------------------------------------------------------------------------------- 1 | package protojure.internal.io; 2 | 3 | import java.io.IOException; 4 | 5 | public interface AsyncInputStream { 6 | public int available () throws IOException; 7 | public int read_int() throws IOException; 8 | public int read_bytes(byte[] b) throws IOException; 9 | public int read_offset(byte[] b, int offset, int len) throws IOException; 10 | } 11 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | # 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | version: 2 # use CircleCI 2.0 6 | jobs: 7 | build: 8 | working_directory: ~/lib 9 | docker: 10 | - image: circleci/clojure:openjdk-14-lein-buster 11 | environment: 12 | LEIN_ROOT: nbd 13 | JVM_OPTS: -Xmx3200m 14 | steps: 15 | - checkout 16 | - run: make all 17 | -------------------------------------------------------------------------------- /modules/io/src/protojure/internal/io/AsyncOutputStream.java: -------------------------------------------------------------------------------- 1 | package protojure.internal.io; 2 | 3 | import java.io.IOException; 4 | 5 | public interface AsyncOutputStream { 6 | public void flush() throws IOException; 7 | public void close() throws IOException; 8 | public void write_int(int b) throws IOException; 9 | public void write_bytes(byte[] b) throws IOException; 10 | public void write_offset(byte[] b, int offset, int len) throws IOException; 11 | } 12 | -------------------------------------------------------------------------------- /modules/core/src/protojure/protobuf/protocol.cljc: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.protobuf.protocol) 7 | 8 | (defprotocol Writer 9 | (serialize [this os])) 10 | 11 | ;; Supports 'Any' type https://developers.google.com/protocol-buffers/docs/proto3#any 12 | (defprotocol TypeReflection 13 | (gettype [this])) -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | # Protojure Contributors 2 | 3 | - Greg Haskins 4 | - Srinivasan (Murali) Muralidharan 5 | - Matt Rkiouak 6 | - Jon Andrews 7 | - George Lindsell 8 | - Sean Harrap 9 | - Enzzo Cavallo 10 | - Ryan Sundberg 11 | - James Vickers 12 | - Janos Meszaros 13 | -------------------------------------------------------------------------------- /test/dev-resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /test/dev-resources/user.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns user 7 | (:require [clojure.tools.namespace.repl :refer [refresh]] 8 | [eftest.runner :refer [find-tests run-tests]])) 9 | 10 | ;; to run one test: `(run-tests (find-tests #'protojure.pedestal-test/edn-check))` 11 | ;; to see output, use (run-tests ... {:capture-output? false}) 12 | 13 | (defn run-all-tests [] 14 | (run-tests (find-tests "test") {:fail-fast? true :multithread? false})) -------------------------------------------------------------------------------- /test/test/resources/addressbook.proto: -------------------------------------------------------------------------------- 1 | 2 | syntax = "proto3"; 3 | package com.example.addressbook; 4 | 5 | message Person { 6 | string name = 1; 7 | int32 id = 2; // Unique ID number for this person. 8 | string email = 3; 9 | 10 | enum PhoneType { 11 | MOBILE = 0; 12 | HOME = 1; 13 | WORK = 2; 14 | } 15 | 16 | message PhoneNumber { 17 | string number = 1; 18 | PhoneType type = 2; 19 | } 20 | 21 | repeated PhoneNumber phones = 4; 22 | } 23 | 24 | // Our address book file is just one of these. 25 | message AddressBook { 26 | repeated Person people = 1; 27 | } 28 | -------------------------------------------------------------------------------- /modules/io/project.clj: -------------------------------------------------------------------------------- 1 | (defproject io.github.protojure/io "2.11.1-SNAPSHOT" 2 | :description "IO library to support io.github.protojure/core" 3 | :url "http://github.com/protojure/lib" 4 | :license {:name "Apache License 2.0" 5 | :url "https://www.apache.org/licenses/LICENSE-2.0" 6 | :year 2022 7 | :key "apache-2.0"} 8 | :plugins [[lein-cljfmt "0.9.0"] 9 | [lein-kibit "0.1.8"] 10 | [lein-bikeshed "0.5.2"] 11 | [lein-set-version "0.4.1"] 12 | [lein-parent "0.3.9"]] 13 | :parent-project {:path "../../project.clj" 14 | :inherit [:managed-dependencies :javac-options]} 15 | :dependencies [[org.clojure/clojure] 16 | [org.clojure/core.async]] 17 | :java-source-paths ["src"]) 18 | -------------------------------------------------------------------------------- /modules/core/src/protojure/protobuf/serdes/stream.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 2 | ;; 3 | ;; SPDX-License-Identifier: Apache-2.0 4 | 5 | (ns protojure.protobuf.serdes.stream 6 | (:import (com.google.protobuf CodedInputStream) 7 | (java.io InputStream) 8 | (java.nio ByteBuffer))) 9 | 10 | (defn end? [is] 11 | (.isAtEnd ^CodedInputStream is)) 12 | 13 | (defn read-tag [is] 14 | (.readTag ^CodedInputStream is)) 15 | 16 | (defmulti new-cis (fn [src] (type src))) 17 | (defmethod new-cis InputStream 18 | [^InputStream src] 19 | (CodedInputStream/newInstance src)) 20 | (defmethod new-cis ByteBuffer 21 | [^ByteBuffer src] 22 | (CodedInputStream/newInstance src)) 23 | (defmethod new-cis (Class/forName "[B") 24 | [^bytes src] 25 | (CodedInputStream/newInstance src)) 26 | -------------------------------------------------------------------------------- /test/test/protojure/test/utils.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.test.utils 7 | (:require [clojure.data :as data] 8 | [io.pedestal.http :as pedestal])) 9 | 10 | (defn get-free-port [] 11 | (let [socket (java.net.ServerSocket. 0)] 12 | (.close socket) 13 | (.getLocalPort socket))) 14 | 15 | (defn data-equal? 16 | "Returns true if the data items have no differences according to clojure.data/diff" 17 | [a b] 18 | (let [[a-diff b-diff _] (data/diff a b)] 19 | (and (nil? a-diff) (nil? b-diff)))) 20 | 21 | (defn start-pedestal-server [desc] 22 | (let [server (pedestal/create-server desc)] 23 | (pedestal/start server) 24 | server)) 25 | -------------------------------------------------------------------------------- /modules/grpc-client/project.clj: -------------------------------------------------------------------------------- 1 | (defproject io.github.protojure/grpc-client "2.11.1-SNAPSHOT" 2 | :description "GRPC client library for protoc-gen-clojure" 3 | :url "http://github.com/protojure/lib" 4 | :license {:name "Apache License 2.0" 5 | :url "https://www.apache.org/licenses/LICENSE-2.0" 6 | :year 2022 7 | :key "apache-2.0"} 8 | :plugins [[lein-cljfmt "0.9.0"] 9 | [lein-kibit "0.1.8"] 10 | [lein-bikeshed "0.5.2"] 11 | [lein-set-version "0.4.1"] 12 | [lein-parent "0.3.9"]] 13 | :parent-project {:path "../../project.clj" 14 | :inherit [:managed-dependencies :javac-options]} 15 | :dependencies [[org.clojure/clojure] 16 | [org.clojure/core.async] 17 | [io.github.protojure/core] 18 | [org.eclipse.jetty.http2/http2-client] 19 | [org.eclipse.jetty/jetty-alpn-java-client] 20 | [lambdaisland/uri]]) 21 | -------------------------------------------------------------------------------- /modules/io/src/protojure/internal/io/ProxyInputStream.java: -------------------------------------------------------------------------------- 1 | package protojure.internal.io; 2 | 3 | import java.io.IOException; 4 | 5 | // adds a Proxy between java.io.InputStream and a core.async-based backend, by way of a reified AsyncInputStream 6 | public class ProxyInputStream extends java.io.InputStream { 7 | 8 | public ProxyInputStream(AsyncInputStream backend) { 9 | m_backend = backend; 10 | } 11 | 12 | private final AsyncInputStream m_backend; 13 | 14 | @Override 15 | public int available () throws IOException { 16 | return m_backend.available(); 17 | } 18 | 19 | @Override 20 | public int read() throws IOException { 21 | return m_backend.read_int(); 22 | } 23 | 24 | @Override 25 | public int read(byte[] bytes) throws IOException { 26 | return m_backend.read_bytes(bytes); 27 | } 28 | 29 | @Override 30 | public int read(byte[] bytes, int offset, int len) throws IOException { 31 | return m_backend.read_offset(bytes, offset, len); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /modules/core/project.clj: -------------------------------------------------------------------------------- 1 | (defproject io.github.protojure/core "2.11.1-SNAPSHOT" 2 | :description "Core protobuf and GRPC utilities for protojure" 3 | :url "http://github.com/protojure/lib" 4 | :license {:name "Apache License 2.0" 5 | :url "https://www.apache.org/licenses/LICENSE-2.0" 6 | :year 2022 7 | :key "apache-2.0"} 8 | :plugins [[lein-cljfmt "0.9.0"] 9 | [lein-kibit "0.1.8"] 10 | [lein-bikeshed "0.5.2"] 11 | [lein-set-version "0.4.1"] 12 | [lein-parent "0.3.9"]] 13 | :parent-project {:path "../../project.clj" 14 | :inherit [:managed-dependencies :javac-options]} 15 | :dependencies [[org.clojure/clojure] 16 | [org.clojure/core.async] 17 | [org.clojure/tools.logging] 18 | [io.github.protojure/io] 19 | [com.google.protobuf/protobuf-java] 20 | [org.apache.commons/commons-compress] 21 | [commons-io/commons-io] 22 | [funcool/promesa]]) 23 | -------------------------------------------------------------------------------- /modules/grpc-server/project.clj: -------------------------------------------------------------------------------- 1 | (defproject io.github.protojure/grpc-server "2.11.1-SNAPSHOT" 2 | :description "GRPC server library for protoc-gen-clojure" 3 | :url "http://github.com/protojure/lib" 4 | :license {:name "Apache License 2.0" 5 | :url "https://www.apache.org/licenses/LICENSE-2.0" 6 | :year 2022 7 | :key "apache-2.0"} 8 | :plugins [[lein-cljfmt "0.9.0"] 9 | [lein-kibit "0.1.8"] 10 | [lein-bikeshed "0.5.2"] 11 | [lein-set-version "0.4.1"] 12 | [lein-parent "0.3.9"]] 13 | :parent-project {:path "../../project.clj" 14 | :inherit [:managed-dependencies :javac-options]} 15 | :dependencies [[org.clojure/clojure] 16 | [org.clojure/core.async] 17 | [io.github.protojure/core] 18 | [javax.servlet/javax.servlet-api] 19 | [io.undertow/undertow-core] 20 | [io.undertow/undertow-servlet] 21 | [io.pedestal/pedestal.log] 22 | [io.pedestal/pedestal.service] 23 | [io.pedestal/pedestal.error]]) 24 | -------------------------------------------------------------------------------- /modules/io/src/protojure/internal/io/ProxyOutputStream.java: -------------------------------------------------------------------------------- 1 | package protojure.internal.io; 2 | 3 | import java.io.IOException; 4 | 5 | // adds a Proxy between java.io.OutputStream and a core.async-based backend, by way of a reified AsyncOutputStream 6 | public class ProxyOutputStream extends java.io.OutputStream { 7 | 8 | public ProxyOutputStream(AsyncOutputStream backend) { 9 | m_backend = backend; 10 | } 11 | 12 | private final AsyncOutputStream m_backend; 13 | 14 | @Override 15 | public void flush() throws IOException { 16 | m_backend.flush(); 17 | } 18 | 19 | @Override 20 | public void close() throws IOException { 21 | m_backend.close(); 22 | } 23 | 24 | @Override 25 | public void write(int b) throws IOException { 26 | m_backend.write_int(b); 27 | } 28 | 29 | @Override 30 | public void write(byte[] bytes) throws IOException { 31 | m_backend.write_bytes(bytes); 32 | } 33 | 34 | @Override 35 | public void write(byte[] bytes, int offset, int len) throws IOException { 36 | m_backend.write_offset(bytes, offset, len); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # protojure [![CircleCI](https://circleci.com/gh/protojure/lib.svg?style=svg)](https://circleci.com/gh/protojure/lib) 2 | 3 | [![Clojars Project](https://img.shields.io/clojars/v/io.github.protojure/io.svg)](https://clojars.org/io.github.protojure/io) 4 | [![Clojars Project](https://img.shields.io/clojars/v/io.github.protojure/core.svg)](https://clojars.org/io.github.protojure/core) 5 | [![Clojars Project](https://img.shields.io/clojars/v/io.github.protojure/grpc-client.svg)](https://clojars.org/io.github.protojure/grpc-client) 6 | [![Clojars Project](https://img.shields.io/clojars/v/io.github.protojure/grpc-server.svg)](https://clojars.org/io.github.protojure/grpc-server) 7 | 8 | Native Clojure support for [Google Protocol Buffer](https://developers.google.com/protocol-buffers/) and [GRPC](https://grpc.io/). 9 | 10 | ## Documentation 11 | 12 | You can read the [SDK CLJ documentation](https://cljdoc.org/d/protojure/protojure) 13 | 14 | ## Usage 15 | 16 | See our [full documentation](https://protojure.readthedocs.io) 17 | 18 | ## License 19 | 20 | This project is licensed under the Apache License 2.0. 21 | 22 | ## Contributing 23 | 24 | Pull requests welcome. Please be sure to include a [DCO](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin) in any commit messages. 25 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | # 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | NAME = protojure 6 | LEIN = $(shell which lein || echo ./lein) 7 | 8 | DEPS = Makefile project.clj 9 | 10 | all: scan test install 11 | 12 | scan: 13 | $(LEIN) sub cljfmt check 14 | cd test && $(LEIN) cljfmt check 15 | 16 | # 'deep-scan' is a target for useful linters that are not conducive to automated checking, 17 | # typically because they present some false positives without an easy mechanism to overrule 18 | # them. So we provide the target to make it easy to run by hand, but leave them out of the 19 | # automated gates. 20 | deep-scan: scan 21 | -$(LEIN) sub bikeshed 22 | -$(LEIN) sub kibit 23 | 24 | .PHONY: test 25 | test: 26 | cd test && $(LEIN) cloverage 27 | 28 | install: 29 | $(LEIN) sub install 30 | 31 | set-version: 32 | sed -i '' 's/def protojure-version \".*\"/def protojure-version \"$(VERSION)\"/' project.clj 33 | $(LEIN) sub set-version $(VERSION) 34 | 35 | clean: 36 | $(LEIN) sub clean 37 | cd test && $(LEIN) clean 38 | $(LEIN) clean 39 | 40 | .PHONY: protos 41 | protos: 42 | mkdir -p test/test/resources 43 | protoc --clojure_out=grpc-client,grpc-server:test/test --proto_path=test/test/resources $(shell find test/test/resources -name "*.proto" | sed 's|test/test/resources/||g') 44 | -------------------------------------------------------------------------------- /modules/core/src/protojure/protobuf.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.protobuf 7 | "Main API entry point for protobuf applications" 8 | (:require [protojure.protobuf.protocol :as p]) 9 | (:import (com.google.protobuf CodedOutputStream) 10 | (java.io OutputStream ByteArrayOutputStream) 11 | (java.nio ByteBuffer))) 12 | 13 | (set! *warn-on-reflection* true) 14 | 15 | (defn- serialize! [msg ^CodedOutputStream os] 16 | (p/serialize msg os) 17 | (.flush os)) 18 | 19 | (defmulti #^{:private true} serialize (fn [msg output] (type output))) 20 | (defmethod serialize OutputStream 21 | [msg ^OutputStream output] 22 | (serialize! msg (CodedOutputStream/newInstance output))) 23 | (defmethod serialize ByteBuffer 24 | [msg ^ByteBuffer output] 25 | (serialize! msg (CodedOutputStream/newInstance output))) 26 | (defmethod serialize (Class/forName "[B") 27 | [msg ^bytes output] 28 | (serialize! msg (CodedOutputStream/newInstance output))) 29 | 30 | (defn ->pb 31 | "Serialize a record implementing the [[Writer]] protocol into protobuf bytes." 32 | ([msg] 33 | (let [os (ByteArrayOutputStream.)] 34 | (->pb msg os) 35 | (.toByteArray os))) 36 | ([msg output] 37 | (serialize msg output))) 38 | -------------------------------------------------------------------------------- /modules/grpc-server/src/protojure/pedestal/ssl.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.pedestal.ssl 7 | (:require [clojure.java.io :as io]) 8 | (:import (javax.net.ssl SSLContext KeyManagerFactory) 9 | (java.security KeyStore) 10 | (java.io InputStream))) 11 | 12 | (set! *warn-on-reflection* true) 13 | 14 | (defn- load-keystore 15 | [keystore ^String password] 16 | (if (instance? KeyStore keystore) 17 | keystore 18 | (with-open [in (io/input-stream keystore)] 19 | (doto (KeyStore/getInstance (KeyStore/getDefaultType)) 20 | (.load in (.toCharArray password)))))) 21 | 22 | (defn- keystore->key-managers 23 | "Return a KeyManager[] given a KeyStore and password" 24 | [keystore ^String password] 25 | (.getKeyManagers 26 | (doto (KeyManagerFactory/getInstance (KeyManagerFactory/getDefaultAlgorithm)) 27 | (.init keystore (.toCharArray password))))) 28 | 29 | (defn keystore-> 30 | "Turn a keystore, which may be either strings denoting file paths or actual KeyStore 31 | instances, into an SSLContext instance" 32 | [{:keys [keystore key-password]}] 33 | (let [ks (load-keystore keystore key-password)] 34 | (doto (SSLContext/getInstance "TLS") 35 | (.init (keystore->key-managers ks key-password) nil nil)))) 36 | -------------------------------------------------------------------------------- /modules/core/src/protojure/protobuf/any.cljc: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 2 | ;; 3 | ;; SPDX-License-Identifier: Apache-2.0 4 | (ns protojure.protobuf.any 5 | "Support for the 'Any' type: https://developers.google.com/protocol-buffers/docs/proto3#any" 6 | (:require [protojure.protobuf.protocol :as protocol] 7 | [protojure.protobuf :as serdes] 8 | [com.google.protobuf :as google])) 9 | 10 | (def default-path "type.googleapis.com/") 11 | 12 | (defn- fqtype [type] 13 | (str default-path type)) 14 | 15 | (defn- load-registry 16 | "Finds all instances of Protojure records via metadata and creates a type-url registry for any->" 17 | [] 18 | (->> (all-ns) 19 | (map ns-name) 20 | (map ns-interns) 21 | (map vals) 22 | (reduce concat) 23 | (filter (fn [x] (contains? (meta x) ::record))) 24 | (map var-get) 25 | (reduce (fn [acc {:keys [type decoder]}] (assoc acc (fqtype type) decoder)) {}))) 26 | 27 | (def registry (memoize load-registry)) 28 | 29 | (defn- find-type [url] 30 | (get (registry) url)) 31 | 32 | (defn ->any 33 | "Encodes a Protojure record that implements the requisite type-reflection protocol to an Any record" 34 | [msg] 35 | (google/new-Any {:type-url (fqtype (protocol/gettype msg)) 36 | :value (serdes/->pb msg)})) 37 | 38 | (defn any-> 39 | "Decodes an Any record to its native type if the type is known to our system" 40 | [{:keys [type-url value]}] 41 | (if (some? type-url) 42 | (if-let [f (find-type type-url)] 43 | (f value) 44 | (throw (ex-info "Any::type-url not found" {:type-url type-url}))) 45 | (throw (ex-info "Any record missing :type-url" {})))) 46 | 47 | (defn pb-> 48 | "Decodes a raw protobuf message/stream encoded as an Any to its native type, if the type is known to our system" 49 | [pb] 50 | (any-> (google/pb->Any pb))) 51 | -------------------------------------------------------------------------------- /modules/grpc-server/src/protojure/pedestal/interceptors/grpc_web.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.pedestal.interceptors.grpc-web 7 | "A [Pedestal](http://pedestal.io/) [interceptor](http://pedestal.io/reference/interceptors) for the [GRPC-WEB](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md) protocol" 8 | (:require [io.pedestal.interceptor :refer [->Interceptor]]) 9 | (:import (org.apache.commons.codec.binary Base64InputStream)) 10 | (:refer-clojure :exclude [proxy])) 11 | 12 | (set! *warn-on-reflection* true) 13 | 14 | (defn- decode-body 15 | [{:keys [body] :as request}] 16 | (assoc request :body (Base64InputStream. body))) 17 | 18 | (def ^{:no-doc true :const true} content-types 19 | #{"application/grpc-web-text" 20 | "application/grpc-web-text+proto"}) 21 | 22 | (defn- web-text? 23 | [{{:strs [content-type]} :headers}] 24 | (contains? content-types content-type)) 25 | 26 | (defn- pred-> 27 | "Threads 'item' through both the predicate and, when 'pred' evaluates true, 'xform' functions. Else, just returns 'item'" 28 | [item pred xform] 29 | (cond-> item (pred item) xform)) 30 | 31 | (defn- enter-handler 32 | [{:keys [request] :as ctx}] 33 | (assoc ctx :request (pred-> request web-text? decode-body))) 34 | 35 | (defn- leave-handler 36 | [ctx] 37 | ;; TODO "Clarify & implement grpc-web trailer behavior" 38 | ctx) 39 | 40 | (defn- exception-handler 41 | [ctx e] 42 | (assoc ctx :io.pedestal.interceptor.chain/error e)) 43 | 44 | (def proxy 45 | "Interceptor that provides a transparent proxy for the [GRPC-WEB](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md) protocol to standard protojure grpc protocol" 46 | (->Interceptor ::proxy enter-handler leave-handler exception-handler)) 47 | -------------------------------------------------------------------------------- /modules/grpc-server/src/protojure/pedestal/routes.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.pedestal.routes 7 | "Utilities for generating GRPC endpoints as [Pedestal Routes](http://pedestal.io/guides/defining-routes)" 8 | (:require [protojure.pedestal.interceptors.grpc :as grpc] 9 | [protojure.pedestal.interceptors.authz :as authz] 10 | [protojure.pedestal.interceptors.grpc-web :as grpc.web] 11 | [io.pedestal.interceptor :as pedestal] 12 | [clojure.core.async :refer [tablesyntax 34 | "Generates routes in [Table Syntax](http://pedestal.io/reference/table-syntax) format" 35 | [{:keys [rpc-metadata interceptors callback-context authorizer] :as options}] 36 | (for [{:keys [pkg service method method-fn] :as rpc} rpc-metadata] 37 | (let [fqs (str pkg "." service) 38 | name (keyword fqs (str method "-handler")) 39 | handler (handler name (partial method-fn callback-context))] 40 | [(str "/" fqs "/" method) 41 | :post (-> (consv grpc/error-interceptor interceptors) 42 | (conj grpc.web/proxy 43 | (grpc/route-interceptor rpc)) 44 | (cond-> (some? authorizer) (conj (authz/interceptor rpc-metadata authorizer))) 45 | (conj handler)) 46 | :route-name name]))) -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (def protojure-version "2.11.1-SNAPSHOT") 2 | 3 | (defproject io.github.protojure/lib-suite "0.0.1" 4 | :description "Support libraries for protoc-gen-clojure, providing native Clojure support for Google Protocol Buffers and GRPC applications" 5 | :url "http://github.com/protojure/lib" 6 | :license {:name "Apache License 2.0" 7 | :url "https://www.apache.org/licenses/LICENSE-2.0" 8 | :year 2022 9 | :key "apache-2.0"} 10 | :plugins [[lein-set-version "0.4.1"] 11 | [lein-sub "0.3.0"]] 12 | :managed-dependencies [[org.clojure/clojure "1.12.0"] 13 | [org.clojure/core.async "1.6.681"] 14 | [org.clojure/tools.logging "1.3.0"] 15 | [com.google.protobuf/protobuf-java "4.28.0"] 16 | [org.apache.commons/commons-compress "1.27.1"] 17 | [commons-io/commons-io "2.16.1"] 18 | [funcool/promesa "9.2.542"] 19 | [javax.servlet/javax.servlet-api "4.0.1"] 20 | [io.undertow/undertow-core "2.3.17.Final"] 21 | [io.undertow/undertow-servlet "2.3.17.Final"] 22 | [io.pedestal/pedestal.log "0.7.0"] 23 | [io.pedestal/pedestal.service "0.7.0"] 24 | [io.pedestal/pedestal.error "0.7.0"] 25 | [org.eclipse.jetty.http2/http2-client "11.0.24"] 26 | [org.eclipse.jetty/jetty-alpn-java-client "12.0.13"] 27 | [lambdaisland/uri "1.19.155"] 28 | [io.github.protojure/io ~protojure-version] 29 | [io.github.protojure/core ~protojure-version] 30 | [io.github.protojure/grpc-client ~protojure-version] 31 | [io.github.protojure/grpc-server ~protojure-version] 32 | [protojure/google.protobuf "1.0.0"]] 33 | :javac-options ["-target" "11" "-source" "11"] 34 | :sub ["modules/io" "modules/core" "modules/grpc-client" "modules/grpc-server"]) 35 | -------------------------------------------------------------------------------- /modules/core/src/protojure/protobuf/serdes/complex.cljc: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.protobuf.serdes.complex 7 | "Serializer/deserializer support for complex protobuf types." 8 | (:require [protojure.protobuf.serdes.core :refer :all] 9 | [protojure.protobuf.serdes.stream :as stream])) 10 | 11 | (set! *warn-on-reflection* true) 12 | 13 | (defn cis->map 14 | "Deserialize a wire format map-type to user format [key val]" 15 | [f is] 16 | (let [{:keys [key value]} (f is)] 17 | (partial into {key value}))) 18 | 19 | (defn cis->repeated 20 | "Deserialize an 'unpacked' repeated type (see [[cis->packablerepeated]])" 21 | [f is] 22 | (fn [coll] 23 | (conj (or coll []) (f is)))) 24 | 25 | (defn- repeated-seq 26 | "Returns a lazy sequence of repeated items on an input-stream" 27 | [f is] 28 | (lazy-seq (when (not (stream/end? is)) 29 | (cons (f is) (repeated-seq f is))))) 30 | 31 | (defn cis->packedrepeated 32 | "Deserialize a 'packed' repeated type (see [[cis->packablerepeated]])" 33 | [f is] 34 | (fn [coll] 35 | (cis->embedded #(reduce conj (or coll []) (repeated-seq f %)) is))) 36 | 37 | (defn cis->packablerepeated 38 | " 39 | Deserialize a repeated type which may optionally support [packed format](https://developers.google.com/protocol-buffers/docs/encoding#packed). 40 | The field type will indicate unpacked (0) vs packed (2). 41 | " 42 | [tag f is] 43 | (let [type (bit-and 0x2 tag)] 44 | (case type 45 | 0 (cis->repeated f is) 46 | 2 (cis->packedrepeated f is) 47 | (cis->undefined tag is)))) 48 | 49 | ;; FIXME: Add support for optimizing packable types 50 | (defn write-repeated 51 | "Serialize a repeated type" 52 | [f tag items os] 53 | (doseq [item items] 54 | (f tag item os))) 55 | 56 | (defn write-map 57 | "Serialize user format [key val] using given map item constructor" 58 | [constructor tag items os] 59 | (write-repeated write-embedded tag (map (fn [[key value]] (constructor {:key key :value value})) items) os)) 60 | 61 | -------------------------------------------------------------------------------- /test/test/resources/grpctest.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package protojure.test.grpc; 3 | 4 | import "google/protobuf/any.proto"; 5 | import "google/protobuf/empty.proto"; 6 | 7 | message CloseDetectRequest { 8 | string id = 1; 9 | } 10 | 11 | message FlowControlRequest { 12 | int32 count = 1; 13 | int32 payload_size = 2; 14 | 15 | } 16 | 17 | message FlowControlPayload { 18 | int32 id = 1; 19 | bytes data = 2; 20 | } 21 | 22 | message SimpleRequest { 23 | string input = 1; 24 | } 25 | 26 | message SimpleResponse { 27 | string msg = 1; 28 | } 29 | 30 | message ErrorRequest { 31 | int32 status = 1; 32 | string message = 2; 33 | } 34 | 35 | message BigPayload { 36 | enum Mode { 37 | MODE_INVALID = 0; 38 | MODE_UPLOAD = 1; 39 | MODE_DOWNLOAD = 2; 40 | MODE_BIDI = 3; 41 | } 42 | 43 | Mode mode = 1; 44 | bytes data = 2; 45 | } 46 | 47 | message AuthzTestRequest { 48 | enum Type { 49 | REQUEST_GOOD = 0; 50 | REQUEST_BAD = 1; 51 | } 52 | 53 | Type type = 1; 54 | } 55 | 56 | service TestService { 57 | rpc ClientCloseDetect (CloseDetectRequest) returns (stream google.protobuf.Any); 58 | rpc ServerCloseDetect (google.protobuf.Empty) returns (stream google.protobuf.Any); 59 | rpc FlowControl (FlowControlRequest) returns (stream FlowControlPayload); 60 | rpc Metadata (google.protobuf.Empty) returns (SimpleResponse); 61 | rpc ShouldThrow (google.protobuf.Empty) returns (google.protobuf.Empty); 62 | rpc Async (google.protobuf.Empty) returns (SimpleResponse); 63 | rpc AllEmpty(google.protobuf.Empty) returns (google.protobuf.Empty); 64 | rpc AsyncEmpty(google.protobuf.Empty) returns (stream google.protobuf.Empty); 65 | rpc DeniedStreamer(google.protobuf.Empty) returns (stream google.protobuf.Empty); 66 | rpc ReturnError(ErrorRequest) returns (google.protobuf.Empty); 67 | rpc ReturnErrorStreaming(ErrorRequest) returns (stream google.protobuf.Empty); 68 | rpc BandwidthTest(BigPayload) returns (BigPayload); 69 | rpc BidirectionalStreamTest(stream SimpleRequest) returns (stream SimpleResponse); 70 | rpc AuthzTest(AuthzTestRequest) returns (google.protobuf.Empty); 71 | } -------------------------------------------------------------------------------- /modules/core/src/protojure/grpc/status.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 2 | ;; 3 | ;; SPDX-License-Identifier: Apache-2.0 4 | 5 | (ns protojure.grpc.status) 6 | 7 | (def -codes 8 | [[0 :ok "success"] 9 | [1 :cancelled "The operation was cancelled, typically by the caller."] 10 | [2 :unknown "Unknown error."] 11 | [3 :invalid-argument "The client specified an invalid argument."] 12 | [4 :deadline-exceeded "The deadline expired before the operation could complete."] 13 | [5 :not-found "Some requested entity (e.g., file or directory) was not found."] 14 | [6 :already-exists "The entity already exists."] 15 | [7 :permission-denied "The caller does not have permission to execute the specified operation."] 16 | [8 :resource-exhausted "Some resource has been exhausted"] 17 | [9 :failed-precondition "The system is not in a state required for the operation's execution"] 18 | [10 :aborted "The operation was aborted, typically due to a concurrency issue."] 19 | [11 :out-of-range "The operation was attempted past the valid range."] 20 | [12 :unimplemented "The operation is not implemented."] 21 | [13 :internal "Invariants expected by the underlying system have been broken"] 22 | [14 :unavailable "The service is currently unavailable."] 23 | [15 :data-loss "Unrecoverable data loss or corruption."] 24 | [16 :unauthenticated "The request does not have valid authentication credentials."]]) 25 | 26 | (def codes 27 | (->> (map (fn [[code type msg]] {:code code :type type :msg msg}) -codes) 28 | (reduce (fn [acc {:keys [type] :as v}] (assoc acc type v)) {}))) 29 | 30 | (def default-error (get codes :unknown)) 31 | 32 | (defn get-desc [type] 33 | (get codes type default-error)) 34 | 35 | (defn get-code [type] 36 | (:code (get-desc type))) 37 | 38 | (defn exception-info 39 | [{:keys [type code msg]}] 40 | (ex-info "grpc error" {:code (if (some? type) (get-code type) code) :msg msg :exception-type ::error})) 41 | 42 | (defn- -error 43 | [{:keys [type] :as x}] 44 | (when-not (= type :ok) 45 | (throw (exception-info x)))) 46 | 47 | (defn error 48 | ([type] 49 | (-error (get-desc type))) 50 | ([type msg] 51 | (-> (get-desc type) 52 | (assoc :msg msg) 53 | (-error)))) 54 | -------------------------------------------------------------------------------- /modules/core/src/protojure/protobuf/serdes/utils.cljc: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.protobuf.serdes.utils 7 | (:require [protojure.protobuf.serdes.stream :as stream])) 8 | 9 | (defn tag-map 10 | " 11 | Returns a lazy sequence consisting of the result of applying f to the set of 12 | protobuf objects delimited by protobuf tags. 13 | 14 | #### Parameters 15 | 16 | | Value | Type | Description | 17 | |----------|--------------------|------------------------------------------------------------------------------------------------| 18 | | **init** | _map_ | A map of initial values | 19 | | **f** | _(fn [tag index])_ | An arity-2 function that accepts a tag and index and returns a [k v] (see _Return type_ below) | 20 | | **is** | [CodedInputStream](https://developers.google.com/protocol-buffers/docs/reference/java/com/google/protobuf/CodedInputStream) | An input stream containing serialized protobuf data | 21 | 22 | #### Return Type 23 | 24 | _f_ should evaluate to a 2-entry vector in the form [key value], where: 25 | 26 | - _key_ is either 27 | - a keyword representing the field name when the index is known 28 | - simply the index value when it is not 29 | - _value_ is either 30 | - a value that will be returned verbatim to be associated to the _key_ 31 | - a function that will take a collection of previously deserialized values with the same tag and update it to incorporate the new value (to support _repeated_ types, etc) 32 | 33 | 34 | #### Example 35 | 36 | ``` 37 | (tag-map 38 | (fn [tag index] 39 | (case index 40 | 1 [:currency_code (cis->String is)] 41 | 2 [:units (cis->Int64 is)] 42 | 3 [:nanos (cis->Int32 is)] 43 | [index (cis->undefined tag is)])) 44 | is)) 45 | ``` 46 | " 47 | ([f is] 48 | (tag-map {} f is)) 49 | ([init f is] 50 | (loop [acc init tag (stream/read-tag is)] 51 | (if (pos? tag) 52 | (let [[k v] (f tag (bit-shift-right tag 3))] 53 | (recur (if (fn? v) 54 | (update acc k v) 55 | (assoc acc k v)) 56 | (stream/read-tag is))) 57 | acc)))) 58 | 59 | (def default-scalar? #(or (nil? %) (zero? %))) 60 | (def default-bytes? empty?) 61 | (def default-bool? #(not (true? %))) 62 | -------------------------------------------------------------------------------- /modules/grpc-client/src/protojure/grpc/client/utils.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.grpc.client.utils 7 | "Functions used for grpc unary calls in clients generated by protojure-protoc-plugin" 8 | (:require [promesa.core :as p] 9 | [protojure.grpc.client.api :as grpc] 10 | [clojure.core.async :as async]) 11 | (:refer-clojure :exclude [take])) 12 | 13 | (set! *warn-on-reflection* true) 14 | 15 | (defn- take [ch] 16 | (p/create 17 | (fn [resolve reject] 18 | (async/take! ch resolve)))) 19 | 20 | (defn- put [ch val] 21 | (p/create 22 | (fn [resolve reject] 23 | (if (some? val) 24 | (async/put! ch val resolve) 25 | (resolve true))))) 26 | 27 | (defn send-unary-params 28 | " 29 | Places an item on a channel and then closes the channel, returning a promise that completes 30 | after the channel is closed. Used in remote procedure calls with unary parameters. 31 | 32 | #### Parameters 33 | 34 | | Value | Type | Description | 35 | |-------------|----------------------|----------------------------------------------------------------------------| 36 | | **ch** | _core.async/channel_ | A core.async channel expected to carry 'params' and be subsequently closed | 37 | | **params** | _any_ | The object to place on the channel | 38 | " 39 | [ch params] 40 | (-> (put ch params) 41 | (p/then (fn [_] (async/close! ch))))) 42 | 43 | (defn invoke-unary 44 | " 45 | Invokes a GRPC operation similar to the invoke operation within [[api/Provider]], but the promise returned 46 | resolves to a decoded result when successful. Used in remote procedure calls with unary return types. 47 | 48 | #### Parameters 49 | 50 | | Value | Type | Description | 51 | |-------------|----------------------|----------------------------------------------------------------------------| 52 | | **client** | _[[api/Provider]]_ | An instance of a client provider | 53 | | **params** | _map_ | See 'params' in the '(invoke ..)' method within [[api/Provider]] | 54 | | **ch** | _core.async/channel_ | A core.async channel expected to carry the response data | 55 | " 56 | [client params ch] 57 | (-> (grpc/invoke client params) 58 | (p/then (fn [_] (take ch))))) -------------------------------------------------------------------------------- /test/test/protojure/iostream_test.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.iostream-test 7 | (:require [clojure.test :refer :all] 8 | [clojure.core.async :as async] 9 | [clojure.java.io :as io] 10 | [clojure.string :as string] 11 | [clojure.core.async :refer [ (.read stream) (= -1))) 21 | (is (-> (.available stream) (= 0)))))) 22 | 23 | (defn bufferify [x] 24 | (ByteBuffer/wrap (byte-array x))) 25 | 26 | (deftest check-input-available 27 | (testing "Verify that our input stream reports available bytes properly" 28 | (let [input (async/chan 64) 29 | count 20 30 | stream (pio/new-inputstream {:ch input :buf (bufferify (repeat count 42))})] 31 | (is (-> (.available stream) (= count)))))) 32 | 33 | (deftest check-timeout 34 | (testing "Verify that our input stream's timeout mechanism works" 35 | (let [input (async/chan 64) 36 | stream (pio/new-inputstream {:ch input :tmo 100})] 37 | (is (thrown? clojure.lang.ExceptionInfo (.read stream))))) 38 | (testing "Verify that our input stream's timeout mechanism works" 39 | (let [input (async/chan 64) 40 | stream (pio/new-inputstream {:ch input :tmo 10000})] 41 | (async/close! input) 42 | (is (= (.read stream) -1))))) 43 | 44 | (deftest check-array-read 45 | (testing "Verify that we can read an array in one call" 46 | (let [ch (async/chan 64) 47 | stream (pio/new-inputstream {:ch ch}) 48 | len 20 49 | output (byte-array len)] 50 | (async/put! ch (ByteBuffer/wrap (byte-array (repeat len 42)))) 51 | (.read stream output) 52 | (is (= (count output) len)) 53 | (doseq [x output] 54 | (is (= x 42)))))) 55 | 56 | (defn- take-available [ch] 57 | (take-while some? (repeatedly #( result count (= repetitions))) 69 | (doseq [v (map (fn [^ByteBuffer x] (-> x .array (String.))) result)] 70 | (is (= phrase v))))))) -------------------------------------------------------------------------------- /test/test/example/hello/Greeter.clj: -------------------------------------------------------------------------------- 1 | ;;;---------------------------------------------------------------------------------- 2 | ;;; Generated by protoc-gen-clojure. DO NOT EDIT 3 | ;;; 4 | ;;; GRPC implementation of Greeter service from package example.hello 5 | ;;;---------------------------------------------------------------------------------- 6 | (ns example.hello.Greeter 7 | (:require [example.hello :refer :all])) 8 | 9 | ;;---------------------------------------------------------------------------------- 10 | ;;---------------------------------------------------------------------------------- 11 | ;; GRPC Implementations 12 | ;;---------------------------------------------------------------------------------- 13 | ;;---------------------------------------------------------------------------------- 14 | 15 | ;----------------------------------------------------------------------------- 16 | ; GRPC Greeter 17 | ;----------------------------------------------------------------------------- 18 | (defprotocol Service 19 | (SayHello [this param]) 20 | (SayRepeatHello [this param]) 21 | (SayHelloAfterDelay [this param]) 22 | (SayHelloOnDemand [this param]) 23 | (SayHelloError [this param]) 24 | (SayNil [this param])) 25 | 26 | (defn- SayHello-dispatch 27 | [ctx request] 28 | (SayHello ctx request)) 29 | (defn- SayRepeatHello-dispatch 30 | [ctx request] 31 | (SayRepeatHello ctx request)) 32 | (defn- SayHelloAfterDelay-dispatch 33 | [ctx request] 34 | (SayHelloAfterDelay ctx request)) 35 | (defn- SayHelloOnDemand-dispatch 36 | [ctx request] 37 | (SayHelloOnDemand ctx request)) 38 | (defn- SayHelloError-dispatch 39 | [ctx request] 40 | (SayHelloError ctx request)) 41 | (defn- SayNil-dispatch 42 | [ctx request] 43 | (SayNil ctx request)) 44 | 45 | (def ^:const rpc-metadata 46 | [{:pkg "example.hello" :service "Greeter" :method "SayHello" :method-fn SayHello-dispatch :server-streaming false :client-streaming false :input pb->HelloRequest :output new-HelloReply} 47 | {:pkg "example.hello" :service "Greeter" :method "SayRepeatHello" :method-fn SayRepeatHello-dispatch :server-streaming true :client-streaming false :input pb->RepeatHelloRequest :output new-HelloReply} 48 | {:pkg "example.hello" :service "Greeter" :method "SayHelloAfterDelay" :method-fn SayHelloAfterDelay-dispatch :server-streaming false :client-streaming false :input pb->HelloRequest :output new-HelloReply} 49 | {:pkg "example.hello" :service "Greeter" :method "SayHelloOnDemand" :method-fn SayHelloOnDemand-dispatch :server-streaming true :client-streaming true :input pb->HelloRequest :output new-HelloReply} 50 | {:pkg "example.hello" :service "Greeter" :method "SayHelloError" :method-fn SayHelloError-dispatch :server-streaming false :client-streaming false :input pb->HelloRequest :output new-HelloReply} 51 | {:pkg "example.hello" :service "Greeter" :method "SayNil" :method-fn SayNil-dispatch :server-streaming false :client-streaming false :input pb->HelloRequest :output new-HelloReply}]) 52 | 53 | -------------------------------------------------------------------------------- /modules/grpc-server/src/protojure/pedestal/interceptors/authz.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2022 Manetu, Inc. All rights reserved 2 | ;; 3 | ;; SPDX-License-Identifier: Apache-2.0 4 | 5 | (ns protojure.pedestal.interceptors.authz 6 | "A [Pedestal](http://pedestal.io/) [interceptor](http://pedestal.io/reference/interceptors) for authorizing Protojure GRPC endpoints" 7 | (:require [clojure.core.async :refer [go context 17 | (cond-> (not grpc?) (assoc :response {:status 401})) 18 | (cond-> grpc? (assoc ::chain/error (grpc.status/exception-info (grpc.status/get-desc :permission-denied)))) 19 | (terminate))) 20 | 21 | (defn- complete [allow? grpc? context] 22 | (if-not allow? 23 | (permission-denied grpc? context) 24 | context)) 25 | 26 | (defn- authz-enter 27 | [m pred {{:keys [path-info] :as request} :request :as context}] 28 | (let [e (get m path-info) 29 | grpc? (some? e) 30 | allow (pred (cond-> request grpc? (assoc :grpc-requestinfo e)))] 31 | (if (instance? clojure.core.async.impl.channels.ManyToManyChannel allow) 32 | (go 33 | (complete (metadata [m] 47 | (->> (cond-> m (nested? m) merge-metadata) 48 | (map #(select-keys % [:pkg :service :method])) 49 | (reduce 50 | (fn [acc entry] 51 | (assoc acc (str "/" (grpc/method-desc entry)) entry)) 52 | {}))) 53 | 54 | (defn interceptor 55 | " 56 | Installs a predicate function to authorize requests. 57 | 58 | Arguments: 59 | 60 | - `m`: rpc-metadata as generated by protojure compiler. Can either be directly submitted as generated, or as a vector 61 | of 1 or more instances to support multiple interfaces. 62 | - 'pred': An arity-1 predicate function that accepts a pedestal request map as input. Evaluating to true signals that the 63 | call is authorized, and should continue. Evaluating to false stops further execution and triggers a permission 64 | denied response. The request-map is augmented with :grpc-requestinfo containing the :pkg, :service, and :method of 65 | the call. Returning a core.async channel indicates that the predicate is asynchronous and will return true/false 66 | on the channel. 67 | 68 | " 69 | [m pred] 70 | (pedestal/interceptor {:name ::interceptor 71 | :enter (partial authz-enter (->metadata m) pred)})) 72 | -------------------------------------------------------------------------------- /test/project.clj: -------------------------------------------------------------------------------- 1 | (defproject io.github.protojure/test "2.0.2-SNAPSHOT" 2 | :description "Test harness for protojure libs" 3 | :url "http://github.com/protojure/lib" 4 | :license {:name "Apache License 2.0" 5 | :url "https://www.apache.org/licenses/LICENSE-2.0" 6 | :year 2022 7 | :key "apache-2.0"} 8 | :plugins [[lein-cljfmt "0.9.2"] 9 | [lein-set-version "0.4.1"] 10 | [lein-cloverage "1.2.4"] 11 | [lein-parent "0.3.9"]] 12 | :parent-project {:path "../project.clj" 13 | :inherit [:managed-dependencies :javac-options]} 14 | :profiles {:dev {:dependencies [[org.clojure/clojure] 15 | [org.clojure/core.async] 16 | [protojure/google.protobuf] 17 | [com.google.protobuf/protobuf-java] 18 | [org.apache.commons/commons-compress] 19 | [commons-io/commons-io] 20 | [funcool/promesa] 21 | [javax.servlet/javax.servlet-api] 22 | [io.undertow/undertow-core] 23 | [io.undertow/undertow-servlet] 24 | [io.pedestal/pedestal.log] 25 | [io.pedestal/pedestal.service] 26 | [io.pedestal/pedestal.error] 27 | [org.eclipse.jetty.http2/http2-client] 28 | [org.eclipse.jetty/jetty-alpn-java-client] 29 | [lambdaisland/uri] 30 | [org.clojure/tools.logging] 31 | [org.clojure/tools.namespace "1.4.4"] 32 | [clj-http "3.12.3"] 33 | [com.taoensso/timbre "6.2.2"] 34 | [com.fzakaria/slf4j-timbre "0.4.0"] 35 | [org.slf4j/jul-to-slf4j "2.0.9"] 36 | [org.slf4j/jcl-over-slf4j "2.0.9"] 37 | [org.slf4j/log4j-over-slf4j "2.0.9"] 38 | [org.clojure/data.codec "0.1.1"] 39 | [org.clojure/data.generators "1.0.0"] 40 | [danlentz/clj-uuid "0.1.9"] 41 | [eftest "0.6.0"] 42 | [criterium "0.4.6"] 43 | [circleci/bond "0.6.0"] 44 | [crypto-random "1.2.1"]] 45 | :resource-paths ["test/resources"]}} 46 | :source-paths ["../modules/io/src" "../modules/core/src" "../modules/grpc-client/src" "../modules/grpc-server/src"] 47 | :java-source-paths ["../modules/io/src"] 48 | :cloverage {:runner :eftest 49 | :runner-opts {:multithread? false 50 | :fail-fast? true} 51 | :fail-threshold 81}) 52 | -------------------------------------------------------------------------------- /modules/grpc-client/src/protojure/grpc/client/providers/http2.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.grpc.client.providers.http2 7 | "Implements the [GRPC-HTTP2](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md) protocol for clients" 8 | (:require [protojure.internal.grpc.client.providers.http2.core :as core] 9 | [protojure.internal.grpc.client.providers.http2.jetty :as jetty] 10 | [protojure.grpc.codec.compression :refer [builtin-codecs]] 11 | [promesa.core :as p] 12 | [lambdaisland.uri :as lambdaisland] 13 | [clojure.string :refer [lower-case]] 14 | [clojure.tools.logging :as log])) 15 | 16 | (set! *warn-on-reflection* true) 17 | 18 | (defn connect 19 | " 20 | Connects the client to a [GRPC-HTTP2](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md) compatible server 21 | 22 | #### Parameters 23 | A map with the following entries: 24 | 25 | | Value | Type | Default | Description | 26 | |-----------------------|---------------|-------------------------------------------------------------------------------------| 27 | | **uri** | _String_ | n/a | The URI of the GRPC server | 28 | | **codecs** | _map_ | [[protojure.grpc.codec.core/builtin-codecs]] | Optional custom codecs | 29 | | **content-coding** | _String_ | nil | The encoding to use on request data | 30 | | **max-frame-size** | _UInt32_ | 16KB | The maximum HTTP2 DATA frame size | 31 | | **input-buffer-size** | _UInt32_ | 1MB | The input-buffer size | 32 | | **insecure?** | _bool_ | false | Disables TLS checks such as host verification and truststore (dev only) | 33 | | **metadata** | _map_ or _fn_ | n/a | Optional [string string] tuples as a map, or a 0-arity fn that returns same that will be submitted as attributes to the request, such as via HTTP headers for GRPC-HTTP2 | 34 | | **on-close** | _fn_ | n/a | Optional zero argument callback function when a connection is closed. | 35 | 36 | #### Return value 37 | A promise that, on success, evaluates to an instance of [[api/Provider]]. 38 | _(api/disconnect)_ should be used to release any resources when the connection is no longer required. 39 | " 40 | [{:keys [uri codecs content-coding max-frame-size input-buffer-size metadata idle-timeout ssl insecure? on-close] 41 | :or {codecs builtin-codecs max-frame-size 16384 input-buffer-size jetty/default-input-buffer insecure? false} 42 | :as params}] 43 | (log/debug "Connecting with GRPC-HTTP2:" params) 44 | (let [{:keys [host port scheme]} (lambdaisland/uri uri) 45 | https? (= "https" (lower-case scheme)) 46 | parsed-port (cond 47 | port (Integer/parseInt port) 48 | https? 443 49 | :else 80)] 50 | (-> (jetty/connect {:host host :port parsed-port :input-buffer-size input-buffer-size :idle-timeout idle-timeout :ssl (or ssl https?) :insecure? insecure? :on-close on-close}) 51 | (p/then #(core/->Http2Provider % uri codecs content-coding max-frame-size input-buffer-size metadata))))) 52 | -------------------------------------------------------------------------------- /modules/core/src/protojure/protobuf/serdes/core.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.protobuf.serdes.core 7 | "Serializer/deserializer support for fundamental protobuf types." 8 | (:require [protojure.protobuf :refer [->pb]] 9 | [protojure.protobuf.serdes.utils :as utils]) 10 | (:import (com.google.protobuf CodedInputStream 11 | CodedOutputStream 12 | WireFormat 13 | UnknownFieldSet 14 | ExtensionRegistry 15 | ByteString))) 16 | 17 | (set! *warn-on-reflection* true) 18 | 19 | (defmacro defparsefn [type] 20 | (let [name (symbol (str "cis->" type)) 21 | sym (symbol (str "read" type)) 22 | doc (format "Deserialize a '%s' type" type)] 23 | `(defn ~name ~doc [^CodedInputStream is#] 24 | (. is# ~sym)))) 25 | 26 | (defmacro defwritefn [type default?] 27 | (let [name (symbol (str "write-" type)) 28 | sym (symbol (str "write" type)) 29 | doc (format "Serialize a '%s' type" type)] 30 | `(defn ~name ~doc 31 | ([tag# value# os#] 32 | (~name tag# {} value# os#)) 33 | ([tag# options# value# ^CodedOutputStream os#] 34 | (when-not (and (get options# :optimize true) (~default? value#)) 35 | (. os# ~sym tag# value#)))))) 36 | 37 | (defmacro defserdes [type default?] 38 | `(do 39 | (defparsefn ~type) 40 | (defwritefn ~type ~default?))) 41 | 42 | (defmacro defscalar [type] 43 | `(defserdes ~type utils/default-scalar?)) 44 | 45 | (defscalar "Double") 46 | (defscalar "Enum") 47 | (defscalar "Fixed32") 48 | (defscalar "Fixed64") 49 | (defscalar "Float") 50 | (defscalar "Int32") 51 | (defscalar "Int64") 52 | (defscalar "SFixed32") 53 | (defscalar "SFixed64") 54 | (defscalar "SInt32") 55 | (defscalar "SInt64") 56 | (defscalar "UInt32") 57 | (defscalar "UInt64") 58 | 59 | (defserdes "String" utils/default-bytes?) 60 | (defserdes "Bool" utils/default-bool?) 61 | 62 | ;; manually implement the "Bytes" scalar so we can properly handle native byte-array import/export 63 | (defn cis->Bytes 64 | "Deserialize 'Bytes' type" 65 | [^CodedInputStream is] 66 | (.toByteArray (.readBytes is))) 67 | 68 | (defn write-Bytes 69 | "Serialize 'Bytes' type" 70 | ([tag value os] 71 | (write-Bytes tag {} value os)) 72 | ([tag {:keys [optimize] :or {optimize true} :as options} value ^CodedOutputStream os] 73 | (when-not (and optimize (empty? value)) 74 | (let [bytestring (ByteString/copyFrom (bytes value))] 75 | (.writeBytes os tag bytestring))))) 76 | 77 | (defn cis->undefined 78 | "Deserialize an unknown type, retaining its tag/type" 79 | [tag ^CodedInputStream is] 80 | (let [num (WireFormat/getTagFieldNumber tag) 81 | type (WireFormat/getTagWireType tag)] 82 | (case type 83 | 0 (.readInt64 is) 84 | 1 (.readFixed64 is) 85 | 2 (.readBytes is) 86 | 3 (.readGroup is num (UnknownFieldSet/newBuilder) (ExtensionRegistry/getEmptyRegistry)) 87 | 4 nil 88 | 5 (.readFixed32 is)))) 89 | 90 | (defn cis->embedded 91 | "Deserialize an embedded type, where **f** is an (fn) that can deserialize the embedded message" 92 | [f ^CodedInputStream is] 93 | (let [len (.readRawVarint32 is) 94 | lim (.pushLimit is len)] 95 | (let [result (f is)] 96 | (.popLimit is lim) 97 | result))) 98 | 99 | (defn write-embedded 100 | "Serialize an embedded type along with tag/length metadata" 101 | [tag item ^CodedOutputStream os] 102 | (when (some? item) 103 | (let [data (->pb item) 104 | len (count data)] 105 | (.writeTag os tag 2);; embedded messages are always type=2 (string) 106 | (.writeUInt32NoTag os len) 107 | (.writeRawBytes os (bytes data))))) 108 | -------------------------------------------------------------------------------- /modules/io/src/protojure/internal/io.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns ^:no-doc protojure.internal.io 7 | (:require [clojure.core.async :refer [> buf (.get) (bit-and 0xff) int) 57 | -1))) 58 | 59 | (defn new-inputstream 60 | ^java.io.InputStream [{:keys [ch tmo buf]}] 61 | (let [ctx {:ch ch :tmo tmo :buf (atom buf)}] 62 | (ProxyInputStream. 63 | (reify AsyncInputStream 64 | (available [_] 65 | (is-available ctx)) 66 | (read_int [_] 67 | (is-read-int ctx)) 68 | (read_bytes [_ b] 69 | (is-read ctx b 0 (count b))) 70 | (read_offset [_ b off len] 71 | (is-read ctx b off len)))))) 72 | 73 | ;;-------------------------------------------------------------------------------------------- 74 | ;; OutputStream 75 | ;;-------------------------------------------------------------------------------------------- 76 | (defn- os-flush 77 | [{:keys [ch frame-size buf]}] 78 | (let [^ByteBuffer _buf @buf] 79 | (when (pos? (.position _buf)) 80 | (async/>!! ch (.flip _buf)) 81 | (reset! buf (ByteBuffer/allocate frame-size))))) 82 | 83 | (defn- os-maybe-flush 84 | [{:keys [buf] :as ctx}] 85 | (let [^ByteBuffer _buf @buf] 86 | (when (zero? (.remaining _buf)) 87 | (os-flush ctx)))) 88 | 89 | (defn- os-close 90 | [{:keys [ch] :as ctx}] 91 | (os-flush ctx) 92 | (async/close! ch)) 93 | 94 | (defn- os-write 95 | [{:keys [buf] :as ctx} b off len] 96 | (os-maybe-flush ctx) 97 | (let [^ByteBuffer _buf @buf 98 | alen (min len (.remaining _buf))] 99 | (when (pos? alen) 100 | (.put _buf b off alen) 101 | (when (< alen len) 102 | (recur ctx b (+ off alen) (- len alen)))))) 103 | 104 | (defn new-outputstream 105 | ^java.io.OutputStream [{:keys [ch max-frame-size] :or {max-frame-size 16384} :as options}] 106 | {:pre [(and (some? max-frame-size) (pos? max-frame-size))]} 107 | (let [ctx {:ch ch :frame-size max-frame-size :buf (atom (ByteBuffer/allocate max-frame-size))}] 108 | (ProxyOutputStream. 109 | (reify AsyncOutputStream 110 | (flush [_] 111 | (os-flush ctx)) 112 | (close [_] 113 | (os-close ctx)) 114 | (write_int [_ b] 115 | (let [b (bit-and 0xff b)] 116 | (os-write ctx (byte-array [b]) 0 1))) 117 | (write_bytes [_ b] 118 | (os-write ctx b 0 (count b))) 119 | (write_offset [_ b off len] 120 | (os-write ctx b off len)))))) 121 | -------------------------------------------------------------------------------- /test/test/protojure/grpc_web_test.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.grpc-web-test 7 | (:require [clojure.test :refer :all] 8 | [io.pedestal.test :refer [response-for]] 9 | [protojure.pedestal.core :as protojure.pedestal] 10 | [protojure.pedestal.interceptors.grpc-web :as grpc-web] 11 | [io.pedestal.http :as pedestal] 12 | [io.pedestal.http.body-params :as body-params] 13 | [example.types :as example] 14 | [protojure.protobuf :as pb] 15 | [clojure.data.codec.base64 :as b64])) 16 | 17 | (defn grpc-echo [{:keys [body] :as request}] 18 | {:status 200 19 | :body (example/pb->Money body) 20 | :trailers {"grpc-status" 0 "grpc-message" "Got it!"}}) 21 | 22 | (def interceptors [(body-params/body-params) 23 | grpc-web/proxy]) 24 | 25 | (def routes [["/" :get (conj interceptors `grpc-echo)]]) 26 | 27 | (def service (let [service-params {:env :prod 28 | ::pedestal/routes (into #{} routes) 29 | ::pedestal/type protojure.pedestal/config 30 | ::pedestal/chain-provider protojure.pedestal/provider}] 31 | (:io.pedestal.http/service-fn (io.pedestal.http/create-servlet service-params)))) 32 | 33 | (deftest grpc-web-text-check 34 | (testing "Check that a round-trip GRPC request works" 35 | (let [input-msg (pb/->pb (example/new-Money {:currency_code (apply str (repeat 20 "foobar")) :units 42 :nanos 750000000}))] 36 | (is 37 | (= 38 | (with-out-str (pr (example/pb->Money input-msg))) 39 | (:body (response-for 40 | service 41 | :get "/" 42 | :headers {"Content-Type" "application/grpc-web-text"} 43 | :body (clojure.java.io/input-stream (b64/encode input-msg))))))))) 44 | 45 | (deftest grpc-web-check 46 | (testing "Check that a round-trip GRPC request works" 47 | (let [input-msg (pb/->pb (example/new-Money {:currency_code (apply str (repeat 20 "foobar")) :units 42 :nanos 750000000}))] 48 | (is 49 | (= 50 | (with-out-str (pr (example/pb->Money input-msg))) 51 | (:body (response-for 52 | service 53 | :get "/" 54 | :headers {"Content-Type" "application/grpc-web"} 55 | :body (clojure.java.io/input-stream input-msg)))))))) 56 | 57 | (deftest grpc-web-proto-check 58 | (testing "Check that a round-trip GRPC request works" 59 | (let [input-msg (pb/->pb (example/new-Money {:currency_code (apply str (repeat 20 "foobar")) :units 42 :nanos 750000000}))] 60 | (is 61 | (= 62 | (with-out-str (pr (example/pb->Money input-msg))) 63 | (:body (response-for 64 | service 65 | :get "/" 66 | :headers {"Content-Type" "application/grpc-web+proto"} 67 | :body (clojure.java.io/input-stream input-msg)))))))) 68 | 69 | (deftest grpc-web-text-proto-check 70 | (testing "Check that a round-trip GRPC request works" 71 | (let [input-msg (pb/->pb (example/new-Money {:currency_code (apply str (repeat 20 "foobar")) :units 42 :nanos 750000000}))] 72 | (is 73 | (= 74 | (with-out-str (pr (example/pb->Money input-msg))) 75 | (:body (response-for 76 | service 77 | :get "/" 78 | :headers {"Content-Type" "application/grpc-web-text+proto"} 79 | :body (clojure.java.io/input-stream (b64/encode input-msg))))))))) 80 | 81 | (deftest grpc-web-no-header-match-check 82 | (testing "Check that a round-trip GRPC request works" 83 | (let [input-msg (pb/->pb (example/new-Money {:currency_code (apply str (repeat 20 "foobar")) :units 42 :nanos 750000000}))] 84 | (is 85 | (= 86 | (with-out-str (pr (example/pb->Money input-msg))) 87 | (:body (response-for 88 | service 89 | :get "/" 90 | :headers {"Content-Type" "application/grpc"} 91 | :body (clojure.java.io/input-stream input-msg)))))))) 92 | -------------------------------------------------------------------------------- /modules/core/src/protojure/grpc/codec/compression.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.grpc.codec.compression 7 | (:import (java.io InputStream OutputStream) 8 | (org.apache.commons.compress.compressors.gzip GzipCompressorInputStream 9 | GzipCompressorOutputStream 10 | GzipParameters) 11 | (org.apache.commons.compress.compressors.snappy FramedSnappyCompressorInputStream 12 | FramedSnappyCompressorOutputStream) 13 | (org.apache.commons.compress.compressors.deflate DeflateCompressorInputStream 14 | DeflateCompressorOutputStream))) 15 | 16 | (set! *warn-on-reflection* true) 17 | 18 | ;;-------------------------------------------------------------------------------------- 19 | ;; compression support 20 | ;;-------------------------------------------------------------------------------------- 21 | (def ^:no-doc _builtin-codecs 22 | [{:name "gzip" 23 | :input #(GzipCompressorInputStream. ^InputStream %) 24 | :output #(let [params (GzipParameters.)] (.setCompressionLevel params 9) (GzipCompressorOutputStream. ^OutputStream % params))} 25 | 26 | {:name "snappy" 27 | :input #(FramedSnappyCompressorInputStream. ^InputStream %) 28 | :output #(FramedSnappyCompressorOutputStream. ^OutputStream %)} 29 | 30 | {:name "deflate" 31 | :input #(DeflateCompressorInputStream. ^InputStream %) 32 | :output #(DeflateCompressorOutputStream. ^OutputStream %)}]) 33 | 34 | (def builtin-codecs 35 | " 36 | A map of built-in compression [codecs](https://en.wikipedia.org/wiki/Codec), keyed by name. 37 | 38 | | Name | Description | 39 | |--------------|--------------------------------------------------------| 40 | | \"gzip\" | [gzip](https://en.wikipedia.org/wiki/Gzip) codec | 41 | | \"deflate\" | [deflate](https://en.wikipedia.org/wiki/DEFLATE) codec | 42 | | \"snappy\" | [snappy](https://github.com/google/snappy) codec | 43 | 44 | These built-in codecs are used by default, unless the caller overrides the codec dictionary. A common use 45 | case would be to augment the built-in codecs with 1 or more custom codecs. 46 | 47 | #### Custom codecs 48 | 49 | ##### Map specification 50 | The codec map consists of a collection of name/value pairs of codec-specifications keyed by a string representing 51 | the name of the codec. 52 | 53 | ``` 54 | [\"mycodec\" {:input inputfn :output outputfn}] 55 | ``` 56 | 57 | where 58 | 59 | - **inputfn**: a (fn) that accepts an InputStream input, and returns an InputStream 60 | - **outputfn**: a (fn) that accepts an OutputStream input, and returns an OutputStream 61 | 62 | ##### Example 63 | 64 | ``` 65 | (assoc builtin-codecs 66 | \"mycodec\" {:input clojure.core/identity :output clojure.core/identity}) 67 | ``` 68 | 69 | **N.B.**: The output stream returned in _outputfn_ will have its (.close) method invoked to finalize 70 | compression. Therefore, the use of `identity` above would be problematic in the real-world since we 71 | may not wish to actually close the underlying stream at that time. Therefore, its use above is only for 72 | simplistic demonstration. A functional \"pass through\" example could be built using something like 73 | [CloseShieldOutputStream](https://commons.apache.org/proper/commons-io/javadocs/api-2.4/org/apache/commons/io/output/CloseShieldOutputStream.html) 74 | " 75 | (->> _builtin-codecs (map #(vector (:name %) %)) (into {}))) 76 | 77 | (defn- get-codec-by-polarity [factory polarity] 78 | (if-let [codec (get factory polarity)] 79 | codec 80 | (throw (ex-info "CODEC polarity not found" {:codec factory :polarity polarity})))) 81 | 82 | (defn- get-codec [codecs type polarity] 83 | (if-let [factory (get codecs type)] 84 | (get-codec-by-polarity factory polarity) 85 | (throw (ex-info "Unknown CODEC name" {:name type :polarity polarity})))) 86 | 87 | (defn ^:no-doc compressor ^OutputStream [codecs type] (get-codec codecs type :output)) 88 | (defn ^:no-doc decompressor ^InputStream [codecs type] (get-codec codecs type :input)) 89 | 90 | -------------------------------------------------------------------------------- /modules/grpc-client/src/protojure/grpc/client/api.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.grpc.client.api 7 | "Provider independent client API for invoking GRPC requests") 8 | 9 | (defprotocol Provider 10 | ;;;;---------------------------------------------------------------------------------------------------- 11 | (invoke [this params] 12 | " 13 | Invokes a GRPC-based remote-procedure against the provider 14 | 15 | #### Parameters 16 | A map with the following entries: 17 | 18 | | Value | Type | Description | 19 | |-----------------|-------------------|---------------------------------------------------------------------------| 20 | | **service** | _String_ | The GRPC service-name of the endpoint | 21 | | **method** | _String_ | The GRPC method-name of the endpoint | 22 | | **metadata** | _map_ | Optional [string string] tuples that will be submitted as attributes to the request, such as via HTTP headers for GRPC-HTTP2 | 23 | | **input** | _map_ | See _Input_ section below | 24 | | **output** | _map_ | See _Output_ section below | 25 | 26 | ##### Unary vs Streaming Input 27 | 28 | Any [GRPC Service endpoint](https://grpc.io/docs/guides/concepts.html#service-definition) can define methods that take either unary or streaming inputs or outputs. 29 | This API assumes core.async channels in either case as a 'streaming first' design. For unary input, simply produce one message before closing the stream. Closing the stream indicates that the input is complete. 30 | 31 | ##### Input 32 | The _input_ parameter is a map with the following fields: 33 | 34 | | Value | Type | Description | 35 | |-----------------|-------------------|---------------------------------------------------------------------------| 36 | | **f** | _(fn [map])_ | A protobuf new-XX function, such as produced by the protoc-gen-clojure compiler, to be applied to any outbound request messages | 37 | | **ch** | _core.async/chan_ | A core.async channel used to send input parameters. Close to complete. | 38 | 39 | ##### Output 40 | The _output_ parameter is a map with the following fields: 41 | 42 | | Value | Type | Description | 43 | |-----------------|-------------------|---------------------------------------------------------------------------| 44 | | **f** | _(fn [is])_ | A protobuf pb->msg function, such as produced by the protoc-gen-clojure compiler, to be applied to any incoming response messages | 45 | | **ch** | _core.async/chan_ | A core.async channel that will be populated with any GRPC return messages. Unary responses arrive as a single message on the channel. Will close when the response is complete. | 46 | 47 | #### Return value 48 | A promise that, on success, evaluates to a map with the following entries: 49 | 50 | | Value | Type | Description | 51 | |-----------------|----------|---------------------------------------------------------------------------| 52 | | **status** | _Int_ | The [GRPC status code](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md) returned from the remote procedure | 53 | | **message** | _String_ | The GRPC message (if any) returned from the remote procedure | 54 | 55 | #### Example 56 | 57 | ``` 58 | (let [{:keys [status message]} @(invoke client {:service \"my.service\" 59 | :method \"MyMethod\" 60 | :input {:f myservice/new-MyRequest :ch input} 61 | :output {:f myservice/pb->MyResponse :ch output}})] 62 | (println \"status:\" status)) 63 | ``` 64 | 65 | ") 66 | 67 | ;;;;---------------------------------------------------------------------------------------------------- 68 | (disconnect [this] 69 | "Disconnects from the GRPC endpoint and releases all resources held by the underlying provider")) -------------------------------------------------------------------------------- /test/test/protojure/test/grpc/TestService/server.cljc: -------------------------------------------------------------------------------- 1 | ;;;---------------------------------------------------------------------------------- 2 | ;;; Generated by protoc-gen-clojure. DO NOT EDIT 3 | ;;; 4 | ;;; GRPC protojure.test.grpc.TestService Service Implementation 5 | ;;;---------------------------------------------------------------------------------- 6 | (ns protojure.test.grpc.TestService.server 7 | (:require [protojure.test.grpc :refer :all] 8 | [com.google.protobuf :as com.google.protobuf])) 9 | 10 | ;----------------------------------------------------------------------------- 11 | ; GRPC TestService 12 | ;----------------------------------------------------------------------------- 13 | (defprotocol Service 14 | (BandwidthTest [this param]) 15 | (BidirectionalStreamTest [this param]) 16 | (ClientCloseDetect [this param]) 17 | (FlowControl [this param]) 18 | (ReturnError [this param]) 19 | (AllEmpty [this param]) 20 | (ServerCloseDetect [this param]) 21 | (Async [this param]) 22 | (DeniedStreamer [this param]) 23 | (AsyncEmpty [this param]) 24 | (Metadata [this param]) 25 | (ReturnErrorStreaming [this param]) 26 | (AuthzTest [this param]) 27 | (ShouldThrow [this param])) 28 | 29 | (def TestService-service-name "protojure.test.grpc.TestService") 30 | 31 | (defn- BandwidthTest-dispatch 32 | [ctx request] 33 | (BandwidthTest ctx request)) 34 | (defn- BidirectionalStreamTest-dispatch 35 | [ctx request] 36 | (BidirectionalStreamTest ctx request)) 37 | (defn- ClientCloseDetect-dispatch 38 | [ctx request] 39 | (ClientCloseDetect ctx request)) 40 | (defn- FlowControl-dispatch 41 | [ctx request] 42 | (FlowControl ctx request)) 43 | (defn- ReturnError-dispatch 44 | [ctx request] 45 | (ReturnError ctx request)) 46 | (defn- AllEmpty-dispatch 47 | [ctx request] 48 | (AllEmpty ctx request)) 49 | (defn- ServerCloseDetect-dispatch 50 | [ctx request] 51 | (ServerCloseDetect ctx request)) 52 | (defn- Async-dispatch 53 | [ctx request] 54 | (Async ctx request)) 55 | (defn- DeniedStreamer-dispatch 56 | [ctx request] 57 | (DeniedStreamer ctx request)) 58 | (defn- AsyncEmpty-dispatch 59 | [ctx request] 60 | (AsyncEmpty ctx request)) 61 | (defn- Metadata-dispatch 62 | [ctx request] 63 | (Metadata ctx request)) 64 | (defn- ReturnErrorStreaming-dispatch 65 | [ctx request] 66 | (ReturnErrorStreaming ctx request)) 67 | (defn- AuthzTest-dispatch 68 | [ctx request] 69 | (AuthzTest ctx request)) 70 | (defn- ShouldThrow-dispatch 71 | [ctx request] 72 | (ShouldThrow ctx request)) 73 | 74 | (def ^:const rpc-metadata 75 | [{:pkg "protojure.test.grpc" :service "TestService" :method "BandwidthTest" :method-fn BandwidthTest-dispatch :server-streaming false :client-streaming false :input pb->BigPayload :output new-BigPayload} 76 | {:pkg "protojure.test.grpc" :service "TestService" :method "BidirectionalStreamTest" :method-fn BidirectionalStreamTest-dispatch :server-streaming true :client-streaming true :input pb->SimpleRequest :output new-SimpleResponse} 77 | {:pkg "protojure.test.grpc" :service "TestService" :method "ClientCloseDetect" :method-fn ClientCloseDetect-dispatch :server-streaming true :client-streaming false :input pb->CloseDetectRequest :output com.google.protobuf/new-Any} 78 | {:pkg "protojure.test.grpc" :service "TestService" :method "FlowControl" :method-fn FlowControl-dispatch :server-streaming true :client-streaming false :input pb->FlowControlRequest :output new-FlowControlPayload} 79 | {:pkg "protojure.test.grpc" :service "TestService" :method "ReturnError" :method-fn ReturnError-dispatch :server-streaming false :client-streaming false :input pb->ErrorRequest :output com.google.protobuf/new-Empty} 80 | {:pkg "protojure.test.grpc" :service "TestService" :method "AllEmpty" :method-fn AllEmpty-dispatch :server-streaming false :client-streaming false :input com.google.protobuf/pb->Empty :output com.google.protobuf/new-Empty} 81 | {:pkg "protojure.test.grpc" :service "TestService" :method "ServerCloseDetect" :method-fn ServerCloseDetect-dispatch :server-streaming true :client-streaming false :input com.google.protobuf/pb->Empty :output com.google.protobuf/new-Any} 82 | {:pkg "protojure.test.grpc" :service "TestService" :method "Async" :method-fn Async-dispatch :server-streaming false :client-streaming false :input com.google.protobuf/pb->Empty :output new-SimpleResponse} 83 | {:pkg "protojure.test.grpc" :service "TestService" :method "DeniedStreamer" :method-fn DeniedStreamer-dispatch :server-streaming true :client-streaming false :input com.google.protobuf/pb->Empty :output com.google.protobuf/new-Empty} 84 | {:pkg "protojure.test.grpc" :service "TestService" :method "AsyncEmpty" :method-fn AsyncEmpty-dispatch :server-streaming true :client-streaming false :input com.google.protobuf/pb->Empty :output com.google.protobuf/new-Empty} 85 | {:pkg "protojure.test.grpc" :service "TestService" :method "Metadata" :method-fn Metadata-dispatch :server-streaming false :client-streaming false :input com.google.protobuf/pb->Empty :output new-SimpleResponse} 86 | {:pkg "protojure.test.grpc" :service "TestService" :method "ReturnErrorStreaming" :method-fn ReturnErrorStreaming-dispatch :server-streaming true :client-streaming false :input pb->ErrorRequest :output com.google.protobuf/new-Empty} 87 | {:pkg "protojure.test.grpc" :service "TestService" :method "AuthzTest" :method-fn AuthzTest-dispatch :server-streaming false :client-streaming false :input pb->AuthzTestRequest :output com.google.protobuf/new-Empty} 88 | {:pkg "protojure.test.grpc" :service "TestService" :method "ShouldThrow" :method-fn ShouldThrow-dispatch :server-streaming false :client-streaming false :input com.google.protobuf/pb->Empty :output com.google.protobuf/new-Empty}]) 89 | -------------------------------------------------------------------------------- /modules/grpc-server/src/protojure/pedestal/interceptors/grpc.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.pedestal.interceptors.grpc 7 | "A [Pedestal](http://pedestal.io/) [interceptor](http://pedestal.io/reference/interceptors) for [GRPC](https://grpc.io/) support" 8 | (:require [clojure.core.async :refer [go protojure.grpc.codec.compression/builtin-codecs (keys) (conj "identity") (set))) 21 | 22 | (defn- determine-output-encoding 23 | [accepted-encodings] 24 | (->> (clojure.string/split accepted-encodings #",") 25 | (filter supported-encodings) 26 | (first))) 27 | 28 | (defn logging-chan [bufsiz id label] 29 | (async/chan bufsiz (map (fn [m] (log/trace (str "GRPC: " id " -> " label) m) m)))) 30 | 31 | (defn- create-req-ctx 32 | [id f {:keys [body-ch] {:strs [grpc-encoding] :or {grpc-encoding "identity"}} :headers :as req}] 33 | (let [in body-ch 34 | out (logging-chan 128 id "rx")] 35 | {:in in 36 | :out out 37 | :encoding grpc-encoding 38 | :status (lpm/decode f in out {:content-coding grpc-encoding})})) 39 | 40 | (defn- create-resp-ctx 41 | [id f {{:strs [grpc-accept-encoding] :or {grpc-accept-encoding ""}} :headers :as req}] 42 | (let [in (logging-chan 128 id "tx") 43 | out (async/chan 128) 44 | encoding (or (determine-output-encoding grpc-accept-encoding) "identity")] 45 | {:in in 46 | :out out 47 | :encoding encoding 48 | :status (lpm/encode f in out {:content-coding encoding :max-frame-size 16383})})) 49 | 50 | (defn- set-params [context params] 51 | (assoc-in context [:request :grpc-params] params)) 52 | 53 | (defn gen-uuid [] 54 | (.toString (UUID/randomUUID))) 55 | 56 | (defn method-desc [{:keys [pkg service method]}] 57 | (str pkg "." service "/" method)) 58 | 59 | (defn- grpc-enter 60 | " interceptor for handling GRPC requests" 61 | [{:keys [server-streaming client-streaming input output] :as rpc-metadata} 62 | {:keys [request] :as context}] 63 | (let [id (gen-uuid) 64 | req-ctx (create-req-ctx id input request) 65 | resp-ctx (create-resp-ctx id output request) 66 | input-ch (:out req-ctx) 67 | context (-> context 68 | (assoc ::ctx {:req-ctx req-ctx :resp-ctx resp-ctx :id id}) 69 | (cond-> server-streaming 70 | (assoc-in [:request :grpc-out] (:in resp-ctx))))] 71 | 72 | (log/trace (str "GRPC: " id " -> start " (method-desc rpc-metadata)) request) 73 | 74 | ;; set :grpc-params 75 | (if client-streaming 76 | (set-params context input-ch) ;; client-streaming means simply pass the channel directly 77 | (if-let [params (async/poll! input-ch)] 78 | (set-params context params) ;; materialize unary params opportunistically, if available 79 | (go (set-params context (trailers 92 | [{:keys [grpc-status grpc-message] :or {grpc-status 0}}] 93 | (-> {"grpc-status" grpc-status} 94 | (cond-> (some? grpc-message) (assoc "grpc-message" grpc-message)))) 95 | 96 | (defn- prepare-trailers [id {:keys [trailers] :as response}] 97 | (let [ch (async/promise-chan)] 98 | [ch (fn [_] (-> (if (some? trailers) 99 | (take-promise trailers) 100 | response) 101 | (p/then ->trailers) 102 | (p/then (fn [r] 103 | (log/trace (str "GRPC: " id " -> trailers") r) 104 | (put ch r)))))])) 105 | 106 | (defn- grpc-leave 107 | " interceptor for handling GRPC responses" 108 | [{:keys [server-streaming] :as rpc-metadata} 109 | {{:keys [body] :as response} :response {:keys [req-ctx resp-ctx id]} ::ctx :as context}] 110 | 111 | (log/trace (str "GRPC: " id " -> leave ") response) 112 | 113 | (let [output-ch (:in resp-ctx) 114 | [trailers-ch trailers-fn] (prepare-trailers id response)] 115 | 116 | (cond 117 | ;; special-case unary return types 118 | (not server-streaming) 119 | (do 120 | (when body 121 | (async/>!! output-ch body)) 122 | (async/close! output-ch)) 123 | 124 | ;; Auto-close the output ch if the user does not signify they have consumed it 125 | ;; by referencing it in the :body 126 | (not= output-ch body) 127 | (async/close! output-ch)) 128 | 129 | ;; defer sending trailers until our IO has completed 130 | (-> (p/all (mapv :status [req-ctx resp-ctx])) 131 | (p/then trailers-fn) 132 | (p/then (fn [_] 133 | (log/trace (str "GRPC: " id " -> complete ") nil))) 134 | (p/timeout 30000) 135 | (p/catch (fn [ex] 136 | (log/error :msg "Pipeline" :exception ex) 137 | (status/error :internal)))) 138 | 139 | (update context :response 140 | #(assoc % 141 | :headers {"Content-Type" "application/grpc+proto" 142 | "grpc-encoding" (:encoding resp-ctx)} 143 | :status 200 ;; always return 200 144 | :body (:out resp-ctx) 145 | :trailers trailers-ch)))) 146 | 147 | (defn route-interceptor 148 | [rpc-metadata] 149 | (pedestal/interceptor {:name ::interceptor 150 | :enter (partial grpc-enter rpc-metadata) 151 | :leave (partial grpc-leave rpc-metadata)})) 152 | 153 | (defn- err-status 154 | [ctx status msg] 155 | (update ctx :response 156 | assoc 157 | :headers {"Content-Type" "application/grpc+proto"} 158 | :status 200 159 | :body "" 160 | :trailers (->trailers {:grpc-status status :grpc-message msg}))) 161 | 162 | (def error-interceptor 163 | (err/error-dispatch 164 | [ctx ex] 165 | 166 | [{:exception-type ::status/error}] 167 | (let [{:keys [code msg]} (ex-data ex)] 168 | (err-status ctx code msg)) 169 | 170 | :else 171 | (err-status ctx (grpc.status/get-code :internal) (ex-message ex)))) 172 | -------------------------------------------------------------------------------- /test/test/example/hello.clj: -------------------------------------------------------------------------------- 1 | ;;;---------------------------------------------------------------------------------- 2 | ;;; Generated by protoc-gen-clojure. DO NOT EDIT 3 | ;;; 4 | ;;; Message Implementation of package com.sttgts.omnia.hello 5 | ;;;---------------------------------------------------------------------------------- 6 | (ns example.hello 7 | (:require [protojure.protobuf.protocol :as pb] 8 | [protojure.protobuf.serdes.core :refer :all] 9 | [protojure.protobuf.serdes.complex :refer :all] 10 | [protojure.protobuf.serdes.utils :refer [tag-map]] 11 | [protojure.protobuf.serdes.stream :as stream] 12 | [clojure.spec.alpha :as s])) 13 | 14 | ;;---------------------------------------------------------------------------------- 15 | ;;---------------------------------------------------------------------------------- 16 | ;; Forward declarations 17 | ;;---------------------------------------------------------------------------------- 18 | ;;---------------------------------------------------------------------------------- 19 | 20 | (declare cis->HelloRequest) 21 | (declare ecis->HelloRequest) 22 | (declare new-HelloRequest) 23 | (declare cis->RepeatHelloRequest) 24 | (declare ecis->RepeatHelloRequest) 25 | (declare new-RepeatHelloRequest) 26 | (declare cis->HelloReply) 27 | (declare ecis->HelloReply) 28 | (declare new-HelloReply) 29 | 30 | ;;---------------------------------------------------------------------------------- 31 | ;;---------------------------------------------------------------------------------- 32 | ;; Message Implementations 33 | ;;---------------------------------------------------------------------------------- 34 | ;;---------------------------------------------------------------------------------- 35 | 36 | ;----------------------------------------------------------------------------- 37 | ; HelloRequest 38 | ;----------------------------------------------------------------------------- 39 | (defrecord HelloRequest [name] 40 | pb/Writer 41 | 42 | (serialize [this os] 43 | (write-String 1 {:optimize true} (:name this) os))) 44 | 45 | (s/def :com.sttgts.omnia.hello.messages.HelloRequest/name string?) 46 | (s/def ::HelloRequest-spec (s/keys :opt-un [:com.sttgts.omnia.hello.messages.HelloRequest/name])) 47 | (def HelloRequest-defaults {:name ""}) 48 | 49 | (defn cis->HelloRequest 50 | "CodedInputStream to HelloRequest" 51 | [is] 52 | (->> (tag-map HelloRequest-defaults 53 | (fn [tag index] 54 | (case index 55 | 1 [:name (cis->String is)] 56 | 57 | [index (cis->undefined tag is)])) 58 | is) 59 | (map->HelloRequest))) 60 | 61 | (defn ecis->HelloRequest 62 | "Embedded CodedInputStream to HelloRequest" 63 | [is] 64 | (cis->embedded cis->HelloRequest is)) 65 | 66 | (defn new-HelloRequest 67 | "Creates a new instance from a map, similar to map->HelloRequest except that 68 | it properly accounts for nested messages, when applicable. 69 | " 70 | [init] 71 | {:pre [(if (s/valid? ::HelloRequest-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::HelloRequest-spec init))))]} 72 | (-> (merge HelloRequest-defaults init) 73 | (map->HelloRequest))) 74 | 75 | (defn pb->HelloRequest 76 | "Protobuf to HelloRequest" 77 | [input] 78 | (-> input 79 | stream/new-cis 80 | cis->HelloRequest)) 81 | 82 | ;----------------------------------------------------------------------------- 83 | ; RepeatHelloRequest 84 | ;----------------------------------------------------------------------------- 85 | (defrecord RepeatHelloRequest [name count] 86 | pb/Writer 87 | 88 | (serialize [this os] 89 | (write-String 1 {:optimize true} (:name this) os) 90 | (write-Int32 2 {:optimize true} (:count this) os))) 91 | 92 | (s/def :com.sttgts.omnia.hello.messages.RepeatHelloRequest/name string?) 93 | (s/def :com.sttgts.omnia.hello.messages.RepeatHelloRequest/count int?) 94 | (s/def ::RepeatHelloRequest-spec (s/keys :opt-un [:com.sttgts.omnia.hello.messages.RepeatHelloRequest/name :com.sttgts.omnia.hello.messages.RepeatHelloRequest/count])) 95 | (def RepeatHelloRequest-defaults {:name "" :count 0}) 96 | 97 | (defn cis->RepeatHelloRequest 98 | "CodedInputStream to RepeatHelloRequest" 99 | [is] 100 | (->> (tag-map RepeatHelloRequest-defaults 101 | (fn [tag index] 102 | (case index 103 | 1 [:name (cis->String is)] 104 | 2 [:count (cis->Int32 is)] 105 | 106 | [index (cis->undefined tag is)])) 107 | is) 108 | (map->RepeatHelloRequest))) 109 | 110 | (defn ecis->RepeatHelloRequest 111 | "Embedded CodedInputStream to RepeatHelloRequest" 112 | [is] 113 | (cis->embedded cis->RepeatHelloRequest is)) 114 | 115 | (defn new-RepeatHelloRequest 116 | "Creates a new instance from a map, similar to map->RepeatHelloRequest except that 117 | it properly accounts for nested messages, when applicable. 118 | " 119 | [init] 120 | {:pre [(if (s/valid? ::RepeatHelloRequest-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::RepeatHelloRequest-spec init))))]} 121 | (-> (merge RepeatHelloRequest-defaults init) 122 | (map->RepeatHelloRequest))) 123 | 124 | (defn pb->RepeatHelloRequest 125 | "Protobuf to RepeatHelloRequest" 126 | [input] 127 | (-> input 128 | stream/new-cis 129 | cis->RepeatHelloRequest)) 130 | 131 | ;----------------------------------------------------------------------------- 132 | ; HelloReply 133 | ;----------------------------------------------------------------------------- 134 | (defrecord HelloReply [message] 135 | pb/Writer 136 | 137 | (serialize [this os] 138 | (write-String 1 {:optimize true} (:message this) os))) 139 | 140 | (s/def :com.sttgts.omnia.hello.messages.HelloReply/message string?) 141 | (s/def ::HelloReply-spec (s/keys :opt-un [:com.sttgts.omnia.hello.messages.HelloReply/message])) 142 | (def HelloReply-defaults {:message ""}) 143 | 144 | (defn cis->HelloReply 145 | "CodedInputStream to HelloReply" 146 | [is] 147 | (->> (tag-map HelloReply-defaults 148 | (fn [tag index] 149 | (case index 150 | 1 [:message (cis->String is)] 151 | 152 | [index (cis->undefined tag is)])) 153 | is) 154 | (map->HelloReply))) 155 | 156 | (defn ecis->HelloReply 157 | "Embedded CodedInputStream to HelloReply" 158 | [is] 159 | (cis->embedded cis->HelloReply is)) 160 | 161 | (defn new-HelloReply 162 | "Creates a new instance from a map, similar to map->HelloReply except that 163 | it properly accounts for nested messages, when applicable. 164 | " 165 | [init] 166 | {:pre [(if (s/valid? ::HelloReply-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::HelloReply-spec init))))]} 167 | (-> (merge HelloReply-defaults init) 168 | (map->HelloReply))) 169 | 170 | (defn pb->HelloReply 171 | "Protobuf to HelloReply" 172 | [input] 173 | (-> input 174 | stream/new-cis 175 | cis->HelloReply)) 176 | 177 | -------------------------------------------------------------------------------- /modules/grpc-client/src/protojure/internal/grpc/client/providers/http2/core.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.internal.grpc.client.providers.http2.core 7 | (:require [clojure.core.async :refer [ (compute-metadata conn-metadata) 42 | (p/then (fn [conn-metadata] 43 | (let [hdrs (-> {"content-type" "application/grpc+proto" 44 | "grpc-encoding" (or content-coding "identity") 45 | "grpc-accept-encoding" (codecs-to-accept codecs) 46 | "te" "trailers"} 47 | (merge conn-metadata metadata)) 48 | url (str uri "/" service "/" method)] 49 | (jetty/send-request context {:method "POST" 50 | :url url 51 | :headers hdrs 52 | :input-ch input-ch 53 | :meta-ch meta-ch 54 | :output-ch output-ch})))))) 55 | 56 | (defn- receive-headers 57 | "Listen on the metadata channel _until_ we receive a status code. We are interested in both 58 | ensuring the call was successful (e.g. :status == 200) and we want to know what :content-coding 59 | may be applied to any response-body LPM protobufs. Therefore, we must gate any further 60 | processing until we have received the \"headers\", and we assume we have fully received them 61 | once we see the :status tag. We also note that the metadata channel is not expected to close 62 | before :status has been received, and nor do we expect it to close even after we've received 63 | :status since we will presumably be receiving trailers in the future. Therefore, we treat 64 | core.async channel closure as an error, and terminate the processing once the response contains 65 | the :status code. 66 | " 67 | [meta-ch] 68 | (p/create 69 | (fn [resolve reject] 70 | (go-loop [response {}] 71 | (if-let [data (status-code [status] 103 | (if (some? status) 104 | (Integer/parseInt status) 105 | 2)) 106 | 107 | (defn- decode-grpc-status [{:strs [grpc-status grpc-message]}] 108 | (let [grpc-status (->status-code grpc-status)] 109 | (cond-> {:status grpc-status} 110 | (some? grpc-message) (assoc :message grpc-message)))) 111 | 112 | (defn- receive-payload 113 | "Handles all remaining response payload, which consists of both response body and trailers. 114 | We process them in parallel since we can't be sure that the server won't interleave HEADER 115 | and DATA frames, even though we don't expect this to be a normal ordering. We _could_ 116 | probably get away with draining the queues serially (data-ch and then meta-ch) but we would 117 | run the risk of stalling the pipeline if the meta-ch were to fill" 118 | [codecs meta-ch data-ch output {:keys [status] :as response}] 119 | (if (-> status (= 200)) 120 | (-> (p/all [(receive-body codecs data-ch output response) 121 | (receive-trailers meta-ch response)]) 122 | (p/then (fn [[_ {:keys [headers] :as response}]] ;; [body-response trailers-response] 123 | (let [{:keys [status] :as resp} (decode-grpc-status headers)] 124 | (if (zero? status) 125 | resp 126 | (p/rejected (ex-info "bad grpc-status response" (assoc resp :meta {:response response})))))))) 127 | (p/rejected (ex-info "bad status response" {:response response})))) 128 | 129 | (defn- client-send [input-ch stream] 130 | (jetty/transmit-data-frames input-ch stream)) 131 | 132 | (defn- client-receive [meta-ch codecs output-ch output] 133 | (-> (receive-headers meta-ch) 134 | (p/then (partial receive-payload codecs meta-ch output-ch output)))) 135 | 136 | (defn- safe-close! [ch] 137 | (some-> ch async/close!)) 138 | 139 | (def #^{:private true} executor (Executors/newCachedThreadPool)) 140 | 141 | ;;----------------------------------------------------------------------------- 142 | ;;----------------------------------------------------------------------------- 143 | ;; External API 144 | ;;----------------------------------------------------------------------------- 145 | ;;----------------------------------------------------------------------------- 146 | 147 | ;;----------------------------------------------------------------------------- 148 | ;; Provider 149 | ;;----------------------------------------------------------------------------- 150 | (deftype Http2Provider [context uri codecs content-coding max-frame-size input-buffer-size metadata] 151 | api/Provider 152 | 153 | (invoke [_ {:keys [input output] :as params}] 154 | (let [input-ch (input-pipeline input codecs content-coding max-frame-size) 155 | meta-ch (async/chan 32) 156 | output-ch (when (some? output) (async/chan (max 32 157 | (/ input-buffer-size max-frame-size))))] 158 | (-> (send-request context uri codecs content-coding metadata params input-ch meta-ch output-ch) 159 | (p/then (fn [^Stream stream] 160 | (p/all [(client-send input-ch stream) 161 | (-> (client-receive meta-ch codecs output-ch output) 162 | (p/catch (fn [ex] 163 | (safe-close! output-ch) 164 | (async/close! meta-ch) 165 | (safe-close! input-ch) 166 | (throw ex))))]))) 167 | (p/then (fn [[_ status]] 168 | (log/trace "GRPC completed:" status) 169 | status) 170 | executor) 171 | (p/catch (fn [ex] 172 | (log/error "GRPC failed:" ex) 173 | (safe-close (:ch output)) 174 | (throw ex)))))) 175 | 176 | (disconnect [_] 177 | (jetty/disconnect context))) 178 | -------------------------------------------------------------------------------- /test/test/protojure/test/grpc/TestService/client.cljc: -------------------------------------------------------------------------------- 1 | ;;;---------------------------------------------------------------------------------- 2 | ;;; Generated by protoc-gen-clojure. DO NOT EDIT 3 | ;;; 4 | ;;; GRPC protojure.test.grpc.TestService Client Implementation 5 | ;;;---------------------------------------------------------------------------------- 6 | (ns protojure.test.grpc.TestService.client 7 | (:require [protojure.test.grpc :refer :all] 8 | [com.google.protobuf :as com.google.protobuf] 9 | [clojure.core.async :as async] 10 | [protojure.grpc.client.utils :refer [send-unary-params invoke-unary]] 11 | [promesa.core :as p] 12 | [protojure.grpc.client.api :as grpc])) 13 | 14 | ;----------------------------------------------------------------------------- 15 | ; GRPC Client Implementation 16 | ;----------------------------------------------------------------------------- 17 | 18 | (def TestService-service-name "protojure.test.grpc.TestService") 19 | 20 | (defn BandwidthTest 21 | ([client params] (BandwidthTest client {} params)) 22 | ([client metadata params] 23 | (let [input (async/chan 1) 24 | output (async/chan 1) 25 | desc {:service "protojure.test.grpc.TestService" 26 | :method "BandwidthTest" 27 | :input {:f protojure.test.grpc/new-BigPayload :ch input} 28 | :output {:f protojure.test.grpc/pb->BigPayload :ch output} 29 | :metadata metadata}] 30 | (-> (send-unary-params input params) 31 | (p/then (fn [_] (invoke-unary client desc output))))))) 32 | 33 | (defn BidirectionalStreamTest 34 | ([client params reply] (BidirectionalStreamTest client {} params reply)) 35 | ([client metadata params reply] 36 | (let [desc {:service "protojure.test.grpc.TestService" 37 | :method "BidirectionalStreamTest" 38 | :input {:f protojure.test.grpc/new-SimpleRequest :ch params} 39 | :output {:f protojure.test.grpc/pb->SimpleResponse :ch reply} 40 | :metadata metadata}] 41 | (grpc/invoke client desc)))) 42 | 43 | (defn ClientCloseDetect 44 | ([client params reply] (ClientCloseDetect client {} params reply)) 45 | ([client metadata params reply] 46 | (let [input (async/chan 1) 47 | desc {:service "protojure.test.grpc.TestService" 48 | :method "ClientCloseDetect" 49 | :input {:f protojure.test.grpc/new-CloseDetectRequest :ch input} 50 | :output {:f com.google.protobuf/pb->Any :ch reply} 51 | :metadata metadata}] 52 | (-> (send-unary-params input params) 53 | (p/then (fn [_] (grpc/invoke client desc))))))) 54 | 55 | (defn FlowControl 56 | ([client params reply] (FlowControl client {} params reply)) 57 | ([client metadata params reply] 58 | (let [input (async/chan 1) 59 | desc {:service "protojure.test.grpc.TestService" 60 | :method "FlowControl" 61 | :input {:f protojure.test.grpc/new-FlowControlRequest :ch input} 62 | :output {:f protojure.test.grpc/pb->FlowControlPayload :ch reply} 63 | :metadata metadata}] 64 | (-> (send-unary-params input params) 65 | (p/then (fn [_] (grpc/invoke client desc))))))) 66 | 67 | (defn ReturnError 68 | ([client params] (ReturnError client {} params)) 69 | ([client metadata params] 70 | (let [input (async/chan 1) 71 | output (async/chan 1) 72 | desc {:service "protojure.test.grpc.TestService" 73 | :method "ReturnError" 74 | :input {:f protojure.test.grpc/new-ErrorRequest :ch input} 75 | :output {:f com.google.protobuf/pb->Empty :ch output} 76 | :metadata metadata}] 77 | (-> (send-unary-params input params) 78 | (p/then (fn [_] (invoke-unary client desc output))))))) 79 | 80 | (defn AllEmpty 81 | ([client params] (AllEmpty client {} params)) 82 | ([client metadata params] 83 | (let [input (async/chan 1) 84 | output (async/chan 1) 85 | desc {:service "protojure.test.grpc.TestService" 86 | :method "AllEmpty" 87 | :input {:f com.google.protobuf/new-Empty :ch input} 88 | :output {:f com.google.protobuf/pb->Empty :ch output} 89 | :metadata metadata}] 90 | (-> (send-unary-params input params) 91 | (p/then (fn [_] (invoke-unary client desc output))))))) 92 | 93 | (defn ServerCloseDetect 94 | ([client params reply] (ServerCloseDetect client {} params reply)) 95 | ([client metadata params reply] 96 | (let [input (async/chan 1) 97 | desc {:service "protojure.test.grpc.TestService" 98 | :method "ServerCloseDetect" 99 | :input {:f com.google.protobuf/new-Empty :ch input} 100 | :output {:f com.google.protobuf/pb->Any :ch reply} 101 | :metadata metadata}] 102 | (-> (send-unary-params input params) 103 | (p/then (fn [_] (grpc/invoke client desc))))))) 104 | 105 | (defn Async 106 | ([client params] (Async client {} params)) 107 | ([client metadata params] 108 | (let [input (async/chan 1) 109 | output (async/chan 1) 110 | desc {:service "protojure.test.grpc.TestService" 111 | :method "Async" 112 | :input {:f com.google.protobuf/new-Empty :ch input} 113 | :output {:f protojure.test.grpc/pb->SimpleResponse :ch output} 114 | :metadata metadata}] 115 | (-> (send-unary-params input params) 116 | (p/then (fn [_] (invoke-unary client desc output))))))) 117 | 118 | (defn DeniedStreamer 119 | ([client params reply] (DeniedStreamer client {} params reply)) 120 | ([client metadata params reply] 121 | (let [input (async/chan 1) 122 | desc {:service "protojure.test.grpc.TestService" 123 | :method "DeniedStreamer" 124 | :input {:f com.google.protobuf/new-Empty :ch input} 125 | :output {:f com.google.protobuf/pb->Empty :ch reply} 126 | :metadata metadata}] 127 | (-> (send-unary-params input params) 128 | (p/then (fn [_] (grpc/invoke client desc))))))) 129 | 130 | (defn AsyncEmpty 131 | ([client params reply] (AsyncEmpty client {} params reply)) 132 | ([client metadata params reply] 133 | (let [input (async/chan 1) 134 | desc {:service "protojure.test.grpc.TestService" 135 | :method "AsyncEmpty" 136 | :input {:f com.google.protobuf/new-Empty :ch input} 137 | :output {:f com.google.protobuf/pb->Empty :ch reply} 138 | :metadata metadata}] 139 | (-> (send-unary-params input params) 140 | (p/then (fn [_] (grpc/invoke client desc))))))) 141 | 142 | (defn Metadata 143 | ([client params] (Metadata client {} params)) 144 | ([client metadata params] 145 | (let [input (async/chan 1) 146 | output (async/chan 1) 147 | desc {:service "protojure.test.grpc.TestService" 148 | :method "Metadata" 149 | :input {:f com.google.protobuf/new-Empty :ch input} 150 | :output {:f protojure.test.grpc/pb->SimpleResponse :ch output} 151 | :metadata metadata}] 152 | (-> (send-unary-params input params) 153 | (p/then (fn [_] (invoke-unary client desc output))))))) 154 | 155 | (defn ReturnErrorStreaming 156 | ([client params reply] (ReturnErrorStreaming client {} params reply)) 157 | ([client metadata params reply] 158 | (let [input (async/chan 1) 159 | desc {:service "protojure.test.grpc.TestService" 160 | :method "ReturnErrorStreaming" 161 | :input {:f protojure.test.grpc/new-ErrorRequest :ch input} 162 | :output {:f com.google.protobuf/pb->Empty :ch reply} 163 | :metadata metadata}] 164 | (-> (send-unary-params input params) 165 | (p/then (fn [_] (grpc/invoke client desc))))))) 166 | 167 | (defn AuthzTest 168 | ([client params] (AuthzTest client {} params)) 169 | ([client metadata params] 170 | (let [input (async/chan 1) 171 | output (async/chan 1) 172 | desc {:service "protojure.test.grpc.TestService" 173 | :method "AuthzTest" 174 | :input {:f protojure.test.grpc/new-AuthzTestRequest :ch input} 175 | :output {:f com.google.protobuf/pb->Empty :ch output} 176 | :metadata metadata}] 177 | (-> (send-unary-params input params) 178 | (p/then (fn [_] (invoke-unary client desc output))))))) 179 | 180 | (defn ShouldThrow 181 | ([client params] (ShouldThrow client {} params)) 182 | ([client metadata params] 183 | (let [input (async/chan 1) 184 | output (async/chan 1) 185 | desc {:service "protojure.test.grpc.TestService" 186 | :method "ShouldThrow" 187 | :input {:f com.google.protobuf/new-Empty :ch input} 188 | :output {:f com.google.protobuf/pb->Empty :ch output} 189 | :metadata metadata}] 190 | (-> (send-unary-params input params) 191 | (p/then (fn [_] (invoke-unary client desc output))))))) 192 | 193 | -------------------------------------------------------------------------------- /test/test/protojure/pedestal_test.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.pedestal-test 7 | (:require [clojure.test :refer :all] 8 | [clojure.core.async :as async :refer [go >! !!]] 9 | [io.pedestal.http :as http] 10 | [io.pedestal.http.body-params :as body-params] 11 | [promesa.exec :as p.exec] 12 | [protojure.pedestal.core :as core] 13 | [protojure.pedestal.interceptors.authz :as authz] 14 | [protojure.test.utils :as test.utils] 15 | [protojure.internal.io :as pio] 16 | [clj-http.client :as client] 17 | [clojure.java.io :as io]) 18 | (:import [java.nio ByteBuffer])) 19 | 20 | ;;----------------------------------------------------------------------------- 21 | ;; Data 22 | ;;----------------------------------------------------------------------------- 23 | (defonce test-svc (atom {})) 24 | 25 | (defn- get-healthz [_] 26 | {:status 200 27 | :headers {"Content-Type" "application/text"} 28 | :trailers {"Meta" "test"} 29 | :body "OK"}) 30 | 31 | (defn- get-bytes [_] 32 | {:status 200 33 | :headers {"Content-Type" "application/text"} 34 | :trailers {"Meta" "test"} 35 | :body (byte-array [(byte 0x43) 36 | (byte 0x6c) 37 | (byte 0x6f) 38 | (byte 0x6a) 39 | (byte 0x75) 40 | (byte 0x72) 41 | (byte 0x65) 42 | (byte 0x21)])}) 43 | 44 | (defn- get-edn [_] 45 | {:status 200 46 | :headers {"Content-Type" "application/text"} 47 | :trailers {"Meta" "test"} 48 | :body {:key "clojure is awesome"}}) 49 | 50 | (defn- echo-params [{{:keys [content]} :params}] 51 | {:status 200 :body content}) 52 | 53 | (defn- echo-body [{:keys [body]}] 54 | {:status 200 :body body}) 55 | 56 | (defn- json-content [{:keys [json-params] :as req}] 57 | {:status 200 :body json-params}) 58 | 59 | (defn- echo-async [{{:keys [content]} :params}] 60 | (let [ch (async/chan 1)] 61 | (go 62 | (>! ch (byte-array (map byte content))) 63 | (async/close! ch)) 64 | {:status 200 :body ch})) 65 | 66 | (defn- testdata-download [_] 67 | {:status 200 :body (io/as-file (io/resource "testdata.txt"))}) 68 | 69 | (defn- get-denied [_] 70 | ;; we should never get here 71 | {:status 200}) 72 | 73 | (defn routes [interceptors] 74 | [["/healthz" :get (conj interceptors `get-healthz)] 75 | ["/echo" :get (conj interceptors `echo-params)] 76 | ["/echo" :post (conj interceptors `echo-body)] 77 | ["/echo/async" :get (conj interceptors `echo-async)] 78 | ["/testdata" :get (conj interceptors `testdata-download)] 79 | ["/bytes" :get (conj interceptors `get-bytes)] 80 | ["/edn" :get (conj interceptors `get-edn)] 81 | ["/json" :post (conj interceptors `json-content)] 82 | ["/denied" :get (conj interceptors `get-denied)]]) 83 | 84 | (defn- sync-authorize? 85 | [{:keys [path-info] :as request}] 86 | ;; This authz predicate will always deny calls to the /denied endpoint. A real implementation would probably 87 | ;; look at other criteria, such as the callers credentials 88 | (not= path-info "/denied")) 89 | 90 | (defn- async-authorize? 91 | [{:keys [path-info] :as request}] 92 | ;; Identical to sync-authorize, except runs asynchronously 93 | (go 94 | (not= path-info "/denied"))) 95 | 96 | ;;----------------------------------------------------------------------------- 97 | ;; Utilities 98 | ;;----------------------------------------------------------------------------- 99 | 100 | (defn service-url 101 | [& rest] 102 | (apply str "http://localhost:" (:port @test-svc) rest)) 103 | 104 | (defn service-url-ssl 105 | [& rest] 106 | (apply str "https://localhost:" (:ssl-port @test-svc) rest)) 107 | 108 | (defn -check-throw 109 | [code f] 110 | (is (thrown? clojure.lang.ExceptionInfo 111 | (try 112 | (f) 113 | (catch Exception e 114 | (let [{:keys [status]} (ex-data e)] 115 | (is (= code status))) 116 | (throw e)))))) 117 | 118 | (defmacro check-throw 119 | [code & body] 120 | `(-check-throw ~code #(do ~@body))) 121 | 122 | ;;----------------------------------------------------------------------------- 123 | ;; Fixtures 124 | ;;----------------------------------------------------------------------------- 125 | (defn create-service [] 126 | (let [port (test.utils/get-free-port) 127 | ssl-port (test.utils/get-free-port) 128 | interceptors [(body-params/body-params) 129 | http/html-body 130 | io.pedestal.http/json-body 131 | (authz/interceptor nil sync-authorize?) 132 | (authz/interceptor nil async-authorize?)] 133 | thread-pool (p.exec/fixed-executor {:parallelism 64}) 134 | desc {:env :prod 135 | ::http/routes (into #{} (routes interceptors)) 136 | ::http/port port 137 | 138 | ::http/type core/config 139 | ::http/chain-provider core/provider 140 | ::core/thread-pool thread-pool 141 | 142 | ::http/container-options {:ssl-port ssl-port 143 | ; keystore may be either string denoting file path (relative or 144 | ; absolute) or actual KeyStore instance 145 | :keystore (io/resource "https/keystore.jks") 146 | :key-password "password"}}] 147 | 148 | (let [server (http/create-server desc)] 149 | (http/start server) 150 | (swap! test-svc assoc :port port :ssl-port ssl-port :server server :thread-pool thread-pool)))) 151 | 152 | (defn destroy-service [] 153 | (swap! test-svc (fn [x] 154 | (-> x 155 | (update :server http/stop) 156 | (update :thread-pool #(.shutdown %)))))) 157 | 158 | (defn wrap-service [test-fn] 159 | (create-service) 160 | (test-fn) 161 | (destroy-service)) 162 | 163 | (use-fixtures :once wrap-service) 164 | 165 | ;;----------------------------------------------------------------------------- 166 | ;; Tests 167 | ;;----------------------------------------------------------------------------- 168 | (deftest healthz-check 169 | (testing "Check that basic connectivity works" 170 | (is (-> (client/get (service-url "/healthz")) :body (= "OK"))))) 171 | 172 | (comment deftest ssl-check ;; FIXME: re-enable after we figure out why it fails on new JDK 173 | (testing "Check that SSL works" 174 | (is (-> (client/get (service-url-ssl "/healthz") {:insecure? true}) :body (= "OK"))))) 175 | 176 | (deftest query-param-check 177 | (testing "Check that query-parameters work" 178 | (is (-> (client/get (service-url "/echo") {:query-params {"content" "FOO"}}) :body (= "FOO"))))) 179 | 180 | (deftest body-check 181 | (testing "Check that response/request body work" 182 | (is (-> (client/post (service-url "/echo") {:body "BAR"}) :body (= "BAR"))))) 183 | 184 | (deftest async-check 185 | (testing "Check that async-body works" 186 | (is (-> (client/get (service-url "/echo/async") {:query-params {"content" "FOO"}}) :body (= "FOO"))))) 187 | 188 | (deftest file-download-check 189 | (testing "Check that we can download a file" 190 | (is (->> (client/get (service-url "/testdata")) :body (re-find #"testdata!") some?)))) 191 | 192 | (deftest bytes-check 193 | (testing "Check that bytes transfer correctly" 194 | (is (-> (client/get (service-url "/bytes")) :body (= "Clojure!"))))) 195 | 196 | (deftest edn-check 197 | (testing "Check that EDN format transfers" 198 | (is (-> (client/get (service-url "/edn")) :body clojure.edn/read-string (= {:key "clojure is awesome"}))))) 199 | 200 | (deftest notfound-check 201 | (testing "Check that a request for an invalid resource correctly propagates the error code" 202 | (check-throw 404 (client/get (service-url "/invalid"))))) 203 | 204 | (deftest read-check 205 | (testing "Check that bytes entered to channel are properly read from InputStream" 206 | (let [test-string "Hello" 207 | test-channel (async/chan 8096) 208 | in-stream (pio/new-inputstream {:ch test-channel}) 209 | buff (byte-array 5)] 210 | (>!! test-channel (ByteBuffer/wrap (.getBytes test-string))) 211 | (async/close! test-channel) 212 | (.read in-stream buff 0 5) 213 | (is (= "Hello" (String. buff)))))) 214 | 215 | (deftest content-type-check 216 | (testing "Check that content-type key is set per [io.pedestal.http.request.map](https://github.com/pedestal/pedestal/blob/master/service/src/io/pedestal/http/request/map.clj) expectations" 217 | (is (as-> (client/post (service-url "/json") {:content-type :json :form-params {"content" "FOO"} :as :json-string-keys}) resp 218 | (= (:body resp) {"content" "FOO"}))))) 219 | 220 | (deftest denied-check 221 | (testing "Check that a request for an explicitly denied" 222 | (check-throw 401 (client/get (service-url "/denied"))))) 223 | -------------------------------------------------------------------------------- /test/test/com/example/empty.cljc: -------------------------------------------------------------------------------- 1 | ;;;---------------------------------------------------------------------------------- 2 | ;;; Generated by protoc-gen-clojure. DO NOT EDIT 3 | ;;; 4 | ;;; Message Implementation of package com.example.empty 5 | ;;;---------------------------------------------------------------------------------- 6 | (ns com.example.empty 7 | (:require [protojure.protobuf.protocol :as pb] 8 | [protojure.protobuf.serdes.core :as serdes.core] 9 | [protojure.protobuf.serdes.complex :as serdes.complex] 10 | [protojure.protobuf.serdes.utils :refer [tag-map]] 11 | [protojure.protobuf.serdes.stream :as serdes.stream] 12 | [clojure.set :as set] 13 | [clojure.spec.alpha :as s])) 14 | 15 | ;;---------------------------------------------------------------------------------- 16 | ;;---------------------------------------------------------------------------------- 17 | ;; Forward declarations 18 | ;;---------------------------------------------------------------------------------- 19 | ;;---------------------------------------------------------------------------------- 20 | 21 | (declare cis->Empty) 22 | (declare ecis->Empty) 23 | (declare new-Empty) 24 | (declare cis->NonEmpty) 25 | (declare ecis->NonEmpty) 26 | (declare new-NonEmpty) 27 | (declare cis->Selection) 28 | (declare ecis->Selection) 29 | (declare new-Selection) 30 | (declare cis->Container) 31 | (declare ecis->Container) 32 | (declare new-Container) 33 | 34 | ;;---------------------------------------------------------------------------------- 35 | ;;---------------------------------------------------------------------------------- 36 | ;; Selection-opt's oneof Implementations 37 | ;;---------------------------------------------------------------------------------- 38 | ;;---------------------------------------------------------------------------------- 39 | 40 | (defn convert-Selection-opt [origkeyval] 41 | (cond 42 | (get-in origkeyval [:opt :e]) (update-in origkeyval [:opt :e] new-Empty) 43 | (get-in origkeyval [:opt :ne]) (update-in origkeyval [:opt :ne] new-NonEmpty) 44 | :default origkeyval)) 45 | 46 | (defn write-Selection-opt [opt os] 47 | (let [field (first opt) 48 | k (when-not (nil? field) (key field)) 49 | v (when-not (nil? field) (val field))] 50 | (case k 51 | :e (serdes.core/write-embedded 1 v os) 52 | :ne (serdes.core/write-embedded 2 v os) 53 | nil))) 54 | 55 | ;;---------------------------------------------------------------------------------- 56 | ;;---------------------------------------------------------------------------------- 57 | ;; Message Implementations 58 | ;;---------------------------------------------------------------------------------- 59 | ;;---------------------------------------------------------------------------------- 60 | 61 | ;----------------------------------------------------------------------------- 62 | ; Empty 63 | ;----------------------------------------------------------------------------- 64 | (defrecord Empty-record [] 65 | pb/Writer 66 | (serialize [this os]) 67 | pb/TypeReflection 68 | (gettype [this] 69 | "com.example.empty.Empty")) 70 | 71 | (s/def ::Empty-spec (s/keys :opt-un [])) 72 | (def Empty-defaults {}) 73 | 74 | (defn cis->Empty 75 | "CodedInputStream to Empty" 76 | [is] 77 | (->> (tag-map Empty-defaults 78 | (fn [tag index] 79 | (case index 80 | [index (serdes.core/cis->undefined tag is)])) 81 | is) 82 | (map->Empty-record))) 83 | 84 | (defn ecis->Empty 85 | "Embedded CodedInputStream to Empty" 86 | [is] 87 | (serdes.core/cis->embedded cis->Empty is)) 88 | 89 | (defn new-Empty 90 | "Creates a new instance from a map, similar to map->Empty except that 91 | it properly accounts for nested messages, when applicable. 92 | " 93 | [init] 94 | {:pre [(if (s/valid? ::Empty-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Empty-spec init))))]} 95 | (-> (merge Empty-defaults init) 96 | (map->Empty-record))) 97 | 98 | (defn pb->Empty 99 | "Protobuf to Empty" 100 | [input] 101 | (cis->Empty (serdes.stream/new-cis input))) 102 | 103 | (def ^:protojure.protobuf.any/record Empty-meta {:type "com.example.empty.Empty" :decoder pb->Empty}) 104 | 105 | ;----------------------------------------------------------------------------- 106 | ; NonEmpty 107 | ;----------------------------------------------------------------------------- 108 | (defrecord NonEmpty-record [i] 109 | pb/Writer 110 | (serialize [this os] 111 | (serdes.core/write-Int32 1 {:optimize true} (:i this) os)) 112 | pb/TypeReflection 113 | (gettype [this] 114 | "com.example.empty.NonEmpty")) 115 | 116 | (s/def :com.example.empty.NonEmpty/i int?) 117 | (s/def ::NonEmpty-spec (s/keys :opt-un [:com.example.empty.NonEmpty/i])) 118 | (def NonEmpty-defaults {:i 0}) 119 | 120 | (defn cis->NonEmpty 121 | "CodedInputStream to NonEmpty" 122 | [is] 123 | (->> (tag-map NonEmpty-defaults 124 | (fn [tag index] 125 | (case index 126 | 1 [:i (serdes.core/cis->Int32 is)] 127 | 128 | [index (serdes.core/cis->undefined tag is)])) 129 | is) 130 | (map->NonEmpty-record))) 131 | 132 | (defn ecis->NonEmpty 133 | "Embedded CodedInputStream to NonEmpty" 134 | [is] 135 | (serdes.core/cis->embedded cis->NonEmpty is)) 136 | 137 | (defn new-NonEmpty 138 | "Creates a new instance from a map, similar to map->NonEmpty except that 139 | it properly accounts for nested messages, when applicable. 140 | " 141 | [init] 142 | {:pre [(if (s/valid? ::NonEmpty-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::NonEmpty-spec init))))]} 143 | (-> (merge NonEmpty-defaults init) 144 | (map->NonEmpty-record))) 145 | 146 | (defn pb->NonEmpty 147 | "Protobuf to NonEmpty" 148 | [input] 149 | (cis->NonEmpty (serdes.stream/new-cis input))) 150 | 151 | (def ^:protojure.protobuf.any/record NonEmpty-meta {:type "com.example.empty.NonEmpty" :decoder pb->NonEmpty}) 152 | 153 | ;----------------------------------------------------------------------------- 154 | ; Selection 155 | ;----------------------------------------------------------------------------- 156 | (defrecord Selection-record [opt] 157 | pb/Writer 158 | (serialize [this os] 159 | (write-Selection-opt (:opt this) os)) 160 | pb/TypeReflection 161 | (gettype [this] 162 | "com.example.empty.Selection")) 163 | 164 | (s/def ::Selection-spec (s/keys :opt-un [])) 165 | (def Selection-defaults {}) 166 | 167 | (defn cis->Selection 168 | "CodedInputStream to Selection" 169 | [is] 170 | (->> (tag-map Selection-defaults 171 | (fn [tag index] 172 | (case index 173 | 1 [:opt {:e (ecis->Empty is)}] 174 | 2 [:opt {:ne (ecis->NonEmpty is)}] 175 | 176 | [index (serdes.core/cis->undefined tag is)])) 177 | is) 178 | (map->Selection-record))) 179 | 180 | (defn ecis->Selection 181 | "Embedded CodedInputStream to Selection" 182 | [is] 183 | (serdes.core/cis->embedded cis->Selection is)) 184 | 185 | (defn new-Selection 186 | "Creates a new instance from a map, similar to map->Selection except that 187 | it properly accounts for nested messages, when applicable. 188 | " 189 | [init] 190 | {:pre [(if (s/valid? ::Selection-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Selection-spec init))))]} 191 | (-> (merge Selection-defaults init) 192 | (convert-Selection-opt) 193 | (map->Selection-record))) 194 | 195 | (defn pb->Selection 196 | "Protobuf to Selection" 197 | [input] 198 | (cis->Selection (serdes.stream/new-cis input))) 199 | 200 | (def ^:protojure.protobuf.any/record Selection-meta {:type "com.example.empty.Selection" :decoder pb->Selection}) 201 | 202 | ;----------------------------------------------------------------------------- 203 | ; Container 204 | ;----------------------------------------------------------------------------- 205 | (defrecord Container-record [e ne] 206 | pb/Writer 207 | (serialize [this os] 208 | (serdes.core/write-embedded 1 (:e this) os) 209 | (serdes.core/write-embedded 2 (:ne this) os)) 210 | pb/TypeReflection 211 | (gettype [this] 212 | "com.example.empty.Container")) 213 | 214 | (s/def ::Container-spec (s/keys :opt-un [])) 215 | (def Container-defaults {}) 216 | 217 | (defn cis->Container 218 | "CodedInputStream to Container" 219 | [is] 220 | (->> (tag-map Container-defaults 221 | (fn [tag index] 222 | (case index 223 | 1 [:e (ecis->Empty is)] 224 | 2 [:ne (ecis->NonEmpty is)] 225 | 226 | [index (serdes.core/cis->undefined tag is)])) 227 | is) 228 | (map->Container-record))) 229 | 230 | (defn ecis->Container 231 | "Embedded CodedInputStream to Container" 232 | [is] 233 | (serdes.core/cis->embedded cis->Container is)) 234 | 235 | (defn new-Container 236 | "Creates a new instance from a map, similar to map->Container except that 237 | it properly accounts for nested messages, when applicable. 238 | " 239 | [init] 240 | {:pre [(if (s/valid? ::Container-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Container-spec init))))]} 241 | (-> (merge Container-defaults init) 242 | (cond-> (some? (get init :e)) (update :e new-Empty)) 243 | (cond-> (some? (get init :ne)) (update :ne new-NonEmpty)) 244 | (map->Container-record))) 245 | 246 | (defn pb->Container 247 | "Protobuf to Container" 248 | [input] 249 | (cis->Container (serdes.stream/new-cis input))) 250 | 251 | (def ^:protojure.protobuf.any/record Container-meta {:type "com.example.empty.Container" :decoder pb->Container}) 252 | 253 | -------------------------------------------------------------------------------- /test/test/com/example/addressbook.cljc: -------------------------------------------------------------------------------- 1 | ;;;---------------------------------------------------------------------------------- 2 | ;;; Generated by protoc-gen-clojure. DO NOT EDIT 3 | ;;; 4 | ;;; Message Implementation of package com.example.addressbook 5 | ;;;---------------------------------------------------------------------------------- 6 | (ns com.example.addressbook 7 | (:require [protojure.protobuf.protocol :as pb] 8 | [protojure.protobuf.serdes.core :as serdes.core] 9 | [protojure.protobuf.serdes.complex :as serdes.complex] 10 | [protojure.protobuf.serdes.utils :refer [tag-map]] 11 | [protojure.protobuf.serdes.stream :as serdes.stream] 12 | [clojure.set :as set] 13 | [clojure.spec.alpha :as s])) 14 | 15 | ;;---------------------------------------------------------------------------------- 16 | ;;---------------------------------------------------------------------------------- 17 | ;; Forward declarations 18 | ;;---------------------------------------------------------------------------------- 19 | ;;---------------------------------------------------------------------------------- 20 | 21 | (declare cis->Person) 22 | (declare ecis->Person) 23 | (declare new-Person) 24 | (declare cis->Person-PhoneNumber) 25 | (declare ecis->Person-PhoneNumber) 26 | (declare new-Person-PhoneNumber) 27 | (declare cis->AddressBook) 28 | (declare ecis->AddressBook) 29 | (declare new-AddressBook) 30 | 31 | ;;---------------------------------------------------------------------------------- 32 | ;;---------------------------------------------------------------------------------- 33 | ;; Enumerations 34 | ;;---------------------------------------------------------------------------------- 35 | ;;---------------------------------------------------------------------------------- 36 | 37 | ;----------------------------------------------------------------------------- 38 | ; Person-PhoneType 39 | ;----------------------------------------------------------------------------- 40 | (def Person-PhoneType-default :mobile) 41 | 42 | (def Person-PhoneType-val2label {0 :mobile 43 | 1 :home 44 | 2 :work}) 45 | 46 | (def Person-PhoneType-label2val (set/map-invert Person-PhoneType-val2label)) 47 | 48 | (defn cis->Person-PhoneType [is] 49 | (let [val (serdes.core/cis->Enum is)] 50 | (get Person-PhoneType-val2label val val))) 51 | 52 | (defn- get-Person-PhoneType [value] 53 | {:pre [(or (int? value) (contains? Person-PhoneType-label2val value))]} 54 | (get Person-PhoneType-label2val value value)) 55 | 56 | (defn write-Person-PhoneType 57 | ([tag value os] (write-Person-PhoneType tag {:optimize false} value os)) 58 | ([tag options value os] 59 | (serdes.core/write-Enum tag options (get-Person-PhoneType value) os))) 60 | 61 | ;;---------------------------------------------------------------------------------- 62 | ;;---------------------------------------------------------------------------------- 63 | ;; Message Implementations 64 | ;;---------------------------------------------------------------------------------- 65 | ;;---------------------------------------------------------------------------------- 66 | 67 | ;----------------------------------------------------------------------------- 68 | ; Person 69 | ;----------------------------------------------------------------------------- 70 | (defrecord Person-record [name id email phones] 71 | pb/Writer 72 | (serialize [this os] 73 | (serdes.core/write-String 1 {:optimize true} (:name this) os) 74 | (serdes.core/write-Int32 2 {:optimize true} (:id this) os) 75 | (serdes.core/write-String 3 {:optimize true} (:email this) os) 76 | (serdes.complex/write-repeated serdes.core/write-embedded 4 (:phones this) os)) 77 | pb/TypeReflection 78 | (gettype [this] 79 | "com.example.addressbook.Person")) 80 | 81 | (s/def :com.example.addressbook.Person/name string?) 82 | (s/def :com.example.addressbook.Person/id int?) 83 | (s/def :com.example.addressbook.Person/email string?) 84 | 85 | (s/def ::Person-spec (s/keys :opt-un [:com.example.addressbook.Person/name :com.example.addressbook.Person/id :com.example.addressbook.Person/email])) 86 | (def Person-defaults {:name "" :id 0 :email "" :phones []}) 87 | 88 | (defn cis->Person 89 | "CodedInputStream to Person" 90 | [is] 91 | (->> (tag-map Person-defaults 92 | (fn [tag index] 93 | (case index 94 | 1 [:name (serdes.core/cis->String is)] 95 | 2 [:id (serdes.core/cis->Int32 is)] 96 | 3 [:email (serdes.core/cis->String is)] 97 | 4 [:phones (serdes.complex/cis->repeated ecis->Person-PhoneNumber is)] 98 | 99 | [index (serdes.core/cis->undefined tag is)])) 100 | is) 101 | (map->Person-record))) 102 | 103 | (defn ecis->Person 104 | "Embedded CodedInputStream to Person" 105 | [is] 106 | (serdes.core/cis->embedded cis->Person is)) 107 | 108 | (defn new-Person 109 | "Creates a new instance from a map, similar to map->Person except that 110 | it properly accounts for nested messages, when applicable. 111 | " 112 | [init] 113 | {:pre [(if (s/valid? ::Person-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Person-spec init))))]} 114 | (-> (merge Person-defaults init) 115 | (cond-> (some? (get init :phones)) (update :phones #(map new-Person-PhoneNumber %))) 116 | (map->Person-record))) 117 | 118 | (defn pb->Person 119 | "Protobuf to Person" 120 | [input] 121 | (cis->Person (serdes.stream/new-cis input))) 122 | 123 | (def ^:protojure.protobuf.any/record Person-meta {:type "com.example.addressbook.Person" :decoder pb->Person}) 124 | 125 | ;----------------------------------------------------------------------------- 126 | ; Person-PhoneNumber 127 | ;----------------------------------------------------------------------------- 128 | (defrecord Person-PhoneNumber-record [number type] 129 | pb/Writer 130 | (serialize [this os] 131 | (serdes.core/write-String 1 {:optimize true} (:number this) os) 132 | (write-Person-PhoneType 2 {:optimize true} (:type this) os)) 133 | pb/TypeReflection 134 | (gettype [this] 135 | "com.example.addressbook.Person-PhoneNumber")) 136 | 137 | (s/def :com.example.addressbook.Person-PhoneNumber/number string?) 138 | (s/def :com.example.addressbook.Person-PhoneNumber/type (s/or :keyword keyword? :int int?)) 139 | (s/def ::Person-PhoneNumber-spec (s/keys :opt-un [:com.example.addressbook.Person-PhoneNumber/number :com.example.addressbook.Person-PhoneNumber/type])) 140 | (def Person-PhoneNumber-defaults {:number "" :type Person-PhoneType-default}) 141 | 142 | (defn cis->Person-PhoneNumber 143 | "CodedInputStream to Person-PhoneNumber" 144 | [is] 145 | (->> (tag-map Person-PhoneNumber-defaults 146 | (fn [tag index] 147 | (case index 148 | 1 [:number (serdes.core/cis->String is)] 149 | 2 [:type (cis->Person-PhoneType is)] 150 | 151 | [index (serdes.core/cis->undefined tag is)])) 152 | is) 153 | (map->Person-PhoneNumber-record))) 154 | 155 | (defn ecis->Person-PhoneNumber 156 | "Embedded CodedInputStream to Person-PhoneNumber" 157 | [is] 158 | (serdes.core/cis->embedded cis->Person-PhoneNumber is)) 159 | 160 | (defn new-Person-PhoneNumber 161 | "Creates a new instance from a map, similar to map->Person-PhoneNumber except that 162 | it properly accounts for nested messages, when applicable. 163 | " 164 | [init] 165 | {:pre [(if (s/valid? ::Person-PhoneNumber-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Person-PhoneNumber-spec init))))]} 166 | (-> (merge Person-PhoneNumber-defaults init) 167 | (map->Person-PhoneNumber-record))) 168 | 169 | (defn pb->Person-PhoneNumber 170 | "Protobuf to Person-PhoneNumber" 171 | [input] 172 | (cis->Person-PhoneNumber (serdes.stream/new-cis input))) 173 | 174 | (def ^:protojure.protobuf.any/record Person-PhoneNumber-meta {:type "com.example.addressbook.Person-PhoneNumber" :decoder pb->Person-PhoneNumber}) 175 | 176 | ;----------------------------------------------------------------------------- 177 | ; AddressBook 178 | ;----------------------------------------------------------------------------- 179 | (defrecord AddressBook-record [people] 180 | pb/Writer 181 | (serialize [this os] 182 | (serdes.complex/write-repeated serdes.core/write-embedded 1 (:people this) os)) 183 | pb/TypeReflection 184 | (gettype [this] 185 | "com.example.addressbook.AddressBook")) 186 | 187 | (s/def ::AddressBook-spec (s/keys :opt-un [])) 188 | (def AddressBook-defaults {:people []}) 189 | 190 | (defn cis->AddressBook 191 | "CodedInputStream to AddressBook" 192 | [is] 193 | (->> (tag-map AddressBook-defaults 194 | (fn [tag index] 195 | (case index 196 | 1 [:people (serdes.complex/cis->repeated ecis->Person is)] 197 | 198 | [index (serdes.core/cis->undefined tag is)])) 199 | is) 200 | (map->AddressBook-record))) 201 | 202 | (defn ecis->AddressBook 203 | "Embedded CodedInputStream to AddressBook" 204 | [is] 205 | (serdes.core/cis->embedded cis->AddressBook is)) 206 | 207 | (defn new-AddressBook 208 | "Creates a new instance from a map, similar to map->AddressBook except that 209 | it properly accounts for nested messages, when applicable. 210 | " 211 | [init] 212 | {:pre [(if (s/valid? ::AddressBook-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::AddressBook-spec init))))]} 213 | (-> (merge AddressBook-defaults init) 214 | (cond-> (some? (get init :people)) (update :people #(map new-Person %))) 215 | (map->AddressBook-record))) 216 | 217 | (defn pb->AddressBook 218 | "Protobuf to AddressBook" 219 | [input] 220 | (cis->AddressBook (serdes.stream/new-cis input))) 221 | 222 | (def ^:protojure.protobuf.any/record AddressBook-meta {:type "com.example.addressbook.AddressBook" :decoder pb->AddressBook}) 223 | 224 | -------------------------------------------------------------------------------- /test/test/example/types.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; 3 | ;; SPDX-License-Identifier: Apache-2.0 4 | 5 | (ns example.types 6 | (:require [protojure.protobuf.protocol :as pb] 7 | [protojure.protobuf.serdes.core :refer :all] 8 | [protojure.protobuf.serdes.complex :refer :all] 9 | [protojure.protobuf.serdes.utils :refer [tag-map]] 10 | [protojure.protobuf.serdes.stream :as stream])) 11 | 12 | ;----------------------------------------------------------------------------- 13 | ; Money 14 | ; 15 | ; implementation of https://github.com/googleapis/googleapis/blob/master/google/type/money.proto 16 | ;----------------------------------------------------------------------------- 17 | (defrecord Money [currency_code units nanos] 18 | pb/Writer 19 | 20 | (serialize [this os] 21 | (write-String 1 (:currency_code this) os) 22 | (write-Int64 2 (:units this) os) 23 | (write-Int32 3 (:nanos this) os))) 24 | 25 | (def Money-defaults {:currency_code "" :units 0 :nanos 0}) 26 | 27 | (defn cis->Money 28 | "CodedInputStream to Money" 29 | [is] 30 | (->> (tag-map 31 | (fn [tag index] 32 | (case index 33 | 1 [:currency_code (cis->String is)] 34 | 2 [:units (cis->Int64 is)] 35 | 3 [:nanos (cis->Int32 is)] 36 | [index (cis->undefined tag is)])) 37 | is) 38 | (merge Money-defaults) 39 | (map->Money))) 40 | 41 | (defn ecis->Money 42 | "Embedded CodedInputStream to Money" 43 | [is] 44 | (cis->embedded cis->Money is)) 45 | 46 | (defn new-Money 47 | "Creates a new instance from a map, similar to map->Money except that 48 | it properly accounts for nested messages, when applicable. 49 | " 50 | [init] 51 | (-> (merge Money-defaults init) 52 | (map->Money))) 53 | 54 | (defn pb->Money 55 | "Protobuf to Money" 56 | [input] 57 | (-> input 58 | stream/new-cis 59 | cis->Money)) 60 | 61 | ;----------------------------------------------------------------------------- 62 | ; SimpleRepeated 63 | ;----------------------------------------------------------------------------- 64 | (defrecord SimpleRepeated [data] 65 | pb/Writer 66 | 67 | (serialize [this os] 68 | (write-repeated write-Int32 1 (:data this) os))) 69 | 70 | (def SimpleRepeated-defaults {:data []}) 71 | 72 | (defn cis->SimpleRepeated 73 | "CodedInputStream to SimpleRepeated" 74 | [is] 75 | (->> (tag-map 76 | (fn [tag index] 77 | (case index 78 | 1 [:data (cis->packablerepeated tag cis->Int32 is)] 79 | 80 | [index (cis->undefined tag is)])) 81 | is) 82 | (merge SimpleRepeated-defaults) 83 | (map->SimpleRepeated))) 84 | 85 | (defn ecis->SimpleRepeated 86 | "Embedded CodedInputStream to SimpleRepeated" 87 | [is] 88 | (cis->embedded cis->SimpleRepeated is)) 89 | 90 | (defn new-SimpleRepeated 91 | "Creates a new instance from a map, similar to map->SimpleRepeated except that 92 | it properly accounts for nested messages, when applicable. 93 | " 94 | [init] 95 | (-> (merge SimpleRepeated-defaults init) 96 | (map->SimpleRepeated))) 97 | 98 | (defn pb->SimpleRepeated 99 | "Protobuf to SimpleRepeated" 100 | [input] 101 | (-> input 102 | stream/new-cis 103 | cis->SimpleRepeated)) 104 | 105 | ;----------------------------------------------------------------------------- 106 | ; SimpleString 107 | ;----------------------------------------------------------------------------- 108 | (defrecord SimpleString [s] 109 | pb/Writer 110 | 111 | (serialize [this os] 112 | (write-String 1 {:optimize true} (:s this) os))) 113 | 114 | (def SimpleString-defaults {:s ""}) 115 | 116 | (defn cis->SimpleString 117 | "CodedInputStream to SimpleString" 118 | [is] 119 | (->> (tag-map 120 | (fn [tag index] 121 | (case index 122 | 1 [:s (cis->String is)] 123 | 124 | [index (cis->undefined tag is)])) 125 | is) 126 | (merge SimpleString-defaults) 127 | (map->SimpleString))) 128 | 129 | (defn ecis->SimpleString 130 | "Embedded CodedInputStream to SimpleString" 131 | [is] 132 | (cis->embedded cis->SimpleString is)) 133 | 134 | (defn new-SimpleString 135 | "Creates a new instance from a map, similar to map->SimpleString except that 136 | it properly accounts for nested messages, when applicable. 137 | " 138 | [init] 139 | (-> (merge SimpleString-defaults init) 140 | (map->SimpleString))) 141 | 142 | (defn pb->SimpleString 143 | "Protobuf to SimpleString" 144 | [input] 145 | (-> input 146 | stream/new-cis 147 | cis->SimpleString)) 148 | 149 | ;----------------------------------------------------------------------------- 150 | ; AllThingsMap-MSimpleEntry 151 | ;----------------------------------------------------------------------------- 152 | (defrecord AllThingsMap-MSimpleEntry [key value] 153 | pb/Writer 154 | 155 | (serialize [this os] 156 | (write-String 1 {:optimize true} (:key this) os) 157 | (write-Int32 2 {:optimize true} (:value this) os))) 158 | 159 | (def AllThingsMap-MSimpleEntry-defaults {:key "" :value 0}) 160 | 161 | (defn cis->AllThingsMap-MSimpleEntry 162 | "CodedInputStream to AllThingsMap-MSimpleEntry" 163 | [is] 164 | (->> (tag-map 165 | (fn [tag index] 166 | (case index 167 | 1 [:key (cis->String is)] 168 | 2 [:value (cis->Int32 is)] 169 | 170 | [index (cis->undefined tag is)])) 171 | is) 172 | (merge AllThingsMap-MSimpleEntry-defaults) 173 | (map->AllThingsMap-MSimpleEntry))) 174 | 175 | (defn ecis->AllThingsMap-MSimpleEntry 176 | "Embedded CodedInputStream to AllThingsMap-MSimpleEntry" 177 | [is] 178 | (cis->embedded cis->AllThingsMap-MSimpleEntry is)) 179 | 180 | (defn new-AllThingsMap-MSimpleEntry 181 | "Creates a new instance from a map, similar to map->AllThingsMap-MSimpleEntry except that 182 | it properly accounts for nested messages, when applicable. 183 | " 184 | [init] 185 | (-> (merge AllThingsMap-MSimpleEntry-defaults init) 186 | (map->AllThingsMap-MSimpleEntry))) 187 | 188 | (defn pb->AllThingsMap-MSimpleEntry 189 | "Protobuf to AllThingsMap-MSimpleEntry" 190 | [input] 191 | (-> input 192 | stream/new-cis 193 | cis->AllThingsMap-MSimpleEntry)) 194 | 195 | ;----------------------------------------------------------------------------- 196 | ; AllThingsMap-MComplexEntry 197 | ;----------------------------------------------------------------------------- 198 | (defrecord AllThingsMap-MComplexEntry [key value] 199 | pb/Writer 200 | 201 | (serialize [this os] 202 | (write-String 1 {:optimize true} (:key this) os) 203 | (write-embedded 2 (:value this) os))) 204 | 205 | (def AllThingsMap-MComplexEntry-defaults {:key ""}) 206 | 207 | (defn cis->AllThingsMap-MComplexEntry 208 | "CodedInputStream to AllThingsMap-MComplexEntry" 209 | [is] 210 | (->> (tag-map 211 | (fn [tag index] 212 | (case index 213 | 1 [:key (cis->String is)] 214 | 2 [:value (ecis->SimpleString is)] 215 | 216 | [index (cis->undefined tag is)])) 217 | is) 218 | (merge AllThingsMap-MComplexEntry-defaults) 219 | (map->AllThingsMap-MComplexEntry))) 220 | 221 | (defn ecis->AllThingsMap-MComplexEntry 222 | "Embedded CodedInputStream to AllThingsMap-MComplexEntry" 223 | [is] 224 | (cis->embedded cis->AllThingsMap-MComplexEntry is)) 225 | 226 | (defn new-AllThingsMap-MComplexEntry 227 | "Creates a new instance from a map, similar to map->AllThingsMap-MComplexEntry except that 228 | it properly accounts for nested messages, when applicable. 229 | " 230 | [init] 231 | (-> (merge AllThingsMap-MComplexEntry-defaults init) 232 | (cond-> (contains? init :value) (update :value new-SimpleString)) 233 | (map->AllThingsMap-MComplexEntry))) 234 | 235 | (defn pb->AllThingsMap-MComplexEntry 236 | "Protobuf to AllThingsMap-MComplexEntry" 237 | [input] 238 | (-> input 239 | stream/new-cis 240 | cis->AllThingsMap-MComplexEntry)) 241 | 242 | ;----------------------------------------------------------------------------- 243 | ; AllThingsMap 244 | ;----------------------------------------------------------------------------- 245 | (defrecord AllThingsMap [s i mSimple mComplex sSimple oe] 246 | pb/Writer 247 | 248 | (serialize [this os] 249 | (write-String 1 {:optimize true} (:s this) os) 250 | (write-Int32 2 {:optimize true} (:i this) os) 251 | (write-map new-AllThingsMap-MSimpleEntry 3 (:mSimple this) os) 252 | (write-map new-AllThingsMap-MComplexEntry 4 (:mComplex this) os) 253 | (write-embedded 5 (:sSimple this) os))) 254 | 255 | (def AllThingsMap-defaults {:s "" :i 0 :mSimple [] :mComplex []}) 256 | 257 | (defn cis->AllThingsMap 258 | "CodedInputStream to AllThingsMap" 259 | [is] 260 | (->> (tag-map 261 | (fn [tag index] 262 | (case index 263 | 1 [:s (cis->String is)] 264 | 2 [:i (cis->Int32 is)] 265 | 3 [:mSimple (cis->map ecis->AllThingsMap-MSimpleEntry is)] 266 | 4 [:mComplex (cis->map ecis->AllThingsMap-MComplexEntry is)] 267 | 5 [:sSimple (ecis->SimpleString is)] 268 | 269 | [index (cis->undefined tag is)])) 270 | is) 271 | (merge AllThingsMap-defaults) 272 | (map->AllThingsMap))) 273 | 274 | (defn ecis->AllThingsMap 275 | "Embedded CodedInputStream to AllThingsMap" 276 | [is] 277 | (cis->embedded cis->AllThingsMap is)) 278 | 279 | (defn new-AllThingsMap 280 | "Creates a new instance from a map, similar to map->AllThingsMap except that 281 | it properly accounts for nested messages, when applicable. 282 | " 283 | [init] 284 | (-> (merge AllThingsMap-defaults init) 285 | (cond-> (contains? init :sSimple) (update :sSimple new-SimpleString)) 286 | (map->AllThingsMap))) 287 | 288 | (defn pb->AllThingsMap 289 | "Protobuf to AllThingsMap" 290 | [input] 291 | (-> input 292 | stream/new-cis 293 | cis->AllThingsMap)) 294 | 295 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /modules/grpc-client/src/protojure/internal/grpc/client/providers/http2/jetty.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.internal.grpc.client.providers.http2.jetty 7 | (:require [promesa.core :as p] 8 | [promesa.exec :as p.exec] 9 | [clojure.core.async :refer [>!! ! go go-loop] :as async] 10 | [clojure.tools.logging :as log]) 11 | (:import (java.net InetSocketAddress) 12 | (java.nio ByteBuffer) 13 | (java.util.concurrent.atomic AtomicBoolean) 14 | (org.eclipse.jetty.http2.client HTTP2Client) 15 | (org.eclipse.jetty.http2.api Stream$Listener 16 | Stream 17 | Session 18 | Session$Listener 19 | Session$Listener$Adapter) 20 | (org.eclipse.jetty.http2.frames HeadersFrame 21 | DataFrame) 22 | (org.eclipse.jetty.util Promise Callback) 23 | (org.eclipse.jetty.util.component LifeCycle 24 | LifeCycle$Listener) 25 | (org.eclipse.jetty.http HttpFields 26 | HttpField 27 | HttpURI 28 | HttpVersion 29 | MetaData$Request 30 | MetaData$Response 31 | MetaData) 32 | (org.eclipse.jetty.util.ssl SslContextFactory SslContextFactory$Client)) 33 | (:refer-clojure :exclude [resolve])) 34 | 35 | (set! *warn-on-reflection* true) 36 | 37 | ;;------------------------------------------------------------------------------------ 38 | ;; Utility functions 39 | ;;------------------------------------------------------------------------------------ 40 | 41 | (defn- jetty-promise 42 | "converts a jetty promise to promesa" 43 | [f] 44 | (p/create 45 | (fn [resolve reject] 46 | (let [p (reify Promise 47 | (succeeded [_ result] 48 | (resolve result)) 49 | (failed [_ error] 50 | (reject error)))] 51 | (f p))))) 52 | 53 | (defn- jetty-callback-promise 54 | "converts a jetty 'callback' to promesa" 55 | [f] 56 | (let [p (async/promise-chan) 57 | cb (reify Callback 58 | (succeeded [_] 59 | (async/put! p true)) 60 | (failed [_ error] 61 | (async/put! p error)))] 62 | (f cb) 63 | p)) 64 | 65 | (defn- ->fields 66 | "converts a map of [string string] name/value attributes to a jetty HttpFields container" 67 | [headers] 68 | (let [fields (HttpFields/build)] 69 | (run! (fn [[k v]] (.put fields ^String k ^String v)) headers) 70 | fields)) 71 | 72 | (defn- fields-> 73 | "converts jetty HttpFields container to a [string string] map" 74 | [^HttpFields fields] 75 | (->> (.iterator fields) 76 | (iterator-seq) 77 | (reduce (fn [acc ^HttpField x] 78 | (assoc acc (.getName x) (.getValue x))) {}))) 79 | 80 | (defn- build-request 81 | "Builds a HEADERFRAME representing our request" 82 | [{:keys [method headers url] :or {method "GET" headers {}} :as request} last?] 83 | (log/trace "Sending request:" request "ENDFRAME=" last?) 84 | (let [_uri (HttpURI/from ^String url)] 85 | (as-> (->fields headers) $ 86 | (MetaData$Request. method _uri HttpVersion/HTTP_2 $) 87 | (HeadersFrame. $ nil last?)))) 88 | 89 | (defn- close-all! [& channels] 90 | (run! (fn [ch] (when (some? ch) (async/close! ch))) channels)) 91 | 92 | (defn- stream-log [sev ^Stream stream & msg] 93 | (log/log sev (apply str (cons (str "STREAM " (.getId stream) ": ") msg)))) 94 | 95 | (defn- receive-listener 96 | "Implements a org.eclipse.jetty.http2.api.Stream.Listener set of callbacks" 97 | [meta-ch data-ch] 98 | (let [end-stream! (fn [stream] (stream-log :trace stream "Closing") (close-all! meta-ch data-ch))] 99 | (reify Stream$Listener 100 | (onHeaders [_ stream frame] 101 | (let [^MetaData metadata (.getMetaData ^HeadersFrame frame) 102 | fields (fields-> (.getFields metadata)) 103 | data (if (.isResponse metadata) 104 | (let [status (.getStatus ^MetaData$Response metadata) 105 | reason (.getReason ^MetaData$Response metadata)] 106 | (cond-> {:headers fields} 107 | (some? status) (assoc :status status) 108 | (some? reason) (assoc :reason reason))) 109 | {:trailers fields}) 110 | last? (.isEndStream ^HeadersFrame frame)] 111 | (stream-log :trace stream "Received HEADER-FRAME: " data " ENDFRAME=" last?) 112 | (>!! meta-ch data) 113 | (when last? 114 | (end-stream! stream)))) 115 | (onData [_ stream frame callback] 116 | (let [data (.getData ^DataFrame frame) 117 | len (.remaining data) 118 | last? (.isEndStream ^DataFrame frame)] 119 | (stream-log :trace stream "Received DATA-FRAME (" len " bytes) ENDFRAME=" last?) 120 | (when (and (some? data-ch) (pos? len)) 121 | (let [clone (ByteBuffer/allocate len)] 122 | (.put clone data) 123 | (async/>!! data-ch (.flip clone)))) 124 | (when last? 125 | (end-stream! stream)) 126 | (.succeeded callback))) 127 | (onFailure [_ stream error reason ex callback] 128 | (stream-log :error stream "FAILURE: code-> " error " message-> " (ex-message ex)) 129 | (>!! meta-ch {:error {:type :failure :code error :reason reason :ex ex}}) 130 | (end-stream! stream) 131 | (.succeeded callback)) 132 | (onReset [_ stream frame] 133 | (stream-log :error stream "Received RST-FRAME") 134 | (let [error (.getError frame)] 135 | (>!! meta-ch {:error {:type :reset :code error}}) 136 | (end-stream! stream))) 137 | (onIdleTimeout [_ stream ex] 138 | (stream-log :error stream "Timeout") 139 | (>!! meta-ch {:error {:type :timeout :error ex}}) 140 | (end-stream! stream) 141 | ;; true: Close the session 142 | true) 143 | (onClosed [_ stream] 144 | (stream-log :trace stream "Closed")) 145 | (onPush [_ stream frame] 146 | (stream-log :trace stream "Received PUSH-FRAME"))))) 147 | 148 | (defn- transmit-data-frame 149 | "Transmits a single DATA frame" 150 | ([stream data] 151 | (transmit-data-frame stream data false 0)) 152 | ([^Stream stream ^ByteBuffer data last? padding] 153 | (stream-log :trace stream "Sending DATA-FRAME with " (.remaining data) " bytes, ENDFRAME=" last?) 154 | (jetty-callback-promise 155 | (fn [cb] 156 | (let [frame (DataFrame. (.getId stream) data last? padding)] 157 | (.data stream frame cb)))))) 158 | 159 | (def empty-data (ByteBuffer/wrap (byte-array 0))) 160 | 161 | (defn- transmit-eof 162 | "Transmits an empty DATA frame with the ENDSTREAM flag set to true, signifying the end of stream" 163 | [stream] 164 | (transmit-data-frame stream empty-data true 0)) 165 | 166 | (defn transmit-data-frames 167 | "Creates DATA frames from the buffers on the channel" 168 | [input stream] 169 | (if (some? input) 170 | (p/create 171 | (fn [resolve reject] 172 | (go-loop [] 173 | (if-let [frame ( (jetty-promise 227 | (fn [p] 228 | (.connect client (when ssl ssl-context-factory) address listener p))) 229 | (p/then (fn [session] 230 | (let [context {:client client :session session}] 231 | (log/debug "Session established:" context) 232 | context))) 233 | (p/catch (fn [e] 234 | (p/create 235 | (fn [resolve reject] 236 | (.stop client) ;; run (.stop) in a different thread, because p/catch will be called from .connect -> reject 237 | (reject e)) 238 | p.exec/*default-executor*)))))) 239 | 240 | (defn send-request 241 | [{:keys [^Session session] :as context} 242 | {:keys [input-ch meta-ch output-ch] :as request}] 243 | (let [request-frame (build-request request (nil? input-ch)) 244 | listener (receive-listener meta-ch output-ch)] 245 | (-> (jetty-promise 246 | (fn [p] 247 | (.newStream session request-frame p listener))) 248 | (p/catch (fn [ex] 249 | (close-all! meta-ch output-ch) 250 | (throw ex)))))) 251 | 252 | (defn disconnect [{:keys [^HTTP2Client client] :as context}] 253 | (log/debug "Disconnecting:" context) 254 | (p/create 255 | (fn [resolve reject] 256 | (let [listener (reify LifeCycle$Listener 257 | (^void lifeCycleFailure [this ^LifeCycle event ^Throwable cause] 258 | (.removeEventListener client this) 259 | (reject cause)) 260 | (^void lifeCycleStopped [this ^LifeCycle event] 261 | (.removeEventListener client this) 262 | (resolve (dissoc context :client :session))))] 263 | (.addEventListener client listener) 264 | (.stop client))))) 265 | -------------------------------------------------------------------------------- /modules/core/src/protojure/grpc/codec/lpm.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.grpc.codec.lpm 7 | "Utility functions for GRPC [length-prefixed-message](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests) encoding." 8 | (:require [clojure.core.async :refer [!!] :as async] 9 | [promesa.core :as p] 10 | [promesa.exec :as p.exec] 11 | [protojure.protobuf :refer [->pb]] 12 | [protojure.grpc.codec.compression :as compression] 13 | [protojure.internal.io :as pio] 14 | [clojure.tools.logging :as log] 15 | [clojure.java.io :as io]) 16 | (:import (java.io InputStream OutputStream ByteArrayOutputStream) 17 | (org.apache.commons.io.input BoundedInputStream)) 18 | (:refer-clojure :exclude [resolve])) 19 | 20 | (set! *warn-on-reflection* true) 21 | 22 | (def lpm-thread-executor (if p.exec/vthreads-supported? p.exec/vthread-executor p.exec/thread-executor)) 23 | 24 | ;;-------------------------------------------------------------------------------------- 25 | ;; integer serdes used for GRPC framing 26 | ;;-------------------------------------------------------------------------------------- 27 | (defn- bytes->num 28 | "Deserializes an integer from a byte-array. 29 | 30 | Shamelessly borrowed from https://gist.github.com/pingles/1235344" 31 | [data] 32 | (->> data 33 | (map-indexed 34 | (fn [i x] 35 | (bit-shift-left (bit-and x 0x0FF) 36 | (* 8 (- (count data) i 1))))) 37 | (reduce bit-or))) 38 | 39 | (defn- num->bytes 40 | "Serializes an integer to a byte-array." 41 | ^bytes [num] 42 | (byte-array (for [i (range 4)] 43 | (-> (unsigned-bit-shift-right num 44 | (* 8 (- 4 i 1))) 45 | (bit-and 0x0FF))))) 46 | 47 | ;;====================================================================================== 48 | ;; GRPC length-prefixed-message (LPM) codec 49 | ;;====================================================================================== 50 | ;; 51 | ;; GRPC encodes protobuf messages as: 52 | ;; 53 | ;; [ 54 | ;; 1b : compressed? (0 = no, 1 = yes) 55 | ;; 4b : length of message 56 | ;; Nb : N = length bytes of optionally compressed protobuf 57 | ;; 58 | ;; ] 59 | ;; 60 | ;; Reference: 61 | ; https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests 62 | ;;====================================================================================== 63 | 64 | ;;-------------------------------------------------------------------------------------- 65 | ;; Decoder 66 | ;;-------------------------------------------------------------------------------------- 67 | (defn- decoder-stream 68 | ^InputStream [is compressed? decompressor] 69 | (if (and compressed? (some? decompressor)) 70 | (decompressor is) 71 | is)) 72 | 73 | (defn- decode-header 74 | "Decodes 5-bytes into a {compressed? len} tuple" 75 | [hdr] 76 | (let [compressed? (first hdr) 77 | lenbuf (rest hdr)] 78 | {:compressed? (pos? compressed?) :len (bytes->num lenbuf)})) 79 | 80 | (defn- blocking-read 81 | [^InputStream is b off len] 82 | (let [alen (.read is b off len)] 83 | (cond 84 | (= alen -1) (throw (ex-info "short-read" {})) 85 | (< alen len) (recur is b (+ off alen) (- len alen)) 86 | :default true))) 87 | 88 | (defn- read-header 89 | [^InputStream is] 90 | (let [hdr (byte-array 5)] 91 | (blocking-read is hdr 0 5) 92 | (decode-header hdr))) 93 | 94 | (defn- decode-body 95 | "Decodes a LPM payload based on a previously decoded header (see [[decode-header]])" 96 | [f is {:keys [compressed? len] :as header} options] 97 | (-> (BoundedInputStream. is len) 98 | (decoder-stream compressed? options) 99 | f)) 100 | 101 | (defn- decode->seq [f ^InputStream is decompressor] 102 | (lazy-seq (when (pos? (.available is)) 103 | (let [hdr (read-header is)] 104 | (cons (decode-body f is hdr decompressor) (decode->seq f is decompressor)))))) 105 | 106 | ;;-------------------------------------------------------------------------------------------- 107 | 108 | (defn decode 109 | " 110 | Takes a parsing function, a pair of input/output [core.async channels](https://clojuredocs.org/clojure.core.async/chan), and an optional codec and 111 | decodes a stream of [length-prefixed-message](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests) (LPM). 112 | 113 | #### Parameters 114 | 115 | | Value | Type | Description | 116 | |-----------------|----------------------|---------------------------------------------------------------------------| 117 | | **f** | _(fn [is])_ | A protobuf decoder function with an arity of 1 that accepts an instance of [InputStream](https://docs.oracle.com/javase/7/docs/api/java/io/InputStream.html), such as the pb-> functions produced by the protoc-gen-clojure compiler | 118 | | **input** | _core.async/channel_ | A core.async channel that carries LPM encoded bytes, one byte per [(take!)](https://clojuredocs.org/clojure.core.async/take!). Closing the input will shut down the pipeline. | 119 | | **output** | _core.async/channel_ | A core.async channel that carries decoded protobuf records. Will be closed when the pipeline shuts down. | 120 | | **options** | _map_ | See _Options_ 121 | 122 | ##### Options 123 | 124 | | Value | Type | Default | Description | 125 | |--------------------|-----------------------------------------------------------------| 126 | | **content-coding** | _string_ | \"identity\" | See _Content-coding_ table | 127 | | **codecs** | _map_ | [[protojure.grpc.codec.compression/builtin-codecs]] | The dictionary of codecs to utilize | 128 | | **tmo** | _unsigned int_ | 5000ms | A timeout, in ms, for receiving the remaining LPM payload bytes once the header has been received. | 129 | 130 | ###### Example 131 | 132 | ``` 133 | {:content-coding \"gzip\" 134 | :codecs mycodecs 135 | :tmo 1000} 136 | ``` 137 | 138 | ##### Content-coding 139 | The value for the **content-coding** option must be one of 140 | 141 | | Value | Comment | 142 | |----------------|-------------------------------------------| 143 | | nil | no compression | 144 | | \"identity\" | no compression | 145 | | _other string_ | retrieves the codec from _codecs_ option | 146 | 147 | 148 | " 149 | [f input output {:keys [codecs content-coding tmo] :or {codecs compression/builtin-codecs tmo 5000} :as options}] 150 | (let [decompressor (when (and (some? content-coding) (not= content-coding "identity")) 151 | (compression/decompressor codecs content-coding))] 152 | (p/create 153 | (fn [resolve reject] 154 | (try 155 | (loop [] 156 | (if-let [buf (seq f is decompressor)] 159 | (when (some? msg) 160 | (log/trace "Decoded: " msg) 161 | (>!! output msg))) 162 | (recur)) 163 | (resolve :ok))) 164 | (catch Exception e 165 | (reject e)) 166 | (finally 167 | (log/trace "closing output channel") 168 | (async/close! output)))) 169 | lpm-thread-executor))) 170 | 171 | ;;-------------------------------------------------------------------------------------- 172 | ;; Encoder 173 | ;;-------------------------------------------------------------------------------------- 174 | (defn- encode-buffer [buf len compressed? ^OutputStream os] 175 | (.write os (int (if compressed? 1 0))) 176 | (.write os (num->bytes len) 0 4) 177 | (when (pos? len) 178 | (.write os buf 0 len))) 179 | 180 | (defn- encode-uncompressed [msg os] 181 | (let [buf (->pb msg) 182 | len (count buf)] 183 | (encode-buffer buf len false os))) 184 | 185 | (defn- compress-buffer [compressor ^bytes buf] 186 | (let [os (ByteArrayOutputStream.)] 187 | (with-open [cos (compressor os)] 188 | (io/copy buf ^OutputStream cos)) 189 | (.toByteArray os))) 190 | 191 | (defn- encode-maybe-compressed 192 | "This function will encode the message either with or without compression, 193 | depending on whichever results in the smaller message" 194 | [msg compressor os] 195 | (let [buf (->pb msg) 196 | len (count buf) 197 | cbuf (compress-buffer compressor buf) 198 | clen (count buf)] 199 | (if (< clen len) 200 | (encode-buffer cbuf clen true os) 201 | (encode-buffer buf len false os)))) 202 | 203 | ;;-------------------------------------------------------------------------------------------- 204 | (defn encode 205 | " 206 | Takes an input and output [core.async channel](https://clojuredocs.org/clojure.core.async/chan), along with an optional codec and 207 | encodes a stream of 0 or more [length-prefixed-message](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests) 208 | messages. 209 | 210 | #### Parameters 211 | 212 | | Value | Type | Description | 213 | |-----------------|----------------------|---------------------------------------------------------------------------| 214 | | **f** | _(fn [init])_ | A protobuf encoder function with an arity of 1 that accepts an init-map, such as the new-XX functions produced by the protoc-gen-clojure compiler | 215 | | **input** | _core.async/channel_ | A core.async channel expected to carry maps that will be transformed to protobuf messages based on _f_ | 216 | | **output** | _core.async/channel_ | A core.async channel that will carry bytes representing the encoded messages. See _max-frame-size_. Will be closed when the pipeline shuts down. | 217 | | **options** | _map_ | See _Options_ | 218 | 219 | ##### Options 220 | 221 | | Value | Type | Default | Description | 222 | |--------------------|---------------------------------------------------------------------------------------------------------------| 223 | | **content-coding** | _String_ | \"identity\" | See the _content-coding_ section of [[decode]] | 224 | | **codecs** | _map_ | [[protojure.grpc.codec.compression/builtin-codecs]] | The dictionary of codecs to utilize | 225 | | **max-frame-size** | _UInt32_ | 0 | Maximum frame-size emitted on _output_ channel. See _Output channel details_ below | 226 | 227 | ##### Output channel details 228 | 229 | The _max-frame-size_ option dictates how bytes are encoded on the _output_ channel: 230 | 231 | - **0**: (Default) Indicates 'stream' mode: Bytes are encoded one byte per [(put!)](https://clojuredocs.org/clojure.core.async/put!). 232 | - **>0**: Indicates 'framed' mode: The specified value will dictate the upper bound on byte-array based frames emitted to the output channel. 233 | 234 | ###### Example 235 | 236 | ``` 237 | {:content-coding \"gzip\" 238 | :codecs mycodecs 239 | :max-frame-size 16384} 240 | ``` 241 | " 242 | [f input output {:keys [codecs content-coding max-frame-size] :or {codecs compression/builtin-codecs} :as options}] 243 | (let [os (pio/new-outputstream (cond-> {:ch output} (some? max-frame-size) (assoc :max-frame-size max-frame-size))) 244 | compressor (when (and (some? content-coding) (not= content-coding "identity")) 245 | (compression/compressor codecs content-coding))] 246 | (p/create 247 | (fn [resolve reject] 248 | (try 249 | (loop [] 250 | (if-let [_msg (&2 echo "Leiningen couldn't find $1 in your \$PATH ($PATH), which is required." 33 | exit 1 34 | } 35 | 36 | function make_native_path { 37 | # ensure we have native paths 38 | if $cygwin && [[ "$1" == /* ]]; then 39 | echo -n "$(cygpath -wp "$1")" 40 | elif [[ "$OSTYPE" == "msys" && "$1" == /?/* ]]; then 41 | echo -n "$(sh -c "(cd $1 2 /dev/null 86 | download_failed_message "$LEIN_URL" "$exit_code" 87 | exit 1 88 | fi 89 | } 90 | 91 | NOT_FOUND=1 92 | ORIGINAL_PWD="$PWD" 93 | while [ ! -r "$PWD/project.clj" ] && [ "$PWD" != "/" ] && [ $NOT_FOUND -ne 0 ] 94 | do 95 | cd .. 96 | if [ "$(dirname "$PWD")" = "/" ]; then 97 | NOT_FOUND=0 98 | cd "$ORIGINAL_PWD" 99 | fi 100 | done 101 | 102 | export LEIN_HOME="${LEIN_HOME:-"$HOME/.lein"}" 103 | 104 | for f in "/etc/leinrc" "$LEIN_HOME/leinrc" ".leinrc"; do 105 | if [ -e "$f" ]; then 106 | source "$f" 107 | fi 108 | done 109 | 110 | if $cygwin; then 111 | export LEIN_HOME=$(cygpath -w "$LEIN_HOME") 112 | fi 113 | 114 | LEIN_JAR="$LEIN_HOME/self-installs/leiningen-$LEIN_VERSION-standalone.jar" 115 | 116 | # normalize $0 on certain BSDs 117 | if [ "$(dirname "$0")" = "." ]; then 118 | SCRIPT="$(which "$(basename "$0")")" 119 | if [ -z "$SCRIPT" ]; then 120 | SCRIPT="$0" 121 | fi 122 | else 123 | SCRIPT="$0" 124 | fi 125 | 126 | # resolve symlinks to the script itself portably 127 | while [ -h "$SCRIPT" ] ; do 128 | ls=$(ls -ld "$SCRIPT") 129 | link=$(expr "$ls" : '.*-> \(.*\)$') 130 | if expr "$link" : '/.*' > /dev/null; then 131 | SCRIPT="$link" 132 | else 133 | SCRIPT="$(dirname "$SCRIPT"$)/$link" 134 | fi 135 | done 136 | 137 | BIN_DIR="$(dirname "$SCRIPT")" 138 | 139 | export LEIN_JVM_OPTS="${LEIN_JVM_OPTS-"-Xverify:none -XX:+TieredCompilation -XX:TieredStopAtLevel=1"}" 140 | 141 | # This needs to be defined before we call HTTP_CLIENT below 142 | if [ "$HTTP_CLIENT" = "" ]; then 143 | if type -p curl >/dev/null 2>&1; then 144 | if [ "$https_proxy" != "" ]; then 145 | CURL_PROXY="-x $https_proxy" 146 | fi 147 | HTTP_CLIENT="curl $CURL_PROXY -f -L -o" 148 | else 149 | HTTP_CLIENT="wget -O" 150 | fi 151 | fi 152 | 153 | 154 | # When :eval-in :classloader we need more memory 155 | grep -E -q '^\s*:eval-in\s+:classloader\s*$' project.clj 2> /dev/null && \ 156 | export LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Xms64m -Xmx512m" 157 | 158 | if [ -r "$BIN_DIR/../src/leiningen/version.clj" ]; then 159 | # Running from source checkout 160 | LEIN_DIR="$(dirname "$BIN_DIR")" 161 | 162 | # Need to use lein release to bootstrap the leiningen-core library (for aether) 163 | if [ ! -r "$LEIN_DIR/leiningen-core/.lein-bootstrap" ]; then 164 | echo "Leiningen is missing its dependencies." 165 | echo "Please run \"lein bootstrap\" in the leiningen-core/ directory" 166 | echo "with a stable release of Leiningen. See CONTRIBUTING.md for details." 167 | exit 1 168 | fi 169 | 170 | # If project.clj for lein or leiningen-core changes, we must recalculate 171 | LAST_PROJECT_CHECKSUM=$(cat "$LEIN_DIR/.lein-project-checksum" 2> /dev/null) 172 | PROJECT_CHECKSUM=$(sum "$LEIN_DIR/project.clj" "$LEIN_DIR/leiningen-core/project.clj") 173 | if [ "$PROJECT_CHECKSUM" != "$LAST_PROJECT_CHECKSUM" ]; then 174 | if [ -r "$LEIN_DIR/.lein-classpath" ]; then 175 | rm "$LEIN_DIR/.lein-classpath" 176 | fi 177 | fi 178 | 179 | # Use bin/lein to calculate its own classpath. 180 | if [ ! -r "$LEIN_DIR/.lein-classpath" ] && [ "$1" != "classpath" ]; then 181 | echo "Recalculating Leiningen's classpath." 182 | ORIG_PWD="$PWD" 183 | cd "$LEIN_DIR" 184 | 185 | LEIN_NO_USER_PROFILES=1 $0 classpath .lein-classpath 186 | sum "$LEIN_DIR/project.clj" "$LEIN_DIR/leiningen-core/project.clj" > \ 187 | .lein-project-checksum 188 | cd "$ORIG_PWD" 189 | fi 190 | 191 | mkdir -p "$LEIN_DIR/target/classes" 192 | export LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Dclojure.compile.path=$LEIN_DIR/target/classes" 193 | add_path CLASSPATH "$LEIN_DIR/leiningen-core/src/" "$LEIN_DIR/leiningen-core/resources/" \ 194 | "$LEIN_DIR/test:$LEIN_DIR/target/classes" "$LEIN_DIR/src" ":$LEIN_DIR/resources" 195 | 196 | if [ -r "$LEIN_DIR/.lein-classpath" ]; then 197 | add_path CLASSPATH "$(cat "$LEIN_DIR/.lein-classpath" 2> /dev/null)" 198 | else 199 | add_path CLASSPATH "$(cat "$LEIN_DIR/leiningen-core/.lein-bootstrap" 2> /dev/null)" 200 | fi 201 | else # Not running from a checkout 202 | add_path CLASSPATH "$LEIN_JAR" 203 | 204 | if [ "$LEIN_USE_BOOTCLASSPATH" != "" ]; then 205 | LEIN_JVM_OPTS="-Xbootclasspath/a:$LEIN_JAR $LEIN_JVM_OPTS" 206 | fi 207 | 208 | if [ ! -r "$LEIN_JAR" -a "$1" != "self-install" ]; then 209 | self_install 210 | fi 211 | fi 212 | 213 | if [ ! -x "$JAVA_CMD" ] && ! type -f java >/dev/null 214 | then 215 | >&2 echo "Leiningen couldn't find 'java' executable, which is required." 216 | >&2 echo "Please either set JAVA_CMD or put java (>=1.6) in your \$PATH ($PATH)." 217 | exit 1 218 | fi 219 | 220 | export LEIN_JAVA_CMD="${LEIN_JAVA_CMD:-${JAVA_CMD:-java}}" 221 | 222 | if [[ -z "${DRIP_INIT+x}" && "$(basename "$LEIN_JAVA_CMD")" == *drip* ]]; then 223 | export DRIP_INIT="$(printf -- '-e\n(require (quote leiningen.repl))')" 224 | export DRIP_INIT_CLASS="clojure.main" 225 | fi 226 | 227 | # Support $JAVA_OPTS for backwards-compatibility. 228 | export JVM_OPTS="${JVM_OPTS:-"$JAVA_OPTS"}" 229 | 230 | # Handle jline issue with cygwin not propagating OSTYPE through java subprocesses: https://github.com/jline/jline2/issues/62 231 | cygterm=false 232 | if $cygwin; then 233 | case "$TERM" in 234 | rxvt* | xterm* | vt*) cygterm=true ;; 235 | esac 236 | fi 237 | 238 | if $cygterm; then 239 | LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Djline.terminal=jline.UnixTerminal" 240 | stty -icanon min 1 -echo > /dev/null 2>&1 241 | fi 242 | 243 | # TODO: investigate http://skife.org/java/unix/2011/06/20/really_executable_jars.html 244 | # If you're packaging this for a package manager (.deb, homebrew, etc) 245 | # you need to remove the self-install and upgrade functionality or see lein-pkg. 246 | if [ "$1" = "self-install" ]; then 247 | if [ -r "$BIN_DIR/../src/leiningen/version.clj" ]; then 248 | echo "Running self-install from a checkout is not supported." 249 | echo "See CONTRIBUTING.md for SNAPSHOT-specific build instructions." 250 | exit 1 251 | fi 252 | echo "Manual self-install is deprecated; it will run automatically when necessary." 253 | self_install 254 | elif [ "$1" = "upgrade" ] || [ "$1" = "downgrade" ]; then 255 | if [ "$LEIN_DIR" != "" ]; then 256 | echo "The upgrade task is not meant to be run from a checkout." 257 | exit 1 258 | fi 259 | if [ $SNAPSHOT = "YES" ]; then 260 | echo "The upgrade task is only meant for stable releases." 261 | echo "See the \"Bootstrapping\" section of CONTRIBUTING.md." 262 | exit 1 263 | fi 264 | if [ ! -w "$SCRIPT" ]; then 265 | echo "You do not have permission to upgrade the installation in $SCRIPT" 266 | exit 1 267 | else 268 | TARGET_VERSION="${2:-stable}" 269 | echo "The script at $SCRIPT will be upgraded to the latest $TARGET_VERSION version." 270 | echo -n "Do you want to continue [Y/n]? " 271 | read RESP 272 | case "$RESP" in 273 | y|Y|"") 274 | echo 275 | echo "Upgrading..." 276 | TARGET="/tmp/lein-${$}-upgrade" 277 | if $cygwin; then 278 | TARGET=$(cygpath -w "$TARGET") 279 | fi 280 | LEIN_SCRIPT_URL="https://github.com/technomancy/leiningen/raw/$TARGET_VERSION/bin/lein" 281 | $HTTP_CLIENT "$TARGET" "$LEIN_SCRIPT_URL" 282 | if [ $? == 0 ]; then 283 | cmp -s "$TARGET" "$SCRIPT" 284 | if [ $? == 0 ]; then 285 | echo "Leiningen is already up-to-date." 286 | fi 287 | mv "$TARGET" "$SCRIPT" && chmod +x "$SCRIPT" 288 | exec "$SCRIPT" version 289 | else 290 | download_failed_message "$LEIN_SCRIPT_URL" 291 | fi;; 292 | *) 293 | echo "Aborted." 294 | exit 1;; 295 | esac 296 | fi 297 | else 298 | if $cygwin; then 299 | # When running on Cygwin, use Windows-style paths for java 300 | ORIGINAL_PWD=$(cygpath -w "$ORIGINAL_PWD") 301 | fi 302 | 303 | # apply context specific CLASSPATH entries 304 | if [ -f .lein-classpath ]; then 305 | add_path CLASSPATH "$(cat .lein-classpath)" 306 | fi 307 | 308 | if [ -n "$DEBUG" ]; then 309 | echo "Leiningen's classpath: $CLASSPATH" 310 | fi 311 | 312 | if [ -r .lein-fast-trampoline ]; then 313 | export LEIN_FAST_TRAMPOLINE='y' 314 | fi 315 | 316 | if [ "$LEIN_FAST_TRAMPOLINE" != "" ] && [ -r project.clj ]; then 317 | INPUTS="$* $(cat project.clj) $LEIN_VERSION $(test -f "$LEIN_HOME/profiles.clj" && cat "$LEIN_HOME/profiles.clj")" 318 | 319 | if command -v shasum >/dev/null 2>&1; then 320 | SUM="shasum" 321 | elif command -v sha1sum >/dev/null 2>&1; then 322 | SUM="sha1sum" 323 | else 324 | command_not_found "sha1sum or shasum" 325 | fi 326 | 327 | export INPUT_CHECKSUM=$(echo "$INPUTS" | $SUM | cut -f 1 -d " ") 328 | # Just don't change :target-path in project.clj, mkay? 329 | TRAMPOLINE_FILE="target/trampolines/$INPUT_CHECKSUM" 330 | else 331 | if hash mktemp 2>/dev/null; then 332 | # Check if mktemp is available before using it 333 | TRAMPOLINE_FILE="$(mktemp /tmp/lein-trampoline-XXXXXXXXXXXXX)" 334 | else 335 | TRAMPOLINE_FILE="/tmp/lein-trampoline-$$" 336 | fi 337 | trap 'rm -f $TRAMPOLINE_FILE' EXIT 338 | fi 339 | 340 | if $cygwin; then 341 | TRAMPOLINE_FILE=$(cygpath -w "$TRAMPOLINE_FILE") 342 | fi 343 | 344 | if [ "$INPUT_CHECKSUM" != "" ] && [ -r "$TRAMPOLINE_FILE" ]; then 345 | if [ -n "$DEBUG" ]; then 346 | echo "Fast trampoline with $TRAMPOLINE_FILE." 347 | fi 348 | exec sh -c "exec $(cat "$TRAMPOLINE_FILE")" 349 | else 350 | export TRAMPOLINE_FILE 351 | "$LEIN_JAVA_CMD" \ 352 | -Dfile.encoding=UTF-8 \ 353 | -Dmaven.wagon.http.ssl.easy=false \ 354 | -Dmaven.wagon.rto=10000 \ 355 | $LEIN_JVM_OPTS \ 356 | -Dleiningen.original.pwd="$ORIGINAL_PWD" \ 357 | -Dleiningen.script="$SCRIPT" \ 358 | -classpath "$CLASSPATH" \ 359 | clojure.main -m leiningen.core.main "$@" 360 | 361 | EXIT_CODE=$? 362 | 363 | if $cygterm ; then 364 | stty icanon echo > /dev/null 2>&1 365 | fi 366 | 367 | if [ -r "$TRAMPOLINE_FILE" ] && [ "$LEIN_TRAMPOLINE_WARMUP" = "" ]; then 368 | TRAMPOLINE="$(cat "$TRAMPOLINE_FILE")" 369 | if [ "$INPUT_CHECKSUM" = "" ]; then # not using fast trampoline 370 | rm "$TRAMPOLINE_FILE" 371 | fi 372 | if [ "$TRAMPOLINE" = "" ]; then 373 | exit $EXIT_CODE 374 | else 375 | exec sh -c "exec $TRAMPOLINE" 376 | fi 377 | else 378 | exit $EXIT_CODE 379 | fi 380 | fi 381 | fi 382 | -------------------------------------------------------------------------------- /test/test/protojure/protobuf_test.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright © 2019 State Street Bank and Trust Company. All rights reserved 2 | ;; Copyright © 2019-2022 Manetu, Inc. All rights reserved 3 | ;; 4 | ;; SPDX-License-Identifier: Apache-2.0 5 | 6 | (ns protojure.protobuf-test 7 | (:require [clojure.test :refer :all] 8 | [clojure.core.async :refer [!! ! go] :as async] 9 | [protojure.protobuf.serdes.core :as serdes] 10 | [protojure.protobuf.serdes.complex :as serdes.complex] 11 | [protojure.protobuf.serdes.utils :refer [tag-map]] 12 | [protojure.protobuf :refer [->pb]] 13 | [protojure.protobuf.any :refer [any-> ->any] :as any] 14 | [protojure.grpc.codec.lpm :as lpm] 15 | [protojure.grpc.codec.compression :as compression] 16 | [protojure.test.utils :refer [data-equal?]] 17 | [com.google.protobuf :as google] 18 | [promesa.core :as p] 19 | [example.types :as example] 20 | [com.example.addressbook :as addressbook] 21 | [com.example.empty :as empty]) 22 | (:import (com.google.protobuf CodedOutputStream 23 | CodedInputStream) 24 | (java.io ByteArrayOutputStream) 25 | (org.apache.commons.io.input CloseShieldInputStream) 26 | (org.apache.commons.io.output CloseShieldOutputStream)) 27 | (:refer-clojure :exclude [resolve])) 28 | 29 | ;;----------------------------------------------------------------------------- 30 | ;; Helper functions 31 | ;;----------------------------------------------------------------------------- 32 | 33 | (defn- fns [type] 34 | (mapv #(clojure.core/resolve (symbol "protojure.protobuf.serdes.core" (str % type))) 35 | ["write-" "cis->"])) 36 | 37 | (defn- resolve-fns [type] 38 | (let [[writefn parsefn] (fns type)] 39 | {:writefn writefn :parsefn parsefn})) 40 | 41 | (defn- pbverify 42 | "End to end serdes testing for a specific message" 43 | [newf pb->f data] 44 | (let [input (newf data) 45 | output (-> input ->pb pb->f)] 46 | (is (data-equal? input output)))) 47 | 48 | (defn- with-buffer 49 | "Invokes 'f' with a fully formed buffered output-stream and returns the bytes" 50 | [f] 51 | (let [os (ByteArrayOutputStream.) 52 | cos (CodedOutputStream/newInstance os)] 53 | (f cos) 54 | (.flush cos) 55 | (.toByteArray os))) 56 | 57 | (defn- size [f] 58 | (count (with-buffer f))) 59 | 60 | (defn- write [writefn tag value] 61 | (with-buffer (partial writefn tag value))) 62 | 63 | (defn- write-embedded [tag item] 64 | (write serdes/write-embedded tag item)) 65 | 66 | (defn- write-repeated [writefn tag items] 67 | (with-buffer (partial serdes.complex/write-repeated writefn tag items))) 68 | 69 | (defn- parse [^bytes buf readfn] 70 | (let [is (CodedInputStream/newInstance buf)] 71 | (.readTag is) 72 | (readfn is))) 73 | 74 | (defn- parse-repeated [^bytes buf readfn packable? tag] 75 | (let [is (CodedInputStream/newInstance buf) 76 | f (if packable? (partial serdes.complex/cis->packablerepeated tag) serdes.complex/cis->repeated)] 77 | (tag-map 78 | (fn [tag index] 79 | [index (f readfn is)]) 80 | is))) 81 | 82 | (defn- byte-string [x] (byte-array (mapv byte x))) 83 | 84 | (defn- test-repeated [data] 85 | (-> (byte-array data) 86 | (example/pb->SimpleRepeated) 87 | (:data))) 88 | 89 | (defn- repeat-num 90 | "Create an range of 'n' contiguous values from 'input'" 91 | [n input] 92 | (take n (iterate inc input))) 93 | 94 | (defn async-seq 95 | "Returns a lazy sequence of items available on a core.async channel" 96 | [ch] 97 | (lazy-seq (when-some [data (async/deserialize cycle" 147 | [{:keys [type input]}] 148 | (let [{:keys [writefn parsefn]} (resolve-fns type) 149 | output (-> (write writefn tag input) 150 | (parse parsefn))] 151 | 152 | (is (data-equal? input output)))) 153 | 154 | (defn- validate-optimizer 155 | "validate optimizer behavior. 'input' items should _never_ be elided, thus 156 | they should always generate a positive length calculation. 'default' fields 157 | however, are fields that are carrying default values. The optimizer should 158 | detect this and elide them from the wire, resulting in a 0 length calc. For 159 | good measure, we also fire up a (writefn) operation to a nil output stream. 160 | A correct functioning optimizer will elide the write, resulting in no errors 161 | even despite our bogus stream." 162 | [{:keys [type input default]}] 163 | (let [{:keys [writefn]} (resolve-fns type)] 164 | (is (pos? (size (partial writefn tag input)))) 165 | (is (zero? (size (partial writefn tag default)))) 166 | (is (pos? (size (partial writefn tag {:optimize false} default)))) 167 | (writefn tag default nil))) 168 | 169 | (defn- validate-repeated 170 | [{:keys [type input packable? repeatfn]}] 171 | (let [{:keys [writefn parsefn]} (resolve-fns type) 172 | items (vec (repeatfn 10 input)) 173 | output (-> (write-repeated writefn tag items) 174 | (parse-repeated parsefn packable? tag) 175 | (get tag))] 176 | 177 | (is (data-equal? items output)))) 178 | 179 | ;; We add a silly codec named "mycustom" that does nothing. We use the CloseShieldXXX family 180 | ;; of proxy stream classes so that we pass the IO through, but bury the (.close) operation. This 181 | ;; codec is only useful for validating that a custom-codec actually works. 182 | (def custom-codecs 183 | (assoc compression/builtin-codecs 184 | "mycustom" {:input #(CloseShieldInputStream. %) :output #(CloseShieldOutputStream. %)})) 185 | 186 | (defn- validate-lpm-msg 187 | [input-ch output-ch input] 188 | (>!! input-ch input) 189 | (let [output ( {:codecs custom-codecs} 198 | (cond-> (some? content-coding) (assoc :content-coding content-coding))) 199 | tasks (p/all [(lpm/encode example/new-Money input wire options) 200 | (lpm/decode example/pb->Money wire output options)])] 201 | 202 | (run! (partial validate-lpm-msg input output) (repeat 10 msg)) 203 | (async/close! input) 204 | @tasks)) 205 | 206 | (defn- validate-bad-codec 207 | [msg content-coding] 208 | (is (thrown? clojure.lang.ExceptionInfo (validate-lpm msg content-coding)))) 209 | 210 | ;;----------------------------------------------------------------------------- 211 | ;; Tests 212 | ;;----------------------------------------------------------------------------- 213 | 214 | (deftest raw-e2e-test 215 | (testing "Test each type round trip serialize->deserialize" 216 | (run! validate-e2e test-data))) 217 | 218 | (deftest optimizer-test 219 | (testing "Check that the optimizer skips default values" 220 | (run! validate-optimizer test-data))) 221 | 222 | (deftest pb-e2e-test 223 | (testing "End-to-end testing by processing arbitrary PB type" 224 | (pbverify example/new-Money 225 | example/pb->Money 226 | test-msg))) 227 | 228 | (deftest repeated-test 229 | (testing "Check that repeated values are handled properly" 230 | (run! validate-repeated test-data))) 231 | 232 | ;; Represent a 'repeated int32' wire representation in both 233 | ;; packed and unpacked format. For more details, see: 234 | ;; https://developers.google.com/protocol-buffers/docs/encoding#packed 235 | (deftest packed-repeated-test 236 | (testing "Testing repeated field decoding of packed structures" 237 | (let [result (test-repeated [0xA 3 21 22 23 0xA 3 24 25 26])] ;; send the data in two chunks 238 | (is (= (count result) 6)) 239 | (is (data-equal? result [21 22 23 24 25 26]))))) 240 | 241 | (deftest packed-repeated-varint-test 242 | (testing "Testing repeated field decoding of variable length values" 243 | (let [result (test-repeated [0xA 0x05 0x27 0x00 0xf4 0x06 0x01])] 244 | (is (= (count result) 4)) 245 | (is (data-equal? result [39 0 884 1]))))) 246 | 247 | (deftest unpacked-repeated-test 248 | (testing "Testing repeated field decoding of unpacked structures" 249 | (let [result (test-repeated [0x8 1 0x8 2 0x8 3])] 250 | (is (= (count result) 3)) 251 | (is (data-equal? result [1 2 3]))))) 252 | 253 | (deftest map-test 254 | (testing "Test map serialization" 255 | (pbverify example/new-AllThingsMap 256 | example/pb->AllThingsMap 257 | {:s "hello" 258 | :i 42 259 | :mSimple {"k1" 42 "k2" 43} 260 | :mComplex {"k1" {:s "v1"} "k2" {:s "v2"}} 261 | :sSimple {:s "simple"}}))) 262 | 263 | (deftest embedded-test 264 | (testing "Verify that we can embed a message in a coded stream" 265 | (let [input (example/new-Money test-msg) 266 | output (-> (write-embedded tag input) 267 | (parse example/ecis->Money))] 268 | (is (data-equal? input output))))) 269 | 270 | (deftest embedded-nil-test 271 | (testing "Check that embedded but unset messages are handled properly" 272 | (is (zero? (size (partial serdes/write-embedded tag nil)))) 273 | (serdes/write-embedded tag nil nil))) 274 | 275 | (deftest grpc-lpm-test 276 | (testing "Verify that we can round-trip through the LPM logic with each compression type" 277 | (let [codecs [nil "identity" "gzip" "deflate" "snappy" "mycustom"]] 278 | (run! (partial validate-lpm long-test-msg) codecs) 279 | (run! (partial validate-lpm test-msg) codecs)))) 280 | 281 | (deftest grpc-lpm-empty-test 282 | (testing "Verify that we encode an Empty message properly" 283 | (let [input (async/chan 64) 284 | output (async/chan 16384) 285 | task (lpm/encode google/new-Empty input output {})] 286 | 287 | (>!! input {}) 288 | (async/close! input) 289 | @task 290 | (let [result (async-seq output)] 291 | (is (= 1 (count result))) 292 | (is (-> result first (.remaining) (= 5))))))) 293 | 294 | (deftest grpc-timeout-test 295 | (testing "Verify that we correctly timeout on a stalled decode" 296 | (let [input (async/chan 16384) 297 | output (async/chan 64)] 298 | (run! (fn [x] (async/put! input (byte x))) [0 0 0 0 4]) 299 | (is (thrown? java.util.concurrent.ExecutionException @(lpm/decode example/pb->Money input output {:tmo 100})))))) 300 | 301 | (deftest grpc-bad-codec 302 | (testing "Verify that we reject invalid codec types" 303 | (run! (partial validate-bad-codec test-msg) ["bad-codec" 42 {}]))) 304 | 305 | (deftest grpc-bad-decode 306 | (testing "Verify that we error decoding an invalid message" 307 | (let [input (async/chan 16384) 308 | output (async/chan 64) 309 | pipeline (lpm/decode example/pb->Money input output {})] 310 | 311 | (go 312 | (doseq [b (repeatedly 256000 #(unchecked-byte (rand-int 256)))] 313 | (>! input b)) 314 | (async/close! input)) 315 | 316 | (loop [] 317 | (if-let [_ (!! input {:nanos "bad data"}) 329 | (async/close! input)) 330 | 331 | (loop [] 332 | (if-let [_ ( {:name input} 340 | addressbook/new-Person 341 | ->any 342 | ->pb 343 | any/pb-> 344 | :name)] 345 | (is (-> input (= output)))))) 346 | 347 | (deftest test-empty-oneof 348 | (testing "Verify that empty messages are serialized/deserialized correctly" 349 | (is (= (empty/new-Empty {}) 350 | (-> {:opt {:e {}}} 351 | empty/new-Selection 352 | ->pb 353 | empty/pb->Selection 354 | :opt 355 | :e)))) 356 | 357 | (testing "Verify that nonempty messages are serialized/deserialized correctly" 358 | (is (= (empty/new-NonEmpty {:i 3}) 359 | (-> {:opt {:ne {:i 3}}} 360 | empty/new-Selection 361 | ->pb 362 | empty/pb->Selection 363 | :opt 364 | :ne)))) 365 | 366 | (testing "Verify that unset messages are serialized/deserialized correctly" 367 | (is (= (empty/new-Selection {:opt nil}) 368 | (-> {:opt {}} 369 | empty/new-Selection 370 | ->pb 371 | empty/pb->Selection))))) 372 | 373 | (deftest test-empty-field 374 | (testing "Verify that empty messages are serialized/deserialized correctly" 375 | (is (= (empty/new-Empty {}) 376 | (-> {:e {}} 377 | empty/new-Container 378 | ->pb 379 | empty/pb->Container 380 | :e)))) 381 | 382 | (testing "Verify that nonempty messages are serialized/deserialized correctly" 383 | (is (= (empty/new-NonEmpty {:i 3}) 384 | (-> {:ne {:i 3}} 385 | empty/new-Container 386 | ->pb 387 | empty/pb->Container 388 | :ne)))) 389 | 390 | (testing "Verify that unset messages are serialized/deserialized correctly" 391 | (is (= (empty/new-Container {:e nil :ne nil}) 392 | (-> {} 393 | empty/new-Container 394 | ->pb 395 | empty/pb->Container))))) 396 | 397 | (deftest test-empty-simple 398 | (testing "Verify that empty messages are serialized/deserialized correctly" 399 | (is (= (empty/new-Empty {}) 400 | (-> {} 401 | empty/new-Empty 402 | ->pb 403 | empty/pb->Empty)))) 404 | 405 | (testing "Verify that nonempty messages are serialized/deserialized correctly" 406 | (is (= (empty/new-NonEmpty {:i 3}) 407 | (-> {:i 3} 408 | empty/new-NonEmpty 409 | ->pb 410 | empty/pb->NonEmpty))))) 411 | 412 | (deftest test-any-bad-encoding 413 | (testing "Verify that we gracefully handle an invalid input to Any encoding" 414 | (is (thrown? java.lang.IllegalArgumentException (->any {:foo "bar"}))))) 415 | 416 | (deftest test-any-bad-decoding 417 | (testing "Verify that we gracefully handle an invalid input to Any decoding" 418 | (is (thrown? clojure.lang.ExceptionInfo (any/pb-> (byte-array [10 8 74 97 110 101 32 68 111 101])))))) --------------------------------------------------------------------------------