├── .gitignore ├── renovate.json ├── project.clj ├── test └── jonotin │ ├── test_helpers.clj │ ├── emulator_test.clj │ └── core_test.clj ├── LICENSE ├── src └── jonotin │ ├── emulator.clj │ └── core.clj └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .lein-repl-history 3 | .lein-failures 4 | pom.xml 5 | pom.xml.asc 6 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:recommended" 4 | ], 5 | "schedule": [ 6 | "at any time" 7 | ], 8 | "packageRules": [ 9 | { 10 | "updateTypes": ["patch"], 11 | "schedule": ["every weekend"] 12 | }, 13 | { 14 | "updateTypes": ["minor", "major"], 15 | "enabled": false 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject jonotin "0.5.0" 2 | :description "Google Pub/Sub Java SDK wrapper" 3 | :url "https://github.com/iprally/jonotin" 4 | :license {:name "The MIT License" 5 | :url "https://opensource.org/licenses/MIT"} 6 | :min-lein-version "2.0.0" 7 | :dependencies [[org.clojure/clojure "1.11.4"] 8 | [com.google.cloud/google-cloud-pubsub "1.132.2"]] 9 | :source-paths ["src"]) 10 | -------------------------------------------------------------------------------- /test/jonotin/test_helpers.clj: -------------------------------------------------------------------------------- 1 | (ns jonotin.test-helpers 2 | (:require [jonotin.emulator :as emulator])) 3 | 4 | (def project-name "jonotin-test-emulator") 5 | 6 | (defn unique-topic-name [] 7 | (str "test-topic-" (random-uuid))) 8 | 9 | (defn unique-subscription-name [] 10 | (str "test-subscription-" (random-uuid))) 11 | 12 | (defn ensure-emulator-host-configured [f] 13 | (emulator/ensure-host-configured) 14 | (f)) 15 | 16 | (defmacro in-parallel [& body] 17 | `(.start (Thread. (fn [] ~@body)))) 18 | -------------------------------------------------------------------------------- /test/jonotin/emulator_test.clj: -------------------------------------------------------------------------------- 1 | (ns jonotin.emulator-test 2 | (:require [clojure.test :refer :all] 3 | [jonotin.emulator :as emulator] 4 | [jonotin.test-helpers :refer :all])) 5 | 6 | (deftest topic-test 7 | (let [topic-name (unique-topic-name)] 8 | (is (= (emulator/create-topic project-name topic-name) 9 | (emulator/get-topic project-name topic-name))) 10 | (is (emulator/delete-topic project-name topic-name)))) 11 | 12 | (deftest subscription-test 13 | (let [topic-name (unique-topic-name) 14 | subscription-name (unique-subscription-name)] 15 | (is (emulator/create-topic project-name topic-name)) 16 | (is (= (emulator/create-subscription project-name topic-name subscription-name) 17 | (emulator/get-subscription project-name subscription-name))) 18 | (is (emulator/delete-subscription project-name subscription-name)))) 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) IPRally Technologies Ltd. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/jonotin/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns jonotin.core-test 2 | (:require [clojure.test :refer :all] 3 | [jonotin.core :as core] 4 | [jonotin.emulator :as emulator] 5 | [jonotin.test-helpers :refer :all])) 6 | 7 | (use-fixtures :once ensure-emulator-host-configured) 8 | 9 | (deftest subscribe!-test 10 | (let [successful-messages (atom 0) 11 | errored-messages (atom 0) 12 | topic-name (unique-topic-name) 13 | subscription-name (unique-subscription-name)] 14 | (emulator/create-topic project-name topic-name) 15 | (emulator/create-subscription project-name topic-name subscription-name) 16 | (in-parallel 17 | (core/subscribe! {:project-name project-name 18 | :subscription-name subscription-name 19 | :handle-msg-fn #(case % 20 | "normal" (swap! successful-messages inc) 21 | "exceptional-retry" (throw (ex-info "" {::retry? true})) 22 | "exceptional-no-retry" (throw (ex-info "" {::retry? false}))) 23 | :handle-error-fn (fn [e] 24 | (swap! errored-messages inc) 25 | (if (::retry? (ex-data e)) 26 | {:ack false} 27 | {:ack true}))})) 28 | (core/publish! {:project-name project-name 29 | :topic-name topic-name 30 | :messages ["normal" 31 | "exceptional-retry" 32 | "normal" 33 | "exceptional-no-retry" 34 | "normal"]}) 35 | (Thread/sleep 500) 36 | (is (= 3 @successful-messages)) 37 | (is (= 6 @errored-messages)))) 38 | 39 | (deftest publish!-test 40 | (let [topic-name (unique-topic-name)] 41 | (emulator/create-topic project-name topic-name) 42 | (is (= {:delivered-messages 2} 43 | (core/publish! {:project-name project-name 44 | :topic-name topic-name 45 | :messages ["hello" "goodbye"]}))))) 46 | -------------------------------------------------------------------------------- /src/jonotin/emulator.clj: -------------------------------------------------------------------------------- 1 | (ns jonotin.emulator 2 | (:import (com.google.api.gax.core NoCredentialsProvider) 3 | (com.google.api.gax.grpc GrpcTransportChannel) 4 | (com.google.api.gax.rpc FixedTransportChannelProvider) 5 | (com.google.cloud.pubsub.v1 SubscriptionAdminClient SubscriptionAdminSettings TopicAdminClient TopicAdminSettings) 6 | (com.google.pubsub.v1 PushConfig SubscriptionName TopicName) 7 | (io.grpc ManagedChannelBuilder))) 8 | 9 | (def pubsub-emulator-host (System/getenv "PUBSUB_EMULATOR_HOST")) 10 | 11 | (def host-configured? (boolean pubsub-emulator-host)) 12 | 13 | (defn ensure-host-configured [] 14 | (when-not host-configured? 15 | (throw (ex-info "PUBSUB_EMULATOR_HOST is required for using Google Cloud Pub/Sub emulator" 16 | {:type :jonotin/emulator-host-not-configured})))) 17 | 18 | (defn build-channel [] 19 | (ensure-host-configured) 20 | (-> pubsub-emulator-host 21 | (ManagedChannelBuilder/forTarget) 22 | (.usePlaintext) 23 | (.build))) 24 | 25 | (defn set-builder-options [builder emulator-channel] 26 | (ensure-host-configured) 27 | (-> builder 28 | (.setCredentialsProvider (NoCredentialsProvider/create)) 29 | (.setChannelProvider (-> emulator-channel 30 | (GrpcTransportChannel/create) 31 | (FixedTransportChannelProvider/create))))) 32 | 33 | (defn- set-admin-client-builder-options [builder emulator-channel] 34 | (ensure-host-configured) 35 | (-> builder 36 | (.setCredentialsProvider (NoCredentialsProvider/create)) 37 | (.setTransportChannelProvider (-> emulator-channel 38 | (GrpcTransportChannel/create) 39 | (FixedTransportChannelProvider/create))))) 40 | 41 | (defn with-topic-admin-client [f] 42 | (let [channel (build-channel) 43 | topic-admin-settings (-> (TopicAdminSettings/newBuilder) 44 | (set-admin-client-builder-options channel) 45 | (.build)) 46 | topic-client (TopicAdminClient/create ^TopicAdminSettings topic-admin-settings)] 47 | (try 48 | (f topic-client) 49 | (finally 50 | (.shutdown topic-client) 51 | (.shutdown channel))))) 52 | 53 | (defn with-subscription-admin-client [f] 54 | (let [channel (build-channel) 55 | subscription-admin-settings (-> (SubscriptionAdminSettings/newBuilder) 56 | (set-admin-client-builder-options channel) 57 | (.build)) 58 | subscription-client (SubscriptionAdminClient/create ^SubscriptionAdminSettings subscription-admin-settings)] 59 | (try 60 | (f subscription-client) 61 | (finally 62 | (.shutdown subscription-client) 63 | (.shutdown channel))))) 64 | 65 | (defn create-topic [project-name topic-name] 66 | (with-topic-admin-client 67 | #(.createTopic % (TopicName/of project-name topic-name)))) 68 | 69 | (defn get-topic [project-name topic-name] 70 | (with-topic-admin-client 71 | #(.getTopic % (TopicName/of project-name topic-name)))) 72 | 73 | (defn delete-topic [project-name topic-name] 74 | (with-topic-admin-client 75 | (fn [client] 76 | (.deleteTopic client (TopicName/of project-name topic-name)) 77 | true))) 78 | 79 | (defn create-subscription [project-name topic-name subscription-name & {:keys [ack-deadline-seconds] 80 | :or {ack-deadline-seconds 10}}] 81 | (with-subscription-admin-client 82 | (fn [client] 83 | (let [project-subscription (SubscriptionName/of project-name subscription-name) 84 | project-topic (TopicName/of project-name topic-name) 85 | push-config (PushConfig/getDefaultInstance)] 86 | (.createSubscription client project-subscription project-topic push-config ack-deadline-seconds))))) 87 | 88 | (defn get-subscription [project-name subscription-name] 89 | (with-subscription-admin-client 90 | #(.getSubscription % (SubscriptionName/of project-name subscription-name)))) 91 | 92 | (defn delete-subscription [project-name subscription-name] 93 | (with-subscription-admin-client 94 | (fn [client] 95 | (.deleteSubscription client (SubscriptionName/of project-name subscription-name)) 96 | true))) 97 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jonotin 2 | 3 | Dead-simple Google Cloud Pub/Sub from Clojure. jonotin is a never used Finnish word for a thing that queues. Read more about jonotin from [IPRally blog](https://www.iprally.com/news/google-cloud-pubsub-with-clojure). 4 | 5 | ## Latest version 6 | 7 | Leiningen/Boot 8 | ```clj 9 | [jonotin "0.5.0"] 10 | ``` 11 | 12 | Clojure CLI/deps.edn 13 | ```clj 14 | jonotin {:mvn/version "0.5.0"} 15 | ``` 16 | 17 | Gradle 18 | ```clj 19 | compile 'jonotin:jonotin:0.5.0' 20 | ``` 21 | 22 | Maven 23 | ```clj 24 | 25 | jonotin 26 | jonotin 27 | 0.5.0 28 | 29 | ``` 30 | 31 | ### Publish! 32 | 33 | Publish messages to topic. Thresholds can be configured through options: 34 | - Delay Threshold: Counting from the time that the first message is queued, once this delay has passed, then send the batch. The default value is 100 millisecond, which is good for large amount of messages. 35 | - Message Count Threshold: Once this many messages are queued, send all of the messages in a single call, even if the delay threshold hasn't elapsed yet. The default value is 10 messages. 36 | - Request Byte Threshold: Once the number of bytes in the batched request reaches this threshold, send all of the messages in a single call, even if neither the delay or message count thresholds have been exceeded yet. The default value is 1000 bytes. 37 | 38 | ```clj 39 | (require '[jonotin.core :as jonotin]) 40 | 41 | (jonotin/publish! {:project-name "my-gcloud-project-id" 42 | :topic-name "my-topic" 43 | :messages ["msg1" "msg2"] 44 | :options {:request-byte-threshold 100 45 | :element-count-threshold 10 46 | :delay-threshold 1000}}) 47 | ``` 48 | 49 | ### Subscribe! 50 | 51 | Subscribe processes messages from the queue concurrently. 52 | ```clj 53 | (require '[jonotin.core :as jonotin]) 54 | 55 | (jonotin/subscribe! {:project-name "my-gcloud-project-id" 56 | :subscription-name "my-subscription-name" 57 | :handle-msg-fn (fn [msg] 58 | (println "Handling" msg)) 59 | :handle-error-fn (fn [e] 60 | (println "Oops!" e))}) 61 | ``` 62 | 63 | Error handler function supports return value to determine if the message should be acknowledged or not. 64 | ```clj 65 | {:ack boolean} 66 | ``` 67 | 68 | Subscribe with concurrency control. 69 | ```clj 70 | (require '[jonotin.core :as jonotin]) 71 | 72 | (jonotin/subscribe! {:project-name "my-gcloud-project-id" 73 | :subscription-name "my-subscription-name" 74 | :options {:parallel-pull-count 2 75 | :executor-thread-count 4} 76 | :handle-msg-fn (fn [msg] 77 | (println "Handling" msg)) 78 | :handle-error-fn (fn [e] 79 | (println "Oops!" e))}) 80 | ``` 81 | 82 | ## Testing your application 83 | 84 | jonotin supports Google Cloud Pub/Sub emulator. When environment variable `PUBSUB_EMULATOR_HOST` is set, then jonotin will use emulator instead of the real GCP Pub/Sub API. 85 | 86 | To set up the emulator, follow [Testing apps locally with the emulator](https://cloud.google.com/pubsub/docs/emulator) guide for setting up the emulator. 87 | 88 | Once your emulator is up-and-running, configure `PUBSUB_EMULATOR_HOST`: 89 | 90 | ```bash 91 | $(gcloud beta emulators pubsub env-init) && echo $PUBSUB_EMULATOR_HOST 92 | # => localhost:8085 93 | ``` 94 | 95 | Now run your application and witness jonotin diligently using the emulator. 96 | 97 | Note that emulator is ephemeral and no topics or subscriptions exist when it starts. jonotin includes helpers for creating/removing those: 98 | 99 | ```clojure 100 | (require '[jonotin.emulator :as emulator]) 101 | 102 | ;; Create a topic 103 | (emulator/create-topic "project-name" "topic-name") 104 | 105 | ;; Get the topic 106 | (emulator/get-topic "project-name" "topic-name") 107 | 108 | ;; Delete the topic 109 | (emulator/delete-topic "project-name" "topic-name") 110 | 111 | ;; Create a subscription 112 | (emulator/create-subscription "project-name" "topic-name" "subscription-name") 113 | 114 | ;; Create a subscription with custom ack-deadline-seconds 115 | (emulator/create-subscription "project-name" "topic-name" "subscription-name" {:ack-deadline-seconds 600}) 116 | 117 | ;; Get the subscription 118 | (emulator/get-subscription "project-name" "subscription-name") 119 | 120 | ;; Delete the subscription 121 | (emulator/delete-subscription "project-name" "subscription-name") 122 | ``` 123 | 124 | To be sure that jonotin in your test suite targets Pub/Sub emulator, use 125 | 126 | ```clojure 127 | (emulator/ensure-host-configured) 128 | ``` 129 | 130 | where appropriate. This function will throw if `PUBSUB_EMULATOR_HOST` is not configured. 131 | 132 | # Development 133 | 134 | ## Testing jonotin 135 | 136 | Once you've set up the Google Cloud Pub/Sub emulator, start the emulator for jonotin test project: 137 | 138 | ```bash 139 | gcloud beta emulators pubsub start --project=jonotin-test-emulator 140 | ``` 141 | 142 | In another shell session, configure `PUBSUB_EMULATOR_HOST` and run the tests: 143 | 144 | ```bash 145 | $(gcloud beta emulators pubsub env-init) && echo $PUBSUB_EMULATOR_HOST 146 | # => localhost:8085 147 | 148 | lein test 149 | ``` 150 | -------------------------------------------------------------------------------- /src/jonotin/core.clj: -------------------------------------------------------------------------------- 1 | (ns jonotin.core 2 | (:require [jonotin.emulator :as emulator]) 3 | (:import (com.google.api.core ApiFutureCallback 4 | ApiFutures 5 | ApiService$Listener) 6 | (com.google.api.gax.batching BatchingSettings) 7 | (com.google.api.gax.core InstantiatingExecutorProvider) 8 | (com.google.cloud.pubsub.v1 MessageReceiver 9 | Publisher 10 | Subscriber) 11 | (com.google.common.util.concurrent MoreExecutors) 12 | (com.google.protobuf ByteString) 13 | (com.google.pubsub.v1 ProjectSubscriptionName 14 | ProjectTopicName 15 | PubsubMessage) 16 | (java.util.concurrent TimeUnit) 17 | (org.threeten.bp Duration))) 18 | 19 | (defn- get-executor-provider [{:keys [executor-thread-count]}] 20 | (let [executor-provider-builder (cond-> (InstantiatingExecutorProvider/newBuilder) 21 | executor-thread-count (.setExecutorThreadCount executor-thread-count))] 22 | (.build executor-provider-builder))) 23 | 24 | (defn subscribe! [{:keys [project-name subscription-name handle-msg-fn handle-error-fn options]}] 25 | (let [subscription-name-obj (ProjectSubscriptionName/format project-name subscription-name) 26 | msg-receiver (reify MessageReceiver 27 | (receiveMessage [_ message consumer] 28 | (let [data (.toStringUtf8 (.getData message))] 29 | (try 30 | (handle-msg-fn data) 31 | (.ack consumer) 32 | (catch Throwable e 33 | (if (some? handle-error-fn) 34 | (let [error-response (handle-error-fn e)] 35 | (if (or (nil? error-response) 36 | (:ack error-response)) 37 | (.ack consumer) 38 | (.nack consumer))) 39 | (do 40 | (.ack consumer) 41 | (throw e)))))))) 42 | emulator-channel (when emulator/host-configured? 43 | (emulator/build-channel)) 44 | subscriber-builder (cond-> (Subscriber/newBuilder subscription-name-obj msg-receiver) 45 | (:parallel-pull-count options) (.setParallelPullCount (:parallel-pull-count options)) 46 | (:executor-thread-count options) (.setExecutorProvider (get-executor-provider options)) 47 | emulator-channel (emulator/set-builder-options emulator-channel)) 48 | subscriber (.build subscriber-builder) 49 | listener (proxy [ApiService$Listener] [] 50 | (failed [from failure] 51 | (println "Jonotin failure with msg handling -" failure)))] 52 | (.addListener subscriber listener (MoreExecutors/directExecutor)) 53 | (.awaitRunning (.startAsync subscriber)) 54 | (.awaitTerminated subscriber) 55 | (some-> emulator-channel .shutdown))) 56 | 57 | (defn publish! [{:keys [project-name topic-name messages options]}] 58 | (when (> (count messages) 10000) 59 | (throw (ex-info "Message count over safety limit" 60 | {:type :jonotin/batch-size-limit 61 | :message-count (count messages)}))) 62 | (let [topic (ProjectTopicName/of project-name topic-name) 63 | batching-settings (-> (BatchingSettings/newBuilder) 64 | (.setRequestByteThreshold (or (:request-byte-threshold options) 1000)) 65 | (.setElementCountThreshold (or (:element-count-threshold options) 10)) 66 | (.setDelayThreshold (Duration/ofMillis (or (:delay-threshold options) 100))) 67 | .build) 68 | emulator-channel (when emulator/host-configured? 69 | (emulator/build-channel)) 70 | publisher-builder (cond-> (Publisher/newBuilder topic) 71 | batching-settings (.setBatchingSettings batching-settings) 72 | emulator-channel (emulator/set-builder-options emulator-channel)) 73 | publisher (.build publisher-builder)] 74 | (try 75 | (let [callback-fn (reify ApiFutureCallback 76 | (onFailure [_ t] 77 | (throw (ex-info "Failed to publish message" 78 | {:type :jonotin/publish-failure 79 | :message t}))) 80 | (onSuccess [_ _result] 81 | ())) 82 | publish-msg-fn (fn [msg-str] 83 | (let [msg-builder (PubsubMessage/newBuilder) 84 | data (ByteString/copyFromUtf8 msg-str) 85 | msg (-> msg-builder 86 | (.setData data) 87 | .build) 88 | msg-future (.publish publisher msg)] 89 | (ApiFutures/addCallback msg-future callback-fn (MoreExecutors/directExecutor)) 90 | msg-future)) 91 | futures (map publish-msg-fn messages) 92 | message-ids (.get (ApiFutures/allAsList futures))] 93 | {:delivered-messages (count message-ids)}) 94 | (finally 95 | (.shutdown publisher) 96 | (.awaitTermination publisher 5 TimeUnit/MINUTES) 97 | (some-> emulator-channel .shutdown))))) 98 | --------------------------------------------------------------------------------