├── .gitignore ├── .idea └── vcs.xml ├── .travis.yml ├── LICENSE ├── README.md ├── build.sh ├── doc ├── design.md ├── pool.md ├── retry.md └── stream.md ├── lint.sh ├── project.clj ├── src ├── clojure │ └── tcp_driver │ │ ├── driver.clj │ │ ├── io │ │ ├── conn.clj │ │ ├── pool.clj │ │ └── stream.clj │ │ └── routing │ │ ├── policy.clj │ │ └── retry.clj └── java │ └── tcpdriver │ └── io │ └── IOUtil.java └── test └── tcp_driver ├── driver_test.clj ├── io ├── conn_test.clj └── pool_test.clj ├── routing ├── policy_test.clj └── retry_test.clj └── test └── util.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | script: ./build.sh 3 | 4 | jdk: 5 | oraclejdk8 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tcp-driver 2 | 3 | The idea is the access TCP client connections like any other product driver code would e.g the cassandra or mondodb driver.
4 | 5 | There are allot of situations where software in the past (my own experience) became unstable because the TCP connections 6 | were not written or treated with the equivalent importance as server connections. 7 | 8 | Writing the TCP connection as if it were a product driver sets a certain design mindset. 9 | 10 | For design decisions see: https://github.com/gerritjvv/tcp-driver/blob/master/doc/design.md 11 | 12 | [![Build Status](https://travis-ci.org/gerritjvv/tcp-driver.svg)](https://travis-ci.org/gerritjvv/tcp-driver) 13 | 14 | [![Clojars Project](https://img.shields.io/clojars/v/tcp-driver.svg)](https://clojars.org/tcp-driver) 15 | 16 | 17 | ## Usage 18 | 19 | ```clojure 20 | (require '[tcp-driver.test.util :as test-util]) 21 | (require '[tcp-driver.io.stream :as tcp-stream]) 22 | (require '[tcp-driver.io.conn :as tcp-conn]) 23 | (require '[tcp-driver.driver :as tcp-driver]) 24 | (require '[clojure.test :refer :all]) 25 | 26 | ;;write a short string to the connection 27 | (defn write-msg [conn msg] 28 | (tcp-stream/write-short-str conn (str msg)) 29 | conn) 30 | 31 | ;;function that reads a short string 32 | (defn read-msg [conn timeout-ms] 33 | (tcp-stream/read-short-str conn timeout-ms)) 34 | 35 | ;;create a tcp driver using the default retries etc and 36 | ;;send the passing io-f function to the driver 37 | (defn send-io-f [io-f] 38 | (test-util/with-echo-server 39 | (fn [server] 40 | (let [driver (tcp-driver/create-default [{:host "localhost" :port (:port server)}]) 41 | ret-msg (tcp-driver/send-f 42 | driver 43 | io-f 44 | 10000)] 45 | ;;use ret-msg to make it clear that io-f return value via tcp-driver/send-f 46 | ret-msg)))) 47 | 48 | ;;write hi message and read the response from the echo server 49 | (= (send-io-f 50 | #(read-msg 51 | (write-msg % "HI") 1000)) 52 | "HI") 53 | ``` 54 | 55 | ## IO Stream Util functions 56 | 57 | The namespace ```tcp-driver.io.stream``` provides helper functions to work with 58 | writing an reading data from the tcp connections returned by the driver 59 | 60 | See: https://github.com/gerritjvv/tcp-driver/blob/master/doc/stream.md 61 | 62 | ## Driver Configuration 63 | 64 | The ```tcp-driver.driver/create-default``` function takes several args 65 | ```[hosts & {:keys [routing-conf pool-conf retry-limit] :or {retry-limit 10 routing-conf {} pool-conf {}}}]``` 66 | 67 | ### Connection Pooling 68 | 69 | 70 | The pool-conf arg must have the schema ```tcp-driver.io.pool/PoolConfSchema``` and can be used to 71 | configure the default tcp pool. 72 | 73 | See: https://github.com/gerritjvv/tcp-driver/blob/master/doc/pool.md 74 | 75 | ### Routing Policy 76 | 77 | The routing-conf arg must have the schema ```{:keys [select-f blacklist-expire]}``` 78 | 79 | The default select-f used is rand-nth. 80 | The function will receive a list of host map items and can select between any of them. 81 | 82 | On any exception in ```send-f``` the default routing policy will blacklist the node. 83 | 84 | #### Adding removing hosts 85 | 86 | 87 | *Usage* 88 | 89 | ```clojure 90 | (require '[tcp-driver.driver :as tcp-driver]) 91 | 92 | (def driver (tcp-driver/create-default [{:host "localhost" :port (:port server)}])) 93 | 94 | ;;; add a host to the driver 95 | (tcp-driver/add-host driver {:host "myhost" :port 123}) 96 | 97 | ;;; remove a host from the driver 98 | (tcp-driver/remove-host driver {:host "myhost" :port 123}) 99 | 100 | ;;; blacklist a host -- this means the host will not be used for a certain amount of time defined in the routing policy 101 | (tcp-driver/blacklist-host driver {:host "myhost" :port 123}) 102 | 103 | ``` 104 | 105 | *Routing Policy* 106 | 107 | The namespace ```tcp-driver.routing.policy``` contains the protocols and defaults for defining routing policies and their behaviours. 108 | 109 | The ```tcp-driver.driver/create-default``` function uses the ```DefaultRountingPolicy``` ```IRoute``` policy which 110 | 111 | * Selects hosts randomly 112 | * Blacklist any nodes if any exception reported 113 | 114 | 115 | To define a custom routing policy create you're own driver constructor method using ```tcp-driver.driver/create-default``` as an example, 116 | and define an implementation of the ```IRoute``` protocol. 117 | 118 | 119 | ### Retry Policy 120 | 121 | The default retry policy is used in the ```tcp-driver.driver/create-default``` function, 122 | to configure a custom retry policy use the ```tcp-driver.driver/create``` function. 123 | 124 | See: https://github.com/gerritjvv/tcp-driver/blob/master/doc/retry.md 125 | 126 | 127 | ## Feature complete/updates/maintenance 128 | 129 | This library is considered complete from what I set out to create, 130 | any pull requests, suggestions or bug reports are welcome, note that if I do not respond 131 | to PR or Issues opened feel free to ping me via email gerritjvv@gmail.com, git hub's notifications 132 | do not always work :) 133 | 134 | 135 | ## License 136 | 137 | Copyright © 2015 gerritjvv 138 | 139 | Distributed under the Eclipse Public License either version 1.0 140 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | 5 | $DIR/lint.sh 6 | 7 | if [ "$?" -ne 0 ]; then 8 | echo "Failed linting" 9 | exit -1 10 | fi 11 | 12 | cd "$DIR"; lein test 13 | 14 | if [ "$?" -ne 0 ]; then 15 | echo "Failed testing" 16 | exit -1 17 | fi 18 | 19 | cd "$DIR"; lein install; -------------------------------------------------------------------------------- /doc/design.md: -------------------------------------------------------------------------------- 1 | # Introduction to tcp-driver 2 | 3 | The idea is the access TCP client connections like any other product driver code would e.g the cassandra or mondodb driver.
4 | 5 | There are allot of situations where software in the past (my own experience) became unstable because the TCP connections 6 | were not written or treated with the equivalent importance as server connections. 7 | 8 | Writing the TCP connection as if it were a product driver sets a certain design mindset. 9 | 10 | # Mindset and expectations: 11 | 12 | ## Making configurable and transparent: 13 | 14 | * Single open close connections 15 | * Connection pooling and re-use. 16 | * Connection testing and reconnecting before use. 17 | * Reconnect and Retry on IO failure 18 | * Retry policy e.g default would be Back-off-retry policy 19 | * Automatic closing of open resources on application shutdown (subcribe to the Java Shutdown Hook) 20 | 21 | ## Multiple endpoints 22 | 23 | * Add/Remove enpoints dynamically. 24 | * Blacklisting enpoints (support blaclisting N milliseconds of a host where its still alive but not usable) 25 | * Support custom routing to select hosts based on custom logic, default should be random. 26 | 27 | ## IO Read timeout 28 | * support timeouts on blocking io reads. 29 | 30 | ## Throttling 31 | 32 | * Support Rate limiting (fancy for some usecases but easy to do with the google guava lib). 33 | 34 | 35 | ## Async and multi threaded 36 | 37 | The code should support multi threading and async usage (not NIO). We can support multi threading by 38 | allowing each thread to have its own connection from a Pool, thus not requiring locking or syncing if 39 | the pool has enough connections. Synchronization is only required once the Pool runs out of free connections 40 | and threads need to compete for resources. 41 | 42 | # What about NIO? 43 | 44 | From my own experience in writing high volume performant client TCP IO code its my opinion that NIO doesn't add 45 | any value in terms of performance. It only adds complications. The reasons are: 46 | 47 | * client code requires back pressure, NIO is async and mostly adds connections to memmory to eventually 48 | give you an OOM. 49 | * error handling and feedback to client calling code is easier/cleaner and less bug prone via direct blocking IO. 50 | * for client code bocking IO can be as fast or even faster than NIO (when using poolable connections). 51 | 52 | 53 | # Design 54 | 55 | 56 | ## Service Provider Interface (public api) 57 | 58 | conn = create [{custom-env conf routing rate-limiter retry-policy} bootstrap-end-points] 59 | send [ conn i-o-f data timeout-ms] : RESP ;; the data is applied to the i-o-f 60 | send [ conn exec-service i-o-f data timeout-ms] : Promise[RESP] 61 | 62 | i-o-f [ conn data timeout-ms] : RESP 63 | 64 | stats = conn-stats [ conn ] 65 | 66 | close [ conn ] 67 | 68 | 69 | ### Routing 70 | IRouting 71 | select-host [ custom-env hosts ] : Host 72 | 73 | ## I/O Timeouts 74 | 75 | Support IO timeout see: https://github.com/gerritjvv/kafka-fast/blob/master/kafka-clj/java/kafka_clj/util/IOUtil.java#L37 76 | 77 | Provide support functions for common java primites (and short strings) 78 | See: https://github.com/gerritjvv/kafka-fast/blob/master/kafka-clj/java/kafka_clj/util/IOUtil.java#L21 79 | 80 | * read-bytes [reader bts-len timeout-ms] : byte-array 81 | * read-int [reader timeout-ms ] : int 82 | * read-long [reader timeout-ms ] : long 83 | * read-double [reader timeout-ms] : double 84 | * read-short [reader timeout-ms ] : short 85 | * read-float [reader timeout-ms ] : short 86 | * read-short-str [reader timeout-ms ] : String 87 | * read-bool [reader timeout-ms] : boolean 88 | 89 | * write-bytes [writer bts from len ] 90 | * write-int [writer int] 91 | * write-double ... 92 | .... for all primitives 93 | * write-short-str [writer string] 94 | 95 | Implemented using ```tcp-driver.io.stream``` 96 | 97 | # Implementation 98 | 99 | create-conn [ bootstrap-hosts pool-conf routing-obj retry-policy conf] 100 | { :pool (pool 101 | :conf conf 102 | :bootstrap-hosts bootstrap-hosts 103 | :retry-policy retry-policy 104 | :routing-policy routing-obj 105 | :routing-env {:hosts (atom (set boostrap-hosts))} 106 | 107 | 108 | 109 | add-host! [ ctx host host-conf] 110 | (assoc-stm! (get-in ctx [:routing-env :hosts]) host) 111 | 112 | remove-host! [ctx host host-conf] 113 | (dissoc-stm! (get-in ctx [:routing-env :hosts]) host) 114 | 115 | send-op [ctx host i-o-f data timeout-ms] 116 | try: 117 | conn = (pool/borrow (:pool ctx) host timeout-ms) 118 | try: 119 | return (i-o-f conn data timeout-ms) 120 | finally: 121 | (pool/return pool-inst conn) 122 | 123 | catch Exception e 124 | (pool/invalidate pool-inst conn) 125 | return e 126 | 127 | 128 | send [ ctx i-o-f data timeout-ms] 129 | 130 | hosts = (:bootstrap-hosts ctx) 131 | routing-policy (:routing-policy ctx) 132 | 133 | 134 | loop [retry-state nil] 135 | host = (routing/select-host routing-policy (:routing-env ctx) hosts) 136 | 137 | if host == null 138 | throw NoHostAvailableException 139 | 140 | ret-val = (send-op ctx host i-o-f data timeout-ms) 141 | 142 | if ret-val == Exception 143 | [retry-state retry-op] = (io-retry/retry-action! (:retry-policy ctx) (:conf ctx) conn host retry-state) 144 | if retry-op == :recur 145 | (recur retry-state) 146 | else 147 | (throw ret-val) 148 | else 149 | ret-val 150 | 151 | NS: routing 152 | Protocol IRouting 153 | select-host 154 | add-host 155 | remove-host 156 | 157 | NS: io-retry 158 | multi-method retry-action! identity 159 | 160 | defmulti retry-action! :retry [_ _ _ _ retry-state] 161 | [retry-state (if (available-tries? retry-state) :retry nil)] 162 | 163 | defmulti retry-action! :blacklist-retry [ctx conn host e retry-state] 164 | (routing/blacklist! (:routing-policy ctx) (:routing-env ctx) conn host) 165 | [retry-state (if (hosts-left? ... ) :retry :nil)] 166 | 167 | defmulti retry-action! :default [ _ _ _ e _ ] nil 168 | 169 | 170 | 171 | 172 | 173 | 174 | Even though the implementation language is Clojure the public api should give first class support for: 175 | 176 | * Java 1.8 177 | * Clojure 178 | 179 | -------------------------------------------------------------------------------- /doc/pool.md: -------------------------------------------------------------------------------- 1 | # Connection pooling 2 | 3 | The namespace ```tcp-driver.io.pool``` provides all the functions required to create a keyed 4 | connection pool. Each key is a host address of type ```{:host :port }``` and 5 | creates a sub pool of TCP connections. 6 | 7 | The default ```IPool``` implementation uses the apache commons object pool, 8 | see https://commons.apache.org/proper/commons-pool/ 9 | 10 | ## Usage 11 | 12 | ```clojure 13 | 14 | (require '[tcp-driver.io.pool :as tcp-pool]) 15 | (require '[tcp-driver.io.conn :as tcp-conn]) 16 | 17 | (def pool (tcp-pool/create-tcp-pool {})) 18 | (def host (tcp-conn/host-address "localhost" 8001)) 19 | (def timeout-ms 1000) 20 | 21 | 22 | ;;manual borrow and return 23 | (let [conn (tcp-pool/borrow pool host timeout-ms)] 24 | ;;do something with conn 25 | (tcp-pool/return pool host)) 26 | 27 | ;; or convenience function 28 | 29 | (tcp-pool/try-conn pool host timeout-ms 30 | (fn [conn] 31 | ;;do something with conn 32 | )) 33 | 34 | ``` 35 | 36 | ### Pool Configuration 37 | 38 | ```clojure (tcp-pool/create-tcp-pool conf)``` 39 | 40 | Conf values are: 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 |
KeyValue/Description
:max-idle-per-keyMax connections idle for a pool to a particular server, default is 2
:min-idle-per-keyMinimum connections idle for a pool to a particular server, default is 0
:max-totalThe maximum number of connections to open for all servers, default is 100
:max-total-per-keySame as :max-total but per key, default is 100
:min-idle-per-keyMaximum idle connections per key, default is 0
:close-pool-jvm-shutdownIf set to true, the pool will be closed for all keys and connections on JVM shutdown
51 | -------------------------------------------------------------------------------- /doc/retry.md: -------------------------------------------------------------------------------- 1 | # Connection Retry Policy 2 | 3 | ## Overview 4 | 5 | A IO Function can at any point throw an exception, the retry policy determines how 6 | on Exception should the function be called, if any or if the exception should be 7 | re thrown 8 | 9 | ## Default Retry Policy 10 | 11 | A ```tcp-driver.routing.retry.DefaultRetryPolicy``` record is provided that implements 12 | ```tcp-driver.routing.retry.IRetry``` protocol, and calls the IO Functions if 13 | and exception is thrown, doing so N times, where N is provided as part of the creation 14 | of DefaultRetryPolicy. 15 | 16 | 17 | ## Usage 18 | 19 | ```clojure 20 | (require '[tcp-driver.routing.retry :as retry]) 21 | 22 | (let [rpolicy (retry/retry-policy 3) 23 | f (fn [] (prn "try-function and throw ex") (throw (Exception. "test")))] 24 | (try 25 | (retry/with-retry rpolicy f) 26 | (catch Exception e 27 | (prn (= (get (ex-data e) :retries) 3))))) 28 | 29 | ``` -------------------------------------------------------------------------------- /doc/stream.md: -------------------------------------------------------------------------------- 1 | #IO Stream Support 2 | 3 | The namespace ```tcp-driver.io.stream``` provides all the functions required to open/write/read 4 | to and from an ```InputStream``` or ```OutputStream```. 5 | 6 | The java class ```tcpdriver.io.IOUtil``` implements the backend of the stream functions for efficiency. 7 | 8 | ## Blocking IO 9 | 10 | Note that for client communication blocking IO is the most convenient and also simplest to reason about, 11 | providing automatic back pressure. The only issue with blocking reads is that there are no timeouts implemented 12 | in the API, this library implements timeouts on blocking reads without any background threads. 13 | 14 | Have a look at the java class ```tcpdriver.io.IOUtil``` to see how its implemented using avialble and partial reads. 15 | 16 | ## Timeouts 17 | 18 | All stream read operations have a timeout in milliseconds argument, and will throw a ```TimeoutException``` if 19 | the required bytes could not be read from the connection's ```InputStream``` in that time. 20 | 21 | 22 | 23 | ## Example 24 | 25 | ```clojure 26 | 27 | (require '[tcp-driver.io.stream :as tcp-stream]) 28 | 29 | ;;get a connection either directly or from a pool 30 | ;;write a short string 31 | (tcp-stream/write-short-str conn "hi") 32 | 33 | ;;read a short string 34 | (tcp-stream/read-short-str conn 1000) 35 | 36 | ``` 37 | 38 | 39 | -------------------------------------------------------------------------------- /lint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | lein eastwood "{:exclude-linters [:unused-ret-vals] :exclude-namespaces [tcp-driver.io.conn-test tcp-driver.io.pool-test tcp-driver.routing.policy-test tcp-driver.routing.retry-test tcp-driver.test.util tcp-driver.driver-test]}" -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject tcp-driver "0.1.2-SNAPSHOT" 2 | :description "Java/Clojure TCP Connections done right" 3 | :url "https://github.com/gerritjvv/tcp-driver" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | 7 | :global-vars {*warn-on-reflection* true 8 | *assert* true} 9 | 10 | :javac-options ["-target" "1.8" "-source" "1.8" "-Xlint:-options"] 11 | :jvm-opts ["-Xmx1g"] 12 | 13 | :source-paths ["src/clojure"] 14 | :java-source-paths ["src/java"] 15 | :dependencies [ 16 | [org.clojure/clojure "1.8.0"] 17 | 18 | [fun-utils "0.6.2"] 19 | [org.apache.commons/commons-pool2 "2.4.2"] 20 | [prismatic/schema "1.1.3"]] 21 | 22 | :plugins [[jonase/eastwood "0.2.3"]]) 23 | -------------------------------------------------------------------------------- /src/clojure/tcp_driver/driver.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc " 3 | 4 | The idea is the access TCP client connections like any other product driver code would e.g the cassandra or mondodb driver. 5 | There are allot of situations where software in the past (my own experience) became unstable because the TCP connections 6 | were not written or treated with the equivalent importance as server connections. 7 | Writing the TCP connection as if it were a product driver sets a certain design mindset. 8 | 9 | This is the main entry point namespace for this project, the other namespaces are: 10 | 11 | tcp-driver.io.conn -> TCP connection abstractions 12 | tcp-driver.io-pool -> Connection pooling and creating object pools 13 | tcp-driver.io-stream -> reading and writing from TCP Connections 14 | 15 | The main idea is that a driver can point at 1,2 or more servers, for each server a Pool of Connections are maintained 16 | using a KeyedObjectPool from the commons pool2 library. 17 | 18 | Pooling connections is done not only for performance but also make connection error handling easier, the connection 19 | is tested and retried before given to the application user, and if you have a connection at least at the moment 20 | of handoff you know that it is connection and ready to go. 21 | 22 | "} 23 | tcp-driver.driver 24 | 25 | (:require 26 | [schema.core :as s] 27 | [clojure.tools.logging :refer [error]] 28 | [tcp-driver.io.pool :as tcp-pool] 29 | [tcp-driver.io.conn :as tcp-conn] 30 | [tcp-driver.routing.policy :as routing] 31 | [tcp-driver.routing.retry :as retry]) (:import (java.io IOException))) 32 | 33 | 34 | ;;;;;;;;;;;;;; 35 | ;;;;;; Schemas and Protocols 36 | 37 | (def IRouteSchema (s/pred #(satisfies? routing/IRoute %))) 38 | 39 | (def IRetrySchema (s/pred #(satisfies? retry/IRetry %))) 40 | 41 | (def IPoolSchema (s/pred #(satisfies? tcp-pool/IPool %))) 42 | 43 | (def DriverRetSchema {:pool tcp-pool/IPoolSchema 44 | :routing-policy IRouteSchema 45 | :retry-policy IRetrySchema}) 46 | 47 | 48 | ;;;;;;;;;;;;;; 49 | ;;;;;; Private functions 50 | 51 | (defn throw-no-connection! [] 52 | (throw (RuntimeException. "No connection is available to perform the send"))) 53 | 54 | 55 | (defn select-send! 56 | "ctx - DriverRetSchema 57 | host-address if specified this host is used, otherwise the routing policy is asked for a host 58 | io-f - function that takes a connection and on error throws an exception 59 | timeout-ms - connection timeout" 60 | ([ctx io-f timeout-ms] 61 | (select-send! ctx nil io-f timeout-ms)) 62 | ([ctx host-address io-f timeout-ms] 63 | {:pre [ctx io-f timeout-ms]} 64 | (loop [i 0] 65 | (if-let [host (if host-address host-address (routing/-select-host (:routing-policy ctx)))] 66 | 67 | (let [pool (:pool ctx)] 68 | 69 | ;;;try the io-f, if an exception then only if we haven't tried (count hosts) already 70 | ;;;loop and retry, its expected that the routing policy blacklist or remove the host on error 71 | 72 | (let [host-key (select-keys host [:host :port]) 73 | res (try 74 | 75 | (if-let [ 76 | conn (tcp-pool/borrow pool host-key timeout-ms)] 77 | 78 | (try 79 | (io-f conn) 80 | (catch Throwable e 81 | ;;any exception will cause invalidation of the connection. 82 | (tcp-pool/invalidate pool host-key conn) 83 | (throw e)) 84 | (finally 85 | (try 86 | (tcp-pool/return pool host-key conn) 87 | (catch Exception e nil)))) 88 | 89 | (throw-no-connection!)) 90 | 91 | (catch Exception t 92 | 93 | ;;blacklist host 94 | (routing/-blacklist! (:routing-policy ctx) host-key) 95 | 96 | (routing/-on-error! (:routing-policy ctx) host-key t) 97 | (ex-info (str "Error while connecting to " host-key) {:throwable t :host host-key :retries i :hosts (routing/-hosts (:routing-policy ctx))})))] 98 | 99 | (if (instance? Throwable res) 100 | (do 101 | (error res) 102 | (if (< i (count (routing/-hosts (:routing-policy ctx)))) 103 | (recur (inc i)) 104 | (throw res))) 105 | res))) 106 | 107 | (throw-no-connection!))))) 108 | 109 | (defn retry-select-send! 110 | "Send with the retry-policy, select-send! will be retried depending on the retry policy" 111 | ([{:keys [retry-policy] :as ctx} host-address io-f timeout-ms] 112 | {:pre [retry-policy]} 113 | (retry/with-retry retry-policy #(select-send! ctx host-address io-f timeout-ms))) 114 | ([{:keys [retry-policy] :as ctx} io-f timeout-ms] 115 | {:pre [retry-policy]} 116 | (retry/with-retry retry-policy #(select-send! ctx io-f timeout-ms)))) 117 | 118 | 119 | ;; routing-policy is a function to which we pass the routing-env atom, which contains {:hosts (set [tcp-conn/HostAddressSchema]) } by default 120 | (s/defn create [pool :- IPoolSchema 121 | routing-policy :- IRouteSchema 122 | retry-policy :- IRetrySchema 123 | ] :- DriverRetSchema 124 | {:pool pool 125 | :routing-policy routing-policy 126 | :retry-policy retry-policy}) 127 | 128 | ;;;;;;;;;;;;;;;; 129 | ;;;;;; Public API 130 | 131 | (defn send-f 132 | " 133 | Apply the io-f with a connection from the connection pool selected based on 134 | the retry policy, and retried if exceptions in the io-f based on the retry policy 135 | ctx - returned from create 136 | io-f - function that should accept the tcp-driver.io.conn/ITCPConn 137 | timeout-ms - the timeout for connection borrow" 138 | ([ctx host-address io-f timeout-ms] 139 | (retry-select-send! ctx host-address io-f timeout-ms)) 140 | ([ctx io-f timeout-ms] 141 | (retry-select-send! ctx io-f timeout-ms))) 142 | 143 | 144 | (defn create-default 145 | "Create a driver with the default settings for tcp-pool, routing and retry-policy 146 | hosts: a vector or seq of {:host :port} maps 147 | return: DriverRetSchema 148 | 149 | pool-conf : tcp-driver.io.pool/PoolConfSchema 150 | 151 | Routing policy: The default routing policy will select hosts at random and on any exception blacklist a particular host. 152 | To add/remove/blacklist a node use the public functions add-host, remove-host and blacklist-host in this namespace. 153 | " 154 | ^{:arg-lists [routing-conf pool-conf retry-limit]} 155 | [hosts & {:keys [routing-conf pool-conf retry-limit] :or {retry-limit 10 routing-conf {} pool-conf {}}}] 156 | {:pre [ 157 | (s/validate tcp-pool/PoolConfSchema pool-conf) 158 | (s/validate [tcp-conn/HostAddressSchema] hosts) 159 | (number? retry-limit) 160 | ]} 161 | (create 162 | (tcp-pool/create-tcp-pool pool-conf) 163 | (apply routing/create-default-routing-policy hosts (mapcat identity routing-conf)) 164 | (retry/retry-policy retry-limit))) 165 | 166 | (defn close 167 | "Close the driver connection pool" 168 | ^{:arg-lists [pool]} 169 | [{:keys [pool]}] 170 | (tcp-pool/close pool)) 171 | 172 | (defn add-host [{:keys [routing-policy]} host] 173 | {:pre [(s/validate tcp-conn/HostAddressSchema host)]} 174 | (routing/-add-host! routing-policy host)) 175 | 176 | (defn remove-host [{:keys [routing-policy]} host] 177 | {:pre [(s/validate tcp-conn/HostAddressSchema host)]} 178 | (routing/-remove-host! routing-policy host)) 179 | 180 | (defn blacklist-host [{:keys [routing-policy]} host] 181 | {:pre [(s/validate tcp-conn/HostAddressSchema host)]} 182 | (routing/-blacklist! routing-policy host)) 183 | 184 | 185 | (defn blacklisted? [{:keys [routing-policy]} host] 186 | {:pre [(s/validate tcp-conn/HostAddressSchema host)]} 187 | (routing/-blacklisted? routing-policy host)) 188 | -------------------------------------------------------------------------------- /src/clojure/tcp_driver/io/conn.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "TCP Connection abstractions and implementations 3 | see host-address and tcp-conn-factory"} 4 | tcp-driver.io.conn 5 | (:require [schema.core :as s]) 6 | (:import 7 | (java.net InetAddress Socket SocketAddress InetSocketAddress) 8 | (org.apache.commons.pool2 BaseKeyedPooledObjectFactory PooledObject KeyedPooledObjectFactory) 9 | (org.apache.commons.pool2.impl DefaultPooledObject) 10 | (java.io InputStream OutputStream))) 11 | 12 | 13 | ;;;;;;;;;;;;;;;;;;;;;;;;; 14 | ;;;;;;;;;;;;Protocol & Data 15 | 16 | (def HostAddressSchema {:host s/Str :port s/Int s/Any s/Any}) 17 | 18 | (defrecord HostAddress [^String host ^int port]) 19 | 20 | (defprotocol ITCPConn 21 | (-input-stream [this]) 22 | (-output-stream [this]) 23 | (-close [this]) 24 | (-valid? [this])) 25 | 26 | (def ITCPConnSchema (s/pred (partial satisfies? ITCPConn))) 27 | 28 | ;;;;;;;;;;;;;;;;;;;;;;;;; 29 | ;;;;;;;;;;;;Private 30 | 31 | (defrecord SocketConn [^Socket socket] 32 | ITCPConn 33 | (-input-stream [_] (.getInputStream socket)) 34 | (-output-stream [_] (.getOutputStream socket)) 35 | (-close [_] (.close socket)) 36 | (-valid? [_] (and 37 | (.isConnected socket) 38 | (not (.isClosed socket))))) 39 | 40 | 41 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 42 | ;;;;;;;;;;;;;;;; Public API 43 | 44 | (defn wrap-tcp-conn 45 | "Wrap the Socket in a ITCPConn" 46 | [^Socket socket] 47 | (->SocketConn socket)) 48 | 49 | (defn create-tcp-conn [{:keys [host port]}] 50 | {:pre [(string? host) (number? port)]} 51 | (->SocketConn 52 | (doto (Socket.) 53 | (.connect (InetSocketAddress. (str host) (int port))) 54 | (.setKeepAlive true)))) 55 | 56 | 57 | (defn 58 | ^InputStream 59 | input-stream [conn] 60 | (-input-stream conn)) 61 | 62 | (defn 63 | ^OutputStream 64 | output-stream [conn] 65 | (-output-stream conn)) 66 | 67 | (defn close! [conn] 68 | (-close conn)) 69 | 70 | (defn valid? [conn] 71 | (-valid? conn)) 72 | 73 | (defn ^HostAddress host-address 74 | "Creates a host address instance using host and port" 75 | [host port] 76 | {:pre [(string? host) (number? port)]} 77 | (->HostAddress host port)) 78 | 79 | (defn ^KeyedPooledObjectFactory tcp-conn-factory 80 | "Return a keyed pool factory that return ITCPConn instances 81 | The keys used should always be instances of HostAddress or implement host and port keys 82 | 83 | post-create-fn: is called after the connection has been created 84 | pre-destroy-fn is called before the connection is destroyed" 85 | ([] 86 | (tcp-conn-factory identity identity)) 87 | ([post-create-fn pre-destroy-fn] 88 | (let [post-create-fn' (if post-create-fn post-create-fn :conn) 89 | pre-destroy-fn' (if pre-destroy-fn pre-destroy-fn :conn)] 90 | 91 | (proxy 92 | [BaseKeyedPooledObjectFactory] 93 | [] 94 | (create [address] 95 | (s/validate HostAddressSchema address) 96 | 97 | (let [conn (create-tcp-conn address)] 98 | (post-create-fn' {:address address :conn conn}))) 99 | 100 | (wrap [v] (DefaultPooledObject. v)) 101 | 102 | (destroyObject [address ^PooledObject v] 103 | (let [conn (.getObject v)] 104 | (pre-destroy-fn' {:address address :conn conn}) 105 | (close! conn))) 106 | 107 | (validateObject [_ ^PooledObject v] 108 | (valid? (.getObject v))))))) -------------------------------------------------------------------------------- /src/clojure/tcp_driver/io/pool.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "TCP connection pools 3 | see: create-tcp-pool 4 | 5 | Pool keys: 6 | Note keys (not keywords) passed are in fact host addresses of the format {:host :port } 7 | The tcp-driver.io.conn/host-address can be used for convinience to return a record 8 | Any other key types will result in a runtime exception on connection creation"} 9 | tcp-driver.io.pool 10 | (:require 11 | [tcp-driver.io.conn :as tcp-conn] 12 | [schema.core :as s]) 13 | (:import 14 | (org.apache.commons.pool2 KeyedObjectPool BaseKeyedPooledObjectFactory) 15 | (java.net SocketAddress) 16 | (org.apache.commons.pool2.impl GenericKeyedObjectPool GenericKeyedObjectPoolConfig))) 17 | 18 | 19 | ;;;;;;;;;;;;;;;;;;;;;;;;; 20 | ;;;;;;;;;;;;Protocols 21 | 22 | (def PoolConfSchema {(s/optional-key :post-create-fn) s/Any ;(s/=> s/Any {:address tcp-conn/HostAddressSchema :conn tcp-conn/ITCPConnSchema}) 23 | (s/optional-key :pre-destroy-fn) s/Any ;(s/=> s/Any {:address tcp-conn/HostAddressSchema :conn tcp-conn/ITCPConnSchema}) 24 | (s/optional-key :max-idle-per-key) s/Int 25 | (s/optional-key :max-total) s/Int 26 | (s/optional-key :max-total-per-key) s/Int 27 | (s/optional-key :min-idle-per-key) s/Int 28 | (s/optional-key :close-pool-jvm-shutdown) s/Bool 29 | s/Any s/Any}) 30 | 31 | (defprotocol IPool 32 | (-borrow [this key timeout-ms]) 33 | (-return [this key obj]) 34 | (-invalidate [this key obj]) 35 | (-close [this]) 36 | (-num-idle [this] [this key]) 37 | (-num-active [this] [this key])) 38 | 39 | 40 | (def IPoolSchema (s/pred (partial extends? IPool))) 41 | 42 | ;;;;;;;;;;;;;;;;;;;;;;;;; 43 | ;;;;;;;;;;;;Private 44 | 45 | 46 | 47 | (defrecord KeyedTCPConnFactory [^GenericKeyedObjectPool pool] 48 | 49 | IPool 50 | (-borrow [_ key timeout-ms] (.borrowObject pool key (long timeout-ms))) 51 | (-return [_ key obj] (.returnObject pool key obj)) 52 | (-invalidate [_ key obj] (.invalidateObject pool key obj)) 53 | (-close [_] (.close pool)) 54 | 55 | (-num-active [_] (.getNumActive pool)) 56 | (-num-active [_ key] (.getNumActive pool key)) 57 | 58 | (-num-idle [_] (.getNumIdle pool)) 59 | (-num-idle [_ key] (.getNumIdle pool key))) 60 | 61 | 62 | (defn keyed-pool-config 63 | "Create a pool config with block when exhausted set to true" 64 | [{:keys [max-idle-per-key 65 | max-total 66 | max-total-per-key 67 | min-idle-per-key] 68 | 69 | :or {max-idle-per-key 2 70 | max-total 100 71 | max-total-per-key 100 72 | min-idle-per-key 0}}] 73 | 74 | (doto 75 | (GenericKeyedObjectPoolConfig.) 76 | (.setBlockWhenExhausted true) 77 | (.setTestOnBorrow true) 78 | (.setMaxIdlePerKey (int max-idle-per-key)) 79 | (.setMaxTotal (int max-total)) 80 | (.setMaxTotalPerKey (int max-total-per-key)) 81 | (.setMinIdlePerKey (int min-idle-per-key)))) 82 | 83 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 84 | ;;;;;;;;;;;;; Public API 85 | ;;;; Remember that all keys should be created by tcp-driver.io.conn/host-address 86 | 87 | (defn borrow 88 | " 89 | Params: 90 | pool an instance of IPool 91 | address an instance of tcp-driver.io.conn.HostAddress or map of :host and :port 92 | timeout-ms long timeout in milliseconds 93 | Exceptions: NoSuchElementException, Exception" 94 | [pool address timeout-ms] 95 | {:pre [(s/validate tcp-conn/HostAddressSchema address)]} 96 | (-borrow pool address timeout-ms)) 97 | 98 | (defn return 99 | "Return conn to the pool 100 | Params: 101 | pool instanceof of IPool 102 | address an instance of tcp-driver.io.conn.HostAddress or map of :host and :port 103 | conn connection to return" 104 | [pool address conn] 105 | {:pre [(s/validate tcp-conn/HostAddressSchema address)]} 106 | (-return pool address conn)) 107 | 108 | (defn invalidate [pool address conn] 109 | {:pre [(s/validate tcp-conn/HostAddressSchema address)]} 110 | (-invalidate pool address conn)) 111 | 112 | (defn close [pool] 113 | (-close pool)) 114 | 115 | (defn num-idle 116 | ([pool] (-num-idle pool)) 117 | ([pool key] (-num-idle pool key))) 118 | 119 | (defn num-active 120 | ([pool] (-num-active pool)) 121 | ([pool key] (-num-active pool key))) 122 | 123 | 124 | (s/defn 125 | create-tcp-pool :- IPoolSchema 126 | [conf :- PoolConfSchema] 127 | ;;create a tcp pool factory where each key is the address to connect to 128 | (let [pool 129 | (->KeyedTCPConnFactory (GenericKeyedObjectPool. (tcp-conn/tcp-conn-factory (get conf :post-create-fn :conn) 130 | (get conf :pre-destroy-create-fn :conn)) 131 | (keyed-pool-config conf)))] 132 | 133 | (when (:close-pool-jvm-shutdown conf) 134 | (.addShutdownHook (Runtime/getRuntime) (Thread. #(close pool)))) 135 | 136 | pool)) 137 | 138 | 139 | (defn try-conn 140 | "Get a connection, call (f conn) and in a finally clause return the connection to the pool" 141 | [pool host timeout-ms f] 142 | (let [conn (borrow pool host timeout-ms)] 143 | (try 144 | (f conn) 145 | (finally 146 | (return pool host conn))))) -------------------------------------------------------------------------------- /src/clojure/tcp_driver/io/stream.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "Helper utilities with Input/Output Streams and common tasks like reading/writing java primitives 3 | using blockign IO but supporting timeouts without the need to create background threads"} 4 | tcp-driver.io.stream 5 | (:require [tcp-driver.io.conn :as tcp-conn]) 6 | (:import (tcpdriver.io IOUtil) 7 | (java.io OutputStream))) 8 | 9 | 10 | ;;;;;;;;;;;;;;;;;;;;;; 11 | ;;;;; Public API 12 | 13 | (defn available ^long [conn] 14 | (long (.available (tcp-conn/input-stream conn)))) 15 | 16 | (defn read-bytes ^"[B" 17 | ([conn ^long len ^long timeout-ms] 18 | (IOUtil/readBytes (tcp-conn/input-stream conn) len timeout-ms)) 19 | ([conn ^long timeout-ms] 20 | (read-bytes conn (available conn) timeout-ms))) 21 | 22 | (defn read-byte ^long 23 | [ conn ^long timeout-ms] 24 | (long (IOUtil/readByte (tcp-conn/input-stream conn) timeout-ms))) 25 | 26 | (defn read-int ^long 27 | [ conn ^long timeout-ms] 28 | (long (IOUtil/readInt (tcp-conn/input-stream conn) timeout-ms))) 29 | 30 | (defn read-long ^long 31 | [ conn ^long timeout-ms] 32 | (IOUtil/readLong (tcp-conn/input-stream conn) timeout-ms)) 33 | 34 | (defn read-short ^long 35 | [ conn ^long timeout-ms] 36 | (long (IOUtil/readShort (tcp-conn/input-stream conn) timeout-ms))) 37 | 38 | (defn read-float ^double 39 | [ conn ^long timeout-ms] 40 | (double (IOUtil/readFloat (tcp-conn/input-stream conn) timeout-ms))) 41 | 42 | (defn read-double ^double 43 | [ conn ^long timeout-ms] 44 | (IOUtil/readDouble (tcp-conn/input-stream conn) timeout-ms)) 45 | 46 | (defn read-short-str ^String 47 | [ conn ^long timeout-ms] 48 | (IOUtil/readShortString (tcp-conn/input-stream conn) timeout-ms)) 49 | 50 | (defn flush-out [conn] 51 | (.flush ^OutputStream (tcp-conn/output-stream conn))) 52 | 53 | (defn write-bytes 54 | ([conn ^"[B" bts ^long from ^long len] 55 | (IOUtil/write (tcp-conn/output-stream conn) bts from len)) 56 | ([ conn ^"[B" bts] 57 | (IOUtil/write (tcp-conn/output-stream conn) bts))) 58 | 59 | (defn write-byte 60 | [conn ^long v] 61 | (IOUtil/write (tcp-conn/output-stream conn) (byte v))) 62 | 63 | (defn write-int 64 | [conn ^long v] 65 | (IOUtil/write (tcp-conn/output-stream conn) (int v))) 66 | 67 | (defn write-short 68 | [ conn ^long v] 69 | (IOUtil/write (tcp-conn/output-stream conn) (short v))) 70 | 71 | (defn write-long 72 | [ conn ^long v] 73 | (IOUtil/write (tcp-conn/output-stream conn) v)) 74 | 75 | (defn write-float 76 | [ conn ^double v] 77 | (IOUtil/write (tcp-conn/output-stream conn) (float v))) 78 | 79 | (defn write-double 80 | [ conn ^double v] 81 | (IOUtil/write (tcp-conn/output-stream conn) v)) 82 | 83 | (defn write-short-str 84 | [ conn ^String v] 85 | (IOUtil/writeShortString (tcp-conn/output-stream conn) v)) 86 | 87 | -------------------------------------------------------------------------------- /src/clojure/tcp_driver/routing/policy.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "Protocol interface and default implementation for routing and blacklisting hosts 3 | The default rounting is done randomly and the blacklisting uses a TTL Cache"} 4 | tcp-driver.routing.policy 5 | (:require [schema.core :as s] 6 | [fun-utils.cache :as cache] 7 | [clojure.set :as clj-set] 8 | [tcp-driver.io.conn :as tcp-conn])) 9 | 10 | 11 | ;;;;;;;;;;;;;;;;; 12 | ;;;;; Protocols and records 13 | 14 | (defprotocol IRoute 15 | (-hosts [this]) ;;return the hosts {:host :port} currently managed by the routing policy 16 | (-add-host! [this host]) ;;"Called from outside the driver to notify addition of a host" 17 | (-remove-host! [this host]) ;;"Called from outside the driver to notify removal of a host" 18 | (-blacklisted? [this host]) ;;"True if the node is blacklisted" 19 | (-blacklist! [this host]) ;;"Node should be blacklisted, called from outside of the driver to notify blacklisting of a host" 20 | (-on-error! [this host throwable]) ;;"Communicate exceptions back to the routing policy, default is to blacklist the host" 21 | (-select-host [this])) ;; "Driver calls to select host from the routing policy" 22 | 23 | 24 | ;;;;;;;;;;;;; 25 | ;;;; Private functions 26 | 27 | (defn ensure-host-address-schema! [host] 28 | (s/validate tcp-conn/HostAddressSchema host)) 29 | 30 | ;;hosts-at contains #{tcp-conn/HostAddressSchema} 31 | ;;black-listed-hosts-at a ttl cache that hosts-at but holds black-listed 32 | ;;select-f is a function 33 | 34 | (defrecord DefaultRountingPolicy [hosts-at black-listed-hosts-cache select-f] 35 | IRoute 36 | 37 | (-hosts [this] @hosts-at) 38 | 39 | (-on-error! [this host throwable] 40 | (prn "DefaultRountingPolicy:-on-error " host) 41 | (-blacklist! this host)) 42 | 43 | (-add-host! [_ host] 44 | (ensure-host-address-schema! host) 45 | (swap! hosts-at conj host)) 46 | 47 | (-remove-host! [_ host] 48 | (ensure-host-address-schema! host) 49 | (swap! hosts-at disj host)) 50 | 51 | (-blacklisted? [_ host] 52 | (ensure-host-address-schema! host) 53 | (get black-listed-hosts-cache host)) 54 | 55 | (-blacklist! [_ host] 56 | (ensure-host-address-schema! host) 57 | (prn "DefaultRountingPolicy:-blacklist " host " in " black-listed-hosts-cache) 58 | (assoc black-listed-hosts-cache host true)) 59 | 60 | (-select-host [_] 61 | (let [available-hosts (into [] (clj-set/difference @hosts-at (set (keys black-listed-hosts-cache))))] 62 | (prn "DefaultRountingPolicy:-select-host " available-hosts) 63 | (when (pos? (count available-hosts)) 64 | (rand-nth available-hosts))))) 65 | 66 | 67 | ;;;;;;;;;;;;;;;;; 68 | ;;;;; Public API 69 | 70 | (defn create-default-routing-policy 71 | "Create a rounting policy instance that manages blacklisted hosts and on select-host 72 | hosts : HostAddressSchema 73 | returns the host that is not blacklisted, select-f is used for this, note that by default rand-nth is used" 74 | [hosts 75 | & {:keys [select-f 76 | blacklist-expire] :or {select-f rand-nth 77 | blacklist-expire 10000}}] 78 | {:pre [(s/validate [tcp-conn/HostAddressSchema] hosts) 79 | (fn? select-f) 80 | (number? blacklist-expire)]} 81 | 82 | (->DefaultRountingPolicy 83 | (atom (into #{} hosts)) 84 | (cache/create-cache :expire-after-write blacklist-expire) 85 | select-f)) 86 | -------------------------------------------------------------------------------- /src/clojure/tcp_driver/routing/retry.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "Retry interfaces and default counted retry protocol"} 3 | tcp-driver.routing.retry 4 | (:require [schema.core :as s])) 5 | 6 | 7 | ;;;;;;;;;;;;;;;;;;; 8 | ;;;;; Protocols and records 9 | 10 | (defprotocol IRetry 11 | (-try-action [this f] "Retry f according to the policy, a failure is marked by an exception")) 12 | 13 | 14 | ;;;;;;;;;;;;;;;;;; 15 | ;;;;;; Private functions 16 | 17 | (defn _try-catch "run f if exception returns the exception" [f] 18 | (try (f) (catch Throwable e e))) 19 | 20 | (defn _retry-ntimes 21 | "Retry the function call limit times only if an exception is thrown" 22 | [f limit] 23 | (loop [n 1] 24 | (let [ret (_try-catch f)] 25 | (if (instance? Throwable ret) 26 | (if (< n (long limit)) 27 | (recur (inc n)) 28 | (throw (ex-info "Retry-exception" {:throwable ret :retries n}))) 29 | ret)))) 30 | 31 | (defrecord DefaultRetryPolicy [^long retry-limit] 32 | IRetry 33 | (-try-action [_ f] 34 | (_retry-ntimes f retry-limit))) 35 | 36 | 37 | 38 | ;;;;;;;;;;;;;;;;; 39 | ;;;;;; Public Functions 40 | 41 | (s/defn with-retry [retry-policy :- (s/pred #(satisfies? IRetry %)) 42 | f :- (s/pred fn?)] 43 | (-try-action retry-policy f)) 44 | 45 | (defn retry-policy [retry-limit] 46 | {:pre [(number? retry-limit)]} 47 | (->DefaultRetryPolicy retry-limit)) -------------------------------------------------------------------------------- /src/java/tcpdriver/io/IOUtil.java: -------------------------------------------------------------------------------- 1 | package tcpdriver.io; 2 | 3 | 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.io.OutputStream; 7 | import java.util.concurrent.TimeoutException; 8 | 9 | /** 10 | * Mutable IO Utility functions used in clojure code. 11 | */ 12 | public class IOUtil { 13 | 14 | /** 15 | * Read 2 bytes for length, and then length bytes, return a String of format UTF-8 16 | * 17 | * @param in 18 | * @param timeoutMs 19 | * @return 20 | * @throws InterruptedException 21 | * @throws IOException 22 | * @throws TimeoutException 23 | */ 24 | public static final String readShortString(InputStream in, long timeoutMs) throws InterruptedException, IOException, TimeoutException { 25 | short len = readShort(in, timeoutMs); 26 | byte[] bts = readBytes(in, len, timeoutMs); 27 | return new String(bts, "UTF-8"); 28 | } 29 | 30 | /** 31 | * Read 8 bytes and return a double 32 | * 33 | * @param in 34 | * @param timeoutMs 35 | * @return 36 | * @throws InterruptedException 37 | * @throws IOException 38 | * @throws TimeoutException 39 | */ 40 | public static final double readDouble(InputStream in, long timeoutMs) throws InterruptedException, IOException, TimeoutException { 41 | return Double.longBitsToDouble(readLong(in, timeoutMs)); 42 | } 43 | 44 | /** 45 | * Read 4 bytes and return a float 46 | * 47 | * @param in 48 | * @param timeoutMs 49 | * @return 50 | * @throws InterruptedException 51 | * @throws IOException 52 | * @throws TimeoutException 53 | */ 54 | public static final float readFloat(InputStream in, long timeoutMs) throws InterruptedException, IOException, TimeoutException { 55 | return Float.intBitsToFloat(readInt(in, timeoutMs)); 56 | } 57 | 58 | /** 59 | * Read 2 bytes and return a short 60 | * 61 | * @param in 62 | * @param timeoutMs 63 | * @return 64 | * @throws InterruptedException 65 | * @throws IOException 66 | * @throws TimeoutException 67 | */ 68 | public static final short readShort(InputStream in, long timeoutMs) throws InterruptedException, IOException, TimeoutException { 69 | byte[] bts = readBytes(in, 2, timeoutMs); 70 | return (short) ((bts[0] << 8) | (bts[1] & 0xff)); 71 | } 72 | 73 | /** 74 | * Read 8 bytes and return a long 75 | * 76 | * @param in 77 | * @param timeoutMs 78 | * @return 79 | * @throws InterruptedException 80 | * @throws IOException 81 | * @throws TimeoutException 82 | */ 83 | public static final long readLong(InputStream in, long timeoutMs) throws InterruptedException, IOException, TimeoutException { 84 | byte[] bts = readBytes(in, 8, timeoutMs); 85 | return (((long) (bts[0] & 0xff) << 56) | 86 | ((long) (bts[1] & 0xff) << 48) | 87 | ((long) (bts[2] & 0xff) << 40) | 88 | ((long) (bts[3] & 0xff) << 32) | 89 | ((long) (bts[4] & 0xff) << 24) | 90 | ((long) (bts[5] & 0xff) << 16) | 91 | ((long) (bts[6] & 0xff) << 8) | 92 | ((long) (bts[7] & 0xff))); 93 | } 94 | 95 | /** 96 | * Read 4 bytes and return an int 97 | * 98 | * @param in 99 | * @param timeoutMs 100 | * @return 101 | * @throws InterruptedException 102 | * @throws IOException 103 | * @throws TimeoutException 104 | */ 105 | public static final int readInt(InputStream in, long timeoutMs) throws InterruptedException, IOException, TimeoutException { 106 | byte[] bts = readBytes(in, 4, timeoutMs); 107 | return (((bts[0] & 0xff) << 24) | ((bts[1] & 0xff) << 16) | 108 | ((bts[2] & 0xff) << 8) | (bts[3] & 0xff)); 109 | } 110 | 111 | public static final byte readByte(InputStream in, long timeoutMs) throws InterruptedException, IOException, TimeoutException { 112 | return readBytes(in, 1, timeoutMs)[0]; 113 | } 114 | 115 | 116 | 117 | 118 | /** 119 | * Read expectedBytes from an input stream and time out if no change on the input stream for more than timeoutMs.
120 | * Bytes are read from the input stream as they become available. 121 | * 122 | * @param in 123 | * @param expectedBytes the total number of bytes to read, a byte array of this size is created 124 | * @param timeoutMs 125 | * @return 126 | * @throws TimeoutException 127 | * @throws IOException 128 | */ 129 | public static final byte[] readBytes(InputStream in, int expectedBytes, long timeoutMs) throws TimeoutException, IOException, InterruptedException { 130 | byte[] bytes = new byte[expectedBytes]; 131 | int pos = 0; 132 | long lastTimeAvailable = System.currentTimeMillis(); 133 | 134 | do { 135 | int avail = in.available(); 136 | if (avail > 0) { 137 | //have an new data, read the available bytes and add to the byte array, note we use expectedBytes-pos, to calculate the remaining bytes 138 | //required to read 139 | int btsRead = in.read(bytes, pos, Math.min(avail, expectedBytes - pos)); 140 | pos += btsRead; 141 | 142 | if (pos >= expectedBytes) //we've read all required bytes, exit the loop 143 | break; 144 | 145 | //save the last time data was available 146 | lastTimeAvailable = System.currentTimeMillis(); 147 | } else if ((System.currentTimeMillis() - lastTimeAvailable) > timeoutMs) { 148 | //check for timeout 149 | throw new TimeoutException("Timeout while reading data from the input stream: got only " + pos + " bytes of " + expectedBytes + " last seen " + lastTimeAvailable + " diff " + (System.currentTimeMillis() - lastTimeAvailable)); 150 | } else { 151 | //sleep to simulate IO blocking and avoid consuming CPU resources on IO wait 152 | Thread.sleep(100); 153 | } 154 | 155 | } while (true); 156 | 157 | return bytes; 158 | } 159 | 160 | public static final void write(OutputStream out, double v) throws IOException { 161 | write(out, Double.doubleToLongBits(v)); 162 | } 163 | 164 | public static final void write(OutputStream out, float v) throws IOException { 165 | write(out, Float.floatToIntBits(v)); 166 | } 167 | 168 | public static final void write(OutputStream out, long v) throws IOException { 169 | write(out, new byte[]{ 170 | (byte) (0xff & (v >> 56)), 171 | (byte) (0xff & (v >> 48)), 172 | (byte) (0xff & (v >> 40)), 173 | (byte) (0xff & (v >> 32)), 174 | (byte) (0xff & (v >> 24)), 175 | (byte) (0xff & (v >> 16)), 176 | (byte) (0xff & (v >> 8)), 177 | (byte) (0xff & v) 178 | }); 179 | } 180 | 181 | public static final void write(OutputStream out, short v) throws IOException { 182 | write(out, new byte[]{ 183 | (byte) (0xff & (v >> 8)), 184 | (byte) (0xff & v) 185 | }); 186 | } 187 | 188 | public static final void write(OutputStream out, int v) throws IOException { 189 | write(out, new byte[]{ 190 | (byte) (0xff & (v >> 24)), 191 | (byte) (0xff & (v >> 16)), 192 | (byte) (0xff & (v >> 8)), 193 | (byte) (0xff & v) 194 | }); 195 | } 196 | 197 | public static final void write(OutputStream out, byte[] bts) throws IOException { 198 | out.write(bts); 199 | } 200 | 201 | public static final void write(OutputStream out, byte[] bts, int from, int len) throws IOException { 202 | out.write(bts, from, len); 203 | } 204 | 205 | public static final void writeShortString(OutputStream out, String str) throws IOException { 206 | write(out, (short) str.length()); 207 | write(out, str.getBytes("UTF-8")); 208 | } 209 | 210 | public static final void write(OutputStream out, byte v) throws IOException { 211 | out.write(v); 212 | } 213 | 214 | } -------------------------------------------------------------------------------- /test/tcp_driver/driver_test.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "Test the full driver send and receive namespace"} 3 | tcp-driver.driver-test 4 | (:require 5 | [schema.core :as s] 6 | [tcp-driver.test.util :as test-util] 7 | [tcp-driver.io.stream :as tcp-stream] 8 | [tcp-driver.io.conn :as tcp-conn] 9 | [tcp-driver.driver :as tcp-driver] 10 | [clojure.test :refer :all])) 11 | 12 | 13 | (defn write-msg [conn msg] 14 | (tcp-stream/write-short-str conn (str msg)) 15 | conn) 16 | 17 | (defn read-msg [conn timeout-ms] 18 | (tcp-stream/read-short-str conn timeout-ms)) 19 | 20 | (defn send-io-f [io-f & {:keys [post-create-fn pre-destroy-fn]}] 21 | (test-util/with-echo-server 22 | (fn [server] 23 | (let [driver (tcp-driver/create-default [{:host "localhost" :port (:port server)}] :pool-conf {:post-create-fn post-create-fn :pre-destroy-fn pre-destroy-fn}) 24 | ret-msg (tcp-driver/send-f 25 | driver 26 | io-f 27 | 10000)] 28 | ;;use ret-msg to make it clear thta io-f return value via tcp-driver/send-f 29 | (tcp-driver/close driver) 30 | ret-msg)))) 31 | 32 | 33 | (deftest test-send-receive 34 | [] 35 | ;;write hi, then read it 36 | (is (= (send-io-f #(read-msg (write-msg % "HI") 1000)) 37 | "HI"))) 38 | 39 | (deftest test-validate-fns 40 | [] 41 | (let [post-create (atom nil)] 42 | 43 | (is (= (send-io-f #(read-msg (write-msg % "HI") 1000) 44 | :post-create-fn (fn [ctx] 45 | (swap! post-create (constantly (System/nanoTime))) 46 | (:conn ctx))) 47 | "HI")) 48 | 49 | 50 | (is (number? @post-create)))) -------------------------------------------------------------------------------- /test/tcp_driver/io/conn_test.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "Test the TCP connection implementation"} 3 | tcp-driver.io.conn-test 4 | (:require 5 | [tcp-driver.io.stream :as tcp-stream] 6 | [tcp-driver.io.conn :as tcp-conn] 7 | [tcp-driver.test.util :as test-util] 8 | [clojure.test :refer :all])) 9 | 10 | 11 | 12 | (defn test-send-receive [] 13 | (test-util/with-echo-server 14 | (fn [server] 15 | (let [conn (tcp-conn/create-tcp-conn {:host "localhost" :port (:port server)})] 16 | 17 | (try 18 | (tcp-stream/write-short-str conn "test-string") 19 | (is (= "test-string" (tcp-stream/read-short-str conn 5000))) 20 | (finally 21 | (tcp-conn/close! conn))))))) 22 | 23 | (deftest send-receive-testcase [] 24 | (test-send-receive)) -------------------------------------------------------------------------------- /test/tcp_driver/io/pool_test.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "Test the pool namespace"} 3 | tcp-driver.io.pool-test 4 | (:require 5 | [tcp-driver.io.pool :as tcp-pool] 6 | [tcp-driver.io.conn :as tcp-conn] 7 | [tcp-driver.test.util :as test-util] 8 | [clojure.test :refer :all] 9 | [tcp-driver.io.stream :as tcp-stream])) 10 | 11 | 12 | 13 | (defn borrow-send-and-return [] 14 | (test-util/with-echo-server 15 | (fn [server] 16 | (let [pool (tcp-pool/create-tcp-pool {:close-pool-jvm-shutdown true}) 17 | host (tcp-conn/host-address "localhost" (:port server)) 18 | 19 | resp-str (tcp-pool/try-conn pool host 1000 20 | (fn [conn] 21 | (tcp-stream/write-short-str conn "hi") 22 | (tcp-stream/read-short-str conn 20000)))] 23 | (is (= resp-str "hi")))))) 24 | 25 | (deftest send-receive-testcase [] 26 | (borrow-send-and-return)) -------------------------------------------------------------------------------- /test/tcp_driver/routing/policy_test.clj: -------------------------------------------------------------------------------- 1 | (ns 2 | ^{:doc "Test the tcp-driver.routing.policy default implementation"} 3 | tcp-driver.routing.policy-test 4 | (:require [tcp-driver.routing.policy :as routing] 5 | [tcp-driver.io.conn :as tcp-conn]) 6 | (:use clojure.test)) 7 | 8 | 9 | 10 | (defn create-routing [host-names] 11 | (routing/create-default-routing-policy (mapv #(tcp-conn/host-address % 123) host-names))) 12 | 13 | (deftest select-test [] 14 | (is (routing/-select-host (create-routing ["a" "b"])))) 15 | 16 | (deftest select-remove-add-test [] 17 | (let [policy (create-routing ["a" "b"])] 18 | (routing/-remove-host! policy (tcp-conn/host-address "a" 123)) 19 | (routing/-remove-host! policy (tcp-conn/host-address "b" 123)) 20 | 21 | (is (nil? (routing/-select-host policy))) 22 | 23 | (routing/-add-host! policy (tcp-conn/host-address "c" 123)) 24 | 25 | (is (= (routing/-select-host policy) (tcp-conn/host-address "c" 123))))) 26 | 27 | 28 | 29 | (deftest blacklist-test [] 30 | (let [policy (routing/create-default-routing-policy [(tcp-conn/host-address "a" 123)] :blacklist-expire 500)] 31 | 32 | (is (routing/-select-host policy)) 33 | 34 | (routing/-blacklist! policy (tcp-conn/host-address "a" 123)) 35 | 36 | (is (nil? (routing/-select-host policy))) 37 | 38 | (Thread/sleep 1000) 39 | 40 | (is (routing/-select-host policy)))) -------------------------------------------------------------------------------- /test/tcp_driver/routing/retry_test.clj: -------------------------------------------------------------------------------- 1 | (ns tcp-driver.routing.retry-test 2 | (:require [tcp-driver.routing.retry :as retry]) 3 | (:use clojure.test)) 4 | 5 | 6 | (deftest test-retry [] 7 | (let [rpolicy (retry/retry-policy 3) 8 | f (fn [] (prn "try-function and throw ex") (throw (Exception. "test")))] 9 | 10 | (try 11 | (retry/with-retry rpolicy f) 12 | (is false) ;should not be run 13 | (catch Exception e 14 | (is (= (get (ex-data e) :retries) 3)))))) -------------------------------------------------------------------------------- /test/tcp_driver/test/util.clj: -------------------------------------------------------------------------------- 1 | (ns tcp-driver.test.util 2 | (:require [tcp-driver.io.conn :as tcp-conn] 3 | [tcp-driver.io.stream :as tcp-stream]) 4 | (:import (java.net ServerSocket SocketException))) 5 | 6 | (defn ^ServerSocket server-socket [] 7 | (ServerSocket. (int 0))) 8 | 9 | (defn ^Long get-port [^ServerSocket socket] 10 | (Long. (.getLocalPort socket))) 11 | 12 | (defn get-connection! [^ServerSocket socket] 13 | (.accept socket)) 14 | 15 | ;;;;;;;;;;;;;;;;;;;; 16 | ;;;;;;;;; Public API 17 | 18 | (defn create-server 19 | "Connects a server socket to an abritrary free port 20 | Params: 21 | handler-f called (handler-f ^ITCPConn conn) 22 | Returns: 23 | {:server-socket ... 24 | :port the binded port 25 | :future an instance of (future that accepts and calls handler-f}" 26 | [handler-f] 27 | (let [socket (server-socket)] 28 | { 29 | :server-socket socket 30 | :port (get-port socket) 31 | :future-loop (future 32 | (while (not (Thread/interrupted)) 33 | (try 34 | (handler-f (tcp-conn/wrap-tcp-conn (get-connection! socket))) 35 | (catch InterruptedException _ nil) 36 | (catch SocketException _ nil) 37 | (catch Exception e (do (prn e) (.printStackTrace e))))))})) 38 | 39 | (defn stop-server [{:keys [server-socket future-loop]}] 40 | (.close ^ServerSocket server-socket) 41 | (future-cancel future-loop)) 42 | 43 | 44 | (defn echo-handler [conn] 45 | (prn "echo-handler: " conn) 46 | (let [bts (tcp-stream/read-bytes conn 10000)] 47 | ;;we need to ensure we have some bytes 48 | (if (pos? (count bts)) 49 | (do 50 | (tcp-stream/write-bytes conn bts) 51 | (tcp-stream/flush-out conn) 52 | (tcp-conn/close! conn)) 53 | (do 54 | (Thread/sleep 1000) 55 | (recur conn))))) 56 | 57 | (defn echo-server 58 | "Return {:port :server-socket and :future-loop}" 59 | [] 60 | (create-server echo-handler)) 61 | 62 | (defn with-echo-server [f] 63 | (let [server (echo-server)] 64 | (try 65 | (f server) 66 | (finally 67 | (stop-server server))))) --------------------------------------------------------------------------------