├── .gitignore ├── 3rd_party_licenses.txt ├── LICENSE ├── NOTICE ├── README.mdown ├── pom.xml └── src ├── main ├── assembly │ └── assembly.xml ├── java │ └── com │ │ └── addthis │ │ └── meshy │ │ ├── AggregateChannelFuture.java │ │ ├── AutoConnectToPeersTask.java │ │ ├── AutoMeshTask.java │ │ ├── ChannelCloseListener.java │ │ ├── ChannelMaster.java │ │ ├── ChannelState.java │ │ ├── HttpServer.java │ │ ├── InputStreamWrapper.java │ │ ├── LocalFileHandler.java │ │ ├── LocalFileHandlerMux.java │ │ ├── LocalFileSystem.java │ │ ├── Main.java │ │ ├── Meshy.java │ │ ├── MeshyClient.java │ │ ├── MeshyClientConnector.java │ │ ├── MeshyClientHelper.java │ │ ├── MeshyConstants.java │ │ ├── MeshyServer.java │ │ ├── MeshyServerGroup.java │ │ ├── NodeInfo.java │ │ ├── SendWatcher.java │ │ ├── ServerStats.java │ │ ├── SessionHandler.java │ │ ├── SourceHandler.java │ │ ├── TargetHandler.java │ │ ├── VirtualFileFilter.java │ │ ├── VirtualFileInput.java │ │ ├── VirtualFileReference.java │ │ ├── VirtualFileSystem.java │ │ └── service │ │ ├── file │ │ ├── DupFilter.java │ │ ├── FileReference.java │ │ ├── FileReferenceFilter.java │ │ ├── FileSource.java │ │ ├── FileStats.java │ │ ├── FileTarget.java │ │ └── Filter.java │ │ ├── host │ │ ├── HostNode.java │ │ ├── HostSource.java │ │ └── HostTarget.java │ │ ├── message │ │ ├── InternalHandler.java │ │ ├── MessageFile.java │ │ ├── MessageFileInput.java │ │ ├── MessageFileListener.java │ │ ├── MessageFileProvider.java │ │ ├── MessageFileSystem.java │ │ ├── MessageListener.java │ │ ├── MessageSource.java │ │ ├── MessageTarget.java │ │ ├── OutputSender.java │ │ ├── SendOnCloseOutputStream.java │ │ ├── TargetListener.java │ │ ├── TopicListener.java │ │ └── TopicSender.java │ │ ├── peer │ │ ├── PeerService.java │ │ ├── PeerSource.java │ │ ├── PeerTarget.java │ │ └── PeerTuple.java │ │ └── stream │ │ ├── SourceInputStream.java │ │ ├── StreamService.java │ │ ├── StreamSource.java │ │ ├── StreamStats.java │ │ └── StreamTarget.java └── resources │ └── meshy_prometheus_metrics.yml └── test ├── files ├── a │ ├── abc.xml │ └── def.xml ├── b │ ├── ghi.xml │ └── jkl.xml ├── c │ └── hosts ├── mux │ ├── mff.conf │ ├── mff.data │ ├── mff.lock │ ├── mfs.conf │ ├── mfs.data │ ├── mfs.lock │ └── out-00000001 └── xyz.xml ├── java └── com │ └── addthis │ └── meshy │ ├── TestMesh.java │ ├── TestMeshyClientConnector.java │ ├── TestPeerService.java │ ├── TestStreamService.java │ └── service │ ├── file │ ├── TestFileService.java │ └── TestVFS.java │ └── message │ ├── TestMessageFileSystem.java │ └── TestMessageService.java └── resources └── simplelogger.properties /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .idea 3 | meshy.iml 4 | -------------------------------------------------------------------------------- /3rd_party_licenses.txt: -------------------------------------------------------------------------------- 1 | 2 | LICENSES FOR THIRD-PARTY COMPONENTS 3 | 4 | =============================================================================== 5 | 6 | The following sections list licensing information for 7 | libraries included with the Meshy source and components 8 | used to test Meshy. 9 | 10 | The following software may be included in this product: 11 | 12 | =============================================================================== 13 | 14 | Netty/All In One 15 | 16 | 17 | 18 | Netty/All In One 4.0.28.Final uses the Apache 2.0 license, shown below. See the License for details about distribution rights, and the specific rights regarding derivate works. 19 | 20 | 21 | 22 | https://github.com/netty/netty/blob/4.0/LICENSE.txt 23 | 24 | 25 | 26 | --------------------------------------------------------------------------- 27 | 28 | Metrics Core Library 29 | 30 | Copyright 2010-2012 Coda Hale and Yammer, Inc. 31 | 32 | Metrics Core Library 2.2.0 uses the Apache 2.0 license, shown below. See the License for details about distribution rights, and the specific rights regarding derivate works. 33 | 34 | 35 | 36 | http://www.apache.org/licenses/LICENSE-2.0.html 37 | 38 | 39 | 40 | --------------------------------------------------------------------------- 41 | 42 | SLF4J Simple Binding 43 | 44 | SLF4J API Module 45 | 46 | Copyright (c) 2004-2007 QOS.ch 47 | 48 | SLF4J Simple Binding 1.7.25 and SLF4J API Module 1.7.25 use MIT license, shown below. See the License for details about distribution rights, and the specific rights regarding derivate works. 49 | 50 | 51 | 52 | https://opensource.org/licenses/mit-license.php 53 | 54 | 55 | 56 | --------------------------------------------------------------------------- 57 | 58 | Jetty :: Servlet Handling 59 | 60 | 61 | 62 | Jetty :: Servlet Handling 9.4.5.v20170502 uses the Apache 2.0 license and Eclipse Public License - Version 1.0, shown below. See the License for details about distribution rights, and the specific rights regarding derivate works. 63 | 64 | 65 | 66 | http://www.apache.org/licenses/LICENSE-2.0 67 | 68 | http://www.eclipse.org/org/documents/epl-v10.php 69 | 70 | 71 | 72 | --------------------------------------------------------------------------- 73 | 74 | Prometheus Java Simpleclient 75 | 76 | Prometheus Java Simpleclient Hotspot 77 | 78 | Prometheus Java Simpleclient Servlet 79 | 80 | 81 | 82 | Prometheus Java Simpleclient 0.0.26, Prometheus Java Simpleclient Hotspot 0.0.26, Prometheus Java Simpleclient Servlet 0.0.26 use the Apache 2.0 license, shown below. See the License for details about distribution rights, and the specific rights regarding derivate works. 83 | 84 | 85 | 86 | http://www.apache.org/licenses/LICENSE-2.0.html 87 | 88 | 89 | 90 | --------------------------------------------------------------------------- 91 | 92 | Collector 93 | 94 | 95 | 96 | Collector 0.10 uses the Apache 2.0 license, shown below. See the License for details about distribution rights, and the specific rights regarding derivate works. 97 | 98 | 99 | 100 | http://www.apache.org/licenses/LICENSE-2.0.html 101 | 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | meshy 2 | Copyright 2013 AddThis 3 | 4 | This product includes software developed by AddThis. 5 | -------------------------------------------------------------------------------- /README.mdown: -------------------------------------------------------------------------------- 1 | # meshy 2 | 3 | ## TCP multiplexing gearwheels 4 | 5 | `meshy` is a java library for creating clustered services over 6 | multiplexed tcp connections. [Netty](http://netty.io/) is used to 7 | handle all the gritty networking bits and UDP broadcast for service 8 | discovery. 9 | 10 | Services are typically hierarchical -- think file systems or URLs -- 11 | and file services and RPC are the two most common use cases. 12 | 13 | Nodes form a mesh once they have discovered the local network 14 | topology. Each mesh node is capable of handling any request that the 15 | mesh can handle by proxying to an appropriate node. 16 | 17 | A typical use case is for each physical node to have one mesh node, 18 | and for local processes to talk to their local mesh node. Multiple 19 | meshes can be overlayed on the same physical hosts by using unique 20 | ports or "secret" keys. 21 | 22 | ## Building 23 | 24 | `meshy` uses [Apache Maven](http://maven.apache.org/) which it is beyond 25 | the scope to detail. The super simple quick start is: 26 | 27 | `mvn test` 28 | 29 | ## Use 30 | 31 | ```xml 32 | 33 | com.addthis 34 | meshy 35 | latest-and-greatest 36 | 37 | ``` 38 | 39 | You can either install locally, or releases will eventually make their 40 | way to maven central. 41 | 42 | 43 | 44 | ## Administrative 45 | 46 | ### Versioning 47 | 48 | It's x.y.z where: 49 | 50 | * x: something major happened 51 | * y: next release 52 | * z: bug fix only 53 | 54 | ### License 55 | 56 | meshy is released under the Apache License Version 2.0. See 57 | [Apache](http://www.apache.org/licenses/LICENSE-2.0) or the LICENSE 58 | for details. -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 4.0.0 16 | 17 | com.addthis.common.build.maven.pom 18 | jar-pom 19 | 3.9.1 20 | 21 | 22 | com.addthis 23 | meshy 24 | meshy 25 | 4.0.2-SNAPSHOT 26 | TCP multiplexing gearwheels 27 | 28 | 29 | Apache License, Version 2.0 30 | http://www.apache.org/licenses/LICENSE-2.0.txt 31 | 32 | 33 | 34 | 35 | 1.8 36 | 37 | 4.0.28.Final 38 | 0.0.26 39 | 40 | 41 | 42 | 43 | 44 | io.netty 45 | netty-all 46 | ${meshy.dep.netty4.version} 47 | 48 | 49 | io.netty 50 | netty-buffer 51 | ${meshy.dep.netty4.version} 52 | 53 | 54 | 55 | 56 | 57 | 58 | com.addthis 59 | muxy 60 | 2.2.2 61 | true 62 | 63 | 64 | com.addthis.basis 65 | basis-core 66 | 4.3.1 67 | 68 | 69 | io.netty 70 | netty-all 71 | 72 | 73 | com.yammer.metrics 74 | metrics-core 75 | 76 | 77 | org.slf4j 78 | slf4j-simple 79 | test 80 | 81 | 82 | org.eclipse.jetty 83 | jetty-servlet 84 | 9.4.5.v20170502 85 | 86 | 87 | io.prometheus 88 | simpleclient 89 | ${dep.prometheus.version} 90 | 91 | 92 | io.prometheus 93 | simpleclient_hotspot 94 | ${dep.prometheus.version} 95 | 96 | 97 | io.prometheus 98 | simpleclient_servlet 99 | ${dep.prometheus.version} 100 | 101 | 102 | io.prometheus.jmx 103 | collector 104 | 0.10 105 | 106 | 107 | 108 | 109 | 110 | 111 | org.apache.maven.plugins 112 | maven-surefire-plugin 113 | 114 | 115 | PARANOID 116 | 117 | 118 | 119 | 120 | org.apache.maven.plugins 121 | maven-shade-plugin 122 | 2.3 123 | 124 | 125 | package 126 | 127 | shade 128 | 129 | 130 | false 131 | exec 132 | true 133 | 134 | 135 | *:* 136 | 137 | META-INF/*.SF 138 | META-INF/*.DSA 139 | META-INF/*.RSA 140 | 141 | 142 | 143 | 144 | 145 | reference.conf 146 | 147 | 148 | application.conf 149 | 150 | 151 | 152 | com.addthis.meshy.Main 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | org.apache.rat 167 | apache-rat-plugin 168 | 169 | 170 | 171 | 172 | 173 | scm:git:git@github.com:addthis/meshy.git 174 | scm:git:git@github.com:addthis/meshy.git 175 | https://github.com/addthis/meshy 176 | HEAD 177 | 178 | 179 | 180 | -------------------------------------------------------------------------------- /src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 4 | 17 | exec 18 | 19 | jar 20 | 21 | false 22 | 23 | 24 | / 25 | true 26 | true 27 | runtime 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/AggregateChannelFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.util.Collection; 17 | import java.util.concurrent.atomic.AtomicInteger; 18 | 19 | import io.netty.channel.ChannelFuture; 20 | import io.netty.channel.ChannelFutureListener; 21 | import io.netty.util.concurrent.DefaultPromise; 22 | import io.netty.util.concurrent.EventExecutor; 23 | import io.netty.util.concurrent.ImmediateEventExecutor; 24 | 25 | class AggregateChannelFuture extends DefaultPromise { 26 | 27 | public final Collection futures; 28 | 29 | private final AtomicInteger complete; 30 | private final ChannelFutureListener aggregatingListener; 31 | 32 | // the group failure will just report any one sub-cause rather than bothering with suppressing (would need CAS) 33 | private volatile Throwable anyCause = null; 34 | 35 | /** the promises collection should not be mutated after construction */ 36 | AggregateChannelFuture(Collection futures, EventExecutor executor) { 37 | super(executor); 38 | this.futures = futures; 39 | this.complete = new AtomicInteger(0); 40 | this.aggregatingListener = future -> { 41 | if (!future.isSuccess()) { 42 | anyCause = future.cause(); 43 | } 44 | if (complete.incrementAndGet() == futures.size()) { 45 | if (anyCause == null) { 46 | super.setSuccess(null); 47 | } else { 48 | super.setFailure(anyCause); 49 | } 50 | } 51 | }; 52 | for (ChannelFuture future : futures) { 53 | future.addListener(aggregatingListener); 54 | } 55 | } 56 | 57 | AggregateChannelFuture(Collection futures) { 58 | this(futures, ImmediateEventExecutor.INSTANCE); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/AutoConnectToPeersTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | import java.net.InetSocketAddress; 19 | import java.util.List; 20 | 21 | class AutoConnectToPeersTask implements Runnable { 22 | protected static final Logger log = LoggerFactory.getLogger(AutoConnectToPeersTask.class); 23 | 24 | private final MeshyServer meshyServer; 25 | private final List addresses; 26 | private final int timeout; 27 | 28 | public AutoConnectToPeersTask(MeshyServer meshyServer, List addresses, int timeout) { 29 | this.meshyServer = meshyServer; 30 | this.addresses = addresses; 31 | this.timeout = timeout; 32 | } 33 | 34 | @Override public void run() { 35 | while (true) { 36 | log.debug("mss will try connection to seeds again in: " + timeout + " milliseconds."); 37 | try { 38 | Thread.sleep(timeout); 39 | } catch (InterruptedException ignored) { 40 | } 41 | for (InetSocketAddress address : addresses) { 42 | meshyServer.connectPeer(address); 43 | } 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/AutoMeshTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.ByteArrayInputStream; 17 | import java.io.ByteArrayOutputStream; 18 | import java.io.IOException; 19 | 20 | import java.net.DatagramPacket; 21 | import java.net.DatagramSocket; 22 | import java.net.InetAddress; 23 | import java.net.InetSocketAddress; 24 | import java.net.SocketException; 25 | import java.net.SocketTimeoutException; 26 | 27 | import java.util.ArrayList; 28 | import java.util.LinkedList; 29 | import java.util.zip.CRC32; 30 | 31 | import com.addthis.basis.util.LessBytes; 32 | import com.addthis.basis.util.Parameter; 33 | 34 | import com.addthis.meshy.service.peer.PeerService; 35 | 36 | import com.google.common.collect.Lists; 37 | 38 | import org.slf4j.Logger; 39 | import org.slf4j.LoggerFactory; 40 | 41 | class AutoMeshTask implements Runnable { 42 | protected static final Logger log = LoggerFactory.getLogger(AutoMeshTask.class); 43 | 44 | private static final String secret = Parameter.value("meshy.secret"); 45 | 46 | private final MeshyServer meshyServer; 47 | private final MeshyServerGroup group; 48 | private final int timeout; 49 | private final int port; 50 | 51 | public AutoMeshTask(MeshyServer meshyServer, MeshyServerGroup group, int timeout, int port) { 52 | this.meshyServer = meshyServer; 53 | this.group = group; 54 | this.timeout = timeout; 55 | this.port = port; 56 | } 57 | 58 | private DatagramSocket newSocket() throws SocketException { 59 | return new DatagramSocket(port); 60 | } 61 | 62 | @Override public void run() { 63 | try (final DatagramSocket server = newSocket()) { 64 | server.setBroadcast(true); 65 | server.setSoTimeout(timeout); 66 | server.setReuseAddress(false); 67 | log.info("{} AutoMesh enabled server={}", meshyServer, server.getLocalAddress()); 68 | long lastTransmit = 0; 69 | while (true) { 70 | long time = System.currentTimeMillis(); 71 | if (time - lastTransmit > timeout) { 72 | if (log.isDebugEnabled()) { 73 | log.debug("{} AutoMesh.xmit {} members", meshyServer, group.getMembers().length); 74 | } 75 | server.send(encode()); 76 | lastTransmit = time; 77 | } 78 | try { 79 | DatagramPacket packet = new DatagramPacket(new byte[4096], 4096); 80 | server.receive(packet); 81 | log.debug("{} AutoMesh.recv from: {} size={}", 82 | meshyServer, packet.getAddress(), packet.getLength()); 83 | if (packet.getLength() > 0) { 84 | for (NodeInfo info : decode(packet)) { 85 | log.debug("{} AutoMesh.recv: {} : {} from {}", 86 | meshyServer, info.uuid, info.address, info.address); 87 | meshyServer.connectToPeer(info.uuid, info.address); 88 | } 89 | } 90 | } catch (SocketTimeoutException sto) { 91 | // expected ... ignore 92 | log.debug("{} AutoMesh listen timeout", meshyServer); 93 | } 94 | } 95 | } catch (Exception e) { 96 | log.error("{} AutoMesh exit on {}", meshyServer, e, e); 97 | } 98 | } 99 | 100 | private DatagramPacket encode() throws IOException { 101 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 102 | MeshyServer[] members = group.getMembers(); 103 | ArrayList readyList = Lists.newArrayList(members); 104 | LessBytes.writeInt(readyList.size(), out); 105 | for (MeshyServer meshy : readyList) { 106 | LessBytes.writeString(meshy.getUUID(), out); 107 | PeerService.encodeAddress(meshy.getLocalAddress(), out); 108 | } 109 | if (secret != null) { 110 | LessBytes.writeString(secret, out); 111 | } 112 | byte[] raw = out.toByteArray(); 113 | CRC32 crc = new CRC32(); 114 | crc.update(raw); 115 | out = new ByteArrayOutputStream(); 116 | LessBytes.writeBytes(raw, out); 117 | LessBytes.writeLength(crc.getValue(), out); 118 | DatagramPacket p = new DatagramPacket(out.toByteArray(), out.size()); 119 | p.setAddress(InetAddress.getByAddress(new byte[]{(byte) 255, (byte) 255, (byte) 255, (byte) 255})); 120 | p.setPort(port); 121 | return p; 122 | } 123 | 124 | private Iterable decode(DatagramPacket packet) throws IOException { 125 | InetAddress remote = packet.getAddress(); 126 | byte[] packed = packet.getData(); 127 | ByteArrayInputStream in = new ByteArrayInputStream(packed); 128 | byte[] raw = LessBytes.readBytes(in); 129 | long crcValue = LessBytes.readLength(in); 130 | CRC32 crc = new CRC32(); 131 | crc.update(raw); 132 | long crcCheck = crc.getValue(); 133 | if (crcCheck != crcValue) { 134 | throw new IOException("CRC mismatch " + crcValue + " != " + crcCheck); 135 | } 136 | in = new ByteArrayInputStream(raw); 137 | LinkedList list = new LinkedList<>(); 138 | int meshies = LessBytes.readInt(in); 139 | while (meshies-- > 0) { 140 | String remoteUuid = LessBytes.readString(in); 141 | InetSocketAddress address = PeerService.decodeAddress(in); 142 | InetAddress ina = address.getAddress(); 143 | if (ina.isAnyLocalAddress() || ina.isLoopbackAddress()) { 144 | address = new InetSocketAddress(remote, address.getPort()); 145 | } 146 | list.add(new NodeInfo(remoteUuid, address)); 147 | } 148 | if (secret != null) { 149 | String compare = in.available() > 0 ? LessBytes.readString(in) : ""; 150 | /* discard peer's list if secret doesn't match */ 151 | if (!secret.equals(compare)) { 152 | list.clear(); 153 | } 154 | } 155 | return list; 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/ChannelCloseListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import io.netty.channel.Channel; 17 | 18 | @FunctionalInterface 19 | public interface ChannelCloseListener { 20 | 21 | public void channelClosed(Channel channel); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/ChannelMaster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.util.Collection; 17 | 18 | 19 | public interface ChannelMaster { 20 | 21 | String getUUID(); 22 | 23 | /** session handler factory */ 24 | TargetHandler createHandler(int type); 25 | 26 | Collection getChannels(String nameFilter); 27 | 28 | int newSession(); 29 | 30 | int targetHandlerId(Class targetHandler); 31 | 32 | // metrics 33 | 34 | void sentBytes(int size); 35 | 36 | void recvBytes(int size); 37 | 38 | long lastEventTime(); 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/HttpServer.java: -------------------------------------------------------------------------------- 1 | package com.addthis.meshy; 2 | 3 | import org.eclipse.jetty.server.Server; 4 | import org.eclipse.jetty.servlet.ServletContextHandler; 5 | import org.eclipse.jetty.servlet.ServletHolder; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import io.prometheus.client.exporter.MetricsServlet; 10 | 11 | public class HttpServer { 12 | private static final Logger log = LoggerFactory.getLogger(HttpServer.class); 13 | 14 | private Server server; 15 | 16 | public HttpServer(int port) { 17 | this.server = new Server(port); 18 | ServletContextHandler context = new ServletContextHandler(); 19 | context.setContextPath("/"); 20 | context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics"); 21 | server.setHandler(context); 22 | } 23 | 24 | public void start() throws Exception { 25 | log.info("Starting metrics server"); 26 | server.start(); 27 | } 28 | 29 | public void stop() { 30 | log.info("Stopping metrics server"); 31 | try { 32 | server.stop(); 33 | } catch (Exception e) { 34 | log.error("Error shutting down metrics server", e); 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/InputStreamWrapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.EOFException; 17 | import java.io.IOException; 18 | import java.io.InputStream; 19 | 20 | import java.util.Arrays; 21 | import java.util.concurrent.TimeUnit; 22 | import java.util.concurrent.atomic.AtomicLong; 23 | 24 | import com.addthis.basis.util.Parameter; 25 | 26 | import com.yammer.metrics.Metrics; 27 | import com.yammer.metrics.core.Meter; 28 | 29 | /** 30 | * simple wrapper. does not attempt to follow nextBytes() wait 31 | * contract. assumption is this is wrapping a file input or 32 | * similar that won't block for long. 33 | */ 34 | public class InputStreamWrapper implements VirtualFileInput { 35 | 36 | private static final Meter shortReadMeter = Metrics.newMeter(InputStreamWrapper.class, "shortReads", "shortReads", TimeUnit.SECONDS); 37 | private static final AtomicLong shortRead = new AtomicLong(0); 38 | private static final int DEFAULT_BUFFER = Parameter.intValue("meshy.input.wrapper.buffer", 16000); 39 | private static final int DEFAULT_MINIMUM = Parameter.intValue("meshy.input.wrapper.bufferMin", 16000); 40 | 41 | public static long getShortReadCount() { 42 | return shortRead.getAndSet(0); 43 | } 44 | 45 | private final byte[] buf; 46 | private final InputStream input; 47 | private boolean done; 48 | 49 | public InputStreamWrapper(InputStream input) { 50 | /* leave overhead for framing when creating wrapper buffers (powers of 2) */ 51 | this(input, DEFAULT_BUFFER); 52 | } 53 | 54 | public InputStreamWrapper(InputStream input, int bufSize) { 55 | this.input = input; 56 | this.buf = new byte[bufSize]; 57 | } 58 | 59 | public byte[] readBytes(InputStream in, byte[] b, int min) throws IOException { 60 | int got = 0; 61 | int read = 0; 62 | while (got < min && (read = in.read(b, got, b.length - got)) >= 0) { 63 | got += read; 64 | } 65 | if (read < 0) { 66 | done = true; 67 | } 68 | if (got < b.length) { 69 | byte[] ret = new byte[got]; 70 | System.arraycopy(b, 0, ret, 0, got); 71 | b = ret; 72 | shortRead.incrementAndGet(); 73 | shortReadMeter.mark(); 74 | } 75 | return b; 76 | } 77 | 78 | @Override 79 | public byte[] nextBytes(long wait) { 80 | if (done) { 81 | return null; 82 | } 83 | try { 84 | if (DEFAULT_MINIMUM > 0) { 85 | return readBytes(input, buf, DEFAULT_MINIMUM); 86 | } 87 | int read = input.read(buf); 88 | if (read < 0) { 89 | done = true; 90 | return null; 91 | } 92 | if (read < buf.length) { 93 | shortReadMeter.mark(); 94 | shortRead.incrementAndGet(); 95 | return Arrays.copyOf(buf, read); 96 | } 97 | return buf; 98 | } catch (EOFException ex) { 99 | done = true; 100 | return null; 101 | } catch (Exception ex) { 102 | ex.printStackTrace(); 103 | done = true; 104 | return null; 105 | } 106 | } 107 | 108 | @Override 109 | public boolean isEOF() { 110 | return done; 111 | } 112 | 113 | @Override 114 | public void close() { 115 | try { 116 | input.close(); 117 | } catch (Exception ex) { 118 | ex.printStackTrace(); 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/LocalFileHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.File; 17 | 18 | import java.util.Iterator; 19 | 20 | import java.nio.file.PathMatcher; 21 | 22 | 23 | public interface LocalFileHandler { 24 | 25 | public boolean canHandleDirectory(File dir); 26 | 27 | public Iterator listFiles(File dir, PathMatcher filter); 28 | 29 | public VirtualFileReference getFile(File dir, String name); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/LocalFileHandlerMux.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.File; 17 | 18 | import java.util.Iterator; 19 | import java.util.LinkedList; 20 | import java.util.Map; 21 | 22 | import java.nio.file.PathMatcher; 23 | import java.nio.file.Paths; 24 | 25 | import com.addthis.muxy.MuxFile; 26 | import com.addthis.muxy.ReadMuxFileDirectory; 27 | import com.addthis.muxy.ReadMuxFileDirectoryCache; 28 | 29 | import com.google.common.base.MoreObjects; 30 | 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | 35 | public class LocalFileHandlerMux implements LocalFileHandler { 36 | 37 | private static final Logger log = LoggerFactory.getLogger(LocalFileHandlerMux.class); 38 | 39 | private static boolean checkForMux() { 40 | try { 41 | Class.forName("com.addthis.muxy.ReadMuxFileDirectory"); 42 | log.info("Muxy class path found and loaded."); 43 | return true; 44 | } catch (ClassNotFoundException cnfe) { 45 | log.warn("Muxy not found in path and not loaded."); 46 | return false; 47 | } 48 | } 49 | 50 | public static final boolean muxEnabled = !Boolean.getBoolean("meshy.muxy.disable") && checkForMux(); 51 | 52 | @Override 53 | public boolean canHandleDirectory(File dir) { 54 | return ReadMuxFileDirectory.isMuxDir(dir.toPath()); 55 | } 56 | 57 | @Override 58 | public Iterator listFiles(File dir, PathMatcher filter) { 59 | try { 60 | LinkedList list = new LinkedList<>(); 61 | for (MuxFile meta : ReadMuxFileDirectoryCache.listFiles(dir)) { 62 | VirtualFileReference ref = new MuxFileReference(meta); 63 | if ((filter == null) || filter.matches(Paths.get(ref.getName()))) { 64 | list.add(ref); 65 | } 66 | } 67 | return list.iterator(); 68 | } catch (Exception ex) { 69 | log.error("Mystery exception we are swallowing", ex); 70 | return null; 71 | } 72 | } 73 | 74 | @Override 75 | public VirtualFileReference getFile(File dir, String name) { 76 | try { 77 | return new MuxFileReference(ReadMuxFileDirectoryCache.getFileMeta(dir, name)); 78 | } catch (Exception ex) { 79 | log.error("Mystery exception we are swallowing", ex); 80 | return null; 81 | } 82 | } 83 | 84 | /** 85 | * multiplexed ptr reference 86 | */ 87 | private static final class MuxFileReference implements VirtualFileReference { 88 | 89 | private final MuxFile meta; 90 | 91 | MuxFileReference(MuxFile meta) { 92 | this.meta = meta; 93 | } 94 | 95 | @Override 96 | public String getName() { 97 | return meta.getName(); 98 | } 99 | 100 | @Override 101 | public long getLastModified() { 102 | return meta.getLastModified(); 103 | } 104 | 105 | @Override 106 | public long getLength() { 107 | return meta.getLength(); 108 | } 109 | 110 | /** mux files cannot have sub-directories */ 111 | @Override 112 | public Iterator listFiles(PathMatcher filter) { 113 | return null; 114 | } 115 | 116 | @Override 117 | public VirtualFileReference getFile(String name) { 118 | return null; 119 | } 120 | 121 | @Override 122 | public VirtualFileInput getInput(Map options) { 123 | try { 124 | return new InputStreamWrapper(meta.read(0)); 125 | } catch (Exception e) { 126 | log.error("Mystery exception we are swallowing", e); 127 | return null; 128 | } 129 | } 130 | 131 | @Override 132 | public String toString() { 133 | return MoreObjects.toStringHelper(this) 134 | .add("meta", meta) 135 | .toString(); 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/LocalFileSystem.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import javax.annotation.Nonnull; 17 | import javax.annotation.Nullable; 18 | 19 | import java.io.File; 20 | import java.io.FileInputStream; 21 | 22 | import java.util.Iterator; 23 | import java.util.LinkedList; 24 | import java.util.Map; 25 | import java.util.stream.Collectors; 26 | import java.util.stream.Stream; 27 | import java.util.Collections; 28 | 29 | import java.nio.file.Files; 30 | import java.nio.file.Path; 31 | import java.nio.file.PathMatcher; 32 | 33 | import com.addthis.basis.util.Parameter; 34 | import com.addthis.basis.util.LessStrings; 35 | 36 | import org.slf4j.Logger; 37 | import org.slf4j.LoggerFactory; 38 | 39 | 40 | public class LocalFileSystem implements VirtualFileSystem { 41 | private static final Logger log = LoggerFactory.getLogger(LocalFileSystem.class); 42 | 43 | private static LocalFileHandler[] handlers; 44 | 45 | static { 46 | reloadHandlers(); 47 | } 48 | 49 | public static void reloadHandlers() { 50 | LinkedList list = new LinkedList<>(); 51 | if (LocalFileHandlerMux.muxEnabled) { 52 | list.add(new LocalFileHandlerMux()); 53 | } 54 | String[] handlerClasses = LessStrings.splitArray(Parameter.value("mesh.local.handlers", ""), ","); 55 | for (String handler : handlerClasses) { 56 | try { 57 | list.add((LocalFileHandler) (Class.forName(handler).newInstance())); 58 | } catch (Exception ex) { 59 | log.warn("unable to load file handler: ", ex); 60 | } 61 | } 62 | handlers = list.toArray(new LocalFileHandler[list.size()]); 63 | } 64 | 65 | private FileReference rootDir; 66 | 67 | public LocalFileSystem(File rootDir) { 68 | this.rootDir = new FileReference(rootDir); 69 | } 70 | 71 | @Override 72 | public String toString() { 73 | return "VFS:" + rootDir; 74 | } 75 | 76 | @Override 77 | public String[] tokenizePath(String path) { 78 | return LessStrings.splitArray(path, "/"); 79 | } 80 | 81 | @Override 82 | public VirtualFileReference getFileRoot() { 83 | return rootDir; 84 | } 85 | 86 | /** 87 | * normal ptr reference 88 | */ 89 | private static final class FileReference implements VirtualFileReference { 90 | 91 | private final File ptr; 92 | 93 | FileReference(final Path path) { 94 | this.ptr = path.toFile(); 95 | } 96 | 97 | FileReference(final File file) { 98 | this.ptr = file; 99 | } 100 | 101 | @Override 102 | public String toString() { 103 | return "VFR:" + ptr; 104 | } 105 | 106 | @Override 107 | public String getName() { 108 | return ptr.getName(); 109 | } 110 | 111 | @Override 112 | public long getLastModified() { 113 | return ptr.lastModified(); 114 | } 115 | 116 | @Override 117 | public long getLength() { 118 | return ptr.length(); 119 | } 120 | 121 | @Nullable @Override 122 | public Iterator listFiles(@Nonnull final PathMatcher filter) { 123 | try { 124 | return listFilesHelper(filter); 125 | } catch (Exception ex) { 126 | log.error("Mystery exception we are swallowing", ex); 127 | return null; 128 | } 129 | } 130 | 131 | @Nullable @Override 132 | public VirtualFileReference getFile(String name) { 133 | for (LocalFileHandler handler : handlers) { 134 | if (handler.canHandleDirectory(ptr)) { 135 | return handler.getFile(ptr, name); 136 | } 137 | } 138 | File next = new File(ptr, name); 139 | return next.exists() ? new FileReference(next) : null; 140 | } 141 | 142 | /** 143 | * unsafe. catch delegated to wrapper 144 | */ 145 | private Iterator listFilesHelper(@Nonnull final PathMatcher filter) throws Exception { 146 | for (LocalFileHandler handler : handlers) { 147 | if (handler.canHandleDirectory(ptr)) { 148 | log.debug("delegate {} to {}", ptr, handler); 149 | return handler.listFiles(ptr, filter); 150 | } 151 | } 152 | if (!Files.isDirectory(ptr.toPath())) { 153 | return Collections.emptyIterator(); 154 | } 155 | try (Stream files = Files.list(ptr.toPath()).filter(file -> filter.matches(file.getFileName()))) { 156 | return files.map(FileReference::new) 157 | .collect(Collectors.toList()) 158 | .iterator(); 159 | } 160 | } 161 | 162 | @Nullable @Override 163 | public VirtualFileInput getInput(final Map options) { 164 | try { 165 | if (ptr.isFile() && ptr.canRead()) { 166 | return new InputStreamWrapper(new FileInputStream(ptr)); 167 | } 168 | } catch (Exception ex) { 169 | log.error("Mystery exception we are swallowing", ex); 170 | } 171 | return null; 172 | } 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/MeshyClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.IOException; 17 | 18 | import java.net.InetSocketAddress; 19 | 20 | import java.util.Collection; 21 | import java.util.Map; 22 | import java.util.concurrent.Semaphore; 23 | import java.util.concurrent.TimeUnit; 24 | import java.util.concurrent.atomic.AtomicBoolean; 25 | 26 | import com.addthis.meshy.service.file.FileReference; 27 | import com.addthis.meshy.service.file.FileSource; 28 | import com.addthis.meshy.service.stream.SourceInputStream; 29 | import com.addthis.meshy.service.stream.StreamSource; 30 | 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | import io.netty.channel.Channel; 35 | import io.netty.channel.ChannelFuture; 36 | import io.netty.util.concurrent.Future; 37 | 38 | 39 | public class MeshyClient extends Meshy { 40 | 41 | private static final Logger log = LoggerFactory.getLogger(MeshyClient.class); 42 | 43 | /** 44 | * client 45 | */ 46 | public MeshyClient(String host, int port) throws IOException { 47 | this(new InetSocketAddress(host, port)); 48 | } 49 | 50 | /** 51 | * client 52 | */ 53 | public MeshyClient(InetSocketAddress address) throws IOException { 54 | super(); 55 | /* block session creation until connection is fully established */ 56 | try { 57 | clientInitGate.acquire(); 58 | } catch (Exception ex) { 59 | throw new RuntimeException(ex); 60 | } 61 | ChannelFuture clientConnect = connect(address); 62 | clientConnect.awaitUninterruptibly(); 63 | if (!clientConnect.isSuccess()) { 64 | close(); 65 | throw new IOException("connection fail to " + address); 66 | } 67 | clientChannelCloseFuture = clientConnect.channel().closeFuture(); 68 | /* re-acquire after connection comes up, which releases the lock */ 69 | try { 70 | clientInitGate.acquire(); 71 | } catch (Exception ex) { 72 | throw new RuntimeException(ex); 73 | } 74 | if (log.isDebugEnabled()) { 75 | log.debug("client [{}] connected to {}", getUUID(), address); 76 | } 77 | } 78 | 79 | @Override 80 | public String toString() { 81 | return "MC:{" + getUUID() + ",sm=" + getChannelCount() + "}"; 82 | } 83 | 84 | /** 85 | * returns a future that notifies of channel closure 86 | */ 87 | public ChannelFuture getClientChannelCloseFuture() { 88 | return clientChannelCloseFuture; 89 | } 90 | 91 | private final ChannelFuture clientChannelCloseFuture; 92 | private final Semaphore clientInitGate = new Semaphore(1); 93 | private final AtomicBoolean closed = new AtomicBoolean(false); 94 | 95 | private ChannelState clientState; 96 | private int bufferSize; 97 | 98 | /** 99 | * @return peer uuid only if this is a pure client 100 | * otherwise returns a null 101 | */ 102 | public String getPeerUUID() { 103 | return clientState != null ? clientState.getName() : null; 104 | } 105 | 106 | @Override 107 | protected void channelConnected(Channel channel, ChannelState channelState) { 108 | super.channelConnected(channel, channelState); 109 | clientState = channelState; 110 | clientInitGate.release(); 111 | } 112 | 113 | @Override public Future closeAsync() { 114 | if (closed.compareAndSet(false, true)) { 115 | return super.closeAsync(); 116 | } else { 117 | return workerGroup.shutdownGracefully(Meshy.QUIET_PERIOD, Meshy.SHUTDOWN_TIMEOUT, TimeUnit.SECONDS); 118 | } 119 | } 120 | 121 | public MeshyClient setBufferSize(int size) { 122 | bufferSize = size; 123 | return this; 124 | } 125 | 126 | /** 127 | * sync version 128 | */ 129 | public Collection listFiles(final String[] paths) throws IOException { 130 | if (closed.get()) { 131 | throw new IOException("client connection closed"); 132 | } 133 | FileSource fileSource = new FileSource(this, paths); 134 | fileSource.waitComplete(); 135 | return fileSource.getFileList(); 136 | } 137 | 138 | /** async version */ 139 | public void listFiles(final String[] paths, final ListCallback callback) throws IOException { 140 | if (closed.get()) { 141 | throw new IOException("client connection closed"); 142 | } 143 | FileSource fileSource = new FileSource(this) { 144 | @Override 145 | public void receiveReference(FileReference ref) { 146 | callback.receiveReference(ref); 147 | } 148 | 149 | @Override 150 | public void receiveComplete() { 151 | callback.receiveReferenceComplete(); 152 | } 153 | }; 154 | fileSource.requestRemoteFiles(paths); 155 | } 156 | 157 | public SourceInputStream readFile(FileReference ref) throws IOException { 158 | return readFile(ref.getHostUUID(), ref.name); 159 | } 160 | 161 | public SourceInputStream readFile(FileReference ref, Map options) throws IOException { 162 | return readFile(ref.getHostUUID(), ref.name, options); 163 | } 164 | 165 | public SourceInputStream readFile(String nodeUuid, String fileName) throws IOException { 166 | if (closed.get()) { 167 | throw new IOException("client connection closed"); 168 | } 169 | return new StreamSource(this, nodeUuid, fileName, bufferSize).getInputStream(); 170 | } 171 | 172 | public SourceInputStream readFile(String nodeUuid, String fileName, Map options) throws IOException { 173 | if (closed.get()) { 174 | throw new IOException("client connection closed"); 175 | } 176 | return new StreamSource(this, nodeUuid, fileName, options, bufferSize).getInputStream(); 177 | } 178 | 179 | public StreamSource getFileSource(String nodeUuid, String fileName, Map options) 180 | throws IOException { 181 | if (closed.get()) { 182 | throw new IOException("client connection closed"); 183 | } 184 | return new StreamSource(this, nodeUuid, fileName, options, bufferSize); 185 | } 186 | 187 | /** */ 188 | public static interface ListCallback { 189 | 190 | /** 191 | * Called each time a new file reference is received 192 | * 193 | * @param ref - the file referecne received 194 | */ 195 | public void receiveReference(FileReference ref); 196 | 197 | /** 198 | * Called when all reference have been completed. This can 199 | * be used to determine when the communication interaction to 200 | * the mesh source has completed, allowing clients to fail 201 | * quickly when no references are found. 202 | */ 203 | public void receiveReferenceComplete(); 204 | } 205 | 206 | } 207 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/MeshyClientConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.IOException; 17 | 18 | import java.util.concurrent.atomic.AtomicBoolean; 19 | import java.util.concurrent.atomic.AtomicReference; 20 | 21 | /** 22 | * attempts to re-establish client connections when they fail 23 | */ 24 | public abstract class MeshyClientConnector extends Thread { 25 | 26 | private final String host; 27 | private final int port; 28 | private final long initDelay; 29 | private final long retryWait; 30 | private final AtomicBoolean done = new AtomicBoolean(false); 31 | private final AtomicReference ref = new AtomicReference<>(null); 32 | 33 | /** 34 | * @param host meshy server host 35 | * @param port meshy server port 36 | * @param initDelay delay before initial connection 37 | * @param retryWait delay between re-connect attempts 38 | */ 39 | public MeshyClientConnector(String host, int port, long initDelay, long retryWait) { 40 | this.host = host; 41 | this.port = port; 42 | this.initDelay = initDelay; 43 | this.retryWait = retryWait; 44 | setDaemon(true); 45 | setName("MeshyClient Re-Connector to " + host + ":" + port); 46 | start(); 47 | } 48 | 49 | public abstract void linkUp(MeshyClient client); 50 | 51 | public abstract void linkDown(MeshyClient client); 52 | 53 | public MeshyClient getClient() { 54 | return ref.get(); 55 | } 56 | 57 | public void terminate() { 58 | done.set(true); 59 | interrupt(); 60 | } 61 | 62 | @Override 63 | public void run() { 64 | if (initDelay > 0) { 65 | try { 66 | Thread.sleep(initDelay); 67 | } catch (Exception ignored) { 68 | } 69 | } 70 | while (!done.get()) { 71 | try { 72 | MeshyClient client = new MeshyClient(host, port); 73 | client.getClientChannelCloseFuture().addListener(future -> { 74 | linkDown(ref.getAndSet(null)); 75 | }); 76 | ref.set(client); 77 | linkUp(client); 78 | while (ref.get() != null) { 79 | Thread.sleep(500); 80 | } 81 | } catch (InterruptedException ex) { 82 | // expected on terminate() 83 | } catch (IOException ex) { 84 | try { 85 | Thread.sleep(retryWait); 86 | } catch (Exception ignored) { 87 | } 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/MeshyClientHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.IOException; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | import java.util.concurrent.atomic.AtomicInteger; 21 | 22 | import io.netty.util.concurrent.Future; 23 | import io.netty.util.concurrent.GlobalEventExecutor; 24 | import io.netty.util.concurrent.SucceededFuture; 25 | 26 | 27 | public final class MeshyClientHelper { 28 | 29 | /** 30 | * static shared connections 31 | */ 32 | static final Map meshyClients = new HashMap<>(); 33 | 34 | private MeshyClientHelper() { 35 | } 36 | 37 | /** 38 | * for Hydra use ... in filters, etc ... to prevent creating many clients 39 | */ 40 | public static MeshyClient getSharedInstance(String host, int port) throws IOException { 41 | synchronized (meshyClients) { 42 | String key = host + ":" + port; 43 | StaticClient client = meshyClients.get(key); 44 | if (client == null) { 45 | client = new StaticClient(key, host, port); 46 | meshyClients.put(key, client); 47 | } 48 | client.incRef(); 49 | return client; 50 | } 51 | } 52 | 53 | /** */ 54 | private static class StaticClient extends MeshyClient { 55 | 56 | private final AtomicInteger refCount = new AtomicInteger(0); 57 | private final String key; 58 | 59 | StaticClient(String key, String host, int port) throws IOException { 60 | super(host, port); 61 | this.key = key; 62 | } 63 | 64 | StaticClient incRef() { 65 | refCount.incrementAndGet(); 66 | return this; 67 | } 68 | 69 | @Override public Future closeAsync() { 70 | synchronized (meshyClients) { 71 | if (refCount.decrementAndGet() == 0) { 72 | meshyClients.remove(key); 73 | return super.closeAsync(); 74 | } else { 75 | return new SucceededFuture<>(GlobalEventExecutor.INSTANCE, null); 76 | } 77 | } 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/MeshyConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | 17 | public interface MeshyConstants { 18 | 19 | int KEY_RESPONSE = 0; 20 | int KEY_EXISTING = Integer.MIN_VALUE; 21 | String LINK_ALL = null; 22 | String LINK_NAMED = ""; 23 | byte[] EMPTY_BYTES = new byte[0]; 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/MeshyServerGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.util.HashMap; 17 | import java.util.HashSet; 18 | import java.util.LinkedList; 19 | import java.util.Map; 20 | import java.util.UUID; 21 | import java.util.concurrent.TimeUnit; 22 | 23 | import com.addthis.basis.util.JitterClock; 24 | import com.addthis.basis.util.Parameter; 25 | 26 | import com.addthis.meshy.service.file.FileStats; 27 | import com.addthis.meshy.service.stream.StreamStats; 28 | import com.addthis.muxy.ReadMuxFileDirectoryCache; 29 | 30 | import com.yammer.metrics.core.VirtualMachineMetrics; 31 | 32 | import org.slf4j.Logger; 33 | import org.slf4j.LoggerFactory; 34 | 35 | 36 | public class MeshyServerGroup { 37 | 38 | private static final Logger log = LoggerFactory.getLogger(MeshyServerGroup.class); 39 | 40 | private static final GCMetrics gcMetrics = new GCMetrics(); 41 | private static final boolean MERGE_METRICS = Parameter.boolValue("meshy.metrics.merge", true); 42 | 43 | private final HashSet byUuid = new HashSet<>(); 44 | private final HashSet byServer = new HashSet<>(); 45 | private final String uuid = Long.toHexString(UUID.randomUUID().getMostSignificantBits()); 46 | private final LinkedList lastStats = new LinkedList<>(); 47 | private volatile int openStreams; 48 | private final Thread statsThread; 49 | 50 | private int statsCountdown = 2; 51 | 52 | // TODO replace with scheduled thread pool 53 | public MeshyServerGroup() { 54 | statsThread = new Thread() { 55 | public void run() { 56 | setName("MeshyStats"); 57 | if (Meshy.STATS_INTERVAL <= 0) { 58 | log.debug("stats thread disabled"); 59 | return; 60 | } 61 | while (true) { 62 | emitStats(); 63 | try { 64 | Thread.sleep(Meshy.STATS_INTERVAL); 65 | } catch (Exception ignored) { 66 | return; 67 | } 68 | } 69 | } 70 | }; 71 | statsThread.setDaemon(true); 72 | statsThread.start(); 73 | Runtime.getRuntime().addShutdownHook(new Thread() { 74 | public void run() { 75 | statsThread.interrupt(); 76 | } 77 | }); 78 | } 79 | 80 | public String[] getLastStats() { 81 | synchronized (lastStats) { 82 | return lastStats.toArray(new String[lastStats.size()]); 83 | } 84 | } 85 | 86 | public Map getLastStatsMap() { 87 | HashMap stats = new HashMap<>(); 88 | stats.put("sO", openStreams); 89 | return stats; 90 | } 91 | 92 | private void emitStats() { 93 | GCSummary gc = gcMetrics.update(Meshy.vmMetrics); 94 | StreamStats ss = new StreamStats(); 95 | FileStats fs = new FileStats(); 96 | 97 | StringBuilder rep = new StringBuilder(); 98 | rep.append("seqReads="); 99 | rep.append(ss.seqRead); // number of sequential nextBytes from the same target 100 | rep.append(" totalReads="); 101 | rep.append(ss.totalRead); // number of total reads across all targets 102 | rep.append(" bytesRead="); 103 | rep.append(ss.readBytes); // number of total bytes read across all targets (does not include rerouting) 104 | rep.append(" sN="); 105 | rep.append(ss.newOpenCount); // newly open streams since last logline 106 | rep.append(" sC="); 107 | rep.append(ss.closedStreams); // closed streams since last logline 108 | rep.append(" sO="); 109 | openStreams = ss.openCount; 110 | rep.append(openStreams); // open streams 111 | rep.append(" sQ="); 112 | rep.append(ss.qSize); // "more" finderQueue size 113 | rep.append(" sR="); 114 | rep.append(ss.readWaitTime); // time spent reading from disk 115 | rep.append(" sW="); 116 | rep.append(Meshy.numbers.format(ss.sendWaiting)); // send buffers bytes waiting to return 117 | rep.append(" sZ="); 118 | rep.append(ss.sleeps); // sleeps b/c over sendWait limit 119 | rep.append(" cZ="); 120 | rep.append(ChannelState.writeSleeps.getAndSet(0)); // sleeps b/c over channel watermark 121 | rep.append(" fQ="); 122 | rep.append(fs.finderQueue); // number of finds waiting in queue 123 | rep.append(" fR="); 124 | rep.append(fs.findsRunning); // calls to find in-progress 125 | rep.append(" fF="); 126 | rep.append(fs.finds); // calls to find command 127 | rep.append(" fO="); 128 | rep.append(fs.found); // number of files returned 129 | rep.append(" fT="); 130 | rep.append(fs.findTime); // time spend in find command 131 | rep.append(" fTL="); 132 | rep.append(fs.findTimeLocal); // time spend in find command locally 133 | rep.append(" iSR="); 134 | rep.append(InputStreamWrapper.getShortReadCount()); // input stream wrapper short reads (bad for perf) 135 | rep.append(" gcR="); 136 | rep.append(gc.runs); // # of gc invocations 137 | rep.append(" gcT="); 138 | rep.append(gc.timeSpent); // ms spent in gc 139 | if (LocalFileHandlerMux.muxEnabled) { 140 | rep.append(" mD="); 141 | rep.append(ReadMuxFileDirectoryCache.getCacheDirSize()); // muxy cached dirs 142 | rep.append(" mF="); 143 | rep.append(ReadMuxFileDirectoryCache.getCacheFileSize()); // muxy cached files 144 | } 145 | 146 | int bin = 0; 147 | int bout = 0; 148 | if (MERGE_METRICS) { 149 | int channelCount = 0; 150 | int peerCount = 0; 151 | for (MeshyServer server : byServer) { 152 | ServerStats stats = server.getStats(); 153 | bin += stats.bin; 154 | bout += stats.bout; 155 | channelCount += stats.channelCount; 156 | peerCount += stats.peerCount; 157 | } 158 | rep.append(" mC=" + channelCount); // total channel count 159 | rep.append(" mS=" + peerCount); // fully connected channels 160 | rep.append(" mBI=" + bin); // total bytes in 161 | rep.append(" mBO=" + bout); // total bytes out 162 | } else { 163 | int index = 0; 164 | for (MeshyServer server : byServer) { 165 | ServerStats stats = server.getStats(); 166 | bin += stats.bin; 167 | bout += stats.bout; 168 | String pre = byServer.size() > 1 ? (" " + index) : " "; 169 | rep.append(pre + "p=" + server.getLocalPort() + "-" + server.getNetIf()); 170 | rep.append(pre + "mS=" + stats.peerCount); // fully connected channels 171 | rep.append(pre + "mBI=" + stats.bin); // total bytes in 172 | rep.append(pre + "mBO=" + stats.bout); // total bytes out 173 | index++; 174 | } 175 | } 176 | 177 | final boolean statsSkip = (bin | bout) == 0; 178 | if (Meshy.THROTTLE_LOG && statsSkip && statsCountdown-- <= 0) { 179 | return; 180 | } 181 | 182 | String report = rep.toString(); 183 | MeshyServer.log.info(report); 184 | synchronized (lastStats) { 185 | lastStats.addLast("t=" + JitterClock.globalTime() + " " + report); 186 | if (lastStats.size() > 10) { 187 | lastStats.removeFirst(); 188 | } 189 | } 190 | if (!statsSkip) { 191 | statsCountdown = 2; 192 | } 193 | if (gc.timeSpent > Meshy.STATS_INTERVAL) { 194 | for (MeshyServer server : byServer) { 195 | synchronized (server.connectedChannels) { 196 | for (ChannelState channelState : server.connectedChannels) { 197 | channelState.debugSessions(); 198 | } 199 | } 200 | } 201 | } 202 | } 203 | 204 | public void join(MeshyServer server) { 205 | byUuid.add(server.getUUID()); 206 | synchronized (byServer) { 207 | byServer.add(server); 208 | } 209 | } 210 | 211 | public boolean hasUuid(String testUuid) { 212 | synchronized (byUuid) { 213 | return byUuid.contains(testUuid); 214 | } 215 | } 216 | 217 | public boolean hasServer(MeshyServer server) { 218 | synchronized (byServer) { 219 | return byServer.contains(server); 220 | } 221 | } 222 | 223 | public MeshyServer[] getMembers() { 224 | synchronized (byServer) { 225 | return byServer.toArray(new MeshyServer[byServer.size()]); 226 | } 227 | } 228 | 229 | public String getGroupUuid() { 230 | return uuid; 231 | } 232 | 233 | private static class GCMetrics { 234 | 235 | private long lastTime; 236 | private long lastRuns; 237 | 238 | GCSummary update(VirtualMachineMetrics vmMetrics) { 239 | long totalTime = 0; 240 | long totalRuns = 0; 241 | for (Map.Entry e : vmMetrics.garbageCollectors().entrySet()) { 242 | VirtualMachineMetrics.GarbageCollectorStats stats = e.getValue(); 243 | totalTime += Math.max(0, stats.getTime(TimeUnit.MILLISECONDS)); 244 | totalRuns += Math.max(0, stats.getRuns()); 245 | } 246 | long newTime = totalTime - lastTime; 247 | long newRuns = totalRuns - lastRuns; 248 | lastTime = totalTime; 249 | lastRuns = totalRuns; 250 | return new GCSummary(newTime, newRuns); 251 | } 252 | } 253 | 254 | private static class GCSummary { 255 | 256 | public final long timeSpent; 257 | public final long runs; 258 | 259 | private GCSummary(long timeSpent, long runs) { 260 | this.runs = runs; 261 | this.timeSpent = timeSpent; 262 | } 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/NodeInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.net.InetSocketAddress; 17 | 18 | class NodeInfo { 19 | 20 | final String uuid; 21 | final InetSocketAddress address; 22 | 23 | NodeInfo(String uuid, InetSocketAddress address) { 24 | this.uuid = uuid; 25 | this.address = address; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/SendWatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | 17 | public interface SendWatcher { 18 | 19 | public void sendFinished(int bytes); 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/ServerStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | /** for cmd-line stats output */ 17 | public class ServerStats { 18 | 19 | final int bin; 20 | final int bout; 21 | final int peerCount; 22 | final int channelCount; 23 | 24 | ServerStats(MeshyServer server) { 25 | bin = server.getAndClearRecv(); 26 | bout = server.getAndClearSent(); 27 | channelCount = server.getChannelCount(); 28 | peerCount = server.getServerPeerCount(); 29 | MeshyServer.peerCountMetric.clear(); 30 | MeshyServer.peerCountMetric.inc(server.getServerPeerCount()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/SessionHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import io.netty.buffer.ByteBuf; 17 | 18 | public interface SessionHandler { 19 | 20 | boolean send(byte[] data, SendWatcher watcher); 21 | 22 | boolean sendComplete(); 23 | 24 | void receive(ChannelState state, int session, int length, ByteBuf buffer) throws Exception; 25 | 26 | void receiveComplete(ChannelState state, int session) throws Exception; 27 | 28 | void waitComplete(); 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/TargetHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.util.concurrent.CountDownLatch; 17 | import java.util.concurrent.atomic.AtomicBoolean; 18 | 19 | import com.google.common.base.MoreObjects; 20 | 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import io.netty.buffer.ByteBuf; 25 | 26 | import static com.addthis.meshy.MeshyConstants.KEY_RESPONSE; 27 | 28 | 29 | public abstract class TargetHandler implements SessionHandler { 30 | 31 | protected static final Logger log = LoggerFactory.getLogger(TargetHandler.class); 32 | private final AtomicBoolean complete = new AtomicBoolean(false); 33 | private final AtomicBoolean waited = new AtomicBoolean(false); 34 | private final CountDownLatch latch = new CountDownLatch(1); 35 | 36 | private MeshyServer master; 37 | private ChannelState channelState; 38 | private int session; 39 | 40 | public TargetHandler() { 41 | } 42 | 43 | public void setContext(MeshyServer master, ChannelState state, int session) { 44 | this.master = master; 45 | this.channelState = state; 46 | this.session = session; 47 | } 48 | 49 | protected MoreObjects.ToStringHelper toStringHelper() { 50 | return MoreObjects.toStringHelper(this) 51 | .add("channelState", channelState.getName()) 52 | .add("session", session) 53 | .add("complete", complete) 54 | .add("waited", waited); 55 | } 56 | 57 | @Override 58 | public String toString() { 59 | return toStringHelper().toString(); 60 | } 61 | 62 | public ChannelState getChannelState() { 63 | return channelState; 64 | } 65 | 66 | public MeshyServer getChannelMaster() { 67 | return master; 68 | } 69 | 70 | public int getSessionId() { 71 | return session; 72 | } 73 | 74 | public boolean send(byte[] data) { 75 | return send(data, null); 76 | } 77 | 78 | public void send(ByteBuf from, int length) { 79 | log.trace("{} send.buf [{}] {}", this, length, from); 80 | channelState.send(channelState.allocateSendBuffer(KEY_RESPONSE, session, from, length), null, length); 81 | } 82 | 83 | @Override public boolean send(byte[] data, SendWatcher watcher) { 84 | log.trace("{} send {}", this, data.length); 85 | return channelState.send(channelState.allocateSendBuffer(KEY_RESPONSE, session, data), 86 | watcher, data.length); 87 | } 88 | 89 | public void send(byte[] data, int off, int len, SendWatcher watcher) { 90 | log.trace("{} send {} o={} l={}", this, data.length, off, len); 91 | channelState.send(channelState.allocateSendBuffer(KEY_RESPONSE, session, data, off, len), 92 | watcher, len); 93 | } 94 | 95 | public ByteBuf getSendBuffer(int length) { 96 | return channelState.allocateSendBuffer(KEY_RESPONSE, session, length); 97 | } 98 | 99 | public int send(ByteBuf buffer, SendWatcher watcher) { 100 | if (log.isTraceEnabled()) { 101 | log.trace("{} send b={} l={}", this, buffer, buffer.readableBytes()); 102 | } 103 | int length = buffer.readableBytes(); 104 | channelState.send(buffer, watcher, length); 105 | return length; 106 | } 107 | 108 | @Override 109 | public boolean sendComplete() { 110 | return send(MeshyConstants.EMPTY_BYTES, null); 111 | } 112 | 113 | @Override 114 | public void receive(ChannelState state, int receivingSession, int length, ByteBuf buffer) throws Exception { 115 | assert this.channelState == state; 116 | assert this.session == session; 117 | log.debug("{} receive [{}] l={}", this, session, length); 118 | receive(length, buffer); 119 | } 120 | 121 | @Override 122 | public void receiveComplete(ChannelState state, int completedSession) throws Exception { 123 | assert this.channelState == state; 124 | assert this.session == completedSession; 125 | log.debug("{} receiveComplete.1 [{}]", this, completedSession); 126 | if (!state.getChannel().isOpen()) { 127 | channelClosed(); 128 | } 129 | receiveComplete(completedSession); 130 | } 131 | 132 | private void receiveComplete(int completedSession) throws Exception { 133 | assert this.session == completedSession; 134 | log.debug("{} receiveComplete.2 [{}]", this, completedSession); 135 | // ensure this is only called once 136 | if (complete.compareAndSet(false, true)) { 137 | receiveComplete(); 138 | latch.countDown(); 139 | } 140 | } 141 | 142 | protected void autoReceiveComplete() { 143 | channelState.sessionComplete(this, MeshyConstants.KEY_EXISTING, session); 144 | } 145 | 146 | @Override 147 | public void waitComplete() { 148 | // this is technically incorrect, but prevents lockups 149 | if (waited.compareAndSet(false, true)) { 150 | try { 151 | latch.await(); 152 | } catch (Exception ex) { 153 | log.error("Swallowing exception while waitComplete() on targetHandler", ex); 154 | } 155 | } 156 | } 157 | 158 | public abstract void channelClosed(); 159 | 160 | public abstract void receive(int length, ByteBuf buffer) throws Exception; 161 | 162 | public abstract void receiveComplete() throws Exception; 163 | } 164 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/VirtualFileFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | 17 | public interface VirtualFileFilter { 18 | 19 | public boolean accept(VirtualFileReference ref); 20 | 21 | public boolean singleMatch(); 22 | 23 | public String getToken(); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/VirtualFileInput.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | 17 | public interface VirtualFileInput { 18 | 19 | /** 20 | * wait up to wait milliseconds for the next available 21 | * byte array. if wait equals 0, then wait forever. wait 22 | * is advisory and not a hard requirement. this call should never 23 | * block indefinitely as in a case where it's backed by a linked- 24 | * blocking finderQueue and starved for input. if wait is less than 1 25 | * then the method should act like a poll and return instantly 26 | * on no data. this may not be possible in cases where it's backed 27 | * by blocking file-based inputs. 28 | * 29 | * @param wait 30 | * @return byte[] array or null if no bytes are available within the timeout period 31 | */ 32 | public byte[] nextBytes(long wait); 33 | 34 | /** 35 | * @return true if no more bytes will ever be available to nextBytes(), false otherwise 36 | */ 37 | public boolean isEOF(); 38 | 39 | /** 40 | * close input. no-op if isEOF() is true. 41 | */ 42 | public void close(); 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/VirtualFileReference.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import javax.annotation.Nonnull; 17 | 18 | import java.util.Iterator; 19 | import java.util.Map; 20 | 21 | import java.nio.file.PathMatcher; 22 | 23 | 24 | public interface VirtualFileReference { 25 | 26 | public String getName(); 27 | 28 | public long getLastModified(); 29 | 30 | public long getLength(); 31 | 32 | public Iterator listFiles(@Nonnull PathMatcher filter); 33 | 34 | public VirtualFileReference getFile(String name); 35 | 36 | public VirtualFileInput getInput(Map options); 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/VirtualFileSystem.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | 17 | public interface VirtualFileSystem { 18 | 19 | public String[] tokenizePath(String path); 20 | 21 | public VirtualFileReference getFileRoot(); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/file/DupFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.file; 15 | 16 | import java.util.HashSet; 17 | 18 | /** 19 | * first-response duplicates filter 20 | */ 21 | public class DupFilter implements FileReferenceFilter { 22 | 23 | private final HashSet keys = new HashSet<>(); 24 | 25 | @Override 26 | public synchronized boolean accept(FileReference ref) { 27 | return keys.add(ref.name + '#' + ref.getHostUUID()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/file/FileReference.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.file; 15 | 16 | import java.io.ByteArrayInputStream; 17 | import java.io.ByteArrayOutputStream; 18 | import java.io.IOException; 19 | 20 | import com.addthis.basis.util.LessBytes; 21 | 22 | import com.addthis.meshy.VirtualFileReference; 23 | 24 | import com.google.common.base.Objects; 25 | 26 | public class FileReference { 27 | 28 | public final String name; 29 | public final long lastModified; 30 | public final long size; 31 | 32 | private String hostUUID; 33 | 34 | public FileReference(final String name, final long last, final long size) { 35 | this.name = name; 36 | this.lastModified = last; 37 | this.size = size; 38 | } 39 | 40 | public FileReference(final String prefix, final VirtualFileReference ref) { 41 | this(prefix + '/' + ref.getName(), ref.getLastModified(), ref.getLength()); 42 | } 43 | 44 | public FileReference(final byte[] data) throws IOException { 45 | ByteArrayInputStream in = new ByteArrayInputStream(data); 46 | name = LessBytes.readString(in); 47 | lastModified = LessBytes.readLength(in); 48 | size = LessBytes.readLength(in); 49 | hostUUID = LessBytes.readString(in); 50 | } 51 | 52 | /** 53 | * should only be used by the test harness 54 | */ 55 | protected FileReference setHostUUID(final String uuid) { 56 | this.hostUUID = uuid; 57 | return this; 58 | } 59 | 60 | public String getHostUUID() { 61 | return hostUUID; 62 | } 63 | 64 | byte[] encode(String uuid) { 65 | try { 66 | ByteArrayOutputStream out = new ByteArrayOutputStream(name.length() * 2 + 12); 67 | LessBytes.writeString(name, out); 68 | LessBytes.writeLength(lastModified, out); 69 | LessBytes.writeLength(size, out); 70 | LessBytes.writeString(uuid != null ? uuid : hostUUID, out); 71 | return out.toByteArray(); 72 | } catch (IOException ie) { 73 | //using ByteArrayOutputStream. Cant actually throw these 74 | return null; 75 | } 76 | } 77 | 78 | @Override 79 | public String toString() { 80 | return "[nm=" + name + ",lm=" + lastModified + ",sz=" + size + ",uu=" + hostUUID + ']'; 81 | } 82 | 83 | @Override 84 | public boolean equals(Object other) { 85 | if (this == other) { 86 | return true; 87 | } 88 | if (!(other instanceof FileReference)) { 89 | return false; 90 | } 91 | FileReference otherReference = (FileReference) other; 92 | if (!Objects.equal(name, otherReference.name)) { 93 | return false; 94 | } 95 | if (lastModified != otherReference.lastModified) { 96 | return false; 97 | } 98 | if (size != otherReference.size) { 99 | return false; 100 | } 101 | if (!Objects.equal(hostUUID, otherReference.hostUUID)) { 102 | return false; 103 | } 104 | return true; 105 | } 106 | 107 | @Override 108 | public int hashCode() { 109 | return Objects.hashCode(name, lastModified, size, hostUUID); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/file/FileReferenceFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.file; 15 | 16 | public interface FileReferenceFilter { 17 | 18 | public boolean accept(FileReference ref); 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/file/FileSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.file; 15 | 16 | import java.util.Arrays; 17 | import java.util.Collection; 18 | import java.util.HashMap; 19 | import java.util.LinkedList; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | import com.addthis.basis.util.LessBytes; 24 | import com.addthis.basis.util.Parameter; 25 | 26 | import com.addthis.meshy.ChannelMaster; 27 | import com.addthis.meshy.ChannelState; 28 | import com.addthis.meshy.Meshy; 29 | import com.addthis.meshy.MeshyConstants; 30 | import com.addthis.meshy.SourceHandler; 31 | 32 | import com.google.common.base.MoreObjects; 33 | 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | import io.netty.buffer.ByteBuf; 38 | 39 | import static com.google.common.base.Preconditions.checkState; 40 | 41 | public class FileSource extends SourceHandler { 42 | protected static final Logger log = LoggerFactory.getLogger(FileSource.class); 43 | 44 | static final int FILE_FIND_WINDOW_SIZE = Parameter.intValue("meshy.finder.window", 50_000); 45 | 46 | // not thread safe, and only used for single-channel cases (eg. clients) 47 | private final LinkedList list = new LinkedList<>(); 48 | private long currentWindow = 0; 49 | 50 | protected List fileRequest; 51 | protected FileReferenceFilter filter; 52 | 53 | public FileSource(ChannelMaster master) { 54 | super(master, FileTarget.class, true); 55 | } 56 | 57 | public FileSource(ChannelMaster master, String[] files) { 58 | this(master); 59 | requestRemoteFiles(files); 60 | } 61 | 62 | public FileSource(ChannelMaster master, String[] files, String scope) { 63 | this(master); 64 | requestFiles(scope, files); 65 | } 66 | 67 | public FileSource(ChannelMaster master, String[] files, FileReferenceFilter filter) { 68 | this(master); 69 | this.filter = filter; 70 | requestRemoteFiles(files); 71 | } 72 | 73 | public void requestRemoteFiles(String... matches) { 74 | requestFiles("local", matches); 75 | } 76 | 77 | public void requestRemoteFilesWithUpdates(String... matches) { 78 | requestFiles("localF", matches); 79 | } 80 | 81 | public void requestLocalFiles(String... matches) { 82 | start(MeshyConstants.LINK_NAMED); 83 | requestFilesPostStart("remote", matches); 84 | } 85 | 86 | public void requestFiles(String scope, String... matches) { 87 | start(); 88 | requestFilesPostStart(scope, matches); 89 | } 90 | 91 | private void requestFilesPostStart(String scope, String... matches) { 92 | checkState(fileRequest == null, "file search request already started"); 93 | this.fileRequest = Arrays.asList(matches); 94 | send(LessBytes.toBytes(scope)); 95 | log.debug("{} scope={}", this, scope); 96 | for (String match : matches) { 97 | log.trace("{} request={}", this, match); 98 | send(LessBytes.toBytes(match)); 99 | } 100 | send(new byte[]{-1}); 101 | sendInitialWindowing(); 102 | } 103 | 104 | protected void sendInitialWindowing() { 105 | increaseClientWindow(FILE_FIND_WINDOW_SIZE); 106 | } 107 | 108 | private void increaseClientWindow(int windowSize) { 109 | this.currentWindow += windowSize; 110 | send(LessBytes.toBytes(windowSize)); 111 | } 112 | 113 | public Collection getFileList() { 114 | return list; 115 | } 116 | 117 | public Map getFileMap() { 118 | HashMap map = new HashMap<>(); 119 | for (FileReference file : getFileList()) { 120 | map.put(file.name, file); 121 | } 122 | return map; 123 | } 124 | 125 | @Override 126 | public void receive(ChannelState state, int length, ByteBuf buffer) throws Exception { 127 | currentWindow -= 1; 128 | if (currentWindow <= (FILE_FIND_WINDOW_SIZE / 2)) { 129 | increaseClientWindow(FILE_FIND_WINDOW_SIZE / 2); 130 | } 131 | /* sync not required b/c overridden in server-server calls */ 132 | FileReference ref = new FileReference(Meshy.getBytes(length, buffer)); 133 | if (filter == null || filter.accept(ref)) { 134 | receiveReference(ref); 135 | } 136 | log.trace("{} recv={}", this, list.size()); 137 | } 138 | 139 | @Override 140 | public void receiveComplete(ChannelState state, int completedSession) throws Exception { 141 | log.debug("recv.complete [{}] {} from {}", completedSession, fileRequest, state.getName()); 142 | super.receiveComplete(state, completedSession); 143 | } 144 | 145 | // override to detect unexpected channel closures 146 | @Override 147 | public void channelClosed(ChannelState state) { 148 | } 149 | 150 | // override in subclasses for async handling 151 | // call super() if you still want the list populated 152 | public void receiveReference(FileReference ref) { 153 | list.add(ref); 154 | } 155 | 156 | // override in subclasses for async handling 157 | @Override 158 | public void receiveComplete() throws Exception { 159 | log.debug("{} recvComplete", this); 160 | } 161 | 162 | @Override public String toString() { 163 | return MoreObjects.toStringHelper(this) 164 | .add("fileRequest", fileRequest) 165 | .add("filter", filter) 166 | .toString(); 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/file/FileStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.file; 15 | 16 | 17 | public class FileStats { 18 | 19 | public final int finds; 20 | public final int found; 21 | public final int findsRunning; 22 | public final int finderQueue; 23 | public final long findTime; 24 | public final long findTimeLocal; 25 | 26 | public FileStats() { 27 | finds = FileTarget.finds.getAndSet(0); 28 | found = FileTarget.found.getAndSet(0); 29 | findsRunning = (int) FileTarget.findsRunning.count(); 30 | finderQueue = FileTarget.finderQueue.size(); 31 | findTime = FileTarget.findTime.getAndSet(0); 32 | findTimeLocal = FileTarget.findTimeLocal.getAndSet(0); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/file/Filter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.file; 15 | 16 | import com.addthis.meshy.VirtualFileFilter; 17 | import com.addthis.meshy.VirtualFileReference; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | /** 23 | * simple matching: exact, all, begins with and ends with 24 | */ 25 | public class Filter implements VirtualFileFilter { 26 | 27 | private static final Logger log = LoggerFactory.getLogger(Filter.class); 28 | 29 | private String token; 30 | private boolean all; 31 | private boolean start; 32 | private boolean end; 33 | 34 | public Filter(final String token, final boolean all, final boolean start, final boolean end) { 35 | this.token = token; 36 | this.all = all; 37 | this.start = start; 38 | this.end = end; 39 | } 40 | 41 | @Override 42 | public String getToken() { 43 | return token; 44 | } 45 | 46 | @Override 47 | public boolean singleMatch() { 48 | return !(all || start || end); 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return "filter[tok=" + token + ",all=" + all + ",start=" + start + ",end=" + end + ']'; 54 | } 55 | 56 | @Override 57 | public boolean accept(final VirtualFileReference ref) { 58 | final String fileName = ref.getName(); 59 | final boolean ret = (all) || 60 | (start && fileName.startsWith(token)) || 61 | (end && fileName.endsWith(token)) || 62 | (fileName.equals(token)); 63 | if (log.isTraceEnabled()) { 64 | log.trace("accept? ({}) = {}", ref, ret); 65 | } 66 | return ret; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/host/HostNode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.host; 15 | 16 | import java.net.InetSocketAddress; 17 | 18 | public class HostNode { 19 | 20 | public final String uuid; 21 | public final InetSocketAddress address; 22 | 23 | HostNode(final String uuid, final InetSocketAddress address) { 24 | this.uuid = uuid; 25 | this.address = address; 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return uuid + "@" + address; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/host/HostSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.host; 15 | 16 | import java.io.ByteArrayInputStream; 17 | import java.io.ByteArrayOutputStream; 18 | 19 | import java.net.InetSocketAddress; 20 | 21 | import java.util.HashMap; 22 | import java.util.HashSet; 23 | import java.util.LinkedList; 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | import com.addthis.basis.util.LessBytes; 28 | 29 | import com.addthis.meshy.ChannelMaster; 30 | import com.addthis.meshy.ChannelState; 31 | import com.addthis.meshy.Meshy; 32 | import com.addthis.meshy.SourceHandler; 33 | import com.addthis.meshy.service.peer.PeerService; 34 | 35 | import io.netty.buffer.ByteBuf; 36 | 37 | public class HostSource extends SourceHandler { 38 | 39 | private final HashMap hostMap = new HashMap<>(); 40 | private final LinkedList hostList = new LinkedList<>(); 41 | private final HashSet peerAdd = new HashSet<>(); 42 | 43 | public HostSource(ChannelMaster master) { 44 | super(master, HostTarget.class); 45 | } 46 | 47 | public void addPeer(String host) { 48 | peerAdd.add(host); 49 | } 50 | 51 | public void sendRequest() { 52 | try { 53 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 54 | LessBytes.writeInt(peerAdd.size(), out); 55 | for (String peer : peerAdd) { 56 | LessBytes.writeString(peer, out); 57 | } 58 | send(out.toByteArray()); 59 | } catch (Exception ex) { 60 | throw new RuntimeException(ex); 61 | } 62 | this.sendComplete(); 63 | } 64 | 65 | public List getHostList() { 66 | return hostList; 67 | } 68 | 69 | public Map getHostMap() { 70 | return hostMap; 71 | } 72 | 73 | @Override 74 | public void channelClosed(ChannelState state) { 75 | } 76 | 77 | @Override 78 | public void receive(ChannelState state, int length, ByteBuf buffer) throws Exception { 79 | ByteArrayInputStream in = new ByteArrayInputStream(Meshy.getBytes(length, buffer)); 80 | int hosts = LessBytes.readInt(in); 81 | while (hosts-- > 0) { 82 | String uuid = LessBytes.readString(in); 83 | InetSocketAddress address = PeerService.decodeAddress(in); 84 | hostList.add(new HostNode(uuid, address)); 85 | hostMap.put(uuid, address); 86 | } 87 | } 88 | 89 | @Override 90 | public void receiveComplete() throws Exception { 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/host/HostTarget.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.host; 15 | 16 | import java.io.ByteArrayInputStream; 17 | import java.io.ByteArrayOutputStream; 18 | 19 | import java.net.InetSocketAddress; 20 | 21 | import java.util.Collection; 22 | 23 | import com.addthis.basis.util.LessBytes; 24 | import com.addthis.basis.util.LessStrings; 25 | 26 | import com.addthis.meshy.ChannelState; 27 | import com.addthis.meshy.Meshy; 28 | import com.addthis.meshy.MeshyConstants; 29 | import com.addthis.meshy.TargetHandler; 30 | import com.addthis.meshy.service.peer.PeerService; 31 | 32 | import io.netty.buffer.ByteBuf; 33 | 34 | public class HostTarget extends TargetHandler { 35 | 36 | boolean canceled = false; 37 | 38 | @Override 39 | public void receive(int length, ByteBuf buffer) throws Exception { 40 | ByteArrayInputStream in = new ByteArrayInputStream(Meshy.getBytes(length, buffer)); 41 | int count = LessBytes.readInt(in); 42 | while (count-- > 0) { 43 | String[] peer = LessStrings.splitArray(LessBytes.readString(in), ":"); 44 | String host = peer[0]; 45 | int port = Integer.parseInt(peer[1]); 46 | getChannelMaster().connectToPeer(null, new InetSocketAddress(host, port)); 47 | } 48 | } 49 | 50 | @Override 51 | public void channelClosed() { 52 | canceled = true; 53 | } 54 | 55 | @Override 56 | public void receiveComplete() throws Exception { 57 | if (canceled) { 58 | return; 59 | } 60 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 61 | Collection links = getChannelMaster().getChannels(MeshyConstants.LINK_ALL); 62 | LessBytes.writeInt(links.size(), out); 63 | for (ChannelState linkState : links) { 64 | InetSocketAddress remote = linkState.getRemoteAddress(); 65 | if (remote == null) { 66 | remote = (InetSocketAddress) linkState.getChannel().remoteAddress(); 67 | log.debug("missing remote for {} @ {}", remote, linkState); 68 | } 69 | LessBytes.writeString(linkState.getName() != null ? linkState.getName() : "", out); 70 | PeerService.encodeAddress(remote, out); 71 | } 72 | send(out.toByteArray()); 73 | sendComplete(); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/InternalHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import javax.annotation.Nullable; 17 | 18 | import java.io.OutputStream; 19 | 20 | import java.util.Map; 21 | 22 | @FunctionalInterface 23 | public interface InternalHandler extends TopicSender { 24 | 25 | @Nullable @Override default OutputStream sendMessage(String topic) { 26 | return null; 27 | } 28 | 29 | abstract byte[] handleMessageRequest(String fileName, Map options); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/MessageFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.util.ArrayList; 17 | import java.util.HashMap; 18 | import java.util.Iterator; 19 | import java.util.LinkedList; 20 | import java.util.Map; 21 | 22 | import java.nio.file.PathMatcher; 23 | import java.nio.file.Paths; 24 | 25 | import com.addthis.basis.util.JitterClock; 26 | 27 | import com.addthis.meshy.VirtualFileInput; 28 | import com.addthis.meshy.VirtualFileReference; 29 | 30 | class MessageFile implements VirtualFileReference { 31 | 32 | private long lastModified; 33 | private final String name; 34 | private final long length; 35 | private final HashMap files = new HashMap<>(); 36 | 37 | MessageFile(String name, long lastModified, long length) { 38 | this.name = name; 39 | this.lastModified = lastModified; 40 | this.length = length; 41 | } 42 | 43 | void addFile(String fileName, MessageFile file) { 44 | synchronized (files) { 45 | files.put(fileName, file); 46 | } 47 | lastModified = JitterClock.globalTime(); 48 | } 49 | 50 | void removeFile(String fileName) { 51 | synchronized (files) { 52 | files.remove(fileName); 53 | } 54 | lastModified = JitterClock.globalTime(); 55 | } 56 | 57 | void removeFiles(final TopicSender target) { 58 | LinkedList names = new LinkedList<>(); 59 | synchronized (files) { 60 | for (Map.Entry e : files.entrySet()) { 61 | MessageFile mf = e.getValue(); 62 | // TODO this probably isn't good 63 | if (mf instanceof MessageFileListener && ((MessageFileListener) mf).target == target) { 64 | names.add(e.getKey()); 65 | } else { 66 | mf.removeFiles(target); 67 | } 68 | } 69 | } 70 | for (String fileName : names) { 71 | removeFile(fileName); 72 | } 73 | } 74 | 75 | @Override 76 | public String getName() { 77 | return name; 78 | } 79 | 80 | @Override 81 | public long getLastModified() { 82 | return lastModified; 83 | } 84 | 85 | @Override 86 | public long getLength() { 87 | return length; 88 | } 89 | 90 | @Override 91 | public Iterator listFiles(PathMatcher filter) { 92 | synchronized (files) { 93 | if (files.isEmpty()) { 94 | return null; 95 | } 96 | ArrayList filtered = new ArrayList<>(); 97 | for (MessageFile file : files.values()) { 98 | if (filter.matches(Paths.get(file.getName()))) { 99 | filtered.add(file); 100 | } 101 | } 102 | return filtered.iterator(); 103 | } 104 | } 105 | 106 | @Override 107 | public VirtualFileReference getFile(String fileName) { 108 | synchronized (files) { 109 | return files.get(fileName); 110 | } 111 | } 112 | 113 | @Override 114 | public VirtualFileInput getInput(Map options) { 115 | return null; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/MessageFileInput.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | import java.io.OutputStream; 19 | 20 | import java.util.Arrays; 21 | import java.util.Map; 22 | import java.util.concurrent.Semaphore; 23 | import java.util.concurrent.TimeUnit; 24 | import java.util.concurrent.atomic.AtomicBoolean; 25 | 26 | import com.addthis.basis.util.LessBytes; 27 | import com.addthis.basis.util.Parameter; 28 | 29 | import com.addthis.meshy.VirtualFileInput; 30 | 31 | class MessageFileInput implements VirtualFileInput, TargetListener { 32 | 33 | private static final long RPC_TIMEOUT = Parameter.longValue("meshy.rpc.timeout", 5000); 34 | 35 | private final String name; 36 | private final Map options; 37 | private final TopicSender target; 38 | private final AtomicBoolean isEOF = new AtomicBoolean(false); 39 | private final Semaphore gate = new Semaphore(1); 40 | private final String topicID = "rpc.reply." + MessageFileSystem.nextReplyID.incrementAndGet(); 41 | private byte[] data; 42 | 43 | MessageFileInput(String name, Map options, TopicSender target) { 44 | this.name = name; 45 | this.options = options; 46 | this.target = target; 47 | } 48 | 49 | /** 50 | * NOTE: this can be optimized to return null on the first call 51 | * or after "wait" is reached thus freeing up Sender threads. subsequent 52 | * calls can retrieve data is available or return null and set EOF is 53 | * max timeout is passed. 54 | *

55 | * in other words, yes, this is not a perfect implementation and under 56 | * sever load could back up senders. again, in the constant game of right 57 | * vs right now, we are choosing right now. 58 | */ 59 | @Override 60 | public byte[] nextBytes(long wait) { 61 | /* enter this method once only */ 62 | if (!isEOF.compareAndSet(false, true)) { 63 | return null; 64 | } 65 | MessageTarget.registerListener(topicID, this); 66 | try { 67 | final OutputStream out = target.sendMessage(name); 68 | if (out == null && target instanceof InternalHandler) { 69 | return ((InternalHandler) target).handleMessageRequest(name, options); 70 | } 71 | LessBytes.writeString(topicID, out); 72 | if (options != null) { 73 | LessBytes.writeInt(options.size(), out); 74 | for (Map.Entry e : options.entrySet()) { 75 | LessBytes.writeString(e.getKey(), out); 76 | LessBytes.writeString(e.getValue(), out); 77 | } 78 | } else { 79 | LessBytes.writeInt(0, out); 80 | } 81 | out.close(); 82 | gate.acquire(); 83 | long maxWait = RPC_TIMEOUT; 84 | if (options != null) { 85 | String altMax = options.get(MessageFileSystem.READ_TIMEOUT); 86 | if (altMax != null) { 87 | maxWait = Long.parseLong(altMax); 88 | } 89 | } 90 | if (gate.tryAcquire(maxWait, TimeUnit.MILLISECONDS)) { 91 | return data; 92 | } 93 | } catch (Exception ex) { 94 | MessageFileSystem.log.warn("MessageFileInput exception", ex); 95 | } finally { 96 | MessageTarget.deregisterListener(topicID); 97 | } 98 | return null; 99 | } 100 | 101 | @Override 102 | public boolean isEOF() { 103 | return isEOF.get(); 104 | } 105 | 106 | @Override 107 | public void close() { 108 | // noop 109 | } 110 | 111 | @Override 112 | public void receiveMessage(TopicSender ignored, String topic, InputStream in) throws IOException { 113 | if (topic.equals(topicID) && data == null) { 114 | data = LessBytes.readFully(in); 115 | gate.release(); 116 | } else { 117 | MessageFileSystem.log.warn("received reply on invalid topic topic={} data={}", topic, 118 | Arrays.toString(data)); 119 | } 120 | } 121 | 122 | @Override 123 | public void linkDown(TopicSender ignored) { 124 | // ignore? 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/MessageFileListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.util.Map; 17 | 18 | import com.addthis.basis.util.JitterClock; 19 | 20 | import com.addthis.meshy.VirtualFileInput; 21 | 22 | class MessageFileListener extends MessageFile { 23 | 24 | final TopicSender target; 25 | private final String path; 26 | 27 | MessageFileListener(String name, String fullPath, TopicSender target) { 28 | super(name, JitterClock.globalTime(), 0); 29 | this.target = target; 30 | this.path = fullPath; 31 | } 32 | 33 | @Override 34 | public VirtualFileInput getInput(Map options) { 35 | return new MessageFileInput(path, options, target); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/MessageFileProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | import java.io.OutputStream; 19 | 20 | import java.util.HashMap; 21 | 22 | import com.addthis.basis.util.LessBytes; 23 | 24 | import com.addthis.meshy.MeshyClient; 25 | 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | 30 | public class MessageFileProvider implements TopicListener, AutoCloseable { 31 | 32 | private static final Logger log = LoggerFactory.getLogger(MessageFileProvider.class); 33 | 34 | private final MessageSource source; 35 | private final HashMap listeners = new HashMap<>(); 36 | 37 | public MessageFileProvider(MeshyClient client) { 38 | this.source = new MessageSource(client, this); 39 | } 40 | 41 | public void setListener(String fileName, MessageListener listener) { 42 | if (listener == null) { 43 | deleteListener(fileName); 44 | return; 45 | } 46 | synchronized (listeners) { 47 | listeners.put(fileName, listener); 48 | OutputStream out = source.sendMessage(MessageFileSystem.MFS_ADD); 49 | try { 50 | LessBytes.writeString(fileName, out); 51 | out.close(); 52 | } catch (Exception ex) { 53 | throw new RuntimeException(ex); 54 | } 55 | } 56 | } 57 | 58 | public void deleteListener(String fileName) { 59 | synchronized (listeners) { 60 | OutputStream out = source.sendMessage(MessageFileSystem.MFS_DEL); 61 | try { 62 | LessBytes.writeString(fileName, out); 63 | out.close(); 64 | } catch (Exception ex) { 65 | throw new RuntimeException(ex); 66 | } 67 | listeners.remove(fileName); 68 | } 69 | } 70 | 71 | @Override 72 | public void receiveMessage(String fileName, InputStream in) throws IOException { 73 | MessageListener listener = null; 74 | synchronized (listeners) { 75 | listener = listeners.get(fileName); 76 | } 77 | if (listener != null) { 78 | String topic = LessBytes.readString(in); 79 | HashMap options = null; 80 | int count = LessBytes.readInt(in); 81 | if (count > 0) { 82 | options = new HashMap<>(count); 83 | while (count > 0) { 84 | count--; 85 | options.put(LessBytes.readString(in), LessBytes.readString(in)); 86 | } 87 | } 88 | listener.requestContents(fileName, options, source.sendMessage(topic)); 89 | } else { 90 | log.info("receive for topic with no listener: {}", fileName); 91 | } 92 | } 93 | 94 | @Override 95 | public void linkDown() { 96 | // subclass and override to hide this message 97 | log.info("link down source={} listeners={}", source, listeners); 98 | } 99 | 100 | @Override public void close() throws Exception { 101 | source.sendComplete(); 102 | source.waitComplete(); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/MessageFileSystem.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | 19 | import java.util.concurrent.atomic.AtomicLong; 20 | 21 | import com.addthis.basis.util.LessBytes; 22 | import com.addthis.basis.util.JitterClock; 23 | import com.addthis.basis.util.LessStrings; 24 | 25 | import com.addthis.meshy.VirtualFileReference; 26 | import com.addthis.meshy.VirtualFileSystem; 27 | 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | 32 | public class MessageFileSystem implements VirtualFileSystem, TargetListener { 33 | 34 | static final Logger log = LoggerFactory.getLogger(MessageFileSystem.class); 35 | 36 | /** 37 | * file open option that overrides system timeout 38 | */ 39 | public static final String READ_TIMEOUT = "mfs.read.timeout"; 40 | 41 | static final String MFS_ADD = "mfs.add"; 42 | static final String MFS_DEL = "mfs.del"; 43 | 44 | static final AtomicLong nextReplyID = new AtomicLong(1); 45 | 46 | 47 | public MessageFileSystem() { 48 | root = new MessageFile("", JitterClock.globalTime(), 0); 49 | MessageTarget.registerListener(MFS_ADD, this); 50 | MessageTarget.registerListener(MFS_DEL, this); 51 | } 52 | 53 | private final MessageFile root; 54 | 55 | @Override 56 | public String[] tokenizePath(String path) { 57 | return LessStrings.splitArray(path, "/"); 58 | } 59 | 60 | @Override 61 | public VirtualFileReference getFileRoot() { 62 | return root; 63 | } 64 | 65 | /** 66 | * for JVM internal implementations 67 | */ 68 | public void addPath(String path, TopicSender sender) { 69 | updatePath(sender, path, true); 70 | } 71 | 72 | public void addPath(String path, InternalHandler sender) { 73 | updatePath(sender, path, true); 74 | } 75 | 76 | /** 77 | * for JVM internal implementations 78 | */ 79 | public void removePath(String path) { 80 | updatePath(null, path, false); 81 | } 82 | 83 | private void updatePath(TopicSender target, String fullPath, boolean add) { 84 | String[] path = LessStrings.splitArray(fullPath, "/"); 85 | MessageFile ptr = root; 86 | for (int i = 0; i < path.length; i++) { 87 | String tok = path[i]; 88 | if (i == path.length - 1) { 89 | if (add) { 90 | ptr.addFile(tok, new MessageFileListener(tok, fullPath, target)); 91 | } else { 92 | ptr.removeFile(tok); 93 | } 94 | return; 95 | } 96 | MessageFile next = (MessageFile) ptr.getFile(tok); 97 | if (next == null) { 98 | if (!add) { 99 | return; 100 | } 101 | next = new MessageFile(tok, JitterClock.globalTime(), 0); 102 | ptr.addFile(tok, next); 103 | } 104 | ptr = next; 105 | } 106 | } 107 | 108 | @Override 109 | public void receiveMessage(TopicSender target, String topic, InputStream in) throws IOException { 110 | boolean add = topic.equals(MFS_ADD); 111 | boolean del = !add && topic.equals(MFS_DEL); 112 | if (add || del) { 113 | String fullPath = LessBytes.readString(in); 114 | updatePath(target, fullPath, add); 115 | } else { 116 | log.warn("unhandled receive for topic={} target={}", topic, target); 117 | } 118 | } 119 | 120 | @Override 121 | public void linkDown(TopicSender target) { 122 | root.removeFiles(target); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/MessageListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.IOException; 17 | import java.io.OutputStream; 18 | 19 | import java.util.Map; 20 | 21 | public interface MessageListener { 22 | 23 | /** 24 | * @param fileName requested file 25 | * @param options optional options (or null if none) 26 | * @param out stream to write to. will ONLY send on close() !! 27 | */ 28 | public void requestContents(String fileName, Map options, OutputStream out) throws IOException; 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/MessageSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.ByteArrayOutputStream; 17 | import java.io.InputStream; 18 | import java.io.OutputStream; 19 | 20 | import com.addthis.basis.util.LessBytes; 21 | 22 | import com.addthis.meshy.ChannelMaster; 23 | import com.addthis.meshy.ChannelState; 24 | import com.addthis.meshy.Meshy; 25 | import com.addthis.meshy.SourceHandler; 26 | 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import io.netty.buffer.ByteBuf; 31 | 32 | /** 33 | * runs in context of mesh client 34 | */ 35 | public class MessageSource extends SourceHandler implements OutputSender, TopicSender { 36 | 37 | private static final Logger log = LoggerFactory.getLogger(MessageSource.class); 38 | 39 | private final TopicListener listener; 40 | 41 | public MessageSource(ChannelMaster master, TopicListener listener) { 42 | super(master, MessageTarget.class); 43 | this.listener = listener; 44 | } 45 | 46 | @Override 47 | public void receive(ChannelState state, int length, ByteBuf buffer) { 48 | InputStream in = Meshy.getInput(length, buffer); 49 | String topic = null; 50 | try { 51 | topic = LessBytes.readString(in); 52 | listener.receiveMessage(topic, in); 53 | } catch (Exception ex) { 54 | log.warn("fail to receive to topic={} listener={} in={} len-{} buf={}", 55 | topic, listener, in, length, buffer, ex); 56 | } 57 | } 58 | 59 | @Override 60 | public void channelClosed(ChannelState state) { 61 | } 62 | 63 | @Override 64 | public void receiveComplete() throws Exception { 65 | listener.linkDown(); 66 | } 67 | 68 | @Override 69 | public OutputStream sendMessage(String topic) { 70 | try { 71 | ByteArrayOutputStream out = new SendOnCloseOutputStream(this, 4096); 72 | LessBytes.writeString(topic, out); 73 | return out; 74 | } catch (Exception ex) { 75 | throw new RuntimeException(ex); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/MessageTarget.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.ByteArrayOutputStream; 17 | import java.io.InputStream; 18 | import java.io.OutputStream; 19 | 20 | import java.util.HashMap; 21 | 22 | import com.addthis.basis.util.LessBytes; 23 | 24 | import com.addthis.meshy.Meshy; 25 | import com.addthis.meshy.TargetHandler; 26 | 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import io.netty.buffer.ByteBuf; 31 | 32 | /** 33 | * MessageService allows transient client connections to offer services and extensions 34 | * to long-running core mesh services. Query is an example of this, but it's not 35 | * currently done that way. RPC and virtual file extensions are likely implementations. 36 | */ 37 | public class MessageTarget extends TargetHandler implements OutputSender, TopicSender { 38 | 39 | private static final Logger log = LoggerFactory.getLogger(MessageTarget.class); 40 | 41 | private static final HashMap targetListeners = new HashMap<>(); 42 | 43 | public static void registerListener(String topic, TargetListener listener) { 44 | synchronized (targetListeners) { 45 | if (targetListeners.put(topic, listener) != null) { 46 | log.warn("WARNING: override listener for {}", topic); 47 | } 48 | } 49 | } 50 | 51 | @Deprecated 52 | public static void deregisterListener(String topic, TargetListener ignored) { 53 | deregisterListener(topic); 54 | } 55 | 56 | public static void deregisterListener(String topic) { 57 | synchronized (targetListeners) { 58 | targetListeners.remove(topic); 59 | } 60 | } 61 | 62 | @Override 63 | public void channelClosed() { 64 | // target listener's only api method that would make sense is link down 65 | // this will be called in a second anyway, so ignore 66 | } 67 | 68 | @Override 69 | public void receive(int length, ByteBuf buffer) throws Exception { 70 | InputStream in = Meshy.getInput(length, buffer); 71 | String topic = LessBytes.readString(in); 72 | synchronized (targetListeners) { 73 | TargetListener listener = targetListeners.get(topic); 74 | if (listener != null) { 75 | listener.receiveMessage(this, topic, in); 76 | } 77 | } 78 | } 79 | 80 | @Override 81 | public void receiveComplete() throws Exception { 82 | synchronized (targetListeners) { 83 | for (TargetListener listener : targetListeners.values()) { 84 | listener.linkDown(this); 85 | } 86 | } 87 | sendComplete(); 88 | } 89 | 90 | @Override 91 | public OutputStream sendMessage(String topic) { 92 | try { 93 | ByteArrayOutputStream out = new SendOnCloseOutputStream(this, 4096); 94 | LessBytes.writeString(topic, out); 95 | return out; 96 | } catch (Exception ex) { 97 | throw new RuntimeException(ex); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/OutputSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | interface OutputSender { 17 | 18 | public boolean send(byte[] data); 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/SendOnCloseOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.ByteArrayOutputStream; 17 | import java.io.IOException; 18 | 19 | class SendOnCloseOutputStream extends ByteArrayOutputStream { 20 | 21 | private final OutputSender sender; 22 | 23 | SendOnCloseOutputStream(OutputSender sender, int estSize) { 24 | super(estSize); 25 | this.sender = sender; 26 | } 27 | 28 | @Override 29 | public void close() throws IOException { 30 | super.close(); 31 | sender.send(toByteArray()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/TargetListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | 19 | public interface TargetListener { 20 | 21 | public void receiveMessage(TopicSender target, String topic, InputStream in) throws IOException; 22 | 23 | public void linkDown(TopicSender target); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/TopicListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | 19 | public interface TopicListener { 20 | 21 | public void receiveMessage(String topic, InputStream in) throws IOException; 22 | 23 | public void linkDown(); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/message/TopicSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.OutputStream; 17 | 18 | public interface TopicSender { 19 | 20 | public OutputStream sendMessage(String topic); 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/peer/PeerService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.peer; 15 | 16 | import java.io.ByteArrayOutputStream; 17 | import java.io.IOException; 18 | import java.io.InputStream; 19 | import java.io.OutputStream; 20 | 21 | import java.net.InetAddress; 22 | import java.net.InetSocketAddress; 23 | 24 | import java.util.concurrent.LinkedBlockingQueue; 25 | 26 | import com.addthis.basis.util.LessBytes; 27 | 28 | import com.addthis.meshy.ChannelState; 29 | import com.addthis.meshy.MeshyConstants; 30 | import com.addthis.meshy.MeshyServer; 31 | 32 | import com.google.common.base.Strings; 33 | 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | /** 38 | * connect to a peer, receive from peer it's uuid and connection map 39 | */ 40 | public final class PeerService { 41 | 42 | static final Logger log = LoggerFactory.getLogger(PeerService.class); 43 | static final LinkedBlockingQueue peerQueue = new LinkedBlockingQueue<>(); 44 | 45 | static final Thread peeringThread = new Thread() { 46 | { 47 | setName("peering finderQueue"); 48 | setDaemon(true); 49 | start(); 50 | } 51 | 52 | public void run() { 53 | while (true) { 54 | try { 55 | peerQueue.take().connect(); 56 | } catch (Exception e) { 57 | log.warn("peer finderQueue exiting on error", e); 58 | return; 59 | } 60 | } 61 | } 62 | }; 63 | 64 | private PeerService() { 65 | } 66 | 67 | static boolean shouldEncode(InetSocketAddress sockAddr) { 68 | InetAddress addr = sockAddr.getAddress(); 69 | return !(addr.isLoopbackAddress() || addr.isAnyLocalAddress()); 70 | } 71 | 72 | public static void encodeAddress(InetSocketAddress addr, OutputStream out) throws IOException { 73 | LessBytes.writeBytes(addr.getAddress().getAddress(), out); 74 | LessBytes.writeInt(addr.getPort(), out); 75 | } 76 | 77 | public static InetSocketAddress decodeAddress(InputStream in) throws IOException { 78 | return new InetSocketAddress(InetAddress.getByAddress(LessBytes.readBytes(in)), LessBytes.readInt(in)); 79 | } 80 | 81 | /** 82 | * send local peer uuid:port and list of peers to remote 83 | */ 84 | public static byte[] encodeExtraPeers(MeshyServer master) { 85 | try { 86 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 87 | for (ChannelState channelState : master.getChannels(MeshyConstants.LINK_NAMED)) { 88 | LessBytes.writeString(channelState.getName(), out); 89 | encodeAddress(channelState.getRemoteAddress(), out); 90 | log.debug("{} encoded {} @ {}", master, channelState.getName(), channelState.getChannelRemoteAddress()); 91 | } 92 | for (MeshyServer member : master.getMembers()) { 93 | if (member != master && shouldEncode(member.getLocalAddress())) { 94 | log.trace("encode MEMBER: {} / {}", member.getUUID(), member.getLocalAddress()); 95 | LessBytes.writeString(member.getUUID(), out); 96 | encodeAddress(member.getLocalAddress(), out); 97 | } 98 | } 99 | LessBytes.writeString("", out); 100 | return out.toByteArray(); 101 | } catch (Exception ex) { 102 | throw new RuntimeException(ex); 103 | } 104 | } 105 | 106 | public static byte[] encodeSelf(MeshyServer master) { 107 | try { 108 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 109 | LessBytes.writeString(master.getUUID(), out); 110 | encodeAddress(master.getLocalAddress(), out); 111 | return out.toByteArray(); 112 | } catch (Exception ex) { 113 | throw new RuntimeException(ex); 114 | } 115 | } 116 | 117 | /** 118 | * receive peer's uuid:port and peer list then connect to ones that are new to us 119 | */ 120 | public static boolean decodePrimaryPeer(MeshyServer master, ChannelState peerState, InputStream in) { 121 | try { 122 | String newName = LessBytes.readString(in); 123 | if (Strings.isNullOrEmpty(newName)) { 124 | log.debug("would-be peer is refusing peerage: sent {} from {} for {}", newName, peerState, master); 125 | return false; 126 | } 127 | boolean shouldBeConnector = master.shouldBeConnector(newName); 128 | boolean isConnector = peerState.getChannel().parent() == null; 129 | boolean promoteToPeer = shouldBeConnector == isConnector; 130 | 131 | InetSocketAddress newAddr = decodeAddress(in); 132 | InetAddress newInetAddr = newAddr.getAddress(); 133 | /* if remote reports loopback or any-net, use peer ip addr + port */ 134 | if (newInetAddr.isAnyLocalAddress() || newInetAddr.isLoopbackAddress()) { 135 | newAddr = new InetSocketAddress(peerState.getChannelRemoteAddress().getAddress(), newAddr.getPort()); 136 | newInetAddr = newAddr.getAddress(); 137 | } 138 | if ((newInetAddr.isAnyLocalAddress() || newInetAddr.isLoopbackAddress()) && isConnector) { 139 | newAddr = new InetSocketAddress(peerState.getChannel().localAddress().getAddress(), newAddr.getPort()); 140 | } 141 | 142 | if (promoteToPeer) { 143 | return master.promoteToNamedServerPeer(peerState, newName, newAddr); 144 | } else if (shouldBeConnector) { 145 | log.info("dropping (and reconnecting as client) backwards connection from: {} @ {} for: {}", 146 | newName, newAddr, master); 147 | master.connectToPeer(newName, newAddr); 148 | } 149 | return promoteToPeer; 150 | } catch (Exception ex) { 151 | throw new RuntimeException(ex); 152 | } 153 | } 154 | 155 | /** 156 | * receive peer's uuid:port and peer list then connect to ones that are new to us 157 | */ 158 | public static void decodeExtraPeers(MeshyServer master, InputStream in) { 159 | try { 160 | while (true) { 161 | String peerUuid = LessBytes.readString(in); 162 | if (peerUuid.isEmpty()) { 163 | break; 164 | } 165 | InetSocketAddress peerAddress = decodeAddress(in); 166 | log.debug("{} decoded {} @ {}", master, peerUuid, peerAddress); 167 | peerQueue.put(new PeerTuple(master, peerUuid, peerAddress)); 168 | } 169 | } catch (Exception ex) { 170 | throw new RuntimeException(ex); 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/peer/PeerSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.peer; 15 | 16 | import com.addthis.meshy.ChannelState; 17 | import com.addthis.meshy.Meshy; 18 | import com.addthis.meshy.MeshyServer; 19 | import com.addthis.meshy.SourceHandler; 20 | 21 | import org.slf4j.Logger; 22 | 23 | import io.netty.buffer.ByteBuf; 24 | 25 | import static com.addthis.meshy.service.peer.PeerService.decodeExtraPeers; 26 | import static com.addthis.meshy.service.peer.PeerService.decodePrimaryPeer; 27 | 28 | public class PeerSource extends SourceHandler { 29 | private static final Logger log = PeerService.log; 30 | 31 | private boolean receivedStateUuid = false; 32 | 33 | public PeerSource(MeshyServer master, String tempUuid) { 34 | super(master, PeerTarget.class, tempUuid); 35 | send(PeerService.encodeSelf(master)); 36 | } 37 | 38 | @Override 39 | public void channelClosed(ChannelState state) { 40 | } 41 | 42 | @Override 43 | public void receive(ChannelState state, int length, ByteBuf buffer) throws Exception { 44 | log.debug("{} decode from {}", this, state); 45 | if (!receivedStateUuid) { 46 | if (decodePrimaryPeer((MeshyServer) getChannelMaster(), state, Meshy.getInput(length, buffer))) { 47 | send(PeerService.encodeExtraPeers((MeshyServer) getChannelMaster())); 48 | sendComplete(); 49 | } else { 50 | sendComplete(); 51 | state.getChannel().close(); 52 | } 53 | receivedStateUuid = true; 54 | } else { 55 | decodeExtraPeers((MeshyServer) getChannelMaster(), Meshy.getInput(length, buffer)); 56 | } 57 | } 58 | 59 | @Override 60 | public void receiveComplete() throws Exception { 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/peer/PeerTarget.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.peer; 15 | 16 | import com.addthis.meshy.Meshy; 17 | import com.addthis.meshy.TargetHandler; 18 | 19 | import io.netty.buffer.ByteBuf; 20 | 21 | import static com.addthis.meshy.service.peer.PeerService.decodeExtraPeers; 22 | import static com.addthis.meshy.service.peer.PeerService.decodePrimaryPeer; 23 | 24 | public class PeerTarget extends TargetHandler { 25 | 26 | private boolean shouldPeer = false; 27 | private boolean receivedStateUuid = false; 28 | 29 | @Override 30 | public void receive(int length, ByteBuf buffer) throws Exception { 31 | log.debug("{} decode from {}", this, getChannelState().getChannelRemoteAddress()); 32 | if (!receivedStateUuid) { 33 | shouldPeer = decodePrimaryPeer(getChannelMaster(), getChannelState(), Meshy.getInput(length, buffer)); 34 | if (shouldPeer) { 35 | log.debug("{} encode to {}", this, getChannelState().getChannelRemoteAddress()); 36 | send(PeerService.encodeSelf(getChannelMaster())); 37 | send(PeerService.encodeExtraPeers(getChannelMaster())); 38 | } else { 39 | log.debug("writing peer cancel from {}", this); 40 | send(new byte[] {0}); // send byte array with a single "0" byte for the empty string 41 | } 42 | receivedStateUuid = true; 43 | } else { 44 | decodeExtraPeers(getChannelMaster(), Meshy.getInput(length, buffer)); 45 | } 46 | } 47 | 48 | @Override 49 | public void channelClosed() { 50 | } 51 | 52 | @Override 53 | public void receiveComplete() throws Exception { 54 | sendComplete(); 55 | if (!shouldPeer) { 56 | getChannelState().getChannel().close(); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/peer/PeerTuple.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.peer; 15 | 16 | import java.net.InetSocketAddress; 17 | 18 | import com.addthis.meshy.MeshyServer; 19 | 20 | class PeerTuple { 21 | 22 | final MeshyServer master; 23 | final String peerUuid; 24 | final InetSocketAddress peerAddress; 25 | 26 | PeerTuple(MeshyServer master, String peerUuid, InetSocketAddress peerAddress) { 27 | this.master = master; 28 | this.peerUuid = peerUuid; 29 | this.peerAddress = peerAddress; 30 | } 31 | 32 | void connect() { 33 | master.connectToPeer(peerUuid, peerAddress); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/stream/SourceInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.stream; 15 | 16 | import javax.annotation.concurrent.NotThreadSafe; 17 | 18 | import java.io.ByteArrayInputStream; 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | import java.io.InterruptedIOException; 22 | 23 | import java.net.SocketTimeoutException; 24 | 25 | import java.util.concurrent.BlockingQueue; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | import com.addthis.basis.util.Parameter; 29 | 30 | import com.google.common.base.Throwables; 31 | 32 | import com.yammer.metrics.Metrics; 33 | import com.yammer.metrics.core.Timer; 34 | 35 | import org.slf4j.Logger; 36 | 37 | @NotThreadSafe 38 | public class SourceInputStream extends InputStream { 39 | 40 | /* max time to wait in seconds for a read response before timing out and closing stream */ 41 | private static final int MAX_READ_WAIT = Parameter.intValue("meshy.stream.timeout", 0) * 1000; 42 | 43 | private static final Logger log = StreamService.log; 44 | 45 | private final StreamSource source; 46 | private final BlockingQueue messageQueue; 47 | 48 | private ByteArrayInputStream current; 49 | private byte[] currentData; 50 | private boolean done = false; 51 | 52 | /* metrics */ 53 | private static final Timer dequePollTimer = Metrics.newTimer(SourceInputStream.class, "dequeTimer"); 54 | 55 | SourceInputStream(StreamSource source) { 56 | this.source = source; 57 | this.messageQueue = source.getMessageQueue(); 58 | } 59 | 60 | private boolean fill(boolean blocking) throws IOException { 61 | return fill(blocking, -1, null); 62 | } 63 | 64 | /** 65 | * Blocking or non blocking (depending on how the source was initialized) call to fill buffer. 66 | *

67 | * If blocking the call will wait until data is available on messageQueue before 68 | * returning if the current buffer is null or empty. 69 | *

70 | * If non-blocking the call will return when data is available or the time limit (wait) has been 71 | * reached. Callers may not assume that a false return from this method means the end of stream 72 | * has been reached in async mode. 73 | * 74 | * @return true if meshy data is available 75 | * @throws java.io.IOException if remote error 76 | */ 77 | private boolean fill(boolean blocking, long wait, TimeUnit timeUnit) throws IOException { 78 | if (done) { 79 | return false; 80 | } 81 | if ((blocking && (current == null || current.available() == 0)) || (!blocking && currentData == null)) { 82 | if (log.isTraceEnabled()) { 83 | log.trace("{} fill c={}", this, current != null ? current.available() : "empty"); 84 | } 85 | byte[] data = null; 86 | try { 87 | if (blocking) { 88 | if (log.isTraceEnabled()) { 89 | log.trace("{} fill from finderQueue={} wait={}", this, messageQueue.size(), MAX_READ_WAIT); 90 | } 91 | long startTime = System.nanoTime(); 92 | data = MAX_READ_WAIT > 0 ? messageQueue.poll(MAX_READ_WAIT, TimeUnit.MILLISECONDS) : messageQueue.take(); 93 | dequePollTimer.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); 94 | if (data == null) { 95 | /* important that we throw SocketTimeoutException so that SourceTracker does not mark this file "dead" */ 96 | throw new SocketTimeoutException(this + " timeout waiting for fill()"); 97 | } 98 | } else { 99 | if (wait > 0) { 100 | data = messageQueue.poll(wait, timeUnit); 101 | } else { 102 | data = messageQueue.poll(); 103 | } 104 | if (data == null) { 105 | return false; 106 | } 107 | } 108 | source.performBufferAccounting(data); 109 | source.throwIfErrorSignal(data); 110 | if (source.isCloseSignal(data)) { 111 | log.trace("{} fill exit on 0 bytes", this); 112 | currentData = data; 113 | close(); 114 | return false; 115 | } 116 | log.trace("{} fill take={}", this, data.length); 117 | } catch (InterruptedException ex) { 118 | log.warn("{} close on stream service interrupted", this); 119 | close(); 120 | /* important that we throw InterruptedIOException so that SourceTracker does not mark this file "dead" */ 121 | throw new InterruptedIOException("stream interrupted"); 122 | } catch (IOException | RuntimeException ex) { 123 | log.warn("{} close on error", this, ex); 124 | close(); 125 | Throwables.propagateIfInstanceOf(ex, IOException.class); 126 | throw ex; 127 | } 128 | if (blocking) { 129 | current = new ByteArrayInputStream(data); 130 | } else { 131 | currentData = data; 132 | } 133 | return true; 134 | } 135 | return true; 136 | } 137 | 138 | /** 139 | * Polls the messageQueue for available data. 140 | * 141 | * @return - a byte array if data is available or null if no data is currently available. 142 | * @throws java.io.IOException 143 | */ 144 | public byte[] poll() throws IOException { 145 | return poll(-1, null); 146 | } 147 | 148 | /** 149 | * Polls the messageQueue for available data. 150 | * 151 | * @param wait - the amount of time to wait before returning null in the case that no data is available yet 152 | * @param timeUnit - the time unit for wait 153 | * @return - a byte array if data is available or null if no data is currently available. 154 | * @throws java.io.IOException 155 | */ 156 | public byte[] poll(long wait, TimeUnit timeUnit) throws IOException { 157 | // response from fill is ignored because in async mode the call is ignored 158 | fill(false, wait, timeUnit); 159 | byte[] data = currentData; 160 | currentData = null; 161 | return data; 162 | } 163 | 164 | @Override 165 | public int read() throws IOException { 166 | if (fill(true)) { 167 | return current.read(); 168 | } else { 169 | return -1; 170 | } 171 | } 172 | 173 | @Override 174 | public int read(byte[] buf) throws IOException { 175 | return read(buf, 0, buf.length); 176 | } 177 | 178 | @Override 179 | public int read(byte[] buf, int off, int len) throws IOException { 180 | if (fill(true)) { 181 | return current.read(buf, off, len); 182 | } else { 183 | return -1; 184 | } 185 | } 186 | 187 | @Override 188 | public int available() { 189 | if (current != null) { 190 | return current.available(); 191 | } 192 | byte[] peek = messageQueue.peek(); 193 | /* length 0 is valid for EOF packets, so returning 1 is wrong in that case */ 194 | return peek != null ? peek.length : 0; 195 | } 196 | 197 | @Override 198 | public void close() { 199 | done = true; 200 | source.requestClose(); 201 | } 202 | 203 | public boolean isEOF() { 204 | return done; 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/stream/StreamService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.stream; 15 | 16 | import java.util.concurrent.TimeUnit; 17 | import java.util.concurrent.atomic.AtomicInteger; 18 | 19 | import com.addthis.basis.util.Parameter; 20 | 21 | import com.addthis.meshy.MeshyConstants; 22 | 23 | import com.yammer.metrics.Metrics; 24 | import com.yammer.metrics.core.Counter; 25 | import com.yammer.metrics.core.Meter; 26 | 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | 31 | public class StreamService { 32 | 33 | protected static final Logger log = LoggerFactory.getLogger(StreamService.class); 34 | 35 | public static final String ERROR_EXCEED_OPEN = "Exceeded Max Open Files"; 36 | public static final String ERROR_CHANNEL_LOST = "Channel Connection Lost"; 37 | public static final String ERROR_REMOTE_CHANNEL_LOST = "Remote Channel Connection Lost"; 38 | public static final String ERROR_UNKNOWN = "no error message available"; 39 | public static final int STREAM_BYTE_OVERHEAD = 1; 40 | 41 | /* not documented */ 42 | static final boolean DIRECT_COPY = Parameter.boolValue("meshy.copy.direct", true); 43 | /* log dropped "more" requests */ 44 | static final boolean LOG_DROP_MORE = Parameter.boolValue("meshy.log.dropmore", false); 45 | /* max time to wait for a VirtualFileInput read() call in sender threads */ 46 | static final long READ_WAIT = Parameter.longValue("meshy.read.wait", 10); 47 | 48 | // internal constants 49 | static final int MODE_START = 0; 50 | static final int MODE_MORE = 1; 51 | static final int MODE_FAIL = 2; 52 | static final int MODE_CLOSE = 3; 53 | static final int MODE_START_2 = 4; // passing options 54 | static final byte[] CLOSE_BYTES = MeshyConstants.EMPTY_BYTES; // not used for reference comparison -- only length 55 | static final byte[] FAIL_BYTES = new byte[0]; // used in same-vm reference pointer comparison 56 | 57 | // metrics -- some are also used in functional logic 58 | /* for enforcing MAX_OPEN_STREAMS */ 59 | static final Counter openStreams = Metrics.newCounter(StreamService.class, "openStreams"); 60 | static final Meter newStreamMeter = Metrics.newMeter(StreamService.class, "newStreams", "newStreams", TimeUnit.SECONDS); 61 | static final AtomicInteger newOpenStreams = new AtomicInteger(0); 62 | static final AtomicInteger closedStreams = new AtomicInteger(0); 63 | static final AtomicInteger readBytes = new AtomicInteger(0); 64 | static final AtomicInteger seqReads = new AtomicInteger(0); 65 | static final AtomicInteger totalReads = new AtomicInteger(0); 66 | static final AtomicInteger readWaitTime = new AtomicInteger(0); 67 | static final AtomicInteger sendWaiting = new AtomicInteger(0); 68 | static final AtomicInteger sleeps = new AtomicInteger(0); 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/addthis/meshy/service/stream/StreamStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.stream; 15 | 16 | public class StreamStats { 17 | 18 | public final int openCount; 19 | public final int newOpenCount; 20 | public final int readWaitTime; 21 | public final int sendWaiting; 22 | public final int sleeps; 23 | public final int qSize; 24 | public final int seqRead; 25 | public final int totalRead; 26 | public final int readBytes; 27 | public final int closedStreams; 28 | 29 | public StreamStats() { 30 | openCount = (int) StreamService.openStreams.count(); 31 | newOpenCount = StreamService.newOpenStreams.getAndSet(0); 32 | readWaitTime = StreamService.readWaitTime.getAndSet(0); 33 | seqRead = StreamService.seqReads.getAndSet(0); 34 | totalRead = StreamService.totalReads.getAndSet(0); 35 | sendWaiting = StreamService.sendWaiting.get(); 36 | sleeps = StreamService.sleeps.getAndSet(0); 37 | qSize = StreamTarget.senderQueue.size(); 38 | readBytes = StreamService.readBytes.getAndSet(0); 39 | closedStreams = StreamService.closedStreams.getAndSet(0); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/resources/meshy_prometheus_metrics.yml: -------------------------------------------------------------------------------- 1 | --- 2 | lowercaseOutputLabelNames: true 3 | lowercaseOutputName: true 4 | 5 | rules: 6 | 7 | # blacklist JMXONLY metrics 8 | - pattern: "^metrics<>Count" 9 | name: "$1_count" 10 | type: COUNTER 11 | 12 | - pattern: "^metrics<>Value" 13 | name: "$1" 14 | type: GAUGE 15 | 16 | - pattern: "^metrics<>(50|75|95|98|99)thPercentile" 17 | name: "$1" 18 | type: GAUGE 19 | labels: 20 | percentile: "$2" 21 | 22 | - pattern: "\"(.*)\"<>Count" 23 | name: "$1_$2_$4_count" 24 | type: COUNTER 25 | 26 | - pattern: "\"(.*)\"<>Count" 27 | name: "$1_$2_$3_count" 28 | type: COUNTER 29 | 30 | - pattern: "\"(.*)\"<>Value" 31 | name: "$1_$2_$4" 32 | type: GAUGE 33 | 34 | - pattern: "\"(.*)\"<>(50|75|95|98|99|999)thPercentile" 35 | name: "$1_$2_$4" 36 | type: GAUGE 37 | labels: 38 | percentile: "$5" 39 | 40 | -------------------------------------------------------------------------------- /src/test/files/a/abc.xml: -------------------------------------------------------------------------------- 1 | 123 2 | -------------------------------------------------------------------------------- /src/test/files/a/def.xml: -------------------------------------------------------------------------------- 1 | 123456 2 | -------------------------------------------------------------------------------- /src/test/files/b/ghi.xml: -------------------------------------------------------------------------------- 1 | 123 2 | -------------------------------------------------------------------------------- /src/test/files/b/jkl.xml: -------------------------------------------------------------------------------- 1 | 123456 2 | -------------------------------------------------------------------------------- /src/test/files/mux/mff.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addthis/meshy/9386a90a9997b3af8945316c07f3f467321f4383/src/test/files/mux/mff.conf -------------------------------------------------------------------------------- /src/test/files/mux/mff.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addthis/meshy/9386a90a9997b3af8945316c07f3f467321f4383/src/test/files/mux/mff.data -------------------------------------------------------------------------------- /src/test/files/mux/mff.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addthis/meshy/9386a90a9997b3af8945316c07f3f467321f4383/src/test/files/mux/mff.lock -------------------------------------------------------------------------------- /src/test/files/mux/mfs.conf: -------------------------------------------------------------------------------- 1 | @� -------------------------------------------------------------------------------- /src/test/files/mux/mfs.data: -------------------------------------------------------------------------------- 1 | ֛$ -------------------------------------------------------------------------------- /src/test/files/mux/mfs.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addthis/meshy/9386a90a9997b3af8945316c07f3f467321f4383/src/test/files/mux/mfs.lock -------------------------------------------------------------------------------- /src/test/files/mux/out-00000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addthis/meshy/9386a90a9997b3af8945316c07f3f467321f4383/src/test/files/mux/out-00000001 -------------------------------------------------------------------------------- /src/test/files/xyz.xml: -------------------------------------------------------------------------------- 1 | not empty 2 | -------------------------------------------------------------------------------- /src/test/java/com/addthis/meshy/TestMesh.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.File; 17 | import java.io.IOException; 18 | 19 | import java.util.LinkedList; 20 | import java.util.Map; 21 | 22 | import java.text.DecimalFormat; 23 | 24 | import com.addthis.basis.util.JitterClock; 25 | 26 | import com.addthis.meshy.service.file.FileReference; 27 | 28 | import org.junit.After; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import static org.junit.Assert.assertEquals; 33 | import static org.junit.Assert.assertNotNull; 34 | 35 | 36 | public class TestMesh { 37 | 38 | protected static final Logger log = LoggerFactory.getLogger(TestMesh.class); 39 | protected static final DecimalFormat num = new DecimalFormat("0,000"); 40 | 41 | static { 42 | System.setProperty("meshy.autoMesh", "false"); 43 | System.setProperty("meshy.peer.local", "true"); 44 | System.setProperty("meshy.stats.time", "0"); 45 | } 46 | 47 | public static void checkFile(Map map, FileReference test) { 48 | FileReference check = map.get(test.name); 49 | assertNotNull("missing " + test.name, check); 50 | assertEquals(check.name, test.name); 51 | assertEquals(check.size, test.size); 52 | assertEquals(check.getHostUUID(), test.getHostUUID()); 53 | } 54 | 55 | private final LinkedList resources = new LinkedList<>(); 56 | 57 | public MeshyClient getClient(MeshyServer server) throws IOException { 58 | return getClient(server.getLocalPort()); 59 | } 60 | 61 | public MeshyClient getClient(int port) throws IOException { 62 | MeshyClient client = new MeshyClient("localhost", port); 63 | resources.add(client); 64 | return client; 65 | } 66 | 67 | public MeshyServer getServer() throws IOException { 68 | return getServer("."); 69 | } 70 | 71 | public MeshyServer getServer(String root) throws IOException { 72 | return getServer(0, root); 73 | } 74 | 75 | public MeshyServer getServer(int port) throws IOException { 76 | return getServer(port, "."); 77 | } 78 | 79 | public MeshyServer getServer(int port, String root) throws IOException { 80 | MeshyServer server = new MeshyServer(port, new File(root), null); 81 | resources.add(server); 82 | return server; 83 | } 84 | 85 | public boolean waitQuiescent() throws InterruptedException { 86 | return waitQuiescent(2000, 30000); 87 | } 88 | 89 | /** 90 | * wait for all resources to quiesce activity for a time 91 | * 92 | * @return true if resources meet quiescent criteria 93 | */ 94 | public boolean waitQuiescent(long quiescentTime, long maxWait) throws InterruptedException { 95 | long mark = JitterClock.globalTime(); 96 | while (true) { 97 | long min = quiescentTime; 98 | for (Meshy meshy : resources) { 99 | min = Math.min(min, JitterClock.globalTime() - meshy.lastEventTime()); 100 | } 101 | if (min == quiescentTime) { 102 | return true; 103 | } 104 | if (JitterClock.globalTime() - mark > maxWait) { 105 | return false; 106 | } 107 | Thread.sleep(100); 108 | } 109 | } 110 | 111 | @After 112 | public void cleanup() { 113 | log.info("closing resources: {}", resources.size()); 114 | for (Meshy meshy : resources) { 115 | try { 116 | meshy.closeAsync(); 117 | } catch (Exception ex) { 118 | log.warn("unable to cleanup", ex); 119 | } 120 | } 121 | for (Meshy meshy : resources) { 122 | try { 123 | meshy.close(); 124 | } catch (Exception ex) { 125 | log.warn("unable to cleanup", ex); 126 | } 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/test/java/com/addthis/meshy/TestMeshyClientConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.util.LinkedList; 17 | import java.util.concurrent.atomic.AtomicReference; 18 | 19 | import org.junit.Test; 20 | 21 | import static org.junit.Assert.assertEquals; 22 | 23 | 24 | public class TestMeshyClientConnector extends TestMesh { 25 | 26 | @Test 27 | public void simple() throws Exception { 28 | final AtomicReference server = new AtomicReference<>(); 29 | final LinkedList sequence = new LinkedList<>(); 30 | final LinkedList observed = new LinkedList<>(); 31 | server.set(getServer()); 32 | sequence.add("up-" + server.get().getUUID()); 33 | MeshyClientConnector connector = new MeshyClientConnector("localhost", server.get().getLocalPort(), 500, 1000) { 34 | @Override 35 | public void linkUp(MeshyClient client) { 36 | observed.add("up-" + server.get().getUUID()); 37 | } 38 | 39 | @Override 40 | public void linkDown(MeshyClient client) { 41 | observed.add("down-" + server.get().getUUID()); 42 | } 43 | }; 44 | try { 45 | Thread.sleep(1000); 46 | server.get().close(); 47 | sequence.add("down-" + server.get().getUUID()); 48 | server.set(getServer(server.get().getLocalPort())); 49 | sequence.add("up-" + server.get().getUUID()); 50 | Thread.sleep(2000); 51 | server.get().close(); 52 | sequence.add("down-" + server.get().getUUID()); 53 | Thread.sleep(1000); 54 | log.info("seq:{}", sequence); 55 | log.info("obs:{}", observed); 56 | assertEquals(sequence.toString(), observed.toString()); 57 | } finally { 58 | connector.terminate(); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/test/java/com/addthis/meshy/TestPeerService.java: -------------------------------------------------------------------------------- 1 | ///* 2 | // * Licensed under the Apache License, Version 2.0 (the "License"); 3 | // * you may not use this file except in compliance with the License. 4 | // * You may obtain a copy of the License at 5 | // * 6 | // * http://www.apache.org/licenses/LICENSE-2.0 7 | // * 8 | // * Unless required by applicable law or agreed to in writing, software 9 | // * distributed under the License is distributed on an "AS IS" BASIS, 10 | // * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // * See the License for the specific language governing permissions and 12 | // * limitations under the License. 13 | // */ 14 | //package com.addthis.meshy; 15 | // 16 | //import java.net.InetSocketAddress; 17 | // 18 | //import java.util.LinkedList; 19 | //import java.util.List; 20 | //import java.util.Map; 21 | // 22 | //import com.addthis.meshy.service.host.HostSource; 23 | // 24 | //import org.junit.Test; 25 | //import org.slf4j.Logger; 26 | //import org.slf4j.LoggerFactory; 27 | // 28 | //import static org.junit.Assert.assertEquals; 29 | //import static org.junit.Assert.assertTrue; 30 | // 31 | // 32 | //public class TestPeerService extends TestMesh { 33 | // private static final Logger log = LoggerFactory.getLogger(TestPeerService.class); 34 | // 35 | // @Test 36 | // public void twoPeers() throws Exception { 37 | // MeshyServer server1 = getServer(); 38 | // MeshyServer server2 = getServer(); 39 | // log.debug("server1: {}, server2: {}", server1, server2); 40 | // server1.connectToPeer(server2.getUUID(), server2.getLocalAddress()); 41 | // waitQuiescent(); 42 | // assertEquals(1, server1.getServerPeerCount()); 43 | // assertEquals(1, server2.getServerPeerCount()); 44 | // } 45 | // 46 | // @Test 47 | // public void threePeersWithDisconnect() throws Exception { 48 | // final MeshyServer server1 = getServer(); 49 | // final MeshyServer server2 = getServer(); 50 | // final MeshyServer server3 = getServer(); 51 | // server1.connectPeer(new InetSocketAddress("localhost", server2.getLocalPort())); 52 | // server1.connectPeer(new InetSocketAddress("localhost", server2.getLocalPort())); 53 | // server1.connectPeer(new InetSocketAddress("localhost", server2.getLocalPort())); 54 | // server1.connectPeer(new InetSocketAddress("localhost", server3.getLocalPort())); 55 | // // allow server connections to establish 56 | // waitQuiescent(); 57 | // assertEquals(2, server1.getServerPeerCount()); 58 | // assertEquals(2, server2.getServerPeerCount()); 59 | // assertEquals(2, server3.getServerPeerCount()); 60 | // Meshy client = getClient(server1); 61 | // HostSource hosts = new HostSource(client); 62 | // hosts.sendRequest(); 63 | // hosts.waitComplete(); 64 | // log.info("host list.1 --> {}", hosts.getHostList()); 65 | // Map hostMap = hosts.getHostMap(); 66 | // assertTrue(hostMap.containsKey(server2.getUUID())); 67 | // assertTrue(hostMap.containsKey(server3.getUUID())); 68 | // assertEquals(hostMap.get(server2.getUUID()).getPort(), server2.getLocalPort()); 69 | // assertEquals(hostMap.get(server3.getUUID()).getPort(), server3.getLocalPort()); 70 | // client.close(); 71 | // 72 | // // have one server drop out 73 | // server2.close(); 74 | // // allow server connections to stabilize 75 | // waitQuiescent(); 76 | // assertEquals(1, server1.getServerPeerCount()); 77 | // assertEquals(1, server3.getServerPeerCount()); 78 | // 79 | // client = getClient(server1); 80 | // hosts = new HostSource(client); 81 | // hosts.sendRequest(); 82 | // hosts.waitComplete(); 83 | // log.info("host list.2 --> {}", hosts.getHostList()); 84 | // hostMap = hosts.getHostMap(); 85 | // assertTrue(!hostMap.containsKey(server2.getUUID())); 86 | // assertTrue(hostMap.containsKey(server3.getUUID())); 87 | // assertEquals(hostMap.get(server3.getUUID()).getPort(), server3.getLocalPort()); 88 | // } 89 | // 90 | // @Test 91 | // public void manyPeers() throws Exception { 92 | // final int serverCount = 20; 93 | // List servers = new LinkedList<>(); 94 | // for (int i = 0; i < serverCount; i++) { 95 | // servers.add(getServer()); 96 | // } 97 | // MeshyServer first = servers.get(0); 98 | // for (MeshyServer server : servers) { 99 | // server.connectToPeer(first.getUUID(), first.getLocalAddress()); 100 | // } 101 | // 102 | // // allow server connections to establish 103 | // log.info("waiting for servers to become idle"); 104 | // log.info("wait successful = {}", waitQuiescent()); 105 | // 106 | // for (MeshyServer server : servers) { 107 | // log.info("check connection count >> {}", server); 108 | // try (Meshy client = getClient(server.getLocalPort())) { 109 | // HostSource hosts = new HostSource(client); 110 | // hosts.sendRequest(); 111 | // hosts.waitComplete(); 112 | // Map hostMap = hosts.getHostMap(); 113 | // for (MeshyServer peer : servers) { 114 | // if (server == peer) { 115 | // continue; 116 | // } 117 | // assertTrue(server + " is missing peer --> " + peer, hostMap.containsKey(peer.getUUID())); 118 | // } 119 | // } 120 | // } 121 | // } 122 | //} 123 | -------------------------------------------------------------------------------- /src/test/java/com/addthis/meshy/TestStreamService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy; 15 | 16 | import java.io.ByteArrayOutputStream; 17 | import java.io.IOException; 18 | 19 | import java.net.InetSocketAddress; 20 | 21 | import java.util.LinkedList; 22 | import java.util.List; 23 | import java.util.UUID; 24 | import java.util.concurrent.TimeUnit; 25 | import java.util.zip.CRC32; 26 | 27 | import java.security.MessageDigest; 28 | import java.security.NoSuchAlgorithmException; 29 | 30 | import com.addthis.basis.util.LessBytes; 31 | 32 | import com.addthis.meshy.service.stream.SourceInputStream; 33 | import com.addthis.meshy.service.stream.StreamSource; 34 | 35 | import org.junit.Ignore; 36 | import org.junit.Test; 37 | 38 | import static org.junit.Assert.assertEquals; 39 | import static org.junit.Assert.assertTrue; 40 | import static org.junit.Assert.fail; 41 | 42 | 43 | public class TestStreamService extends TestMesh { 44 | 45 | private static final long MD5HOSTS = -1621285313438006658L; 46 | 47 | @Test 48 | public void testReadPressure() throws Exception { 49 | final int serverCount = 20; 50 | List servers = new LinkedList<>(); 51 | for (int i = 0; i < serverCount; i++) { 52 | servers.add(getServer("src/test/files")); 53 | } 54 | MeshyServer first = servers.get(0); 55 | for (MeshyServer server : servers) { 56 | server.connectToPeer(first.getUUID(), first.getLocalAddress()); 57 | } 58 | // allow server connections to establish 59 | waitQuiescent(); 60 | /* read the same file from each local server */ 61 | log.info("-- local read --"); 62 | long time = System.nanoTime(); 63 | for (int i = 0; i < serverCount; i++) { 64 | new StreamReader(servers.get(i), servers.get(i), "c/hosts").kick().join(); 65 | } 66 | long mark = System.nanoTime(); 67 | log.info("... done in {}ns ...", num.format(mark - time)); 68 | /* read the same file proxied through another server */ 69 | log.info("-- remote read --"); 70 | time = System.nanoTime(); 71 | for (int i = 0; i < serverCount; i++) { 72 | new StreamReader(servers.get(i), servers.get((i + 1) % servers.size()), "c/hosts").kick().join(); 73 | } 74 | mark = System.nanoTime(); 75 | log.info("... done in {}ns ...", num.format(mark - time)); 76 | /* concurrently read the same file proxied through another server */ 77 | log.info("-- remote concurrent read --"); 78 | time = System.nanoTime(); 79 | LinkedList threads = new LinkedList<>(); 80 | for (int i = 0; i < serverCount; i++) { 81 | threads.add(new StreamReader(servers.get(i), servers.get((i + 1) % servers.size()), "c/hosts").kick()); 82 | } 83 | for (Thread thread : threads) { 84 | thread.join(); 85 | } 86 | mark = System.nanoTime(); 87 | log.info("... done in {}ns ...", num.format(mark - time)); 88 | /* concurrently read the same multiplexed file proxied through another server */ 89 | log.info("-- remote concurrent multiplexed read --"); 90 | time = System.nanoTime(); 91 | threads = new LinkedList<>(); 92 | for (int i = 0; i < serverCount; i++) { 93 | threads.add(new StreamReader(servers.get(i), servers.get((i + 1) % servers.size()), "mux/hosts").kick()); 94 | } 95 | for (Thread thread : threads) { 96 | thread.join(); 97 | } 98 | mark = System.nanoTime(); 99 | log.info("... done in {}ns ...", num.format(mark - time)); 100 | } 101 | 102 | /** 103 | * reader helper for read pressure 104 | */ 105 | private static class StreamReader extends Thread { 106 | 107 | private final MeshyServer connectTo; 108 | private final MeshyServer readFrom; 109 | private final String path; 110 | private final boolean async; 111 | 112 | StreamReader(final MeshyServer connectTo, final MeshyServer readFrom, String path, boolean async) { 113 | this.connectTo = connectTo; 114 | this.readFrom = readFrom; 115 | this.path = path; 116 | setName("StreamReader " + connectTo.getLocalPort() + "-" + readFrom.getLocalPort() + " @ " + path); 117 | this.async = async; 118 | } 119 | 120 | StreamReader(final MeshyServer connectTo, final MeshyServer readFrom, String path) { 121 | this(connectTo, readFrom, path, false); 122 | } 123 | 124 | public StreamReader kick() { 125 | start(); 126 | return this; 127 | } 128 | 129 | @Override 130 | public void run() { 131 | Meshy client = null; 132 | try { 133 | client = new MeshyClient("localhost", connectTo.getLocalAddress().getPort()); 134 | StreamSource stream = new StreamSource(client, readFrom.getUUID(), path, 0); 135 | long time = System.nanoTime(); 136 | byte[] raw; 137 | if (async) { 138 | SourceInputStream sourceInputStream = stream.getInputStream(); 139 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 140 | boolean done = false; 141 | while (!done) { 142 | byte[] data = sourceInputStream.poll(100, TimeUnit.MILLISECONDS); 143 | if (data != null) { 144 | if (data.length == 0) { 145 | done = true; 146 | } else { 147 | bos.write(data); 148 | } 149 | } 150 | 151 | } 152 | raw = bos.toByteArray(); 153 | } else { 154 | raw = LessBytes.readFully(stream.getInputStream()); 155 | } 156 | CRC32 crc = new CRC32(); 157 | crc.update(raw); 158 | log.info("[{}] read [{}] = [{}:{}] in ({}ns)", 159 | connectTo.getLocalAddress().getPort(), readFrom.getLocalAddress().getPort(), 160 | raw.length, crc.getValue(), num.format(System.nanoTime() - time)); 161 | assertEquals(593366, raw.length); 162 | assertEquals(4164197206L, crc.getValue()); 163 | stream.waitComplete(); 164 | } catch (Exception ex) { 165 | log.warn("FAIL {} -- {}", connectTo, readFrom, ex); 166 | } finally { 167 | if (client != null) { 168 | client.close(); 169 | } 170 | } 171 | } 172 | } 173 | 174 | 175 | @Ignore @Test 176 | public void testPeerLocalStream() throws Exception { 177 | localStreamTest(false, "read sync"); 178 | } 179 | 180 | @Ignore @Test 181 | public void testPeerLocalStreamAsync() throws Exception { 182 | localStreamTest(true, "read async"); 183 | } 184 | 185 | private void localStreamTest(boolean async, String logPrefix) throws Exception { 186 | final MeshyServer server1 = getServer("src/test/files"); 187 | final MeshyServer server2 = getServer("src/test/files/a"); 188 | final MeshyServer server3 = getServer("src/test/files/b"); 189 | final MeshyServer server4 = getServer("src/test/files/c"); 190 | server1.connectPeer(new InetSocketAddress("localhost", server2.getLocalPort())); 191 | server1.connectPeer(new InetSocketAddress("localhost", server3.getLocalPort())); 192 | server1.connectPeer(new InetSocketAddress("localhost", server4.getLocalPort())); 193 | /** wait for network chatter to calm down */ 194 | waitQuiescent(); 195 | 196 | MeshyClient client = getClient(server1); 197 | 198 | /* simple direct test */ 199 | StreamSource stream = new StreamSource(client, server1.getUUID(), "/a/abc.xml", 1024 * 10); 200 | SourceInputStream in = stream.getInputStream(); 201 | byte[] data = async ? readAsync(in) : LessBytes.readFully(in); 202 | assertEquals(data.length, 4); 203 | log.info("{} server1:/a/abc.xml [{}]", logPrefix, data.length); 204 | stream.waitComplete(); 205 | 206 | /* multi-part "meshy" test */ 207 | stream = new StreamSource(client, server1.getUUID(), "/c/hosts", 1024 * 10); 208 | in = stream.getInputStream(); 209 | data = async ? readAsync(in) : LessBytes.readFully(in); 210 | assertEquals(data.length, 593366); 211 | assertEquals(MD5HOSTS, md5(data)); 212 | log.info("{} server1:/c/hosts [{}]", logPrefix, data.length); 213 | stream.waitComplete(); 214 | 215 | /* remote error test */ 216 | final String randomString = UUID.randomUUID().toString(); 217 | stream = new StreamSource(client, server1.getUUID(), "/" + randomString, 1024 * 10); 218 | try { 219 | in = stream.getInputStream(); 220 | data = async ? readAsync(in) : LessBytes.readFully(in); 221 | fail(logPrefix + " should not exist"); 222 | } catch (Exception ex) { 223 | assertTrue(ex.getMessage().contains(randomString)); 224 | } 225 | stream.waitComplete(); 226 | log.info("{} pass non-existent-file test", logPrefix); 227 | 228 | /* proxied file test */ 229 | stream = new StreamSource(client, server2.getUUID(), "/abc.xml", 1024 * 10); 230 | in = stream.getInputStream(); 231 | data = async ? readAsync(in) : LessBytes.readFully(in); 232 | assertEquals(data.length, 4); 233 | log.info("{} server2:/abc.xml [{}]", logPrefix, data.length); 234 | stream.waitComplete(); 235 | 236 | /* proxied "meshy" file test */ 237 | stream = new StreamSource(client, server4.getUUID(), "/hosts", 1024 * 10); 238 | in = stream.getInputStream(); 239 | data = async ? readAsync(in) : LessBytes.readFully(in); 240 | assertEquals(data.length, 593366); 241 | assertEquals(MD5HOSTS, md5(data)); 242 | log.info("{} server4:/hosts [{}]", logPrefix, data.length); 243 | stream.waitComplete(); 244 | 245 | /* mux'd file test */ 246 | stream = new StreamSource(client, server1.getUUID(), "/mux/hosts", 1024 * 10); 247 | in = stream.getInputStream(); 248 | data = async ? readAsync(in) : LessBytes.readFully(in); 249 | assertEquals(data.length, 593366); 250 | assertEquals(MD5HOSTS, md5(data)); 251 | log.info("{} server1:/mux/hosts [{}]", logPrefix, data.length); 252 | stream.waitComplete(); 253 | } 254 | 255 | private static long md5(final byte[] data) { 256 | try { 257 | MessageDigest md = MessageDigest.getInstance("MD5"); 258 | md.digest(data); 259 | long val = 0; 260 | for (byte b : md.digest()) { 261 | val <<= 8; 262 | val |= (b & 0xff); 263 | } 264 | return val; 265 | } catch (NoSuchAlgorithmException e) { 266 | throw new RuntimeException(e); 267 | } 268 | } 269 | 270 | private static byte[] readAsync(SourceInputStream in) throws IOException { 271 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 272 | boolean done = false; 273 | while (!done) { 274 | byte[] data = in.poll(100, TimeUnit.MILLISECONDS); 275 | if (data != null) { 276 | if (data.length == 0) { 277 | done = true; 278 | } else { 279 | bos.write(data); 280 | } 281 | } 282 | 283 | } 284 | return bos.toByteArray(); 285 | } 286 | 287 | } 288 | -------------------------------------------------------------------------------- /src/test/java/com/addthis/meshy/service/file/TestFileService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.file; 15 | 16 | import java.net.InetSocketAddress; 17 | 18 | import java.util.Map; 19 | 20 | import com.addthis.meshy.Meshy; 21 | import com.addthis.meshy.MeshyClient; 22 | import com.addthis.meshy.MeshyServer; 23 | import com.addthis.meshy.TestMesh; 24 | 25 | import org.junit.Test; 26 | 27 | import static org.junit.Assert.assertEquals; 28 | import static org.junit.Assert.assertFalse; 29 | import static org.junit.Assert.assertNotEquals; 30 | 31 | 32 | public class TestFileService extends TestMesh { 33 | 34 | @Test 35 | public void singlePeer() throws Exception { 36 | final MeshyServer server = getServer("src/test/files"); 37 | final MeshyClient client = getClient(server); 38 | FileSource files = new FileSource(client, new String[]{ 39 | "*/*.xml", 40 | "*/hosts", 41 | }); 42 | files.waitComplete(); 43 | log.info("file.list --> {}", files.getFileList()); 44 | Map map = files.getFileMap(); 45 | checkFile(map, new FileReference("/a/abc.xml", 0, 4).setHostUUID(server.getUUID())); 46 | checkFile(map, new FileReference("/a/def.xml", 0, 7).setHostUUID(server.getUUID())); 47 | checkFile(map, new FileReference("/b/ghi.xml", 0, 4).setHostUUID(server.getUUID())); 48 | checkFile(map, new FileReference("/b/jkl.xml", 0, 7).setHostUUID(server.getUUID())); 49 | checkFile(map, new FileReference("/c/hosts", 0, 593366).setHostUUID(server.getUUID())); 50 | checkFile(map, new FileReference("/mux/hosts", 0, 593366).setHostUUID(server.getUUID())); 51 | /** second test exercises the cache */ 52 | files = new FileSource(client, new String[]{ 53 | "*/*.xml", 54 | "*/hosts", 55 | }); 56 | files.waitComplete(); 57 | log.info("file.list --> {}", files.getFileList()); 58 | map = files.getFileMap(); 59 | checkFile(map, new FileReference("/a/abc.xml", 0, 4).setHostUUID(server.getUUID())); 60 | checkFile(map, new FileReference("/a/def.xml", 0, 7).setHostUUID(server.getUUID())); 61 | checkFile(map, new FileReference("/b/ghi.xml", 0, 4).setHostUUID(server.getUUID())); 62 | checkFile(map, new FileReference("/b/jkl.xml", 0, 7).setHostUUID(server.getUUID())); 63 | checkFile(map, new FileReference("/c/hosts", 0, 593366).setHostUUID(server.getUUID())); 64 | checkFile(map, new FileReference("/mux/hosts", 0, 593366).setHostUUID(server.getUUID())); 65 | } 66 | 67 | @Test 68 | public void globSyntax() throws Exception { 69 | final MeshyServer server = getServer("src/test/files"); 70 | final MeshyClient client = getClient(server); 71 | FileSource files = new FileSource(client, new String[]{ 72 | "/{a,c}/*", 73 | "*/h?sts", 74 | }); 75 | files.waitComplete(); 76 | log.info("file.list --> {}", files.getFileList()); 77 | Map map = files.getFileMap(); 78 | checkFile(map, new FileReference("/a/abc.xml", 0, 4).setHostUUID(server.getUUID())); 79 | checkFile(map, new FileReference("/a/def.xml", 0, 7).setHostUUID(server.getUUID())); 80 | assertFalse(map.containsKey("/b/ghi.xml")); 81 | assertFalse(map.containsKey("/b/jkl.xml")); 82 | checkFile(map, new FileReference("/c/hosts", 0, 593366).setHostUUID(server.getUUID())); 83 | checkFile(map, new FileReference("/mux/hosts", 0, 593366).setHostUUID(server.getUUID())); 84 | } 85 | 86 | @Test 87 | public void multiPeer() throws Exception { 88 | final MeshyServer server1 = getServer("src/test/files/a"); 89 | final MeshyServer server2 = getServer("src/test/files/b"); 90 | final MeshyServer server3 = getServer("src/test/files"); 91 | server1.connectPeer(new InetSocketAddress("localhost", server2.getLocalPort())); 92 | server1.connectPeer(new InetSocketAddress("localhost", server3.getLocalPort())); 93 | // allow server connections to establish 94 | waitQuiescent(); 95 | Meshy client = getClient(server1); 96 | FileSource files = new FileSource(client, new String[]{"*.xml"}); 97 | files.waitComplete(); 98 | log.info("file.list --> {}", files.getFileList()); 99 | Map map = files.getFileMap(); 100 | log.info("file map --> {}", map); 101 | checkFile(map, new FileReference("/abc.xml", 0, 4).setHostUUID(server1.getUUID())); 102 | checkFile(map, new FileReference("/def.xml", 0, 7).setHostUUID(server1.getUUID())); 103 | checkFile(map, new FileReference("/ghi.xml", 0, 4).setHostUUID(server2.getUUID())); 104 | checkFile(map, new FileReference("/jkl.xml", 0, 7).setHostUUID(server2.getUUID())); 105 | checkFile(map, new FileReference("/xyz.xml", 0, 10).setHostUUID(server3.getUUID())); 106 | } 107 | 108 | @Test 109 | public void testEquals() throws Exception { 110 | FileReference[] refs = new FileReference[5]; 111 | 112 | refs[0] = new FileReference("/a/abc.xml", 0, 4); 113 | refs[1] = new FileReference("/a/def.xml", 0, 7); 114 | refs[2] = new FileReference(null, 0, 4); 115 | 116 | // refs[0] and refs[3] are equal 117 | refs[3] = new FileReference("/a/abc.xml", 0, 4); 118 | 119 | // refs[2] and refs[4] are equal 120 | refs[4] = new FileReference(null, 0, 4); 121 | 122 | refs[0].setHostUUID("a"); 123 | refs[1].setHostUUID("b"); 124 | // verify that equals() handles hostUUID of null 125 | refs[2].setHostUUID(null); 126 | 127 | refs[3].setHostUUID("a"); // equal to refs[0] 128 | refs[4].setHostUUID(null); // equal to refs[2] 129 | 130 | for (int i = 0; i < 3; i++) { 131 | for (int j = 0; j < 3; j++) { 132 | if (i == j) { 133 | assertEquals(refs[i], refs[j]); 134 | } else { 135 | assertNotEquals(refs[i], refs[j]); 136 | } 137 | } 138 | } 139 | 140 | assertEquals(refs[0], refs[3]); 141 | assertEquals(refs[2], refs[4]); 142 | 143 | refs[3].setHostUUID("b"); 144 | refs[4].setHostUUID("b"); 145 | 146 | assertFalse(refs[0].equals(refs[3])); 147 | assertFalse(refs[2].equals(refs[4])); 148 | } 149 | 150 | @Test 151 | public void testHashCode() throws Exception { 152 | FileReference[] refs = new FileReference[4]; 153 | 154 | refs[0] = new FileReference("/a/abc.xml", 0, 4); 155 | refs[1] = new FileReference("/a/abc.xml", 0, 4); 156 | refs[2] = new FileReference(null, 0, 4); 157 | refs[3] = new FileReference(null, 0, 4); 158 | 159 | refs[0].setHostUUID("a"); 160 | refs[1].setHostUUID("a"); 161 | refs[2].setHostUUID(null); 162 | refs[3].setHostUUID(null); 163 | 164 | assertEquals(refs[0].hashCode(), refs[1].hashCode()); 165 | assertEquals(refs[2].hashCode(), refs[3].hashCode()); 166 | } 167 | 168 | } 169 | -------------------------------------------------------------------------------- /src/test/java/com/addthis/meshy/service/file/TestVFS.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.file; 15 | 16 | import java.io.File; 17 | 18 | import java.util.Iterator; 19 | import java.util.LinkedList; 20 | import java.util.Map; 21 | 22 | import java.nio.file.PathMatcher; 23 | 24 | import com.addthis.meshy.LocalFileHandler; 25 | import com.addthis.meshy.LocalFileSystem; 26 | import com.addthis.meshy.MeshyClient; 27 | import com.addthis.meshy.MeshyServer; 28 | import com.addthis.meshy.TestMesh; 29 | import com.addthis.meshy.VirtualFileInput; 30 | import com.addthis.meshy.VirtualFileReference; 31 | 32 | import org.junit.After; 33 | import org.junit.Before; 34 | import org.junit.Test; 35 | 36 | 37 | public class TestVFS extends TestMesh { 38 | 39 | @Before 40 | public void setup() { 41 | System.setProperty("mesh.local.handlers", DummyHandler.class.getName()); 42 | LocalFileSystem.reloadHandlers(); 43 | MeshyServer.resetFileSystems(); 44 | } 45 | 46 | @After @Override 47 | public void cleanup() { 48 | super.cleanup(); 49 | System.setProperty("mesh.local.handlers", ""); 50 | LocalFileSystem.reloadHandlers(); 51 | MeshyServer.resetFileSystems(); 52 | } 53 | 54 | @Test 55 | public void testVFS() throws Exception { 56 | final MeshyServer server = getServer("src/test"); 57 | final MeshyClient client = getClient(server); 58 | FileSource files = new FileSource(client, new String[]{"*"}); 59 | files.waitComplete(); 60 | Map map = files.getFileMap(); 61 | log.info("map={}", map); 62 | checkFile(map, new FileReference("/dummy", 0, 0).setHostUUID(server.getUUID())); 63 | } 64 | 65 | public static class DummyHandler implements LocalFileHandler { 66 | 67 | LinkedList list = new LinkedList<>(); 68 | VirtualFileReference ref = new DummyReference(); 69 | 70 | public DummyHandler() { 71 | list.add(ref); 72 | } 73 | 74 | @Override 75 | public boolean canHandleDirectory(File dir) { 76 | return true; 77 | } 78 | 79 | @Override 80 | public Iterator listFiles(File dir, PathMatcher filter) { 81 | return list.iterator(); 82 | } 83 | 84 | @Override 85 | public VirtualFileReference getFile(File dir, String name) { 86 | return ref; 87 | } 88 | } 89 | 90 | static class DummyReference implements VirtualFileReference { 91 | 92 | @Override 93 | public String getName() { 94 | return "dummy"; 95 | } 96 | 97 | @Override 98 | public long getLastModified() { 99 | return 0; 100 | } 101 | 102 | @Override 103 | public long getLength() { 104 | return 0; 105 | } 106 | 107 | @Override 108 | public Iterator listFiles(PathMatcher filter) { 109 | return null; 110 | } 111 | 112 | @Override 113 | public VirtualFileReference getFile(String name) { 114 | return null; 115 | } 116 | 117 | @Override 118 | public VirtualFileInput getInput(Map options) { 119 | return null; 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/test/java/com/addthis/meshy/service/message/TestMessageFileSystem.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.InputStream; 17 | 18 | import java.util.Map; 19 | 20 | import com.addthis.basis.util.LessBytes; 21 | 22 | import com.addthis.meshy.service.file.FileReference; 23 | import com.addthis.meshy.service.file.FileSource; 24 | import com.addthis.meshy.MeshyClient; 25 | import com.addthis.meshy.MeshyServer; 26 | import com.addthis.meshy.TestMesh; 27 | 28 | import org.junit.Test; 29 | 30 | import static org.junit.Assert.assertEquals; 31 | import static org.junit.Assert.assertTrue; 32 | 33 | 34 | public class TestMessageFileSystem extends TestMesh { 35 | 36 | @Test 37 | public void basicTest() throws Exception { 38 | final MeshyServer server = getServer("/tmp"); 39 | final MeshyClient client = getClient(server); 40 | 41 | /* 42 | * client registers rpc endpoint in mesh filespace: /rpc.test/one.rpc 43 | */ 44 | MessageFileProvider provider = new MessageFileProvider(client); 45 | provider.setListener("/rpc.test/one.rpc", (fileName, options, out) -> { 46 | /* this is the client rpc reply endpoint implementation */ 47 | LessBytes.writeString("rpc.reply", out); 48 | /* bytes are accumulated and sent on close */ 49 | out.close(); 50 | }); 51 | 52 | /* 53 | * any client can then discover the rpc/file endpoint via normal FileService calls 54 | */ 55 | FileSource files = new FileSource(client, new String[]{"/rpc.test/*.rpc"}); 56 | files.waitComplete(); 57 | Map map = files.getFileMap(); 58 | 59 | log.info("files = {}", map); 60 | 61 | assertTrue(map.containsKey("/rpc.test/one.rpc")); 62 | 63 | /* 64 | * rpc is called/read as a normal file 65 | */ 66 | FileReference ref = map.get("/rpc.test/one.rpc"); 67 | InputStream in = client.readFile(ref); 68 | String str = LessBytes.readString(in); 69 | in.close(); 70 | 71 | assertEquals("rpc.reply", str); 72 | provider.close(); 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/test/java/com/addthis/meshy/service/message/TestMessageService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.addthis.meshy.service.message; 15 | 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | import java.io.OutputStream; 19 | 20 | import java.util.concurrent.atomic.AtomicBoolean; 21 | 22 | import com.addthis.basis.util.LessBytes; 23 | 24 | import com.addthis.meshy.MeshyClient; 25 | import com.addthis.meshy.MeshyServer; 26 | import com.addthis.meshy.TestMesh; 27 | 28 | import org.junit.Test; 29 | 30 | import static org.junit.Assert.assertEquals; 31 | import static org.junit.Assert.assertTrue; 32 | 33 | 34 | public class TestMessageService extends TestMesh { 35 | 36 | @Test 37 | public void basic() throws Exception { 38 | final MeshyServer server = getServer("src/test/files"); 39 | final MeshyClient client = getClient(server.getLocalPort()); 40 | final AtomicBoolean clientRecv = new AtomicBoolean(false); 41 | final AtomicBoolean serverRecv = new AtomicBoolean(false); 42 | /** connect client and set up listener */ 43 | MessageSource mss = new MessageSource(client, new TopicListener() { 44 | @Override 45 | public void receiveMessage(String topic, InputStream message) throws IOException { 46 | log.info("client recv: {}", topic); 47 | assertEquals("def", topic); 48 | assertEquals("67890", LessBytes.readString(message)); 49 | clientRecv.set(true); 50 | } 51 | 52 | @Override 53 | public void linkDown() { 54 | log.info("client linkdown"); 55 | } 56 | }); 57 | /** register server-side listener */ 58 | MessageTarget.registerListener("abc", new TargetListener() { 59 | @Override 60 | public void receiveMessage(TopicSender target, String topic, InputStream message) throws IOException { 61 | log.info("server recv: {}", topic); 62 | assertEquals("abc", topic); 63 | assertEquals("12345", LessBytes.readString(message)); 64 | OutputStream out = target.sendMessage("def"); 65 | LessBytes.writeString("67890", out); 66 | out.close(); 67 | serverRecv.set(true); 68 | } 69 | 70 | @Override 71 | public void linkDown(TopicSender target) { 72 | log.info("server linkdown: {}", target); 73 | } 74 | }); 75 | /** ping test */ 76 | OutputStream out = mss.sendMessage("abc"); 77 | LessBytes.writeString("12345", out); 78 | out.close(); 79 | /** wait for quiet */ 80 | waitQuiescent(); 81 | assertTrue(clientRecv.get()); 82 | assertTrue(serverRecv.get()); 83 | mss.sendComplete(); 84 | mss.waitComplete(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/test/resources/simplelogger.properties: -------------------------------------------------------------------------------- 1 | org.slf4j.simpleLogger.showShortLogName = true 2 | org.slf4j.simpleLogger.showDateTime = true 3 | org.slf4j.simpleLogger.dateTimeFormat = yyyy-MM-dd'T'HH:mm:ss,SSS 4 | 5 | org.slf4j.simpleLogger.log.com.addthis.meshy.MeshyServer = warn 6 | org.slf4j.simpleLogger.log.com.addthis.meshy.service.peer.PeerService = warn --------------------------------------------------------------------------------