├── attic ├── resource │ ├── local.centipede.bench.yaml │ ├── local.centipede.structbench.yaml │ ├── structbench.yaml │ ├── latency.yaml │ ├── bigtest.yaml │ ├── bench.yaml │ ├── pingpong.yaml │ └── unordered.yaml ├── bigtest │ ├── testcluster.txt │ ├── services │ │ ├── BinaryReceiver.java │ │ ├── HTListener.java │ │ ├── RequestServer.java │ │ └── HTHost.java │ ├── HashHostNode.java │ ├── RequestProcessorNode.java │ ├── PureHTListener.java │ ├── ClusterClientNode.java │ ├── TableDataQuery.java │ └── DataMutatingClient.java ├── UnreliableReceiver.java ├── PongService.java ├── UnorderedSender.java ├── PingService.java ├── UnorderedReceiver.java └── LatencyTest.java ├── release.properties ├── diagnostics ├── iperfudpserv.sh ├── iperfudpclient.sh └── nicstat.sh ├── examples ├── src │ └── main │ │ └── java │ │ └── org │ │ └── nustaq │ │ └── fastcast │ │ └── examples │ │ ├── latency │ │ ├── FCPongServer.java │ │ ├── FCPingClientAsync.java │ │ ├── async │ │ │ ├── AsyncLatMessage.java │ │ │ ├── allocfree │ │ │ │ ├── AsyncLatMessageStruct.java │ │ │ │ └── AsyncPubStruct.java │ │ │ ├── AsyncLatReceiver.java │ │ │ ├── fc.kson │ │ │ └── AsyncLatPublisher.java │ │ ├── pingponglat.kson │ │ └── ShareMemPingPong.java │ │ ├── programmatic_configuration │ │ ├── same_using_config_file │ │ │ ├── config.kson │ │ │ ├── ConfigFileSubscriber.java │ │ │ └── ConfigFilePublisher.java │ │ ├── ProgrammaticConfiguredSubscriber.java │ │ └── ProgrammaticConfiguredPublisher.java │ │ ├── structencoding │ │ ├── StructSubscriber.java │ │ ├── StructPublisher.java │ │ └── Protocol.java │ │ └── multiplestructs │ │ ├── MPSubscriber.java │ │ ├── MultipleProtocol.java │ │ └── MPPublisher.java ├── readme.txt └── pom.xml ├── src ├── test │ └── java │ │ └── basic │ │ ├── stuff │ │ ├── sendreceive.kson │ │ ├── Issue4.java │ │ └── Issue4_Slow.java │ │ └── sendreceive.kson └── main │ └── java │ └── org │ └── nustaq │ └── fastcast │ ├── impl │ ├── ByteArrayReceiver.java │ ├── ControlPacket.java │ ├── RetransEntry.java │ ├── Packet.java │ ├── Defragmenter.java │ ├── RetransPacket.java │ ├── DataPacket.java │ ├── Topic.java │ ├── ReceiveBufferDispatcher.java │ └── BatchingController.java │ ├── util │ ├── Sleeper.java │ ├── RateMeasure.java │ ├── FCUtils.java │ └── FCLog.java │ ├── transport │ └── PhysicalTransport.java │ ├── config │ ├── TopicConf.java │ ├── ClusterConf.java │ ├── PublisherConf.java │ └── SubscriberConf.java │ └── api │ ├── util │ ├── ObjectPublisher.java │ ├── ByteArraySubscriber.java │ └── ObjectSubscriber.java │ ├── FCSubscriber.java │ └── FCPublisher.java ├── .gitignore └── pom.xml /attic/resource/local.centipede.bench.yaml: -------------------------------------------------------------------------------- 1 | interfaces: 2 | if0: eth0 3 | -------------------------------------------------------------------------------- /attic/resource/local.centipede.structbench.yaml: -------------------------------------------------------------------------------- 1 | interfaces: 2 | if0: eth0 3 | -------------------------------------------------------------------------------- /release.properties: -------------------------------------------------------------------------------- 1 | #release configuration 2 | #Mon Dec 02 21:46:17 CET 2013 3 | preparationGoals=clean verify 4 | pushChanges=true 5 | scm.commentPrefix=[maven-release-plugin] 6 | remoteTagging=true 7 | exec.additionalArguments=-Psonatype-oss-release 8 | completedPhase=check-poms 9 | scm.url=scm\:svn\:https\://fast-cast.googlecode.com/svn/trunk/ 10 | -------------------------------------------------------------------------------- /diagnostics/iperfudpserv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$1" ]; then 4 | echo 5 | echo usage: $0 [network-interface] 6 | echo 7 | echo e.g. $0 eth0 8 | echo 9 | echo starts iperf udp bench server 10 | exit 11 | fi 12 | 13 | route add -host 239.255.1.3 $1 14 | iperf -s -B 239.255.1.3 -u -f m -i 5 15 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/FCPongServer.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.latency; 2 | 3 | /** 4 | * Created by ruedi on 07.12.14. 5 | */ 6 | public class FCPongServer { 7 | 8 | public static void main(String arg[]) throws InterruptedException { 9 | new FCPing().runPongServer(); 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/FCPingClientAsync.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.latency; 2 | 3 | /** 4 | * Created by ruedi on 09.12.14. 5 | */ 6 | public class FCPingClientAsync { 7 | 8 | public static void main(String arg[]) throws InterruptedException { 9 | new FCPing().runPingClientASync(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /attic/bigtest/testcluster.txt: -------------------------------------------------------------------------------- 1 | client nodes send requests to several request servers. 2 | The request result is sent to HTHosts and stored there. 3 | HtHosts promote the changes to HTlisteners which build a mirror. 4 | Binary receiver receives random byte arrays with a checksum and promotes them back to client nodes 5 | Cyclically all HTHosts deliver their complete content and it is compared with the clients mirror. -------------------------------------------------------------------------------- /attic/bigtest/services/BinaryReceiver.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest.services; 2 | 3 | import org.nustaq.fastcast.api.FCTopicService; 4 | 5 | /** 6 | * Created with IntelliJ IDEA. 7 | * User: ruedi 8 | * Date: 9/14/13 9 | * Time: 2:16 PM 10 | * To change this template use File | Settings | File Templates. 11 | */ 12 | public class BinaryReceiver extends FCTopicService { 13 | } 14 | -------------------------------------------------------------------------------- /diagnostics/iperfudpclient.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # use nicstat.sh to monitor pps 4 | 5 | if [ -z "$1" ]; then 6 | echo 7 | echo usage: $0 [network-interface] [sendrate e.g. 1000] 8 | echo 9 | echo e.g. $0 eth0 10 | echo 11 | echo starts iperf udp bench server 12 | exit 13 | fi 14 | 15 | route add -host 239.255.1.3 $1 16 | iperf -c 239.255.1.3 -u -b $2m -f m -i 5 -t 60 17 | -------------------------------------------------------------------------------- /attic/bigtest/HashHostNode.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest; 2 | 3 | import org.nustaq.fastcast.api.FCRemoting; 4 | import org.nustaq.fastcast.api.FastCast; 5 | 6 | import java.io.IOException; 7 | 8 | /** 9 | * Created with IntelliJ IDEA. 10 | * User: ruedi 11 | * Date: 9/14/13 12 | * Time: 5:07 PM 13 | * To change this template use File | Settings | File Templates. 14 | */ 15 | public class HashHostNode { 16 | 17 | public static void main( String arg[] ) throws IOException { 18 | FCRemoting rem = FastCast.getFastCast(); 19 | rem.joinCluster("shared/bigtest.yaml", "HTHost", null); 20 | rem.startSending("htlisten"); 21 | rem.start("hthost"); 22 | System.out.println("started "+rem.getNodeId()); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /examples/readme.txt: -------------------------------------------------------------------------------- 1 | some samples. 2 | 3 | main classes expect to be run from within the examples directory (some require to locate config files). 4 | 5 | Latency test: 6 | ============= 7 | 8 | run FCPongServer, FCPing for synchronous RTT round trip measurement. The asynchronous FCPingClientAsync can be faulty 9 | because nanoTime is grabbed from different threads (=cores, cpu-sockets). 10 | 11 | all tests default to localhost. Attention: on some OS'es/OS configurations localhost can perform particulary bad 12 | (worse than a network card). E.g. clean CentOS 7 install localhost has like 400 micros (??). Ubuntu 14.04 LTS had ~20. 13 | Pimped scientific Linux (RH 6.x based) had 10..12 micros RTT on localhost. 14 | 15 | Best is to test on real ll network hardware. 16 | 17 | Throughput Test 18 | =============== 19 | 20 | On older cpu's or "Bad localhost"-OS'es, throughput test might disconnect receiver. Lower packet rate limit then 21 | (see topic configuration). -------------------------------------------------------------------------------- /attic/bigtest/RequestProcessorNode.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest; 2 | 3 | import de.ruedigermoeller.fastcast.bigtest.services.RequestServer; 4 | import org.nustaq.fastcast.api.FCRemoting; 5 | import org.nustaq.fastcast.api.FastCast; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * Created with IntelliJ IDEA. 11 | * User: ruedi 12 | * Date: 9/14/13 13 | * Time: 5:08 PM 14 | * To change this template use File | Settings | File Templates. 15 | */ 16 | public class RequestProcessorNode { 17 | 18 | public static void main( String arg[] ) throws IOException { 19 | FCRemoting rem = FastCast.getFastCast(); 20 | rem.joinCluster("shared/bigtest.yaml", "Processor", null); 21 | rem.start("rqserver"); 22 | System.out.println("started "+rem.getNodeId()); 23 | if ( arg.length == 2 ) { 24 | RequestServer rqserver = (RequestServer) rem.getService("rqserver"); 25 | rqserver.setFilter( Integer.parseInt(arg[0]), Integer.parseInt(arg[1]) ); 26 | } 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/basic/stuff/sendreceive.kson: -------------------------------------------------------------------------------- 1 | ClusterConf { 2 | 3 | transports: { 4 | { 5 | name: default 6 | 7 | dgramsize: 1200 8 | interfaceAddr: 127.0.0.1 9 | multicastAddr: 229.9.9.9 10 | port: 45555 11 | 12 | autoFlushMS: 1 13 | # default is blocking IO, don't do latency tests with this configuration 14 | # uncomment below for spin looping 15 | # spinLoopMicros: 10_000_000 16 | # idleParkMicros: 1000 17 | } 18 | } 19 | 20 | topics: { 21 | 22 | { 23 | id: 1 24 | name: sendreceive 25 | 26 | pub: { 27 | numPacketHistory: 30_000 28 | pps: 5_000 29 | heartbeatInterval: 1000 30 | } 31 | 32 | sub: { 33 | receiveBufferPackets: 10_000 34 | senderHBTimeout: 30000 35 | # senderHBTimeout: 500000 # avoid crashes during debug 36 | } 37 | } 38 | 39 | } 40 | 41 | } -------------------------------------------------------------------------------- /diagnostics/nicstat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | INTERVAL="1" # update interval in seconds 4 | 5 | if [ -z "$1" ]; then 6 | echo 7 | echo usage: $0 [network-interface] 8 | echo 9 | echo e.g. $0 eth0 10 | echo 11 | echo shows packets-per-second 12 | exit 13 | fi 14 | 15 | IF=$1 16 | 17 | while true 18 | do 19 | R1=`cat /sys/class/net/$1/statistics/rx_packets` 20 | T1=`cat /sys/class/net/$1/statistics/tx_packets` 21 | RB1=`cat /sys/class/net/$1/statistics/rx_bytes` 22 | TB1=`cat /sys/class/net/$1/statistics/tx_bytes` 23 | sleep $INTERVAL 24 | R2=`cat /sys/class/net/$1/statistics/rx_packets` 25 | T2=`cat /sys/class/net/$1/statistics/tx_packets` 26 | RB2=`cat /sys/class/net/$1/statistics/rx_bytes` 27 | TB2=`cat /sys/class/net/$1/statistics/tx_bytes` 28 | TXPPS=`expr $T2 - $T1` 29 | RXPPS=`expr $R2 - $R1` 30 | TXBPS=`expr $TB2 - $TB1` 31 | RXBPS=`expr $RB2 - $RB1` 32 | TXBPS=`expr $TXBPS / 1000` 33 | RXBPS=`expr $RXBPS / 1000` 34 | echo "TX $1: $TXPPS pkts/s RX $1: $RXPPS pkts/s TB $1: $TXBPS kb/s RB $1: $RXBPS kb/s" 35 | done 36 | -------------------------------------------------------------------------------- /attic/resource/structbench.yaml: -------------------------------------------------------------------------------- 1 | clusterName: bm 2 | logLevel: 2 3 | 4 | interfaces: 5 | if0: eth1 6 | 7 | transports: 8 | 9 | - name: default 10 | # address of network adapter, 11 | ifacAdr: if0 12 | mcastAdr: 230.9.9.10 13 | port: 45556 14 | 15 | dgramsize: 8000 16 | 17 | # in java we like large buffers to minimize packet loss due to GC jitter 18 | socketReceiveBufferSize: 25000000 19 | sendBufferSize: 640000 20 | 21 | #misc 22 | trafficClass: 8 23 | ttl: 2 24 | 25 | - name: control 26 | ifacAdr: if0 27 | mcastAdr: 230.9.9.11 28 | port: 45557 29 | 30 | topics: 31 | 32 | - name: structbench 33 | transport: default #name of transport defined above 34 | topicId: 0 #topicId on the transport associated with this 35 | serviceClass: de.nustaq.fastcast.test.StructBench 36 | 37 | numPacketOffHeapHistory: 30000 38 | sendPauseMicros: 140 39 | numPacketHistory: 2000 40 | maxSendPacketQueueSize: 500 41 | receiveBufferPackets: 5000 42 | 43 | - name: membership 44 | autoStart: true 45 | transport: control 46 | topicId: 1 47 | serviceClass: FCMembership 48 | 49 | 50 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/programmatic_configuration/same_using_config_file/config.kson: -------------------------------------------------------------------------------- 1 | # 2 | # note: this file structure directly maps to the ClusterConf class structure defined in org/nustaq/fastcast/config 3 | # check these classes for avaiable parameters and documenation 4 | # 5 | 6 | ClusterConf { 7 | 8 | # hard limited configuration (only 12.5 MByte per second) 9 | 10 | transports: { 11 | { 12 | name: default 13 | 14 | dgramsize: 2500 15 | interfaceAddr: 127.0.0.1 16 | multicastAddr: 239.9.9.9 17 | port: 46789 18 | ttl: 1 19 | 20 | # default is blocking IO, don't do latency tests with this configuration 21 | # uncomment below for spin looping 22 | # spinLoopMicros: 10_000_000 23 | # idleParkMicros: 1000 24 | } 25 | 26 | # .. next transport goes here .. 27 | } 28 | 29 | topics: { 30 | { 31 | id: 1 32 | name: oneAndOnlyTopic 33 | 34 | pub: { 35 | numPacketHistory: 30_000 36 | pps: 10000 37 | } 38 | sub: { 39 | receiveBufferPackets: 10_000 40 | } 41 | } 42 | 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /examples/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | de.ruedigermoeller 8 | fc-examples 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | 14 | org.apache.maven.plugins 15 | maven-compiler-plugin 16 | 17 | 1.7 18 | 1.7 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | de.ruedigermoeller 28 | fast-cast 29 | 3.08 30 | 31 | 32 | 33 | org.hdrhistogram 34 | HdrHistogram 35 | 2.0.3 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /attic/bigtest/services/HTListener.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest.services; 2 | 3 | /** 4 | * Created with IntelliJ IDEA. 5 | * User: ruedi 6 | * Date: 9/14/13 7 | * Time: 2:11 PM 8 | * To change this template use File | Settings | File Templates. 9 | */ 10 | 11 | import org.nustaq.fastcast.api.FCTopicService; 12 | import org.nustaq.fastcast.api.RemoteMethod; 13 | 14 | import java.util.concurrent.ConcurrentHashMap; 15 | 16 | /** 17 | * listenes to HT changes. Note: configure as threadPerSender 18 | */ 19 | public class HTListener extends FCTopicService { 20 | 21 | ConcurrentHashMap mirror = new ConcurrentHashMap(); 22 | DataListener listener; 23 | 24 | public DataListener getListener() { 25 | return listener; 26 | } 27 | 28 | public void setListener(DataListener listener) { 29 | this.listener = listener; 30 | } 31 | 32 | public static interface DataListener { 33 | public void changeReceived(int id, Object prev, Object newVal); 34 | } 35 | 36 | @RemoteMethod(1) 37 | public void receiveChange( int id, Object previousValue, Object newValue) { 38 | mirror.put(id,newValue); 39 | if ( listener != null ) { 40 | listener.changeReceived(id,previousValue,newValue); 41 | } 42 | } 43 | 44 | public ConcurrentHashMap getMirror() { 45 | return mirror; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /attic/resource/latency.yaml: -------------------------------------------------------------------------------- 1 | # name of cluster 2 | clusterName: test 3 | 4 | ########################################################################################################### 5 | # define technical transport topics 6 | # a transport is defined by a network adapter, a listening address and a port. google 'crosstalking' in case u run unix 7 | 8 | transports: 9 | 10 | - name: default 11 | # address of network adapter, 12 | ifacAdr: 127.0.0.1 13 | mcastAdr: 230.9.9.10 14 | port: 45556 15 | 16 | dgramsize: 1000 17 | # needs to be true in order to run several processes on one machine 18 | loopBack: true 19 | 20 | # in java we like large buffers to minimize packet loss due to GC jitter 21 | socketReceiveBufferSize: 25000000 22 | sendBufferSize: 640000 23 | 24 | #misc 25 | trafficClass: 16 26 | ttl: 2 27 | transportType: MCAST_NIO_SOCKET 28 | 29 | - name: lolat 30 | dgramsize: 500 31 | # size of shared mem buffer 32 | socketReceiveBufferSize: 25000000 33 | 34 | #misc 35 | transportType: MCAST_IPC 36 | queueFile: /test/queue1.bin 37 | 38 | topics: 39 | 40 | - name: latency 41 | transport: lolat 42 | topicId: 0 43 | serviceClass: de.nustaq.fastcast.test.LatencyTest$LatencyService 44 | sendPauseMicros: 0 45 | maxSendPacketQueueSize: 0 46 | decodeInTransportThread: true 47 | optForLatency: true 48 | useSpinlockInSendQueue: true -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/ByteArrayReceiver.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | import org.nustaq.offheap.bytez.Bytez; 26 | 27 | /** 28 | * Created with IntelliJ IDEA. 29 | * User: ruedi 30 | * Date: 8/11/13 31 | * Time: 1:07 PM 32 | * To change this template use File | Settings | File Templates. 33 | */ 34 | public interface ByteArrayReceiver { 35 | public void receiveChunk(long sequence, Bytez b, int off, int len, boolean complete); 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/util/Sleeper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.util; 24 | 25 | /** 26 | * Created with IntelliJ IDEA. 27 | * User: moelrue 28 | * Date: 8/12/13 29 | * Time: 2:28 PM 30 | */ 31 | public class Sleeper { 32 | 33 | public static void spinMicros(int micro) { 34 | if ( micro <= 0 ) 35 | return; 36 | long now = System.nanoTime(); 37 | while( System.nanoTime()-now < micro*1000 ) { 38 | // spin 39 | } 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /attic/UnreliableReceiver.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.test; 2 | 3 | import org.nustaq.fastcast.api.FastCast; 4 | import org.nustaq.fastcast.api.Unreliable; 5 | 6 | import java.io.IOException; 7 | 8 | /** 9 | * Copyright (c) 2012, Ruediger Moeller. All rights reserved. 10 | * 11 | * This library is free software; you can redistribute it and/or 12 | * modify it under the terms of the GNU Lesser General Public 13 | * License as published by the Free Software Foundation; either 14 | * version 2.1 of the License, or (at your option) any later version. 15 | * 16 | * This library is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 19 | * Lesser General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU Lesser General Public 22 | * License along with this library; if not, write to the Free Software 23 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 24 | * MA 02110-1301 USA 25 | * 26 | * Date: 22.08.13 27 | * Time: 21:21 28 | * To change this template use File | Settings | File Templates. 29 | */ 30 | @Unreliable 31 | public class UnreliableReceiver extends UnorderedReceiver { 32 | public UnreliableReceiver() { 33 | } 34 | 35 | public static void main( String arg[] ) throws IOException { 36 | FastCast fc = new FastCast(); 37 | fc.joinCluster("test/unreliablereceiver.yaml", "Bench", null); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/async/AsyncLatMessage.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.latency.async; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by ruedi on 23/01/15. 7 | */ 8 | public class AsyncLatMessage implements Serializable { 9 | 10 | long sendTimeStampNanos; 11 | 12 | double bidPrc,askPrc; 13 | int bidQty, askQty; 14 | 15 | public AsyncLatMessage(long sendTimeStampNanos, double bidPrc, double askPrc, int bidQty, int askQty) { 16 | this.sendTimeStampNanos = sendTimeStampNanos; 17 | this.bidPrc = bidPrc; 18 | this.askPrc = askPrc; 19 | this.bidQty = bidQty; 20 | this.askQty = askQty; 21 | } 22 | 23 | public long getSendTimeStampNanos() { 24 | return sendTimeStampNanos; 25 | } 26 | 27 | public void setSendTimeStampNanos(long sendTimeStampNanos) { 28 | this.sendTimeStampNanos = sendTimeStampNanos; 29 | } 30 | 31 | public double getBidPrc() { 32 | return bidPrc; 33 | } 34 | 35 | public void setBidPrc(double bidPrc) { 36 | this.bidPrc = bidPrc; 37 | } 38 | 39 | public double getAskPrc() { 40 | return askPrc; 41 | } 42 | 43 | public void setAskPrc(double askPrc) { 44 | this.askPrc = askPrc; 45 | } 46 | 47 | public int getBidQty() { 48 | return bidQty; 49 | } 50 | 51 | public void setBidQty(int bidQty) { 52 | this.bidQty = bidQty; 53 | } 54 | 55 | public int getAskQty() { 56 | return askQty; 57 | } 58 | 59 | public void setAskQty(int askQty) { 60 | this.askQty = askQty; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/ControlPacket.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | /** 26 | * Created with IntelliJ IDEA. 27 | * User: ruedi 28 | * Date: 15.08.13 29 | * Time: 01:30 30 | * 31 | * Control packet to signal drops/heartbeats. These packets are always unreliable 32 | * ATTENTION: struct class 33 | */ 34 | public class ControlPacket extends Packet { 35 | public static final short DROPPED = 0; 36 | public static final short HEARTBEAT = 1; // NOT SENT AS CONTROL MESSAGE (required in stream to manage bootstrap) 37 | 38 | protected short type; 39 | 40 | public short getType() { 41 | return type; 42 | } 43 | 44 | public void setType(short type) { 45 | this.type = type; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/async/allocfree/AsyncLatMessageStruct.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.latency.async.allocfree; 2 | 3 | import org.nustaq.offheap.structs.FSTStruct; 4 | 5 | /** 6 | * Created by ruedi on 26/01/15. 7 | */ 8 | public class AsyncLatMessageStruct extends FSTStruct { 9 | 10 | protected long sendTimeStampNanos; 11 | 12 | protected double bidPrc,askPrc; 13 | protected int bidQty, askQty; 14 | 15 | // required 16 | public AsyncLatMessageStruct() { 17 | } 18 | 19 | public AsyncLatMessageStruct(long sendTimeStampNanos, double bidPrc, double askPrc, int bidQty, int askQty) { 20 | this.sendTimeStampNanos = sendTimeStampNanos; 21 | this.bidPrc = bidPrc; 22 | this.askPrc = askPrc; 23 | this.bidQty = bidQty; 24 | this.askQty = askQty; 25 | } 26 | 27 | public long getSendTimeStampNanos() { 28 | return sendTimeStampNanos; 29 | } 30 | 31 | public void setSendTimeStampNanos(long sendTimeStampNanos) { 32 | this.sendTimeStampNanos = sendTimeStampNanos; 33 | } 34 | 35 | public double getBidPrc() { 36 | return bidPrc; 37 | } 38 | 39 | public void setBidPrc(double bidPrc) { 40 | this.bidPrc = bidPrc; 41 | } 42 | 43 | public double getAskPrc() { 44 | return askPrc; 45 | } 46 | 47 | public void setAskPrc(double askPrc) { 48 | this.askPrc = askPrc; 49 | } 50 | 51 | public int getBidQty() { 52 | return bidQty; 53 | } 54 | 55 | public void setBidQty(int bidQty) { 56 | this.bidQty = bidQty; 57 | } 58 | 59 | public int getAskQty() { 60 | return askQty; 61 | } 62 | 63 | public void setAskQty(int askQty) { 64 | this.askQty = askQty; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/transport/PhysicalTransport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.transport; 24 | 25 | import org.nustaq.fastcast.config.PhysicalTransportConf; 26 | 27 | import java.io.IOException; 28 | import java.net.*; 29 | import java.nio.ByteBuffer; 30 | 31 | /** 32 | */ 33 | public interface PhysicalTransport { 34 | 35 | public boolean receive(ByteBuffer pack) throws IOException; 36 | public void sendControl(byte[] bytes, int off, int len) throws IOException; 37 | public void sendControl(ByteBuffer b) throws IOException; 38 | public void send(byte[] bytes, int off, int len) throws IOException; 39 | public void send(ByteBuffer b) throws IOException; 40 | public void join() throws IOException; 41 | public PhysicalTransportConf getConf(); 42 | 43 | void close(); 44 | 45 | boolean isBlocking(); 46 | } 47 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/programmatic_configuration/same_using_config_file/ConfigFileSubscriber.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.programmatic_configuration.same_using_config_file; 2 | 3 | import org.nustaq.fastcast.api.FastCast; 4 | import org.nustaq.fastcast.api.util.ObjectSubscriber; 5 | 6 | /** 7 | * Created by ruedi on 15.12.14. 8 | */ 9 | public class ConfigFileSubscriber { 10 | 11 | public static void main( String arg[] ) throws Exception { 12 | FastCast.getFastCast().setNodeId("csub"); // 5 chars MAX !! 13 | FastCast.getFastCast() 14 | .loadConfig(ConfigFilePublisher.CFG_FILE) 15 | .onTransport("default").subscribe("oneAndOnlyTopic", 16 | new ObjectSubscriber() { 17 | long lastMsg = System.currentTimeMillis(); 18 | int msgReceived = 0; 19 | 20 | @Override 21 | protected void objectReceived(String sender, long sequence, Object msg) { 22 | msgReceived++; 23 | if ( System.currentTimeMillis()-lastMsg > 1000 ) { 24 | System.out.println("received from "+sender+" number of msg "+msgReceived); 25 | System.out.println("current: "+msg); 26 | lastMsg = System.currentTimeMillis(); 27 | msgReceived = 0; 28 | } 29 | } 30 | 31 | @Override 32 | public boolean dropped() { 33 | System.out.println("Fatal: could not keep up with send rate. exiting"); 34 | System.exit(0); 35 | return false; // do not attempt resync 36 | } 37 | } 38 | ); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/programmatic_configuration/same_using_config_file/ConfigFilePublisher.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.programmatic_configuration.same_using_config_file; 2 | 3 | import org.nustaq.fastcast.api.FCPublisher; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.api.util.ObjectPublisher; 6 | import org.nustaq.fastcast.util.RateMeasure; 7 | 8 | import java.io.File; 9 | 10 | /** 11 | * Created by ruedi on 15.12.14. 12 | */ 13 | public class ConfigFilePublisher { 14 | 15 | public static String CFG_FILE = "./src/main/java/org/nustaq/fastcast/examples/programmatic_configuration/same_using_config_file/config.kson"; 16 | 17 | public static void main( String arg[] ) throws Exception { 18 | FastCast.getFastCast().setNodeId("CPUB"); // max 5 chars ! 19 | // note this configuration is far below possible limits regarding throughput and rate 20 | if ( ! new File(CFG_FILE).exists() ) { 21 | CFG_FILE = "./examples/"+CFG_FILE; 22 | } 23 | FastCast fc = FastCast.getFastCast().loadConfig(CFG_FILE); 24 | 25 | FCPublisher pub = FastCast.getFastCast().onTransport("default").publish("oneAndOnlyTopic"); 26 | 27 | // use a helper for fast-serialized messages 28 | ObjectPublisher opub = new ObjectPublisher(pub); 29 | RateMeasure measure = new RateMeasure("msg/s"); 30 | while( true ) { 31 | measure.count(); 32 | opub.sendObject( 33 | null, // all listeners should receive this (by specifying a nodeId, a specific subscriber can be targeted) 34 | "Hello "+System.currentTimeMillis(), // serializable object 35 | false // allow for 'batching' several messages into one (will create slight latency) 36 | ); 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/programmatic_configuration/ProgrammaticConfiguredSubscriber.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.programmatic_configuration; 2 | 3 | import org.nustaq.fastcast.api.FastCast; 4 | import org.nustaq.fastcast.config.SubscriberConf; 5 | import org.nustaq.fastcast.api.util.ObjectSubscriber; 6 | 7 | /** 8 | * Created by ruedi on 14.12.14. 9 | */ 10 | public class ProgrammaticConfiguredSubscriber { 11 | 12 | public static void main( String arg[] ) { 13 | FastCast.getFastCast().setNodeId("SUBS"); // 5 chars MAX !! 14 | ProgrammaticConfiguredPublisher.configureFastCast(); 15 | FastCast.getFastCast().onTransport("default").subscribe( 16 | new SubscriberConf(1) // listen to topic 1 17 | .receiveBufferPackets(20000), // how many packets to buffer in case of a loss+retransmission 18 | new ObjectSubscriber() { 19 | long lastMsg = System.currentTimeMillis(); 20 | int msgReceived = 0; 21 | 22 | @Override 23 | protected void objectReceived(String sender, long sequence, Object msg) { 24 | msgReceived++; 25 | if ( System.currentTimeMillis()-lastMsg > 1000 ) { 26 | System.out.println("received from "+sender+" number of msg "+msgReceived); 27 | System.out.println("current: "+msg); 28 | lastMsg = System.currentTimeMillis(); 29 | msgReceived = 0; 30 | } 31 | } 32 | 33 | @Override 34 | public boolean dropped() { 35 | System.out.println("Fatal: could not keep up with send rate. exiting"); 36 | System.exit(0); 37 | return false; // do not attempt resync 38 | } 39 | } 40 | ); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/RetransEntry.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | import org.nustaq.offheap.structs.FSTStruct; 26 | 27 | /** 28 | * Created with IntelliJ IDEA. 29 | * User: ruedi 30 | * Date: 8/12/13 31 | * Time: 8:34 PM 32 | * To change this template use File | Settings | File Templates. 33 | */ 34 | public class RetransEntry extends FSTStruct { 35 | protected long from; 36 | protected long to; 37 | 38 | public long getFrom() { 39 | return from; 40 | } 41 | 42 | public void setFrom(long from) { 43 | this.from = from; 44 | } 45 | 46 | public long getTo() { 47 | return to; 48 | } 49 | 50 | public void setTo(long to) { 51 | this.to = to; 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return "RetransEntry{" + 57 | "from=" + from + 58 | ", to=" + to + 59 | '}'; 60 | } 61 | 62 | public long getNumPackets() { 63 | return to-from; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /attic/bigtest/PureHTListener.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest; 2 | 3 | import de.ruedigermoeller.fastcast.bigtest.services.HTListener; 4 | import org.nustaq.fastcast.api.FCRemoting; 5 | import org.nustaq.fastcast.api.FastCast; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * Created with IntelliJ IDEA. 11 | * User: ruedi 12 | * Date: 9/16/13 13 | * Time: 9:38 PM 14 | * To change this template use File | Settings | File Templates. 15 | */ 16 | public class PureHTListener implements HTListener.DataListener { 17 | HTListener localHTListener; 18 | 19 | public PureHTListener() { 20 | } 21 | 22 | public void start() { 23 | FCRemoting rem = FastCast.getFastCast(); 24 | try { 25 | rem.joinCluster("shared/bigtest.yaml", "Listen", null); 26 | } catch (IOException e) { 27 | e.printStackTrace(); 28 | System.exit(0); 29 | } 30 | 31 | // install htlisten in this node 32 | rem.startReceiving("htlisten"); 33 | localHTListener = (HTListener) rem.getService("htlisten"); 34 | localHTListener.setListener(this); 35 | 36 | System.out.println("started "+rem.getNodeId()); 37 | } 38 | 39 | long tim = System.currentTimeMillis(); 40 | int responseCount; 41 | 42 | @Override 43 | public void changeReceived(int id, Object prev, Object newVal) { 44 | responseCount++; 45 | if ( id % 10 == 0 ) { 46 | synchronized (this) { 47 | long now = System.currentTimeMillis(); 48 | if ( now - tim > 1000 ) { 49 | System.out.println(""+Thread.currentThread().getName()+" resp "+responseCount+" in "+(now-tim)+" millis"); 50 | System.out.println("mirror size " + localHTListener.getMirror().size()); 51 | tim = now; 52 | responseCount = 0; 53 | } 54 | } 55 | } 56 | } 57 | 58 | 59 | public static void main( String arg[] ) throws IOException { 60 | PureHTListener clusterClientNode = new PureHTListener(); 61 | clusterClientNode.start(); 62 | } 63 | } -------------------------------------------------------------------------------- /attic/PongService.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.test; 2 | 3 | import org.nustaq.fastcast.api.FCRemoting; 4 | import org.nustaq.fastcast.api.FCTopicService; 5 | import org.nustaq.fastcast.api.FastCast; 6 | import org.nustaq.fastcast.api.RemoteMethod; 7 | 8 | import java.io.IOException; 9 | 10 | /** 11 | * Copyright (c) 2012, Ruediger Moeller. All rights reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU Lesser General Public 15 | * License as published by the Free Software Foundation; either 16 | * version 2.1 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | * Lesser General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public 24 | * License along with this library; if not, write to the Free Software 25 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 26 | * MA 02110-1301 USA 27 | * 28 | * Date: 01.09.13 29 | * Time: 22:12 30 | * To change this template use File | Settings | File Templates. 31 | */ 32 | 33 | /** 34 | * receives ping and sends pong 35 | */ 36 | public class PongService extends FCTopicService { 37 | 38 | PingService ping; 39 | 40 | /** 41 | * override to do init and stuff 42 | */ 43 | @Override 44 | public void init() { 45 | ping = (PingService) getRemoting().getRemoteService("ping"); 46 | } 47 | 48 | @RemoteMethod(1) 49 | public void ping(long timeStamp) { 50 | if ( timeStamp % 1000 == 0 ) { 51 | System.out.println("Ping rec "+(System.nanoTime()-timeStamp)/1000000); 52 | } 53 | ping.pong(timeStamp, System.nanoTime()); 54 | } 55 | 56 | public static void main( String arg[] ) throws IOException { 57 | FCRemoting fc = FastCast.getFastCast(); 58 | fc.joinCluster("test/pingpong.yaml", "Pong", null); 59 | fc.startSending("ping"); 60 | fc.startReceiving("pong"); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /attic/UnorderedSender.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.test; 2 | 3 | import org.nustaq.fastcast.api.FCRemoting; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.api.Unordered; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * Created with IntelliJ IDEA. 11 | * User: ruedi 12 | * Date: 19.08.13 13 | * Time: 13:26 14 | * To change this template use File | Settings | File Templates. 15 | */ 16 | @Unordered 17 | public class UnorderedSender { 18 | 19 | String products[] = { "ALV", "BMW", "ODAX", "FDAX", "ZUAN", "OGBL" }; 20 | int sequences[] = new int[products.length]; 21 | private UnorderedReceiver remote; 22 | static FCRemoting fc; 23 | 24 | public UnorderedSender(int stream) { 25 | try { 26 | if ( fc == null ) { 27 | fc = FastCast.getFastCast(); 28 | fc.joinCluster("test/unordered.yaml", "Bench", null); 29 | } 30 | String topic = "stream_" + stream; 31 | fc.startSending(topic); 32 | remote = (UnorderedReceiver) fc.getRemoteService(topic); 33 | sendLoop(); 34 | } catch (IOException e) { 35 | e.printStackTrace(); 36 | } 37 | } 38 | 39 | public void sendLoop() { 40 | new Thread("sendloop") { 41 | public void run () { 42 | try { 43 | Thread.sleep(1000); 44 | } catch (InterruptedException e) { 45 | e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. 46 | } 47 | while (true) { 48 | sendStep(); 49 | } 50 | } 51 | }.start(); 52 | } 53 | 54 | private void sendStep() { 55 | for (int i = 0; i < products.length; i++) { 56 | String product = products[i]; 57 | sequences[i]++; 58 | remote.receiveMarketData(product, sequences[i], 10 + (Math.random() * 6 - 3), 1000 * (int) (Math.random() * 10 + 1)); 59 | } 60 | } 61 | 62 | public static void main( String arg[] ) throws IOException { 63 | new UnorderedSender(0); 64 | new UnorderedSender(1); 65 | } 66 | 67 | } -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/pingponglat.kson: -------------------------------------------------------------------------------- 1 | # optimized for low latency. Will consume 2 cores per process caused by spin locks. 2 | # Gains like half to a third lower latency compared to lock based configuraton. 3 | # Gain depends on OS and hardware (e.g. 54 micros RTT => 20 micros RTT, 35 micros RTT => 12 micros RTT) 4 | ClusterConf { 5 | 6 | transports: { 7 | { 8 | name: ping 9 | 10 | dgramsize: 1500 11 | interfaceAddr: lo 12 | multicastAddr: 229.9.9.10 13 | port: 45556 14 | 15 | autoFlushMS: 1 16 | # comment these 2 lines for lock based mode (also below (**) 2 lines) 17 | spinLoopMicros: 3_000_000 18 | idleParkMicros: 300 19 | } 20 | 21 | { 22 | name: pong 23 | 24 | dgramsize: 1500 25 | interfaceAddr: lo 26 | multicastAddr: 229.9.9.11 27 | port: 45557 28 | 29 | autoFlushMS: 10 30 | # (**) see above 31 | spinLoopMicros: 3_000_000 32 | idleParkMicros: 300 33 | } 34 | 35 | } 36 | 37 | topics: { 38 | # for ping pong test. 39 | { 40 | id: 1 41 | name: pingtopic 42 | 43 | pub: { 44 | numPacketHistory: 500_000 45 | pps: 100_000 46 | ppsWindow: 100 47 | heartbeatInterval: 500 48 | } 49 | 50 | sub: { 51 | receiveBufferPackets: 500_000 52 | # senderHBTimeout: 5000 53 | senderHBTimeout: 500000 # avoid crashes during debug 54 | } 55 | } 56 | 57 | { 58 | id: 2 59 | name: pongtopic 60 | 61 | pub: { 62 | numPacketHistory: 500_000 63 | pps: 100_000 64 | ppsWindow: 100 65 | heartbeatInterval: 500 66 | } 67 | 68 | sub: { 69 | receiveBufferPackets: 500_000 70 | # senderHBTimeout: 5000 71 | senderHBTimeout: 500000 # avoid crashes during debug 72 | } 73 | } 74 | } 75 | 76 | } -------------------------------------------------------------------------------- /attic/PingService.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.test; 2 | 3 | import org.nustaq.fastcast.api.FCRemoting; 4 | import org.nustaq.fastcast.api.FCTopicService; 5 | import org.nustaq.fastcast.api.FastCast; 6 | import org.nustaq.fastcast.api.RemoteMethod; 7 | 8 | import java.io.IOException; 9 | 10 | /** 11 | * Copyright (c) 2012, Ruediger Moeller. All rights reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU Lesser General Public 15 | * License as published by the Free Software Foundation; either 16 | * version 2.1 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | * Lesser General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public 24 | * License along with this library; if not, write to the Free Software 25 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 26 | * MA 02110-1301 USA 27 | * 28 | * Date: 01.09.13 29 | * Time: 21:30 30 | * To change this template use File | Settings | File Templates. 31 | */ 32 | public class PingService extends FCTopicService { 33 | 34 | @RemoteMethod(1) 35 | public void pong(long sendTime, long receiveTime) { 36 | long pongRec = System.nanoTime(); 37 | if ( pongRec % 1000 == 0 ) { 38 | System.out.println("Ping Pong send "+(receiveTime-sendTime)/1000+" round "+(pongRec-sendTime)/1000); 39 | } 40 | } 41 | 42 | public static void main( String arg[] ) throws IOException { 43 | FCRemoting fc = FastCast.getFastCast(); 44 | fc.joinCluster("test/pingpong.yaml", "Ping", null); 45 | 46 | fc.startReceiving("ping"); 47 | fc.startSending("pong"); 48 | 49 | final PongService pong = (PongService) fc.getRemoteService("pong"); 50 | 51 | new Thread() { 52 | public void run() { 53 | while(true) { 54 | long timeStamp = System.nanoTime(); 55 | pong.ping(timeStamp); 56 | } 57 | } 58 | }.start(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /attic/resource/bigtest.yaml: -------------------------------------------------------------------------------- 1 | logLevel: 2 2 | 3 | interfaces: 4 | 5 | if0: 127.0.0.1 6 | 7 | transports: 8 | 9 | - name: default 10 | 11 | ifacAdr: if0 12 | 13 | mcastAdr: 231.1.1.1 14 | port: 49900 15 | 16 | # needs to be true in order to run several processes on one machine 17 | loopBack: true 18 | 19 | - name: datawrite 20 | 21 | ifacAdr: if0 22 | 23 | mcastAdr: 231.1.1.2 24 | port: 49901 25 | 26 | # needs to be true in order to run several processes on one machine 27 | loopBack: true 28 | 29 | - name: datalisten 30 | 31 | ifacAdr: if0 32 | 33 | mcastAdr: 231.1.1.3 34 | port: 49902 35 | 36 | # needs to be true in order to run several processes on one machine 37 | loopBack: true 38 | 39 | - name: reqresp 40 | 41 | ifacAdr: if0 42 | 43 | mcastAdr: 231.1.1.4 44 | port: 49903 45 | 46 | # needs to be true in order to run several processes on one machine 47 | loopBack: true 48 | 49 | topics: 50 | 51 | - name: hthost 52 | transport: datawrite 53 | topicId: 1 54 | serviceClass: de.nustaq.fastcast.bigtest.services.HTHost 55 | DGramRate: 1000 56 | numPacketHistory: 1000 57 | numPacketOffHeapHistory: 30000 58 | perSenderThread: true 59 | flowControlClass: null 60 | receiveBufferPackets: 5000 61 | 62 | - name: htlisten 63 | transport: datalisten 64 | topicId: 2 65 | serviceClass: de.nustaq.fastcast.bigtest.services.HTListener 66 | sendPauseMicros: 1400 67 | numPacketHistory: 1000 68 | numPacketOffHeapHistory: 30000 69 | perSenderThread: true 70 | flowControlClass: null 71 | receiveBufferPackets: 5000 72 | 73 | - name: rqserver 74 | transport: reqresp 75 | topicId: 3 76 | serviceClass: de.nustaq.fastcast.bigtest.services.RequestServer 77 | sendPauseMicros: 1000 78 | numPacketHistory: 1000 79 | numPacketOffHeapHistory: 30000 80 | perSenderThread: true 81 | flowControlClass: null 82 | receiveBufferPackets: 5000 83 | 84 | - name: binary 85 | transport: reqresp 86 | topicId: 4 87 | serviceClass: de.nustaq.fastcast.bigtest.services.BinaryReceiver 88 | sendPauseMicros: 400 89 | 90 | - name: membership 91 | autoStart: true 92 | transport: default 93 | topicId: 5 94 | serviceClass: FCMembership 95 | numPacketHistory: 100 96 | numPacketOffHeapHistory: 1000 97 | receiveBufferPackets: 50 98 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/structencoding/StructSubscriber.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.structencoding; 2 | 3 | import org.nustaq.fastcast.api.FCSubscriber; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.config.SubscriberConf; 6 | import org.nustaq.fastcast.util.RateMeasure; 7 | import org.nustaq.offheap.bytez.Bytez; 8 | import org.nustaq.offheap.structs.unsafeimpl.FSTStructFactory; 9 | import static org.nustaq.fastcast.examples.structencoding.Protocol.*; 10 | 11 | /** 12 | * Created by moelrue on 12/15/14. 13 | */ 14 | public class StructSubscriber { 15 | 16 | public static void main( String arg[] ) { 17 | Protocol.initStructFactory(); 18 | 19 | FastCast.getFastCast().setNodeId("SUB"); // 5 chars MAX !! 20 | StructPublisher.configureFastCast(); 21 | final RateMeasure rateMeasure = new RateMeasure("receive rate"); 22 | 23 | FastCast.getFastCast().onTransport("default").subscribe( 24 | new SubscriberConf(1).receiveBufferPackets(33_000), 25 | new FCSubscriber() { 26 | 27 | PriceUpdateStruct msg = FSTStructFactory.getInstance().createEmptyStructPointer(PriceUpdateStruct.class); 28 | 29 | @Override 30 | public void messageReceived(String sender, long sequence, Bytez b, long off, int len) { 31 | msg.baseOn(b, (int) off); 32 | rateMeasure.count(); 33 | // instanceof'ing in case of various messages 34 | // Class msgStruct = msg.getPointedClass(); otherStructClass = msg.detachTo(otherStructClass); 35 | } 36 | 37 | @Override 38 | public boolean dropped() { 39 | System.out.println("fatal, could not keep up. exiting"); 40 | System.exit(0); 41 | return false; 42 | } 43 | 44 | @Override 45 | public void senderTerminated(String senderNodeId) { 46 | System.out.println("sender died "+senderNodeId); 47 | } 48 | 49 | @Override 50 | public void senderBootstrapped(String receivesFrom, long seqNo) { 51 | System.out.println("bootstrap "+receivesFrom); 52 | } 53 | } 54 | ); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /attic/UnorderedReceiver.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.test; 2 | 3 | import org.nustaq.fastcast.api.*; 4 | 5 | import java.io.IOException; 6 | import java.util.HashMap; 7 | 8 | /** 9 | * Created with IntelliJ IDEA. 10 | * User: ruedi 11 | * Date: 19.08.13 12 | * Time: 13:45 13 | * To change this template use File | Settings | File Templates. 14 | */ 15 | @Unordered 16 | public class UnorderedReceiver extends FCTopicService { 17 | 18 | 19 | HashMap sequences = new HashMap(); 20 | HashMap currentPrc = new HashMap(); 21 | HashMap currentQty = new HashMap(); 22 | 23 | int recCount = 0; 24 | int drop = 0; 25 | private long lastTime; 26 | 27 | @RemoteMethod(1) 28 | public void receiveMarketData(String product, int newSequence, double prc, int qty) { 29 | if ( product.length() > 4 ) { 30 | System.out.println("pok "+product); 31 | } 32 | Integer oldSequence = sequences.get(product); 33 | if (oldSequence==null) { 34 | sequences.put(product,newSequence); 35 | } else { 36 | if (newSequence<=oldSequence.intValue()) { 37 | // System.out.println("old detected "+product+" oldSequence "+newSequence+" received oldSequence "+oldSequence); 38 | return; 39 | } 40 | if ( newSequence != oldSequence.intValue()+1 ) { 41 | // System.out.println("drop detected "+product+" oldSequence "+newSequence+" received oldSequence "+oldSequence); 42 | drop++; 43 | } 44 | } 45 | recCount++; 46 | currentPrc.put(product,prc); 47 | currentQty.put(product,qty); 48 | sequences.put(product,newSequence); 49 | if ( System.currentTimeMillis() - lastTime > 1000 ) { 50 | System.out.println("received/s "+recCount+" dropped:"+drop+" "+getTopicName()); 51 | recCount = 0; 52 | drop = 0; 53 | lastTime = System.currentTimeMillis(); 54 | } 55 | } 56 | 57 | public static void main( String arg[] ) throws IOException { 58 | FCRemoting fc = FastCast.getFastCast(); 59 | fc.joinCluster("test/unordered.yaml", "Bench", null); 60 | fc.startReceiving("stream_0"); 61 | fc.startReceiving("stream_1"); 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/config/TopicConf.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.config; 24 | 25 | /** 26 | * Created by ruedi on 04.12.14. 27 | */ 28 | public class TopicConf { 29 | 30 | private int id; // will be promoted to pub/sub conf [is redundant] 31 | private String name; // symbolic, for lookup 32 | 33 | private PublisherConf pub; 34 | private SubscriberConf sub; 35 | 36 | public int getId() { 37 | return id; 38 | } 39 | 40 | public TopicConf id(int id) { 41 | this.id = id; 42 | validateAfterRead(); 43 | return this; 44 | } 45 | 46 | public String getName() { 47 | return name; 48 | } 49 | 50 | public TopicConf name(String name) { 51 | this.name = name; 52 | return this; 53 | } 54 | 55 | public PublisherConf getPublisher() { 56 | return pub; 57 | } 58 | 59 | public TopicConf publisher(PublisherConf pub) { 60 | this.pub = pub; 61 | validateAfterRead(); 62 | return this; 63 | } 64 | 65 | public SubscriberConf getSub() { 66 | return sub; 67 | } 68 | 69 | public TopicConf subscriber(SubscriberConf sub) { 70 | this.sub = sub; 71 | validateAfterRead(); 72 | return this; 73 | } 74 | 75 | public void validateAfterRead() { 76 | if ( pub != null ) 77 | pub.topicId = id; 78 | if ( sub != null ) 79 | sub.topicId = id; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /attic/bigtest/services/RequestServer.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest.services; 2 | 3 | import org.nustaq.fastcast.api.FCFutureResultHandler; 4 | import org.nustaq.fastcast.api.FCTopicService; 5 | import org.nustaq.fastcast.api.RemoteMethod; 6 | 7 | import java.io.Serializable; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | /** 12 | * Created with IntelliJ IDEA. 13 | * User: ruedi 14 | * Date: 9/14/13 15 | * Time: 2:12 PM 16 | * To change this template use File | Settings | File Templates. 17 | */ 18 | public class RequestServer extends FCTopicService { 19 | 20 | int fi = 0; int numNo = 0; 21 | 22 | public void setFilter(int filter, int numNodes) { 23 | fi = filter; numNo = numNodes; 24 | System.out.println("set filter "+fi+" of "+numNodes); 25 | } 26 | 27 | public static class TestOne implements Serializable { 28 | String name; 29 | int age; 30 | List children; 31 | 32 | public TestOne(String name, int age) { 33 | this.name = name; 34 | this.age = age; 35 | } 36 | 37 | public boolean equals( Object other ) { 38 | if ( other instanceof TestOne ) { 39 | TestOne to = (TestOne) other; 40 | return name.equals(to.name) && age == to.age; 41 | } 42 | return false; 43 | } 44 | 45 | @Override 46 | public int hashCode() { 47 | return name.hashCode()^age; 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return "TestOne{" + 53 | "name='" + name + '\'' + 54 | ", age=" + age + 55 | ", children=" + children + 56 | '}'; 57 | } 58 | } 59 | 60 | public static TestOne createTest(int random, int children) { 61 | TestOne res = new TestOne("Emil"+(random%10)+" Moeller"+(random%10), 7+random%10); 62 | res.children = new ArrayList(); 63 | for ( int i = 0; i 0 ) { 72 | if ( random%numNo == fi ) { 73 | TestOne test = createTest(random, random % 5); 74 | res.sendResult(test); 75 | } 76 | } 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/api/util/ObjectPublisher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.api.util; 24 | 25 | import org.nustaq.fastcast.api.FCPublisher; 26 | import org.nustaq.serialization.simpleapi.DefaultCoder; 27 | import org.nustaq.serialization.simpleapi.FSTCoder; 28 | 29 | /** 30 | * Created by ruedi on 13.12.14. 31 | */ 32 | public class ObjectPublisher { 33 | 34 | protected FCPublisher pub; 35 | protected FSTCoder coder; 36 | 37 | public ObjectPublisher(FCPublisher pub) { 38 | this.pub = pub; 39 | coder = new DefaultCoder(); 40 | } 41 | 42 | public ObjectPublisher(FCPublisher pub, Class ... preregister) { 43 | this.pub = pub; 44 | coder = new DefaultCoder(preregister); 45 | } 46 | 47 | public void sendObject( String receiverNodeIdOrNull, Object toSend, boolean flush ) { 48 | byte[] bytes = coder.toByteArray(toSend);// fixme: performance. Need zerocopy variant in DefaultCoder 49 | // long tim = 0; 50 | while( ! pub.offer(receiverNodeIdOrNull,bytes,0,bytes.length,flush) ) { 51 | // spin 52 | // if ( tim == 0 ) 53 | // tim = System.nanoTime(); 54 | } 55 | // long dur = System.nanoTime()-tim; 56 | // if ( tim != 0 && dur > 1000 ) 57 | // System.out.println("spinned "+dur); 58 | } 59 | 60 | public FCPublisher getPub() { 61 | return pub; 62 | } 63 | 64 | public ObjectPublisher batchOnLimit(boolean doBatch) { 65 | pub.batchOnLimit(doBatch); 66 | return this; 67 | } 68 | 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/util/RateMeasure.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.util; 24 | 25 | import java.util.Date; 26 | 27 | /** 28 | * Created with IntelliJ IDEA. 29 | * User: ruedi 30 | * Date: 10/15/13 31 | * Time: 7:47 PM 32 | * To change this template use File | Settings | File Templates. 33 | */ 34 | public class RateMeasure { 35 | 36 | int count; 37 | long lastStats; 38 | int checkEachMask = 127; 39 | long statInterval = 1000; 40 | long lastRatePersecond; 41 | 42 | String name = "none"; 43 | 44 | public RateMeasure(String name, long statInterval) { 45 | this.name = name; 46 | this.statInterval = statInterval; 47 | } 48 | 49 | public RateMeasure(String name) { 50 | this.name = name; 51 | } 52 | 53 | public void count() { 54 | count++; 55 | if ( (count & ~checkEachMask) == count ) { 56 | checkStats(); 57 | } 58 | } 59 | 60 | private void checkStats() { 61 | long now = System.currentTimeMillis(); 62 | long diff = now-lastStats; 63 | if ( diff > statInterval ) { 64 | lastRatePersecond = count*1000l/diff; 65 | lastStats = now; 66 | count = 0; 67 | statsUpdated(lastRatePersecond); 68 | } 69 | } 70 | 71 | /** 72 | * override this 73 | * @param lastRatePersecond 74 | */ 75 | protected void statsUpdated(long lastRatePersecond) { 76 | System.out.println( new Date()+" ***** Stats for "+name+": "+lastRatePersecond+" per second *********"); 77 | } 78 | 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/util/FCUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.util; 24 | 25 | import org.nustaq.fastcast.impl.Packet; 26 | 27 | import java.net.InetAddress; 28 | import java.net.UnknownHostException; 29 | import java.util.concurrent.*; 30 | 31 | /** 32 | */ 33 | public class FCUtils { 34 | 35 | public static String createNodeId( String addendum ) { 36 | final String name = addendum + "-" + toUnsignedString((int) (Math.random()*0xfffff), 5); 37 | if ( name.length() > Packet.MAX_NODE_NAME_LEN) 38 | throw new RuntimeException("node name '"+name+"' too long. Max length is "+Packet.MAX_NODE_NAME_LEN); 39 | return name; 40 | } 41 | 42 | private static String toUnsignedString(int i, int shift) { 43 | char[] buf = new char[32]; 44 | int charPos = 32; 45 | int radix = 1 << shift; 46 | int mask = radix - 1; 47 | do { 48 | buf[--charPos] = digits[i & mask]; 49 | i >>>= shift; 50 | } while (i != 0); 51 | 52 | return new String(buf, charPos, (32 - charPos)); 53 | } 54 | 55 | final static char[] digits = { 56 | '0' , '1' , '2' , '3' , '4' , '5' , 57 | '6' , '7' , '8' , '9' , 'a' , 'b' , 58 | 'c' , 'd' , 'e' , 'f' , 'g' , 'h' , 59 | 'i' , 'j' , 'k' , 'l' , 'm' , 'n' , 60 | 'o' , 'p' , 'q' , 'r' , 's' , 't' , 61 | 'u' , 'v' , 'w' , 'x' , 'y' , 'z' 62 | }; 63 | 64 | public static boolean isWindows() { 65 | return System.getProperty("os.name","").indexOf("indows") > 0; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/async/AsyncLatReceiver.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.latency.async; 2 | 3 | import org.HdrHistogram.Histogram; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.api.util.ObjectPublisher; 6 | import org.nustaq.fastcast.api.util.ObjectSubscriber; 7 | import org.nustaq.fastcast.util.RateMeasure; 8 | 9 | import java.util.concurrent.TimeUnit; 10 | 11 | /** 12 | * Created by ruedi on 23/01/15. 13 | * 14 | * see AsyncLatPublisher for doc 15 | * 16 | */ 17 | public class AsyncLatReceiver { 18 | FastCast fastCast; 19 | private ObjectPublisher backPub; // pong for rtt measurement 20 | 21 | public void initFastCast() throws Exception { 22 | fastCast = FastCast.getFastCast(); 23 | fastCast.setNodeId("SUB"); 24 | fastCast.loadConfig(AsyncLatPublisher.CFG_FILE_PATH); 25 | 26 | backPub = new ObjectPublisher( 27 | fastCast.onTransport("back").publish("back"), 28 | AsyncLatMessage.class 29 | ); 30 | 31 | final RateMeasure measure = new RateMeasure("receive rate"); 32 | fastCast.onTransport("default").subscribe( "stream", 33 | new ObjectSubscriber(AsyncLatMessage.class) { 34 | int count = 0; 35 | @Override 36 | protected void objectReceived(String s, long l, Object o) { 37 | if ( "END".equals(o) ) { 38 | backPub.sendObject(null,o,true); 39 | return; 40 | } 41 | AsyncLatReceiver.this.objectReceived(s,l,o); 42 | if ( ++count == 10 ) { // backtalk only 10% 43 | backPub.sendObject(null, o, true); 44 | count = 0; 45 | } 46 | measure.count(); 47 | } 48 | @Override 49 | public boolean dropped() { 50 | System.exit(-2); 51 | return false; 52 | } 53 | }); 54 | } 55 | 56 | protected void objectReceived(String sender, long sequence, Object message) { 57 | // MarketEvent ev = (MarketEvent) message; 58 | } 59 | 60 | public static void main(String arg[]) throws Throwable { 61 | AsyncLatReceiver rec = new AsyncLatReceiver(); 62 | 63 | rec.initFastCast(); 64 | while( true ) 65 | Thread.sleep(10_000_000l); 66 | 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/async/fc.kson: -------------------------------------------------------------------------------- 1 | # 2 | # note: this file structure directly maps to the ClusterConf class structure defined in org/nustaq/fastcast/config 3 | # check these classes for avaiable parameters and documenation 4 | # 5 | 6 | ClusterConf { 7 | 8 | # hard limited configuration (only 12.5 MByte per second) 9 | 10 | transports: { 11 | { 12 | name: default 13 | 14 | dgramsize: 1400 15 | 16 | interfaceAddr: 127.0.0.1 17 | ttl: 1 18 | 19 | multicastAddr: 229.9.9.10 20 | port: 46789 21 | # retransGroupAddr: 229.9.9.11 22 | # retransGroupPort: 46790 23 | 24 | 25 | # spin looping 26 | spinLoopMicros: 10_000_000 27 | idleParkMicros: 1000 28 | 29 | socketReceiveBufferSize: 4_000_000 30 | socketSendBufferSize: 128_000 31 | } 32 | 33 | { 34 | name: back 35 | 36 | dgramsize: 1400 37 | 38 | interfaceAddr: 127.0.0.1 39 | ttl: 1 40 | 41 | multicastAddr: 230.10.10.11 42 | port: 57789 43 | # retransGroupAddr: 230.10.10.12 44 | # retransGroupPort: 57790 45 | 46 | # spin looping 47 | spinLoopMicros: 10_000_000 48 | idleParkMicros: 1000 49 | 50 | socketReceiveBufferSize: 4_000_000 51 | socketSendBufferSize: 128_000 52 | } 53 | 54 | # .. next transport goes here .. 55 | } 56 | 57 | topics: { 58 | { 59 | id: 1 60 | name: stream 61 | 62 | pub: { 63 | numPacketHistory: 800_000 64 | 65 | # try increasing this to 20k .. 50k to reduce latency on high end network cards + kernel bypass (10GBit low lat) 66 | # if you get lat in the millis (caused by retransmissions) => reduce to 10k [*must* also adjust pps below for bounce back topic] 67 | pps: 10_000 # 20-50_000 tuned linux, 120_000 to 150_000 open onload 68 | } 69 | sub: { 70 | receiveBufferPackets: 300_000 71 | senderHBTimeout: 5000 72 | } 73 | } 74 | 75 | { 76 | id: 2 77 | name: back 78 | 79 | pub: { 80 | numPacketHistory: 800_000 81 | pps: 10_000 # 20-50_000 tuned linux, 120_000 to 150_000 open onload 82 | } 83 | sub: { 84 | receiveBufferPackets: 300_000 85 | senderHBTimeout: 5000 86 | } 87 | } 88 | } 89 | 90 | } -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/config/ClusterConf.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.config; 24 | 25 | import org.nustaq.fastcast.impl.Topic; 26 | import org.nustaq.fastcast.transport.PhysicalTransport; 27 | import org.nustaq.kson.Kson; 28 | import org.nustaq.kson.KsonTypeMapper; 29 | 30 | import java.io.File; 31 | 32 | /** 33 | * Created by ruedi on 04.12.14. 34 | */ 35 | public class ClusterConf { 36 | public PhysicalTransportConf transports[]; 37 | public TopicConf topics[]; 38 | 39 | public PhysicalTransportConf[] getTransports() { 40 | return transports; 41 | } 42 | 43 | public ClusterConf transports(PhysicalTransportConf ... transports) { 44 | this.transports = transports; 45 | return this; 46 | } 47 | 48 | public TopicConf[] getTopics() { 49 | return topics; 50 | } 51 | 52 | public ClusterConf topics(TopicConf ... topics) { 53 | this.topics = topics; 54 | return this; 55 | } 56 | 57 | public TopicConf getTopic(String name) { 58 | for (int i = 0; i < topics.length; i++) { 59 | TopicConf topic = topics[i]; 60 | if ( topic.getName() == null ) 61 | throw new RuntimeException("unnamed topic. Please ensure each topic has a name assigned"); 62 | if ( name.equalsIgnoreCase(topic.getName())) { 63 | topic.validateAfterRead(); 64 | return topic; 65 | } 66 | } 67 | return null; 68 | } 69 | 70 | public static synchronized ClusterConf readFrom( String filePath ) throws Exception { 71 | return (ClusterConf) new Kson() 72 | .map(PublisherConf.class, SubscriberConf.class, TopicConf.class, ClusterConf.class) 73 | .readObject(new File(filePath)); 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/Packet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | 26 | import org.nustaq.offheap.structs.FSTStruct; 27 | import org.nustaq.offheap.structs.structtypes.StructString; 28 | 29 | /** 30 | * Created with IntelliJ IDEA. 31 | * User: ruedi 32 | * Date: 8/10/13 33 | * Time: 11:43 PM 34 | * superclass of all packets sent. 35 | * ATTENTION: This is a struct class, runtime byte code instrumentation will layout this in a flat manner 36 | * on top a Bytez instance. 37 | */ 38 | public class Packet extends FSTStruct { 39 | 40 | public static final int MAX_NODE_NAME_LEN = 10; 41 | protected StructString receiver = new StructString(MAX_NODE_NAME_LEN); 42 | protected StructString sender = new StructString(MAX_NODE_NAME_LEN); 43 | protected int topic; 44 | protected long seqNo; 45 | 46 | public StructString getSender() { 47 | return sender; 48 | } 49 | 50 | public void setSender(StructString sender) { 51 | this.sender = sender; 52 | } 53 | 54 | public StructString getReceiver() { 55 | return receiver; 56 | } 57 | 58 | public void setReceiver(StructString receiver) { 59 | this.receiver = receiver; 60 | } 61 | 62 | public int getTopic() { 63 | return topic; 64 | } 65 | 66 | public void setTopic(int topic) { 67 | this.topic = topic; 68 | } 69 | 70 | public long getSeqNo() { 71 | return seqNo; 72 | } 73 | 74 | public void setSeqNo(long seqNo) { 75 | this.seqNo = seqNo; 76 | } 77 | 78 | @Override 79 | public String toString() { 80 | return "Packet{" + 81 | "seqNo=" + seqNo + 82 | ", topic=" + topic + 83 | ", sender=" + sender + 84 | ", receiver=" + receiver + 85 | '}'; 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/api/FCSubscriber.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.api; 24 | 25 | import org.nustaq.offheap.bytez.Bytez; 26 | 27 | /** 28 | * interface to be implemented by a subscriber to a topic. Note there is a serialization based util implementation ObjectSubscriber. 29 | * 30 | * Created by ruedi on 29.11.2014. 31 | */ 32 | public interface FCSubscriber { 33 | 34 | /** 35 | * a message has been defragmented and received. This method is called directly in the packet receiving 36 | * thread, so processing should be delegated to a worker or be extremely quick. else packet loss might occur. 37 | * 38 | * one can use 'Bytez.getArr(off, tmpBuf, 0, len);' to copy the message contents to a byte array 'tmpBuf'. 39 | * 40 | * @param sender - nodeId of sending process 41 | * @param sequence - internal sequence number (of interest for unreliable variants) 42 | * @param b - bytesource containing the message at off and with len 43 | * @param off - offset of message 44 | * @param len - len of message 45 | */ 46 | public void messageReceived(String sender, long sequence, Bytez b, long off, int len); 47 | 48 | /** 49 | * called in case the receiver was too slow in processing messages 50 | * and therefore got dropped (unrecoverable message loss). 51 | * return true in order to let fast-cast automatically reconnect and resync 52 | */ 53 | public boolean dropped(); 54 | 55 | /** 56 | * a sender stopped sending or terminated 57 | * @param senderNodeId 58 | */ 59 | public void senderTerminated( String senderNodeId ); 60 | 61 | /** 62 | * called upon the first message regulary received from a sender 63 | * @param receivesFrom 64 | * @param seqNo 65 | */ 66 | public void senderBootstrapped(String receivesFrom, long seqNo); 67 | 68 | // public void resync(); FIXME 69 | } 70 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/programmatic_configuration/ProgrammaticConfiguredPublisher.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.programmatic_configuration; 2 | 3 | import org.nustaq.fastcast.api.FCPublisher; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.config.PhysicalTransportConf; 6 | import org.nustaq.fastcast.config.PublisherConf; 7 | import org.nustaq.fastcast.api.util.ObjectPublisher; 8 | import org.nustaq.fastcast.util.RateMeasure; 9 | 10 | /** 11 | * Created by ruedi on 14.12.14. 12 | */ 13 | public class ProgrammaticConfiguredPublisher { 14 | 15 | public static void main(String arg[]) { 16 | 17 | FastCast.getFastCast().setNodeId("PUB"); // 5 chars MAX !! 18 | configureFastCast(); 19 | 20 | FCPublisher pub = FastCast.getFastCast().onTransport("default").publish( 21 | new PublisherConf(1) // unique-per-transport topic id 22 | .numPacketHistory(40_000) // how long packets are kept for retransmission requests 23 | // beware: memory usage (kept offheap) = dgramsize * history = 2500*40000 = 100MB (GC safe for 8 seconds) 24 | .pps(5000) // packets per second rate limit. So max traffic for topic = 5000*2500 = 12.5 MB/second 25 | ); 26 | 27 | // could directly send raw on publisher 28 | // while( ! pub.offer(..) ) { /* spin */ } 29 | 30 | // or use a helper for fast-serialized messages 31 | ObjectPublisher opub = new ObjectPublisher(pub); 32 | RateMeasure measure = new RateMeasure("msg/s"); 33 | while( true ) { 34 | measure.count(); 35 | opub.sendObject( 36 | null, // all listeners should receive (by specifying a nodeId, a specific subscriber can be targeted) 37 | "Hello "+System.currentTimeMillis(), // serializable object 38 | false // allow for 'batching' several messages into one (will create slight latency) 39 | ); 40 | } 41 | } 42 | 43 | public static void configureFastCast() { 44 | // note this configuration is far below possible limits regarding throughput and rate 45 | FastCast fc = FastCast.getFastCast(); 46 | fc.addTransport( 47 | new PhysicalTransportConf("default") 48 | .interfaceAdr("127.0.0.1") // define the interface 49 | .port(42042) // port is more important than address as some OS only test for ports ('crosstalking') 50 | .mulitcastAdr("229.9.9.9") // ip4 multicast address 51 | .setDgramsize(2500) // datagram size. Small sizes => lower latency, better retransmission behaviour, large sizes => better throughput 52 | .socketReceiveBufferSize(4_000_000) // as large as possible .. however avoid hitting system limits in example 53 | .socketSendBufferSize(2_000_000) 54 | ); 55 | 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /attic/LatencyTest.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.test; 2 | 3 | import org.nustaq.fastcast.api.*; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * Created with IntelliJ IDEA. 9 | * User: moelrue 10 | * Date: 8/6/13 11 | * Time: 5:34 PM 12 | * To change this template use File | Settings | File Templates. 13 | */ 14 | public class LatencyTest { 15 | 16 | public static class LatencyService extends FCTopicService { 17 | 18 | volatile boolean stop = false; 19 | long startTime = System.currentTimeMillis(); 20 | LatencyService remote; 21 | 22 | @Override 23 | public void init() { 24 | remote = (LatencyService) getRemoting().getRemoteService(getTopicName()); 25 | remote.myStartTime(startTime); 26 | new Thread("Pinger") { 27 | @Override 28 | public void run() { 29 | int count = 0; 30 | while( ! stop ) { 31 | try { 32 | if ( count % 3 == 0 ) 33 | Thread.sleep(1); 34 | count++; 35 | } catch (InterruptedException e) { 36 | e.printStackTrace(); 37 | } 38 | remote.unreliablePing(System.nanoTime()); 39 | } 40 | } 41 | } 42 | .start(); 43 | } 44 | 45 | @RemoteMethod(1) 46 | public void roundTrip( long timeNanos, FCFutureResultHandler res ) { 47 | res.sendResult(timeNanos); 48 | } 49 | 50 | @RemoteMethod(3) 51 | public void myStartTime( long otherStartTime ) { 52 | System.out.println("Received myStartTime "+otherStartTime); 53 | if (startTime > otherStartTime ) { 54 | System.out.println("----- STOPPING "); 55 | stop = true; 56 | } 57 | else { 58 | System.out.println("----- ANSWERING "); 59 | remote.myStartTime(startTime); 60 | } 61 | } 62 | 63 | int cnt = 0; 64 | @RemoteMethod(2) 65 | public void unreliablePing( long timeSent ) { 66 | if (cnt++%1000 == 0) { 67 | if ( FCReceiveContext.get().getSender().equals(nodeId) ) { 68 | System.out.println("Loopback ping received "+(System.nanoTime()-timeSent)/1000); 69 | } else { 70 | System.out.println("ping received from "+FCReceiveContext.get().getSender()+" "+(System.nanoTime()-timeSent)/1000); 71 | } 72 | } 73 | } 74 | 75 | } 76 | 77 | public static void main(String arg[]) throws IOException, InterruptedException { 78 | FCRemoting fc = FastCast.getFastCast(); 79 | fc.joinCluster("test/latency.yaml", "LatTest", null); 80 | fc.startSending("latency"); 81 | fc.startReceiving("latency"); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /attic/bigtest/services/HTHost.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest.services; 2 | 3 | import org.nustaq.fastcast.api.FCFutureResultHandler; 4 | import org.nustaq.fastcast.api.FCTopicService; 5 | import org.nustaq.fastcast.api.RemoteMethod; 6 | import org.nustaq.fastcast.service.FCMembership; 7 | import org.nustaq.fastcast.util.FCUtils; 8 | 9 | import java.util.Iterator; 10 | import java.util.Map; 11 | import java.util.concurrent.ConcurrentHashMap; 12 | import java.util.concurrent.Executor; 13 | 14 | /** 15 | * Created with IntelliJ IDEA. 16 | * User: ruedi 17 | * Date: 9/14/13 18 | * Time: 2:10 PM 19 | * To change this template use File | Settings | File Templates. 20 | */ 21 | 22 | /** 23 | * holds a Hashtable and forwards all changes to ListenerNodes 24 | * 25 | */ 26 | public class HTHost extends FCTopicService { 27 | 28 | ConcurrentHashMap myData = new ConcurrentHashMap(); 29 | int idFilter; // only record id's for which id%idFilter == 0 30 | 31 | HTListener remoteListeners; 32 | FCMembership localMembership; 33 | String hostNodeAddresses[]; 34 | 35 | Executor listenerNot = FCUtils.createBoundedSingleThreadExecutor("listeners", 20000); 36 | 37 | @Override 38 | public void init() { 39 | remoteListeners = (HTListener) getRemoting().getRemoteService("htlisten"); 40 | localMembership = (FCMembership) getRemoting().getService("membership"); 41 | } 42 | 43 | @RemoteMethod(1) 44 | public void syncHostNumbers() { 45 | hostNodeAddresses = localMembership.getActiveNodeAdressesOrderDeterministic("HTHost"); 46 | for (int i = 0; i < hostNodeAddresses.length; i++) { 47 | if ( hostNodeAddresses[i].equals(getNodeId()) ) { 48 | idFilter = i; 49 | System.out.println("set filter to "+i+" num hosts "+hostNodeAddresses.length); 50 | } 51 | } 52 | } 53 | 54 | @RemoteMethod(2) 55 | public void putData(final int id, final Object data) { 56 | if ( hostNodeAddresses == null || hostNodeAddresses.length == 0 ) 57 | return; 58 | if ( (id%hostNodeAddresses.length) == idFilter ) { 59 | final Object old = myData.put(id, data); 60 | remoteListeners.receiveChange(id, old, data); 61 | } 62 | } 63 | 64 | 65 | /** 66 | * Caller probably needs to use extendTimeOut in order to get all entries ! 67 | * @param res 68 | */ 69 | @RemoteMethod(3) 70 | public void getTableData(final FCFutureResultHandler res) { 71 | new Thread("TableDataSender") { 72 | public void run() { 73 | for (Iterator iterator = myData.entrySet().iterator(); iterator.hasNext(); ) { 74 | Map.Entry next = iterator.next(); 75 | res.sendResult(new Object[] { next.getKey(), next.getValue() }); 76 | } 77 | } 78 | }.start(); 79 | } 80 | 81 | @RemoteMethod(4) 82 | public void printTableSize() { 83 | System.out.println("table size"+myData.size()); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /attic/resource/bench.yaml: -------------------------------------------------------------------------------- 1 | # name of cluster 2 | clusterName: bm 3 | 4 | interfaces: 5 | if0: eth0 6 | 7 | ########################################################################################################### 8 | # define technical transport topics 9 | # a transport is defined by a network adapter, a listening address and a port. google 'crosstalking' in case u run unix 10 | 11 | transports: 12 | 13 | - name: default 14 | # address of network adapter, 15 | ifacAdr: if0 16 | mcastAdr: 230.9.9.10 17 | port: 45556 18 | 19 | dgramsize: 8000 20 | # needs to be true in order to run several processes on one machine 21 | loopBack: true 22 | 23 | # in java we like large buffers to minimize packet loss due to GC jitter 24 | socketReceiveBufferSize: 25000000 25 | sendBufferSize: 640000 26 | 27 | #misc 28 | trafficClass: 8 29 | ttl: 2 30 | transportType: MCAST_NIO_SOCKET 31 | 32 | ########################################################################################################### 33 | # define separate address+port transport for heartbeats+stats as hi volume might delay heartbeats resulting 34 | # in false drops 35 | # 36 | - name: control 37 | # address of network adapter, 38 | ifacAdr: if0 39 | mcastAdr: 230.9.9.11 40 | port: 45557 41 | 42 | dgramsize: 8000 43 | loopBack: true 44 | 45 | # in java we like large buffers to minimize packet loss due to GC :( 46 | socketReceiveBufferSize: 25000000 47 | sendBufferSize: 640000 48 | 49 | #misc 50 | trafficClass: 8 51 | ttl: 2 52 | transportType: MCAST_NIO_SOCKET 53 | 54 | ########################################################################################################### 55 | # ipc transport 56 | # 57 | - name: ipc 58 | dgramsize: 32000 59 | # size of shared mem buffer 60 | socketReceiveBufferSize: 225000000 61 | 62 | #misc 63 | transportType: MCAST_IPC 64 | queueFile: /tmp/queue.bin 65 | 66 | 67 | ######################################################################################## 68 | # topics bind service classes to a transport+topicId (~topicId). think of a topicId being a 69 | # separate independent communication of a transport. 70 | # Each transport can be divided into MCastSocketTransportQueue.MAX_SCOPE separate topics. 71 | # Separation has the benefit, that in case of retransmission (oversaturation/delays) on one topicId 72 | # other scopes of the same transport are not affected. 73 | topics: 74 | 75 | - name: bench #name of 76 | transport: default #name of transport defined above 77 | # transport: ipc #name of transport defined above 78 | topicId: 0 #topicId on the transport associated with this 79 | serviceClass: de.nustaq.fastcast.test.ThroughputBench$BenchService 80 | 81 | DGramRate: 10000 82 | numPacketHistory: 1000 83 | numPacketOffHeapHistory: 15000 # 3000*5 seconds 84 | receiveBufferPackets: 1000 85 | 86 | perSenderThread: true 87 | 88 | - name: membership 89 | autoStart: true 90 | transport: control 91 | topicId: 1 92 | serviceClass: FCMembership 93 | 94 | 95 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/api/util/ByteArraySubscriber.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | 24 | package org.nustaq.fastcast.api.util; 25 | 26 | import org.nustaq.fastcast.api.FCSubscriber; 27 | import org.nustaq.offheap.bytez.Bytez; 28 | 29 | import java.util.concurrent.Executor; 30 | import java.util.concurrent.Executors; 31 | 32 | /** 33 | * Created by ruedi on 14.12.2014. 34 | * 35 | * subscriber implementation that handles conversion to byte array and by default provides a 36 | * dedicated thread for message processing. Note this is *not* allocation free, so not well suited for 37 | * low latency stuff. 38 | */ 39 | public abstract class ByteArraySubscriber implements FCSubscriber { 40 | 41 | protected Executor executor; 42 | 43 | public ByteArraySubscriber( boolean dedicatedThread) { 44 | if ( dedicatedThread ) { 45 | executor = Executors.newSingleThreadExecutor(); 46 | } 47 | } 48 | 49 | public ByteArraySubscriber() { 50 | this(true); 51 | } 52 | 53 | protected byte tmpBuf[] = new byte[0]; 54 | @Override 55 | public void messageReceived(final String sender, final long sequence, Bytez b, long off, final int len) { 56 | if ( executor != null ) { 57 | final byte[] bytes = b.toBytes(off, len); 58 | executor.execute(new Runnable() { 59 | @Override 60 | public void run() { 61 | messageReceived(sender,sequence,bytes, 0, len); 62 | } 63 | }); 64 | } else { 65 | // directly decode. saves tmp byte array alloc for each message 66 | if ( tmpBuf.length < len ) { 67 | tmpBuf = new byte[len]; 68 | } 69 | b.getArr(off, tmpBuf, 0, len); 70 | messageReceived(sender,sequence,tmpBuf,0,len); 71 | } 72 | } 73 | 74 | protected abstract void messageReceived(String sender, long sequence, byte[] msg, int off, int len); 75 | 76 | @Override 77 | public boolean dropped() { 78 | return true; // resync 79 | } 80 | 81 | @Override 82 | public void senderTerminated(String senderNodeId) { 83 | // do nothing 84 | } 85 | 86 | @Override 87 | public void senderBootstrapped(String receivesFrom, long seqNo) { 88 | // do nothing 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/basic/stuff/Issue4.java: -------------------------------------------------------------------------------- 1 | package basic.stuff; 2 | 3 | import org.nustaq.fastcast.api.FCPublisher; 4 | import org.nustaq.fastcast.api.FCSubscriber; 5 | import org.nustaq.fastcast.api.FastCast; 6 | import org.nustaq.fastcast.api.util.ObjectPublisher; 7 | import org.nustaq.fastcast.api.util.ObjectSubscriber; 8 | 9 | /** 10 | * Created by moelrue on 02.11.2015. 11 | */ 12 | public class Issue4 { 13 | 14 | FastCast fc; 15 | ObjectPublisher publisher; 16 | 17 | public FastCast setupFC(String nodeId, String config) { 18 | System.setProperty("java.net.preferIPv4Stack", "true"); 19 | try { 20 | FastCast fc = new FastCast(); //FastCast.getFastCast(); 21 | fc.setNodeId(nodeId); 22 | fc.loadConfig("./src/test/java/basic/"+config); 23 | return fc; 24 | } catch (Exception e) { 25 | e.printStackTrace(); 26 | } 27 | return null; 28 | } 29 | 30 | public void initFC() { 31 | if ( fc == null ) { 32 | 33 | fc = setupFC("t"+(int)(1000*Math.random()), "stuff/sendreceive.kson"); 34 | 35 | FCPublisher rawPublisher = fc.onTransport("default").publish(fc.getPublisherConf("sendreceive")); 36 | publisher = new ObjectPublisher(rawPublisher); 37 | 38 | FCSubscriber sub = new ObjectSubscriber() { 39 | int count = 0; 40 | 41 | @Override 42 | protected void objectReceived(String sender, long sequence, Object msg) { 43 | if ( msg instanceof String ) { 44 | System.out.println(fc.getNodeId()+" received: "+count); 45 | count = 0; 46 | } else { 47 | count++; 48 | } 49 | } 50 | 51 | @Override 52 | public boolean dropped() { 53 | System.out.println("FATAL ERROR. Enlarge send history"); 54 | System.exit(0); 55 | return false; 56 | } 57 | 58 | @Override 59 | public void senderTerminated(String senderNodeId) { 60 | System.out.println(senderNodeId+" terminated"); 61 | } 62 | 63 | @Override 64 | public void senderBootstrapped(String receivesFrom, long seqNo) { 65 | System.out.println("bootstrap "+receivesFrom); 66 | } 67 | }; 68 | fc.onTransport("default").subscribe(fc.getSubscriberConf("sendreceive"), sub); 69 | } 70 | } 71 | 72 | private void run() { 73 | new Thread(new Runnable() { 74 | @Override 75 | public void run() { 76 | initFC(); 77 | while( true ) { 78 | for (int i = 0; i < 1_000_000; i++) { 79 | publisher.sendObject(null,i, false); 80 | } 81 | publisher.sendObject(null,"million", false); 82 | } 83 | } 84 | }).start(); 85 | } 86 | 87 | 88 | public static void main(String[] args) { 89 | Issue4 i4 = new Issue4(); 90 | i4.run(); 91 | // Issue4 i4other = new Issue4(); // uncomment this for multi-process test 92 | // i4other.run(); 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/multiplestructs/MPSubscriber.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.multiplestructs; 2 | 3 | import org.nustaq.fastcast.api.FCSubscriber; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.config.SubscriberConf; 6 | import org.nustaq.fastcast.examples.multiplestructs.MultipleProtocol.*; 7 | import org.nustaq.fastcast.examples.structencoding.StructPublisher; 8 | import org.nustaq.fastcast.util.RateMeasure; 9 | import org.nustaq.offheap.bytez.Bytez; 10 | import org.nustaq.offheap.structs.FSTStruct; 11 | import org.nustaq.offheap.structs.unsafeimpl.FSTStructFactory; 12 | 13 | /** 14 | * Created by ruedi on 17.12.14. 15 | */ 16 | public class MPSubscriber { 17 | 18 | public static void main( String arg[] ) { 19 | MultipleProtocol.initStructFactory(); 20 | 21 | FastCast.getFastCast().setNodeId("MSUB"); // 5 chars MAX !! 22 | MPPublisher.configureFastCast(); 23 | final RateMeasure rateMeasure = new RateMeasure("receive rate"); 24 | 25 | FastCast.getFastCast().onTransport("default").subscribe( 26 | new SubscriberConf(1).receiveBufferPackets(33_000), 27 | new FCSubscriber() { 28 | 29 | FSTStruct msg = FSTStructFactory.getInstance().createEmptyStructPointer(FSTStruct.class); 30 | int count = 0; 31 | 32 | @Override 33 | public void messageReceived(String sender, long sequence, Bytez b, long off, int len) { 34 | rateMeasure.count(); 35 | 36 | msg.baseOn(b, (int) off); 37 | 38 | Class type = msg.getPointedClass(); 39 | if ( type == AMessage.class ) { 40 | AMessage am = msg.cast(); 41 | // am is valid until another pointer is used. use "am = am.detach" in order to get a non-cached pointer 42 | // note msg data is valid only inside this method (reused). You need to copy data in case processing should 43 | // be done in a different thread (usually do basic filtering in-thread, then do processing on a dedicated thread) 44 | } else if ( type == OtherMessage.class ) { 45 | OtherMessage om = msg.cast(); 46 | // .. 47 | } else if ( type == ComposedMessage.class ) { 48 | ComposedMessage cm = msg.cast(); 49 | if ( count++ == 500_000 ) { 50 | System.out.println("Other"+cm); 51 | count = 0; 52 | } 53 | } 54 | 55 | } 56 | 57 | @Override 58 | public boolean dropped() { 59 | System.out.println("fatal, could not keep up. exiting"); 60 | System.exit(0); 61 | return false; 62 | } 63 | 64 | @Override 65 | public void senderTerminated(String senderNodeId) { 66 | System.out.println("sender died "+senderNodeId); 67 | } 68 | 69 | @Override 70 | public void senderBootstrapped(String receivesFrom, long seqNo) { 71 | System.out.println("bootstrap "+receivesFrom); 72 | } 73 | } 74 | ); 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/Defragmenter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | 26 | import org.nustaq.offheap.bytez.Bytez; 27 | import org.nustaq.offheap.bytez.onheap.HeapBytez; 28 | 29 | /* 30 | * Created with IntelliJ IDEA. 31 | * User: ruedi 32 | * Date: 8/11/13 33 | * Time: 1:20 PM 34 | * To change this template use File | Settings | File Templates. 35 | */ 36 | 37 | /** 38 | * Manages defragmentation of large messages + splitting of multiple messages per packet 39 | */ 40 | public class Defragmenter implements ByteArrayReceiver { 41 | final int STATE_CHAIN = 1; 42 | final int STATE_COMPLETE = 0; 43 | 44 | int state = STATE_COMPLETE; 45 | 46 | byte buf[] = new byte[500]; 47 | HeapBytez bufBytez = new HeapBytez(buf,0,0); 48 | int bufIndex = 0; 49 | 50 | @Override 51 | public void receiveChunk(long sequence, Bytez b, int off, int len, boolean complete) { 52 | switch ( state ) { 53 | case STATE_COMPLETE: 54 | if ( complete ) { 55 | msgDone(sequence, b, off, len); 56 | break; 57 | } else { 58 | state = STATE_CHAIN; 59 | } 60 | // fall through Fixme: can do zerocopy in this case 61 | case STATE_CHAIN: 62 | if ( len+bufIndex >= buf.length ) { 63 | byte old[] = buf; 64 | buf = new byte[Math.max(len+bufIndex,buf.length*2)]; 65 | System.arraycopy(old,0,buf,0,bufIndex); 66 | // System.out.println("POK increase buffer to"+buf.length); 67 | } 68 | try { 69 | if ( len > 0 ) { 70 | //System.arraycopy(b,off,buf,bufIndex,len); 71 | b.getArr(off,buf,bufIndex,len); 72 | // System.out.println("POK chain copy "+len); 73 | } 74 | } catch (Throwable t) { 75 | t.printStackTrace(); 76 | } 77 | bufIndex+=len; 78 | if ( complete ) { 79 | bufBytez.setBase(buf,0,bufIndex); 80 | msgDone(sequence,bufBytez,0,bufIndex); 81 | state = STATE_COMPLETE; 82 | bufIndex = 0; 83 | } 84 | break; 85 | } 86 | } 87 | 88 | public void msgDone(long sequence, Bytez b, int off, int len) { 89 | System.out.println("received byte array "+len); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/basic/sendreceive.kson: -------------------------------------------------------------------------------- 1 | # Warning: 2 | # settings here might be way to agressive for some network stacks 3 | # especially local host performance and behaviour varies wildly depending on OS, OS versione etc. 4 | # also note that some new security interceptors (e.g. reverse packet filtering) might have 5 | # severe drawbacks on multicast. E.g. stock centos 7 localhost can perform very poor both throughput and 6 | # latency wise. 7 | # 8 | # default strat is to start with low pps (like 1000, 2000) and low dgramsize (e.g. 1200). 9 | # to increase throughput then increase pps (=packet per second) stepwise and/or dgramsize. 10 | # throughput/bytes = dgramsize * pps. 11 | # depending on os/network stack, either increasing pps or dgramsize yield to better and more stable results. 12 | # To get real throughput/lowlat use a kernel bypass driver and don't test on localhost. 13 | 14 | ClusterConf { 15 | 16 | transports: { 17 | { 18 | name: default 19 | 20 | dgramsize: 4000 21 | 22 | interfaceAddr: 127.0.0.1 23 | multicastAddr: 229.9.9.9 24 | port: 45555 25 | 26 | autoFlushMS: 1 27 | # default is blocking IO, don't do latency tests with this configuration 28 | # uncomment below for spin looping 29 | # spinLoopMicros: 10_000_000 30 | # idleParkMicros: 1000 31 | } 32 | } 33 | 34 | topics: { 35 | 36 | { 37 | id: 1 38 | name: test 39 | 40 | pub: { 41 | numPacketHistory: 30_000 42 | pps: 5_000 43 | heartbeatInterval: 1000 44 | } 45 | 46 | sub: { 47 | receiveBufferPackets: 10_000 48 | # senderHBTimeout: 5000 49 | senderHBTimeout: 500000 # avoid crashes during debug 50 | } 51 | } 52 | 53 | { 54 | id: 2 55 | name: echo 56 | 57 | pub: { 58 | numPacketHistory: 50_000 59 | pps: 5_000 60 | heartbeatInterval: 1000 61 | } 62 | 63 | sub: { 64 | receiveBufferPackets: 50_000 65 | # senderHBTimeout: 5000 66 | senderHBTimeout: 500000 # avoid crashes during debug 67 | } 68 | } 69 | 70 | { 71 | id: 3 72 | name: echoresp 73 | 74 | pub: { 75 | numPacketHistory: 50_000 76 | pps: 20_000 77 | ppsWindow: 100 78 | heartbeatInterval: 10 # testing 79 | } 80 | 81 | sub: { 82 | receiveBufferPackets: 50_000 83 | # senderHBTimeout: 5000 84 | senderHBTimeout: 500000 # avoid crashes during debug 85 | } 86 | } 87 | 88 | { 89 | id: 4 90 | name: unreliable 91 | pub: { 92 | numPacketHistory: 1_000 # only smallish buffer required for unreliable 93 | pps: 10_000 94 | ppsWindow: 100 95 | heartbeatInterval: 1000 96 | } 97 | 98 | sub: { 99 | receiveBufferPackets: 1_000 # small buffer for unreliable 100 | # senderHBTimeout: 5000 101 | senderHBTimeout: 500000 # avoid crashes during debug 102 | unreliable: true 103 | } 104 | } 105 | } 106 | 107 | } -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/structencoding/StructPublisher.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.structencoding; 2 | 3 | import org.nustaq.fastcast.api.FCPublisher; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.config.PhysicalTransportConf; 6 | import org.nustaq.fastcast.config.PublisherConf; 7 | import org.nustaq.fastcast.api.util.ObjectPublisher; 8 | import org.nustaq.fastcast.util.RateMeasure; 9 | import org.nustaq.offheap.structs.FSTStructAllocator; 10 | 11 | import java.util.concurrent.ThreadLocalRandom; 12 | 13 | /** 14 | * Created by moelrue on 12/15/14. 15 | * 16 | * Demonstrates high throughput (up to 7 million msg/second) allocation free messaging 17 | * using struct encoding. 18 | * 19 | */ 20 | public class StructPublisher { 21 | 22 | public static void main(String arg[]) { 23 | 24 | FastCast.getFastCast().setNodeId("PUB"); // 5 chars MAX !! 25 | configureFastCast(); 26 | 27 | FCPublisher pub = FastCast.getFastCast().onTransport("default").publish( 28 | new PublisherConf(1) // unique-per-transport topic id 29 | .numPacketHistory(33_000) // how many packets are kept for retransmission requests 30 | .pps(10_000) // packets per second rate limit. 31 | ); 32 | 33 | Protocol.initStructFactory(); 34 | 35 | Protocol.PriceUpdateStruct template = new Protocol.PriceUpdateStruct(); 36 | FSTStructAllocator onHeapAlloc = new FSTStructAllocator(0); 37 | 38 | Protocol.PriceUpdateStruct msg = onHeapAlloc.newStruct(template); // speed up instantiation 39 | 40 | ThreadLocalRandom current = ThreadLocalRandom.current(); 41 | // could directly send raw on publisher 42 | RateMeasure measure = new RateMeasure("msg/s"); 43 | while( true ) { 44 | measure.count(); 45 | 46 | // fill in data 47 | Protocol.InstrumentStruct instrument = msg.getInstrument(); 48 | instrument.getMnemonic().setString("BMW"); 49 | instrument.setInstrumentId(13); 50 | msg.setPrc(99.0+current.nextDouble(10.0)-5); 51 | msg.setQty(100+current.nextInt(10)); 52 | 53 | // send message 54 | while( ! pub.offer(null,msg.getBase(),msg.getOffset(),msg.getByteSize(),false) ) { 55 | /* spin */ 56 | } 57 | 58 | } 59 | } 60 | 61 | public static void configureFastCast() { 62 | // note this configuration is far below possible limits regarding throughput and rate 63 | FastCast fc = FastCast.getFastCast(); 64 | fc.addTransport( 65 | new PhysicalTransportConf("default") 66 | .interfaceAdr("127.0.0.1") // define the interface 67 | .port(42043) // port is more important than address as some OS only test for ports ('crosstalking') 68 | .mulitcastAdr("229.9.9.9") // ip4 multicast address 69 | .setDgramsize(64_000) // datagram size. Small sizes => lower latency, large sizes => better throughput [range 1200 to 64_000 bytes] 70 | .socketReceiveBufferSize(4_000_000) // as large as possible .. however avoid hitting system limits in example 71 | .socketSendBufferSize(2_000_000) 72 | // uncomment this to enable spin looping. Will increase throughput once datagram size is lowered below 8kb or so 73 | // .idleParkMicros(1) 74 | // .spinLoopMicros(100_000) 75 | ); 76 | 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/multiplestructs/MultipleProtocol.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.multiplestructs; 2 | 3 | import org.nustaq.offheap.structs.FSTStruct; 4 | import org.nustaq.offheap.structs.Templated; 5 | import org.nustaq.offheap.structs.structtypes.StructByteString; 6 | import org.nustaq.offheap.structs.unsafeimpl.FSTStructFactory; 7 | 8 | import java.util.Arrays; 9 | 10 | /** 11 | * Created by ruedi on 17.12.14. 12 | */ 13 | public class MultipleProtocol { 14 | 15 | public static void initStructFactory() { 16 | FSTStructFactory.getInstance().registerClz(AMessage.class,OtherMessage.class,ComposedMessage.class); 17 | } 18 | 19 | public static class AMessage extends FSTStruct { 20 | 21 | protected int l; 22 | 23 | @Templated 24 | protected StructByteString stringArray[] = new StructByteString[] { 25 | new StructByteString(20), // will be copied to other elemenrs because of @Templated 26 | null, null, null, null, 27 | }; 28 | 29 | public StructByteString stringArray(int index) { 30 | return stringArray[index]; 31 | } 32 | 33 | public void stringArray(int index, StructByteString s ) { 34 | stringArray[index] = s; 35 | } 36 | 37 | public int stringArrayLen() { 38 | return stringArray.length; 39 | } 40 | 41 | public int getL() { 42 | return l; 43 | } 44 | 45 | public void setL(int l) { 46 | this.l = l; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return "AMessage{" + 52 | "l=" + l + 53 | ", stringArray=" + stringArray(0) + ", " +stringArray(1) + ", " +stringArray(2) + ", " +stringArray(3) + ", " +stringArray(4) +"]"+ 54 | '}'; 55 | } 56 | } 57 | 58 | public static class OtherMessage extends FSTStruct { 59 | 60 | protected int quantity; 61 | protected double value; 62 | 63 | public int getQuantity() { 64 | return quantity; 65 | } 66 | 67 | public void setQuantity(int quantity) { 68 | this.quantity = quantity; 69 | } 70 | 71 | public double getValue() { 72 | return value; 73 | } 74 | 75 | public void setValue(double value) { 76 | this.value = value; 77 | } 78 | 79 | @Override 80 | public String toString() { 81 | return "OtherMessage{" + 82 | "quantity=" + quantity + 83 | ", value=" + value + 84 | '}'; 85 | } 86 | } 87 | 88 | public static class ComposedMessage extends FSTStruct { 89 | 90 | protected AMessage megA = new AMessage(); // templating ! 91 | protected OtherMessage msgB = new OtherMessage(); 92 | 93 | public AMessage getMegA() { 94 | return megA; 95 | } 96 | 97 | public void setMegA(AMessage megA) { 98 | this.megA = megA; 99 | } 100 | 101 | public OtherMessage getMsgB() { 102 | return msgB; 103 | } 104 | 105 | public void setMsgB(OtherMessage msgB) { 106 | this.msgB = msgB; 107 | } 108 | 109 | @Override 110 | public String toString() { 111 | return "ComposedMessage{" + 112 | "megA=" + megA + 113 | ", msgB=" + msgB + 114 | '}'; 115 | } 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /attic/bigtest/ClusterClientNode.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest; 2 | 3 | import de.ruedigermoeller.fastcast.bigtest.services.HTHost; 4 | import de.ruedigermoeller.fastcast.bigtest.services.HTListener; 5 | import de.ruedigermoeller.fastcast.bigtest.services.RequestServer; 6 | import org.nustaq.fastcast.api.FCFutureResultHandler; 7 | import org.nustaq.fastcast.api.FCRemoting; 8 | import org.nustaq.fastcast.api.FastCast; 9 | import org.nustaq.fastcast.service.FCMembership; 10 | 11 | import java.io.IOException; 12 | 13 | /** 14 | * Created with IntelliJ IDEA. 15 | * User: ruedi 16 | * Date: 9/14/13 17 | * Time: 5:08 PM 18 | * To change this template use File | Settings | File Templates. 19 | */ 20 | 21 | /** 22 | * sends request to requestprocessor, the takes the result and sends to HTHosts, receives mirror notification 23 | * from HTHosts. 24 | */ 25 | public class ClusterClientNode { 26 | 27 | RequestServer remoteServer; 28 | HTHost remoteHTHost; 29 | 30 | HTListener localHTListener; 31 | 32 | 33 | public ClusterClientNode() { 34 | } 35 | 36 | public void start() { 37 | FCRemoting rem = FastCast.getFastCast(); 38 | try { 39 | rem.joinCluster("shared/bigtest.yaml", "Client", null); 40 | } catch (IOException e) { 41 | e.printStackTrace(); 42 | System.exit(0); 43 | } 44 | 45 | // just talk remotely to those (no local instance) 46 | remoteServer = (RequestServer) rem.startSending("rqserver"); 47 | remoteHTHost = (HTHost) rem.startSending("hthost"); 48 | 49 | // install htlisten in this node 50 | rem.startReceiving("htlisten"); 51 | localHTListener = (HTListener) rem.getService("htlisten"); 52 | 53 | // let data host nodes assign modulo (~sharding) 54 | remoteHTHost.syncHostNumbers(); 55 | 56 | System.out.println("started "+rem.getNodeId()); 57 | processingLoop(); 58 | } 59 | 60 | int responseCount; int requestCount; long tim = System.currentTimeMillis(); 61 | private void processingLoop() { 62 | while( true ) { 63 | final int random = (int) (Math.random() * 1000000); 64 | requestCount++; 65 | remoteServer.receiveRequest(random, new FCFutureResultHandler() { 66 | @Override 67 | public void resultReceived(Object obj, String sender) { 68 | done(); 69 | responseCount++; 70 | if ( random % 10 == 0 ) { 71 | long now = System.currentTimeMillis(); 72 | if ( now - tim > 1000 ) { 73 | System.out.println(""+Thread.currentThread().getName()+" resp "+responseCount+" req:"+requestCount+ " in "+(now-tim)+" millis"); 74 | System.out.println(obj + " mirror size " + localHTListener.getMirror().size()); 75 | requestCount = responseCount = 0; 76 | tim = now; 77 | FCMembership ship = (FCMembership) FastCast.getFastCast().getService("membership"); 78 | System.out.println(ship.dumpToString()); 79 | } 80 | } 81 | remoteHTHost.putData(random,obj); 82 | } 83 | }); 84 | // try { 85 | // Thread.sleep(0,5000); 86 | // } catch (InterruptedException e) { 87 | // e.printStackTrace(); 88 | // } 89 | } 90 | } 91 | 92 | public static void main( String arg[] ) throws IOException { 93 | ClusterClientNode clusterClientNode = new ClusterClientNode(); 94 | clusterClientNode.start(); 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/RetransPacket.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | 26 | import org.nustaq.offheap.structs.Templated; 27 | 28 | /** 29 | * Created with IntelliJ IDEA. 30 | * User: ruedi 31 | * Date: 8/10/13 32 | * Time: 11:50 PM 33 | * Retransmission request. Contains an array of sequence intervals. 34 | * 35 | * ATTENTION: this is a struct class. 36 | */ 37 | public class RetransPacket extends Packet { 38 | 39 | // 21 entries struct template 40 | @Templated 41 | protected RetransEntry[] retransEntries = { 42 | new RetransEntry(), 43 | null,null,null,null,null, 44 | null,null,null,null,null, 45 | 46 | null,null,null,null,null, 47 | null,null,null,null,null, 48 | 49 | // null,null,null,null,null, 50 | // null,null,null,null,null, 51 | // 52 | // null,null,null,null,null, 53 | // null,null,null,null,null, 54 | }; 55 | 56 | protected int retransIndex; 57 | 58 | public int retransEntriesLen() { 59 | return retransEntries.length; 60 | } 61 | 62 | public RetransEntry retransEntries(int i) { 63 | return retransEntries[i]; 64 | } 65 | 66 | public RetransEntry current() { 67 | return retransEntries(retransIndex); 68 | } 69 | 70 | public void nextEntry() { 71 | retransIndex++; 72 | } 73 | 74 | public int getRetransIndex() { 75 | return retransIndex; 76 | } 77 | 78 | public void setRetransIndex(int retransIndex) { 79 | this.retransIndex = retransIndex; 80 | } 81 | 82 | public void clear() { 83 | setRetransIndex(0); 84 | } 85 | 86 | public boolean isFull() { 87 | return getRetransIndex() >= retransEntriesLen()-1; 88 | } 89 | 90 | @Override 91 | public String toString() { 92 | return "RetransPacket{" + 93 | "seqNo=" + seqNo + 94 | ", topic=" + topic + 95 | ", sender=" + sender + 96 | ", receiver=" + receiver + 97 | ", retransEntries=" + entriesString() + 98 | ", retransIndex=" + retransIndex + 99 | '}'; 100 | } 101 | 102 | protected String entriesString() { 103 | String res = "["; 104 | for ( int n = 0; n < getRetransIndex(); n++ ) { 105 | RetransEntry retransEntry = retransEntries(n); 106 | res+="[ "+ retransEntry.getFrom()+","+retransEntry.getTo()+"] "; 107 | } 108 | return res+"]"; 109 | } 110 | 111 | public int computeNumPackets() { 112 | int res = 0; 113 | for (int i=0; i < retransIndex; i++) 114 | res += retransEntries(i).getNumPackets(); 115 | return res; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/config/PublisherConf.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.config; 24 | 25 | /** 26 | * Created by ruedi on 29.11.2014. 27 | */ 28 | public class PublisherConf { 29 | 30 | int topicId; 31 | 32 | // Buffer sizes are very important. For high volume senders, the send buffer must be large 33 | // these defaults hold for moderate traffic 34 | 35 | /////////////////////////////////////////////////////////////////////////////// 36 | // 37 | // buffers 38 | // 39 | /////////////////////////////////////////////////////////////////////////////// 40 | 41 | // overall send history 42 | int numPacketHistory = 500_000; 43 | 44 | /////////////////////////////////////////////////////////////////////////////// 45 | // 46 | // timings 47 | // 48 | /////////////////////////////////////////////////////////////////////////////// 49 | 50 | long heartbeatInterval = 500; // sent per topicId, ms. detects senderTimeoutMillis 51 | 52 | int pps = 10_000_000; // rate limit datagram per second 53 | // unused, deprecated 54 | int ppsWindow = 1000; // time window rate limit is checked. e.g. ppsWindow = 10 => 1 sec/10 = 100ms another: ppsWindow = 1000 => 1 ms 55 | 56 | public PublisherConf() { 57 | } 58 | 59 | public PublisherConf(int topicId) { 60 | this.topicId = topicId; 61 | } 62 | 63 | @Deprecated 64 | public int getPpsWindow() { 65 | return ppsWindow; 66 | } 67 | 68 | public PublisherConf ppsWindow(int ppsWindow) { 69 | this.ppsWindow = ppsWindow; 70 | return this; 71 | } 72 | 73 | /////////////////////////////////////////////////////////////////////////////// 74 | // 75 | // sender misc (threading, locking 76 | // 77 | /////////////////////////////////////////////////////////////////////////////// 78 | 79 | public int getNumPacketHistory() { 80 | return numPacketHistory; 81 | } 82 | 83 | public PublisherConf numPacketHistory(int numPacketHistory) { 84 | this.numPacketHistory = numPacketHistory; 85 | return this; 86 | } 87 | 88 | public int getTopicId() { 89 | return topicId; 90 | } 91 | 92 | public PublisherConf topicId(int topicId) { 93 | this.topicId = topicId; 94 | return this; 95 | } 96 | 97 | public long getHeartbeatInterval() { 98 | return heartbeatInterval; 99 | } 100 | 101 | public PublisherConf heartbeatInterval(long heartbeatInterval) { 102 | this.heartbeatInterval = heartbeatInterval; 103 | return this; 104 | } 105 | 106 | public int getPps() { 107 | return pps; 108 | } 109 | 110 | public PublisherConf pps(int pps) { 111 | this.pps = pps; 112 | return this; 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################# 2 | ## Eclipse 3 | ################# 4 | 5 | *.pydevproject 6 | .project 7 | .metadata 8 | bin/ 9 | tmp/ 10 | *.tmp 11 | *.bak 12 | *.swp 13 | *~.nib 14 | local.properties 15 | .classpath 16 | .settings/ 17 | .loadpath 18 | 19 | # External tool builders 20 | .externalToolBuilders/ 21 | 22 | # Locally stored "Eclipse launch configurations" 23 | *.launch 24 | 25 | # CDT-specific 26 | .cproject 27 | 28 | # PDT-specific 29 | .buildpath 30 | 31 | *.class 32 | .out 33 | 34 | ################# 35 | ## Visual Studio 36 | ################# 37 | 38 | ## Ignore Visual Studio temporary files, build results, and 39 | ## files generated by popular Visual Studio add-ons. 40 | 41 | # User-specific files 42 | *.suo 43 | *.user 44 | *.sln.docstates 45 | 46 | # Build results 47 | 48 | [Dd]ebug/ 49 | [Rr]elease/ 50 | x64/ 51 | build/ 52 | [Bb]in/ 53 | [Oo]bj/ 54 | 55 | # MSTest test Results 56 | [Tt]est[Rr]esult*/ 57 | [Bb]uild[Ll]og.* 58 | 59 | *_i.c 60 | *_p.c 61 | *.ilk 62 | *.meta 63 | *.obj 64 | *.pch 65 | *.pdb 66 | *.pgc 67 | *.pgd 68 | *.rsp 69 | *.sbr 70 | *.tlb 71 | *.tli 72 | *.tlh 73 | *.tmp 74 | *.tmp_proj 75 | *.log 76 | *.vspscc 77 | *.vssscc 78 | .builds 79 | *.pidb 80 | *.log 81 | *.scc 82 | 83 | # Visual C++ cache files 84 | ipch/ 85 | *.aps 86 | *.ncb 87 | *.opensdf 88 | *.sdf 89 | *.cachefile 90 | 91 | # Visual Studio profiler 92 | *.psess 93 | *.vsp 94 | *.vspx 95 | 96 | # Guidance Automation Toolkit 97 | *.gpState 98 | 99 | # ReSharper is a .NET coding add-in 100 | _ReSharper*/ 101 | *.[Rr]e[Ss]harper 102 | 103 | # TeamCity is a build add-in 104 | _TeamCity* 105 | 106 | # DotCover is a Code Coverage Tool 107 | *.dotCover 108 | 109 | # NCrunch 110 | *.ncrunch* 111 | .*crunch*.local.xml 112 | 113 | # Installshield output folder 114 | [Ee]xpress/ 115 | 116 | # DocProject is a documentation generator add-in 117 | DocProject/buildhelp/ 118 | DocProject/Help/*.HxT 119 | DocProject/Help/*.HxC 120 | DocProject/Help/*.hhc 121 | DocProject/Help/*.hhk 122 | DocProject/Help/*.hhp 123 | DocProject/Help/Html2 124 | DocProject/Help/html 125 | 126 | # Click-Once directory 127 | publish/ 128 | 129 | # Publish Web Output 130 | *.Publish.xml 131 | *.pubxml 132 | 133 | # NuGet Packages Directory 134 | ## TODO: If you have NuGet Package Restore enabled, uncomment the next line 135 | #packages/ 136 | 137 | # Windows Azure Build Output 138 | csx 139 | *.build.csdef 140 | 141 | # Windows Store app package directory 142 | AppPackages/ 143 | 144 | # Others 145 | sql/ 146 | *.Cache 147 | ClientBin/ 148 | [Ss]tyle[Cc]op.* 149 | ~$* 150 | *~ 151 | *.dbmdl 152 | *.[Pp]ublish.xml 153 | *.pfx 154 | *.publishsettings 155 | 156 | # RIA/Silverlight projects 157 | Generated_Code/ 158 | 159 | # Backup & report files from converting an old project file to a newer 160 | # Visual Studio version. Backup files are not needed, because we have git ;-) 161 | _UpgradeReport_Files/ 162 | Backup*/ 163 | UpgradeLog*.XML 164 | UpgradeLog*.htm 165 | 166 | # SQL Server files 167 | App_Data/*.mdf 168 | App_Data/*.ldf 169 | 170 | ############# 171 | ## Windows detritus 172 | ############# 173 | 174 | # Windows image file caches 175 | Thumbs.db 176 | ehthumbs.db 177 | 178 | # Folder config file 179 | Desktop.ini 180 | 181 | # Recycle Bin used on file shares 182 | $RECYCLE.BIN/ 183 | 184 | # Mac crap 185 | .DS_Store 186 | 187 | 188 | ############# 189 | ## Python 190 | ############# 191 | 192 | *.py[co] 193 | 194 | # Packages 195 | *.egg 196 | *.egg-info 197 | dist/ 198 | build/ 199 | eggs/ 200 | parts/ 201 | var/ 202 | sdist/ 203 | develop-eggs/ 204 | .installed.cfg 205 | 206 | # Installer logs 207 | pip-log.txt 208 | 209 | # Unit test / coverage reports 210 | .coverage 211 | .tox 212 | 213 | #Translations 214 | *.mo 215 | 216 | #Mr Developer 217 | .mr.developer.cfg 218 | target/ 219 | *.bin 220 | .gradle/ 221 | .idea/ 222 | *.iml -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/ShareMemPingPong.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.latency; 2 | 3 | import org.HdrHistogram.Histogram; 4 | import sun.misc.Unsafe; 5 | import sun.nio.ch.DirectBuffer; 6 | 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.io.RandomAccessFile; 10 | import java.lang.reflect.Field; 11 | import java.nio.MappedByteBuffer; 12 | import java.nio.channels.FileChannel; 13 | import java.util.concurrent.TimeUnit; 14 | 15 | /** 16 | * Copied from peter lawreys blog + added HdrHistogram + emulation of echoing a 40 byte msg. 17 | * 18 | * Results are a good indication on how "quiet" a machine is latency wise 19 | * 20 | */ 21 | public class ShareMemPingPong { 22 | 23 | public static void main(String... args) throws IOException { 24 | boolean odd; 25 | switch (args.length < 1 ? "usage" : args[0].toLowerCase()) { 26 | case "odd": 27 | odd = true; 28 | break; 29 | case "even": 30 | odd = false; 31 | break; 32 | default: 33 | System.err.println("Usage: java PingPongMain [odd|even]"); 34 | return; 35 | } 36 | System.out.println("odd:"+odd); 37 | int runs = 10_000_000; 38 | long start = 0; 39 | System.out.println("Waiting for the other odd/even"); 40 | File counters = new File(System.getProperty("java.io.tmpdir"), "counters.deleteme"); 41 | counters.deleteOnExit(); 42 | 43 | oneRun(odd, runs, start, counters); 44 | oneRun(odd, runs, start, counters); 45 | } 46 | 47 | public static void oneRun(boolean odd, int runs, long start, File counters) throws IOException { 48 | Histogram histo = new Histogram(TimeUnit.SECONDS.toNanos(10),3); 49 | byte msg[] = new byte[40]; 50 | try (FileChannel fc = new RandomAccessFile(counters, "rw").getChannel()) { 51 | MappedByteBuffer mbb = fc.map(FileChannel.MapMode.READ_WRITE, 0, 1024); 52 | long address = ((DirectBuffer) mbb).address(); 53 | for (int i = -1; i < runs; i++) { 54 | for (; ; ) { 55 | long tim = System.nanoTime(); 56 | long value = UNSAFE.getLongVolatile(null, address); 57 | boolean isOdd = (value & 1) != 0; 58 | if (isOdd != odd) 59 | // wait for the other side. 60 | continue; 61 | 62 | // simulate reading message 63 | mbb.position(8); 64 | mbb.get(msg,0,msg.length); 65 | 66 | // simulate writing msg 67 | msg[12] = (byte) i; 68 | mbb.position(8); 69 | mbb.put(msg, 0, msg.length); 70 | 71 | histo.recordValue(System.nanoTime()-tim); 72 | // make the change atomic, just in case there is more than one odd/even process 73 | if (UNSAFE.compareAndSwapLong(null, address, value, value + 1)) 74 | break; 75 | } 76 | if (i == 0) { 77 | System.out.println("Started"); 78 | start = System.nanoTime(); 79 | } 80 | } 81 | } 82 | System.out.printf("... Finished, average ping/pong took %,d ns%n", 83 | (System.nanoTime() - start) / runs); 84 | histo.outputPercentileDistribution(System.out, 1000.0); 85 | } 86 | 87 | static final Unsafe UNSAFE; 88 | 89 | static { 90 | try { 91 | Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe"); 92 | theUnsafe.setAccessible(true); 93 | UNSAFE = (Unsafe) theUnsafe.get(null); 94 | } catch (Exception e) { 95 | throw new AssertionError(e); 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/test/java/basic/stuff/Issue4_Slow.java: -------------------------------------------------------------------------------- 1 | package basic.stuff; 2 | 3 | import org.nustaq.fastcast.api.FCPublisher; 4 | import org.nustaq.fastcast.api.FCSubscriber; 5 | import org.nustaq.fastcast.api.FastCast; 6 | import org.nustaq.fastcast.api.util.ObjectPublisher; 7 | import org.nustaq.fastcast.api.util.ObjectSubscriber; 8 | 9 | import java.util.Date; 10 | 11 | /** 12 | * Created by moelrue on 02.11.2015. 13 | */ 14 | public class Issue4_Slow { 15 | 16 | FastCast fc; 17 | ObjectPublisher publisher; 18 | 19 | public FastCast setupFC(String nodeId, String config) { 20 | System.setProperty("java.net.preferIPv4Stack", "true"); 21 | try { 22 | FastCast fc = new FastCast(); //FastCast.getFastCast(); 23 | fc.setNodeId(nodeId); 24 | fc.loadConfig("./src/test/java/basic/"+config); 25 | return fc; 26 | } catch (Exception e) { 27 | e.printStackTrace(); 28 | } 29 | return null; 30 | } 31 | 32 | public void initFC() { 33 | if ( fc == null ) { 34 | fc = setupFC("t"+(int)(1000*Math.random()), "stuff/sendreceive.kson"); 35 | FCSubscriber sub = new ObjectSubscriber() { 36 | int count = 0; 37 | 38 | @Override 39 | protected void objectReceived(String sender, long sequence, Object msg) { 40 | if ( msg instanceof String ) { 41 | System.out.println(fc.getNodeId()+" received: "+count); 42 | count = 0; 43 | } else { 44 | Object i_time[] = (Object[]) msg; 45 | System.out.println( fc.getNodeId()+" received "+i_time[0]+" delay: "+(System.currentTimeMillis()-(Long)i_time[1])); 46 | count++; 47 | } 48 | } 49 | 50 | @Override 51 | public boolean dropped() { 52 | System.out.println("FATAL ERROR. Enlarge send history"); 53 | System.exit(0); 54 | return false; 55 | } 56 | 57 | @Override 58 | public void senderTerminated(String senderNodeId) { 59 | System.out.println(senderNodeId+" terminated"); 60 | } 61 | 62 | @Override 63 | public void senderBootstrapped(String receivesFrom, long seqNo) { 64 | System.out.println("bootstrap "+receivesFrom); 65 | } 66 | }; 67 | fc.onTransport("default").subscribe(fc.getSubscriberConf("sendreceive"), sub); 68 | FCPublisher rawPublisher = fc.onTransport("default").publish(fc.getPublisherConf("sendreceive")); 69 | publisher = new ObjectPublisher(rawPublisher); 70 | } 71 | } 72 | 73 | private void run() { 74 | new Thread(new Runnable() { 75 | @Override 76 | public void run() { 77 | initFC(); 78 | while( true ) { 79 | for (int i = 0; i < 10; i++) { 80 | try { 81 | Thread.sleep((long) (Math.random()*5000)); 82 | } catch (InterruptedException e) { 83 | e.printStackTrace(); 84 | } 85 | long now = System.currentTimeMillis(); 86 | System.out.println(fc.getNodeId()+" sending "+i+" "+ now); 87 | publisher.sendObject(null,new Object[] { i, now}, true); 88 | } 89 | publisher.sendObject(null,"ten", true); 90 | } 91 | } 92 | }).start(); 93 | } 94 | 95 | 96 | public static void main(String[] args) { 97 | Issue4_Slow i4 = new Issue4_Slow(); 98 | i4.run(); 99 | Issue4_Slow i4other = new Issue4_Slow(); // uncomment this for multi-process test 100 | i4other.run(); 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /attic/resource/pingpong.yaml: -------------------------------------------------------------------------------- 1 | # name of cluster 2 | clusterName: pipo 3 | 4 | ########################################################################################################### 5 | # define technical transport topics 6 | # a transport is defined by a network adapter, a listening address and a port. google 'crosstalking' in case u run unix 7 | 8 | transports: 9 | 10 | - name: default 11 | # address of network adapter, 12 | ifacAdr: 127.0.0.1 13 | mcastAdr: 230.9.9.10 14 | port: 45556 15 | 16 | dgramsize: 8000 17 | # needs to be true in order to run several processes on one machine 18 | loopBack: true 19 | 20 | # in java we like large buffers to minimize packet loss due to GC jitter 21 | socketReceiveBufferSize: 25000000 22 | sendBufferSize: 640000 23 | 24 | #misc 25 | trafficClass: 8 26 | ttl: 2 27 | 28 | ########################################################################################################### 29 | # define separate address+port transport for heartbeats+stats as hi volume might delay heartbeats resulting 30 | # in false drops 31 | # 32 | - name: control 33 | # address of network adapter, 34 | ifacAdr: 127.0.0.1 35 | mcastAdr: 230.9.9.11 36 | port: 45557 37 | 38 | dgramsize: 8000 39 | loopBack: true 40 | 41 | # in java we like large buffers to minimize packet loss due to GC :( 42 | socketReceiveBufferSize: 25000000 43 | sendBufferSize: 640000 44 | 45 | #misc 46 | trafficClass: 8 47 | ttl: 2 48 | transportType: MCAST_NIO_SOCKET 49 | 50 | ########################################################################################################### 51 | # ipc transport 52 | # 53 | - name: ipc 54 | dgramsize: 8000 55 | # size of shared mem buffer 56 | socketReceiveBufferSize: 100000000 57 | 58 | #misc 59 | transportType: MCAST_IPC 60 | queueFile: /test/queue.bin 61 | 62 | 63 | ######################################################################################## 64 | # topics bind service classes to a transport+topicId (~topicId). think of a topicId being a 65 | # separate independent communication of a transport. 66 | # Each transport can be divided into MCastSocketTransportQueue.MAX_SCOPE separate topics. 67 | # Separation has the benefit, that in case of retransmission (oversaturation/delays) on one topicId 68 | # other scopes of the same transport are not affected. 69 | topics: 70 | 71 | - name: ping #name of 72 | #transport: default #name of transport defined above 73 | transport: ipc #name of transport defined above 74 | 75 | topicId: 0 #topicId on the transport associated with this 76 | serviceClass: de.nustaq.fastcast.test.PingService 77 | 78 | numPacketHistory: 20000 #combined size of send history buffer + send queue 79 | maxSendPacketQueueSize: 1 #how many slots of packageHistory are used as send queue 80 | 81 | receiveBufferPackets: 20000 #number of received packets pre-buffered in case of a gap 82 | sendPauseMicros: 50 #the smaller the faster, the larger buffers are required (send+receive), the more retransmissions will occur 83 | #if retransmission get to high, throuput will worsen. Tune conservatively 84 | 85 | - name: pong #name of 86 | #transport: default #name of transport defined above 87 | transport: ipc #name of transport defined above 88 | 89 | topicId: 1 #topicId on the transport associated with this 90 | serviceClass: de.nustaq.fastcast.test.PongService 91 | 92 | numPacketHistory: 20000 #combined size of send history buffer + send queue 93 | maxSendPacketQueueSize: 1 #how many slots of packageHistory are used as send queue 94 | 95 | receiveBufferPackets: 20000 #number of received packets pre-buffered in case of a gap 96 | sendPauseMicros: 200 #the smaller the faster, the larger buffers are required (send+receive), the more retransmissions will occur 97 | #if retransmission get to high, throuput will worsen. Tune conservatively 98 | 99 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/api/util/ObjectSubscriber.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.api.util; 24 | 25 | import org.nustaq.offheap.bytez.Bytez; 26 | import org.nustaq.serialization.simpleapi.DefaultCoder; 27 | import org.nustaq.serialization.simpleapi.FSTCoder; 28 | 29 | /** 30 | * Created by ruedi on 13.12.14. 31 | * 32 | * A subscriber implementation receiving and automatically decoding serialized objects. 33 | * By default it uses a dedicated thread for delivery to avoid blocking the receiverThread 34 | * pulling data packets from the network. 35 | * 36 | */ 37 | public abstract class ObjectSubscriber extends ByteArraySubscriber { 38 | 39 | protected FSTCoder coder; 40 | protected boolean decodeInReceiver = false; 41 | 42 | public ObjectSubscriber( boolean dedicatedThread, Class ... preregister) { 43 | super(dedicatedThread); 44 | coder = new DefaultCoder(true,preregister); 45 | } 46 | 47 | public ObjectSubscriber( Class ... preregister) { 48 | this(true,preregister); 49 | } 50 | 51 | public ObjectSubscriber(boolean dedicatedThread) { 52 | this(dedicatedThread,new Class[0]); 53 | } 54 | 55 | public ObjectSubscriber() { 56 | this(true); 57 | } 58 | 59 | /** 60 | * default is false. If true, deserialization is done directly inside the messaging thread 61 | * saving a tmp byte array alloc for each incoming message. Downside is possible slowdown / packet loss 62 | * in case of large messages coming in. 63 | * 64 | * @param b 65 | * @return 66 | */ 67 | public ObjectSubscriber decodeInReceiverThread(boolean b) { 68 | decodeInReceiver = b; 69 | return this; 70 | } 71 | 72 | @Override 73 | public void messageReceived(final String sender, final long sequence, Bytez b, long off, int len) { 74 | if ( ! decodeInReceiver && executor != null ) 75 | super.messageReceived(sender, sequence, b, off, len); 76 | else { 77 | // directly decode. saves tmp byte array alloc for each message 78 | if ( tmpBuf.length < len ) { 79 | tmpBuf = new byte[len]; 80 | } 81 | b.getArr(off, tmpBuf, 0, len); // fixme: could be zerocopy using OffHeapCoder 82 | final Object objectMessage = coder.toObject(tmpBuf, 0, tmpBuf.length); 83 | if ( executor != null ) { 84 | executor.execute(new Runnable() { 85 | @Override 86 | public void run() { 87 | objectReceived(sender, sequence, objectMessage); 88 | } 89 | }); 90 | } else { 91 | objectReceived(sender, sequence, objectMessage); 92 | } 93 | } 94 | } 95 | 96 | /** 97 | * note: this is not called in case decodeInReceiver is true or no dedicated thread is pressent. 98 | * So take care when overriding 99 | */ 100 | @Override 101 | public void messageReceived(final String sender, final long sequence, byte bytes[], int off, int len) { 102 | Object msg = coder.toObject(bytes, off, len); 103 | objectReceived(sender,sequence,msg); 104 | } 105 | 106 | protected abstract void objectReceived(String sender, long sequence, Object msg); 107 | 108 | } 109 | -------------------------------------------------------------------------------- /attic/resource/unordered.yaml: -------------------------------------------------------------------------------- 1 | # name of cluster 2 | clusterName: bm 3 | 4 | # define technical transport topics 5 | # a transport is defined by a network adapter, a listening address and a port. google 'crosstalking' in case u run unix 6 | 7 | transports: 8 | 9 | - name: s0 10 | # address of network adapter, 11 | ifacAdr: 127.0.0.1 12 | mcastAdr: 230.9.10.10 13 | port: 45566 14 | 15 | dgramsize: 8000 16 | # needs to be true in order to run several processes on one machine 17 | loopBack: true 18 | 19 | # in java we like large buffers to minimize packet loss due to GC jitter 20 | socketReceiveBufferSize: 25000000 21 | sendBufferSize: 640000 22 | 23 | #misc 24 | trafficClass: 8 25 | ttl: 2 26 | 27 | - name: s1 28 | # address of network adapter, 29 | ifacAdr: 127.0.0.1 30 | mcastAdr: 230.9.10.11 31 | port: 45567 32 | 33 | dgramsize: 8000 34 | # needs to be true in order to run several processes on one machine 35 | loopBack: true 36 | 37 | # in java we like large buffers to minimize packet loss due to GC jitter 38 | socketReceiveBufferSize: 25000000 39 | sendBufferSize: 640000 40 | 41 | #misc 42 | trafficClass: 8 43 | ttl: 2 44 | 45 | - name: ip0 46 | socketReceiveBufferSize: 25000000 47 | #misc 48 | transportType: MCAST_IPC 49 | queueFile: /tmp/market0.bin 50 | 51 | # for some reason on linux only one IPC transport is possible (file locking works different) 52 | - name: ip1 53 | socketReceiveBufferSize: 25000000 54 | #misc 55 | transportType: MCAST_IPC 56 | queueFile: /tmp/market1.bin 57 | 58 | ######################################################################################## 59 | # topics bind service classes to a transport+topicId (~topicId). think of a topicId being a 60 | # separate independent communication of a transport. 61 | # Each transport can be divided into MCastSocketTransportQueue.MAX_SCOPE separate topics. 62 | # Separation has the benefit, that in case of retransmission (oversaturation/delays) on one topicId 63 | # other scopes of the same transport are not affected. 64 | topics: 65 | 66 | - name: stream_0 #name of 67 | flowControlClass: null #turn off flow control 68 | 69 | transport: s0 #name of transport defined above 70 | topicId: 0 #topicId on the transport associated with this 71 | serviceClass: de.nustaq.fastcast.test.UnorderedReceiver 72 | 73 | maxOpenRespondedCalls: 500 # how many unanswered calls with results (request-response) can be unanswered at a time. 74 | # this is costly as callbackmaps are held. Only set high if needed. 75 | responseMethodsTimeout: 1000 76 | 77 | numPacketHistory: 15000 #combined size of send history buffer + send queue 78 | maxSendPacketQueueSize: 500 #how many slots of packageHistory are used as send queue 79 | 80 | receiveBufferPackets: 100 #number of received packets pre-buffered in case of a gap 81 | sendPauseMicros: 30 #the smaller the faster, the larger buffers are required (send+receive), the more retransmissions will occur 82 | #if retransmission get to high, throuput will worsen. Tune conservatively 83 | 84 | - name: stream_1 #name of 85 | flowControlClass: null #turn off flow control 86 | 87 | transport: s1 #name of transport defined above 88 | topicId: 1 #topicId on the transport associated with this 89 | serviceClass: de.nustaq.fastcast.test.UnorderedReceiver 90 | 91 | maxOpenRespondedCalls: 500 # how many unanswered calls with results (request-response) can be unanswered at a time. 92 | # this is costly as callbackmaps are held. Only set high if needed. 93 | responseMethodsTimeout: 1000 94 | 95 | numPacketHistory: 15000 #combined size of send history buffer + send queue 96 | maxSendPacketQueueSize: 500 #how many slots of packageHistory are used as send queue 97 | 98 | receiveBufferPackets: 100 #number of received packets pre-buffered in case of a gap 99 | sendPauseMicros: 30 #the smaller the faster, the larger buffers are required (send+receive), the more retransmissions will occur 100 | #if retransmission get to high, throuput will worsen. Tune conservatively 101 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/structencoding/Protocol.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.structencoding; 2 | 3 | import org.nustaq.offheap.structs.FSTStruct; 4 | import org.nustaq.offheap.structs.FSTStructAllocator; 5 | import org.nustaq.offheap.structs.Templated; 6 | import org.nustaq.offheap.structs.structtypes.StructByteString; 7 | import org.nustaq.offheap.structs.structtypes.StructString; 8 | import org.nustaq.offheap.structs.unsafeimpl.FSTStructFactory; 9 | 10 | /** 11 | * Created by moelrue on 12/15/14. 12 | */ 13 | public class Protocol { 14 | 15 | public static class InstrumentStruct extends FSTStruct { 16 | 17 | protected StructByteString mnemonic = new StructByteString(5); 18 | protected long instrumentId; 19 | 20 | public StructByteString getMnemonic() { 21 | return mnemonic; 22 | } 23 | 24 | public void setMnemonic(StructByteString mnemonic) { 25 | this.mnemonic = mnemonic; 26 | } 27 | 28 | public long getInstrumentId() { 29 | return instrumentId; 30 | } 31 | 32 | public void setInstrumentId(long instrumentId) { 33 | this.instrumentId = instrumentId; 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return "InstrumentStruct{" + 39 | "mnemonic=" + mnemonic + 40 | ", instrumentId=" + instrumentId + 41 | '}'; 42 | } 43 | } 44 | 45 | public static class PriceUpdateStruct extends FSTStruct { 46 | 47 | protected InstrumentStruct instrument = new InstrumentStruct(); 48 | protected int qty = 0; 49 | protected double prc; 50 | 51 | public InstrumentStruct getInstrument() { 52 | return instrument; 53 | } 54 | 55 | public void setInstrument(InstrumentStruct instrument) { 56 | this.instrument = instrument; 57 | } 58 | 59 | public int getQty() { 60 | return qty; 61 | } 62 | 63 | public void setQty(int qty) { 64 | this.qty = qty; 65 | } 66 | 67 | public double getPrc() { 68 | return prc; 69 | } 70 | 71 | public void setPrc(double prc) { 72 | this.prc = prc; 73 | } 74 | 75 | @Override 76 | public String toString() { 77 | return "PriceUpdateStruct{" + 78 | "instrument=" + instrument + 79 | ", qty=" + qty + 80 | ", prc=" + prc + 81 | '}'; 82 | } 83 | } 84 | 85 | public static void do20Millions(byte[] networkBuffer, PriceUpdateStruct msg, int sizeOf) { 86 | long tim = System.currentTimeMillis(); 87 | for ( int i = 0; i < 20_000_000; i++ ) { 88 | 89 | InstrumentStruct instrument = msg.getInstrument(); 90 | instrument.getMnemonic().setString("BMW"); 91 | instrument.setInstrumentId(13); 92 | msg.setPrc(99.0); 93 | msg.setQty(100); 94 | 95 | // emulate network sending by copying to buffer 96 | msg.getBase().getArr(msg.getOffset(),networkBuffer,0,sizeOf); 97 | } 98 | System.out.println("tim: "+(System.currentTimeMillis()-tim)); 99 | } 100 | 101 | public static void initStructFactory() { 102 | FSTStructFactory.getInstance().registerClz(InstrumentStruct.class,PriceUpdateStruct.class); 103 | } 104 | 105 | public static void main(String s[]) { 106 | 107 | initStructFactory(); 108 | 109 | PriceUpdateStruct template = new PriceUpdateStruct(); 110 | FSTStructAllocator onHeapAlloc = new FSTStructAllocator(0); 111 | 112 | template = onHeapAlloc.newStruct(template); // speed up instantiation by moving template also off heap 113 | 114 | 115 | // demonstrates that theoretical send rate is >20 millions messages per second on 116 | // an I7 box 117 | byte networkBuffer[] = new byte[template.getByteSize()]; 118 | PriceUpdateStruct msg = onHeapAlloc.newStruct(template); 119 | int sizeOf = msg.getByteSize(); 120 | while ( true ) { 121 | do20Millions(networkBuffer, msg, sizeOf); 122 | } 123 | 124 | // System.out.println(msg); 125 | // System.out.println("size:" + msg.getByteSize()); 126 | 127 | } 128 | 129 | 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/util/FCLog.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.util; 24 | 25 | import java.io.PrintStream; 26 | 27 | /** 28 | * Created with IntelliJ IDEA. 29 | * User: ruedi 30 | * Date: 9/8/13 31 | * Time: 3:01 PM 32 | * 33 | * Loggerwrapper. Override out( int level, String msg, Throwable th) and use setInstance in order to redirect logging to 34 | * other loggers. Defaults to sysout 35 | * 36 | */ 37 | public class FCLog { 38 | 39 | public static final int DEBUG = 0; 40 | public static final int INFO = 1; 41 | public static final int WARN = 2; 42 | public static final int SEVER = 3; 43 | public static final int FATAL = 4; 44 | 45 | static FCLog instance; 46 | 47 | public static FCLog get() { 48 | if ( instance == null ) { 49 | instance = new FCLog(); 50 | } 51 | return instance; 52 | } 53 | 54 | public static void setInstance(FCLog instance) { 55 | FCLog.instance = instance; 56 | } 57 | 58 | public static void log(String s) { 59 | get().info(s); 60 | } 61 | 62 | public static void log(String s, Throwable th) { 63 | get().info(s,th); 64 | } 65 | 66 | public static void log(Throwable th) { 67 | get().warn(th); 68 | } 69 | 70 | int logLevel = INFO; 71 | 72 | public int getLogLevel() { 73 | return logLevel; 74 | } 75 | 76 | public void setLogLevel(int logLevel) { 77 | this.logLevel = logLevel; 78 | } 79 | 80 | void internal_out(int level, String msg, Throwable th) { 81 | out(level, msg, th); 82 | } 83 | 84 | protected void out(int level, String msg, Throwable th) { 85 | if ( level >= getLogLevel() ) { 86 | if ( msg != null ) 87 | System.out.println(msg); 88 | if ( th != null ) 89 | th.printStackTrace( new PrintStream(System.out)); 90 | } 91 | } 92 | 93 | public void info( String msg ) { 94 | internal_out(INFO, msg, null); 95 | } 96 | 97 | public void info( String msg, Throwable e ) { 98 | internal_out(INFO, msg, e); 99 | } 100 | 101 | public void info( Throwable e ) { 102 | internal_out(INFO, null, e); 103 | } 104 | 105 | public void warn( String msg ) { 106 | internal_out(WARN, msg, null); 107 | } 108 | 109 | public void warn( String msg, Throwable e ) { 110 | internal_out(WARN, msg, e); 111 | } 112 | 113 | public void warn( Throwable e ) { 114 | internal_out(WARN, null, e); 115 | } 116 | 117 | public void severe( String msg, Throwable e ) { 118 | internal_out(SEVER, msg, e); 119 | } 120 | 121 | public void fatal( String msg) { 122 | internal_out(FATAL, msg, null); 123 | } 124 | 125 | public void fatal( String msg, Throwable e ) { 126 | internal_out(FATAL, msg, e); 127 | } 128 | 129 | public void debug( String msg, Throwable e ) { 130 | internal_out(DEBUG, msg, e); 131 | } 132 | 133 | public void debug( Throwable e ) { 134 | internal_out(DEBUG, null, e); 135 | } 136 | 137 | public void debug( String msg ) { 138 | internal_out(DEBUG, msg, null); 139 | } 140 | 141 | /** 142 | * called for RUDP low level logging (retransmission etc) 143 | * @param s 144 | */ 145 | public void net(String s) { 146 | internal_out(INFO, s, null); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/DataPacket.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | 26 | import org.nustaq.offheap.structs.FSTStruct; 27 | import org.nustaq.offheap.structs.FSTStructAllocator; 28 | 29 | /** 30 | * Created with IntelliJ IDEA. 31 | * User: ruedi 32 | * Date: 8/10/13 33 | * Time: 11:50 PM 34 | * To change this template use File | Settings | File Templates. 35 | */ 36 | public class DataPacket extends Packet { 37 | 38 | public static DataPacket getTemplate(int packetSize) { 39 | int emptyPackSize = new FSTStructAllocator(1).newStruct(new DataPacket()).getByteSize(); 40 | DataPacket template = new DataPacket(); 41 | int payMaxLen = packetSize - emptyPackSize - 2; 42 | template.data = new byte[payMaxLen]; 43 | return template; 44 | } 45 | 46 | public static final short COMPLETE = 1; // payload fits inpacket 47 | public static final short CHAINED = 2; // payload is chained. len then denotes len in current packet 48 | public static final short EOP = 3; // end of packet 49 | public static final short MAX_CODE = EOP; // 50 | 51 | public static final int HEADERLEN = 4; 52 | 53 | protected boolean isDecoded; 54 | protected boolean isRetrans = false; 55 | protected int bytesLeft; 56 | protected byte[] data = new byte[0]; 57 | 58 | public void data(int index, byte val) { 59 | data[index] = val; 60 | } 61 | 62 | public byte data(int index) { 63 | return data[index]; 64 | } 65 | 66 | public int dataLen() { 67 | return data.length; 68 | } 69 | 70 | public int dataIndex() { 71 | // generated 72 | return -1; 73 | } 74 | 75 | /** 76 | * internal flag, anyway transmitted FIXME: not required anymore (relict of FC 2.x) ! 77 | * @return 78 | */ 79 | public boolean isDecoded() { 80 | return isDecoded; 81 | } 82 | 83 | public void setDecoded(boolean decoded) { 84 | isDecoded = decoded; 85 | } 86 | 87 | public void dataPointer(FSTStruct pointer) { 88 | // generated 89 | } 90 | 91 | public FSTStruct dataPointer() { 92 | // generated 93 | return null; 94 | } 95 | 96 | public boolean isRetrans() { 97 | return isRetrans; 98 | } 99 | 100 | public void setRetrans(boolean isRetrans) { 101 | this.isRetrans = isRetrans; 102 | } 103 | 104 | public int getBytesLeft() { 105 | return bytesLeft; 106 | } 107 | 108 | public int getDGramSize() { 109 | return getByteSize()-bytesLeft; 110 | } 111 | 112 | public void setBytesLeft(int bytesLeft) { 113 | this.bytesLeft = bytesLeft; 114 | } 115 | 116 | public void dumpBytes() { 117 | for ( int n = 0; n < dataLen()-getBytesLeft(); n++ ) { 118 | System.out.print(" [" + n + "], " + data(n)); 119 | } 120 | System.out.println("-"); 121 | } 122 | @Override 123 | public String toString() { 124 | return "DataPacket{" + 125 | "seqNo=" + seqNo + 126 | ", retr=" + isRetrans + 127 | ", topic=" + topic + 128 | ", sender=" + sender + 129 | ", receiver=" + receiver + 130 | ", left=" + bytesLeft + 131 | ", datalen=" + dataLen() + 132 | ", dgsize="+getDGramSize()+ 133 | ", decoded="+isDecoded()+ 134 | '}'; 135 | } 136 | 137 | 138 | } 139 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/multiplestructs/MPPublisher.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.multiplestructs; 2 | 3 | import org.nustaq.fastcast.api.FCPublisher; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.config.PhysicalTransportConf; 6 | import org.nustaq.fastcast.config.PublisherConf; 7 | import org.nustaq.fastcast.util.RateMeasure; 8 | import org.nustaq.offheap.structs.FSTStruct; 9 | import org.nustaq.offheap.structs.FSTStructAllocator; 10 | 11 | import java.util.concurrent.ThreadLocalRandom; 12 | import static org.nustaq.fastcast.examples.multiplestructs.MultipleProtocol.*; 13 | 14 | /** 15 | * Created by ruedi on 17.12.14. 16 | */ 17 | public class MPPublisher { 18 | 19 | public static void main(String arg[]) { 20 | 21 | FastCast.getFastCast().setNodeId("MPUB"); // 5 chars MAX !! 22 | configureFastCast(); 23 | 24 | FCPublisher pub = FastCast.getFastCast().onTransport("default").publish( 25 | new PublisherConf(1) // unique-per-transport topic id 26 | .numPacketHistory(33_000) // how many packets are kept for retransmission requests 27 | .pps(10_000) // packets per second rate limit. 28 | ); 29 | 30 | MultipleProtocol.initStructFactory(); 31 | FSTStructAllocator onHeapAlloc = new FSTStructAllocator(10_000); 32 | 33 | AMessage aMsg = onHeapAlloc.newStruct(new AMessage()); 34 | OtherMessage other = onHeapAlloc.newStruct(new OtherMessage()); 35 | ComposedMessage composed = onHeapAlloc.newStruct(new ComposedMessage()); 36 | 37 | ThreadLocalRandom random = ThreadLocalRandom.current(); 38 | // could directly send raw on publisher 39 | RateMeasure measure = new RateMeasure("msg/s"); 40 | int count = 0; 41 | FSTStruct msg = null; 42 | 43 | while( true ) { 44 | 45 | measure.count(); 46 | 47 | // fill in data 48 | switch( random.nextInt(3) ) { 49 | case 0: 50 | for ( int i = 0; i < aMsg.stringArrayLen(); i++ ) { 51 | aMsg.stringArray(i).setString("Hello "+i); 52 | } 53 | aMsg.setL(count++); 54 | msg = aMsg; 55 | break; 56 | case 1: 57 | other.setQuantity(count++); 58 | other.setValue(random.nextDouble()); 59 | msg = other; 60 | break; 61 | case 2: 62 | aMsg.stringArray(0).setString("Hello !"); 63 | aMsg.stringArray(1).setString("Hello !!"); 64 | aMsg.setL(count++); 65 | other.setQuantity(count++); 66 | other.setValue(random.nextDouble()); 67 | 68 | composed.setMegA(aMsg); // does a copy ! 69 | composed.setMsgB(other); // does a copy ! 70 | msg = composed; 71 | break; 72 | } 73 | // send message 74 | while( ! pub.offer(null,msg.getBase(),msg.getOffset(),msg.getByteSize(),false) ) { 75 | /* spin */ 76 | } 77 | 78 | } 79 | } 80 | 81 | public static void configureFastCast() { 82 | // note this configuration is far below possible limits regarding throughput and rate 83 | FastCast fc = FastCast.getFastCast(); 84 | fc.addTransport( 85 | new PhysicalTransportConf("default") 86 | .interfaceAdr("127.0.0.1") // define the interface 87 | .port(42044) // port is more important than address as some OS only test for ports ('crosstalking') 88 | .mulitcastAdr("229.9.9.13") // ip4 multicast address 89 | .setDgramsize(16_000) // datagram size. Small sizes => lower latency, large sizes => better throughput [range 1200 to 64_000 bytes] 90 | .socketReceiveBufferSize(4_000_000) // as large as possible .. however avoid hitting system limits in example 91 | .socketSendBufferSize(2_000_000) 92 | // uncomment this to enable spin looping. Will increase throughput once datagram size is lowered below 8kb or so 93 | // .idleParkMicros(1) 94 | // .spinLoopMicros(100_000) 95 | ); 96 | 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | org.sonatype.oss 6 | oss-parent 7 | 7 8 | 9 | 10 | 4.0.0 11 | de.ruedigermoeller 12 | fast-cast 13 | 3.10 14 | a fast brokerless messaging library based on reliable UDP multicast 15 | 16 | https://code.google.com/p/fast-cast/ 17 | 18 | 19 | Apache 2.0 20 | http://www.apache.org/licenses/LICENSE-2.0 21 | repo 22 | 23 | 24 | 25 | scm:svn:https://fast-cast.googlecode.com/svn/trunk/fast-cast-read-only 26 | scm:svn:https://fast-cast.googlecode.com/svn/trunk/ 27 | https://fast-cast.googlecode.com/svn/trunk/ 28 | 29 | 30 | 31 | 32 | de.ruedigermoeller 33 | fst 34 | 2.42 35 | 36 | 37 | 38 | test 39 | org.hdrhistogram 40 | HdrHistogram 41 | 2.0.3 42 | 43 | 44 | 45 | test 46 | junit 47 | junit 48 | 4.12 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | org.apache.maven.plugins 58 | maven-javadoc-plugin 59 | 60 | -Xdoclint:none 61 | 62 | 63 | 64 | 65 | 66 | 67 | attach-javadocs 68 | 69 | jar 70 | 71 | 72 | 73 | 74 | 75 | 76 | org.apache.maven.plugins 77 | maven-source-plugin 78 | 79 | 80 | attach-sources 81 | 82 | jar 83 | 84 | 85 | 86 | 87 | 88 | 89 | org.apache.maven.plugins 90 | maven-compiler-plugin 91 | 92 | 93 | 1.7 94 | 1.7 95 | true 96 | lines,vars,source 97 | false 98 | true 99 | 100 | 101 | 102 | default-testCompile 103 | test-compile 104 | 105 | testCompile 106 | 107 | 108 | true 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/api/FCPublisher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.api; 24 | 25 | import org.nustaq.offheap.bytez.ByteSource; 26 | 27 | /** 28 | * interface used to send messages. obtained like 'fastCast.onTransport("default").publish("stream");' 29 | * 30 | * Created by ruedi on 29.11.2014. 31 | */ 32 | public interface FCPublisher { 33 | 34 | /** 35 | * send a byte[] message 36 | * 37 | * @param receiverNodeId - null to send to all subscribers, aNodeId to target a specific node 38 | * @param b - bytes to send 39 | * @param start - start offset 40 | * @param len - number of bytes to send 41 | * @param doFlush - if true, the currently written packet is sent immediately EXCEPT your send rate is > pps limit (packets per second) 42 | * only set to true if you care mostly about latency instead of throughput. fast cast automatically flushes after one ms. So a 43 | * value of true only makes sense if you care about micros. 44 | * @return true if the message has been sent, false if limits/buffers are full. 45 | */ 46 | public boolean offer(String receiverNodeId, byte b[], int start, int len, boolean doFlush); 47 | /** 48 | * send a byte[] message 49 | * 50 | * @param receiverNodeId - null to send to all subscribers, aNodeId to target a specific node 51 | * @param msg - byte source containing the message bytes 52 | * @param start - start offset 53 | * @param len - number of bytes to send 54 | * @param doFlush - if true, the currently written packet is sent immediately EXCEPT your send rate is > pps limit (packets per second) 55 | * only set to true if you care mostly about latency instead of throughput. fast cast automatically flushes after one ms. So a 56 | * value of true only makes sense if you care about micros. 57 | * @return true if the message has been sent, false if limits/buffers are full. 58 | */ 59 | public boolean offer(String receiverNodeId, ByteSource msg, long start, int len, boolean doFlush); 60 | 61 | /** 62 | * send a byte[] message 63 | * 64 | * @param receiverNodeId - null to send to all subscribers, aNodeId to target a specific node 65 | * @param msg - byte source containing the message bytes. 66 | * @param doFlush - if true, the currently written packet is sent immediately EXCEPT your send rate is > pps limit (packets per second) 67 | * only set to true if you care mostly about latency instead of throughput. fast cast automatically flushes after one ms. So a 68 | * value of true only makes sense if you care about micros. 69 | * @return true if the message has been sent, false if limits/buffers are full. 70 | */ 71 | public boolean offer(String receiverNodeId, ByteSource msg, boolean doFlush); 72 | 73 | public int getTopicId(); 74 | public void flush(); 75 | 76 | // FIXME: add listener interface 77 | // public void retransMissionRequestReceived(int percentageOfHistory, String nodeId); 78 | 79 | public void setPacketRateLimit(int limit); 80 | public int getPacketRateLimit(); 81 | 82 | /** 83 | * decides wether to start batching once rate limit is reached. Default is true. 84 | * Note this must be set to false in case there are unreliable or unordered receivers amongst 85 | * subscribers. 86 | * Batching means to pack several small messages into one datagram packet. 87 | * 88 | * @param doBatch 89 | * @return 90 | */ 91 | public FCPublisher batchOnLimit(boolean doBatch); 92 | public boolean isBatchOnLimit(boolean doBatch); 93 | 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/Topic.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | import org.nustaq.fastcast.config.PublisherConf; 26 | import org.nustaq.fastcast.config.SubscriberConf; 27 | import org.nustaq.fastcast.api.*; 28 | import org.nustaq.fastcast.transport.PhysicalTransport; 29 | import java.util.List; 30 | 31 | /* 32 | * Created with IntelliJ IDEA. 33 | * User: ruedi 34 | * Date: 23.08.13 35 | * Time: 02:04 36 | * To change this template use File | Settings | File Templates. 37 | */ 38 | 39 | /** 40 | * Combines publisher+subscriber configuration, topic stats .. 41 | */ 42 | public class Topic { 43 | 44 | PublisherConf publisherConf; 45 | SubscriberConf subscriberConf; 46 | 47 | TransportDriver channelDispatcher; 48 | PacketSendBuffer sender; 49 | 50 | private FCSubscriber subscriber; 51 | int topicId = -1; 52 | private long hbTimeoutMS = 3000; 53 | 54 | public Topic(SubscriberConf subscriberConf, PublisherConf publisherConf) { 55 | this.subscriberConf = subscriberConf; 56 | this.publisherConf = publisherConf; 57 | if ( subscriberConf != null ) { 58 | hbTimeoutMS = subscriberConf.getSenderHBTimeout(); 59 | } 60 | } 61 | 62 | public List getTimedOutSenders(List res, long now, long timeout) { 63 | ReceiveBufferDispatcher receiver = channelDispatcher.getReceiver(topicId); 64 | if ( receiver != null ) { 65 | receiver.getTimedOutSenders(now,timeout,res); 66 | } 67 | return res; 68 | } 69 | 70 | public SubscriberConf getSubscriberConf() { 71 | return subscriberConf; 72 | } 73 | 74 | public PhysicalTransport getTrans() { 75 | return channelDispatcher.trans; 76 | } 77 | 78 | public void setSubscriberConf(SubscriberConf subscriberConf) { 79 | this.subscriberConf = subscriberConf; 80 | if ( subscriberConf != null ) { 81 | hbTimeoutMS = subscriberConf.getSenderHBTimeout(); 82 | } 83 | } 84 | 85 | public boolean isUnordered() { 86 | // if (subscriberConf != null ) 87 | // return subscriberConf.isUnreliable(); 88 | return false; 89 | } 90 | 91 | public TransportDriver getChannelDispatcher() { 92 | return channelDispatcher; 93 | } 94 | 95 | public void setChannelDispatcher(TransportDriver channelDispatcher) { 96 | this.channelDispatcher = channelDispatcher; 97 | } 98 | 99 | public boolean isUnreliable() { 100 | if (subscriberConf != null ) 101 | return subscriberConf.isUnreliable(); 102 | return false; 103 | } 104 | 105 | public FCSubscriber getSubscriber() { 106 | return subscriber; 107 | } 108 | 109 | public void setSubscriber(FCSubscriber subs) { 110 | this.subscriber = subs; 111 | } 112 | 113 | public void setSender(PacketSendBuffer sender) { 114 | this.sender = sender; 115 | } 116 | 117 | public PacketSendBuffer getSender() { 118 | return sender; 119 | } 120 | 121 | public int getTopicId() { 122 | if ( topicId < 0 ) { 123 | if ( subscriberConf != null ) { 124 | topicId = subscriberConf.getTopicId(); 125 | } else if ( publisherConf != null ) { 126 | topicId = publisherConf.getTopicId(); 127 | } 128 | } 129 | return topicId; 130 | } 131 | 132 | public void setPublisherConf(PublisherConf publisherConf) { 133 | this.publisherConf = publisherConf; 134 | } 135 | 136 | public PublisherConf getPublisherConf() { 137 | return publisherConf; 138 | } 139 | 140 | public long getHbTimeoutMS() { 141 | return hbTimeoutMS; 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/config/SubscriberConf.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.config; 24 | 25 | /** 26 | * Created with IntelliJ IDEA. 27 | * User: moelrue 28 | * Date: 8/5/13 29 | * Time: 5:18 PM 30 | * To change this template use File | Settings | File Templates. 31 | */ 32 | public class SubscriberConf { 33 | 34 | int topicId; 35 | 36 | /////////////////////////////////////////////////////////////////////////////// 37 | // 38 | // buffers 39 | // 40 | /////////////////////////////////////////////////////////////////////////////// 41 | 42 | // in case of gaps, buffer that many received packets (=datagram in fastcast context). 43 | // KEPT PER SENDER PER TOPIC !. So value of 10 with 10 senders on 2 topics = 20000 = 160MB with 8kb packets 44 | // increase for high volume receivers causing retransmissions (the larger, the fewer retransmissions will be there) 45 | int receiveBufferPackets = 5000; 46 | 47 | /////////////////////////////////////////////////////////////////////////////// 48 | // 49 | // timings 50 | // 51 | /////////////////////////////////////////////////////////////////////////////// 52 | 53 | // time interval until a receiver sends a retransmission request after a gap 54 | long maxDelayRetransMS = 1; 55 | // time until a retransrequest is sent again if sender does not fulfill 56 | long maxDelayNextRetransMS = 5; 57 | 58 | // time until a sender is lost+deallocated if it stops sending heartbeats 59 | // on overload i had crashes from false overload induced timeouts and receive of packets after buffer dealloc 60 | // pls report once you observe crashes ! 61 | long senderHBTimeout = 10000; 62 | 63 | /////////////////////////////////////////////////////////////////////////////// 64 | // 65 | // receiver misc 66 | // 67 | /////////////////////////////////////////////////////////////////////////////// 68 | 69 | // accept packet loss. don't send retransmissions. Note that this only works for messages < datagramsize (see transport conf) 70 | boolean unreliable = false; 71 | 72 | public SubscriberConf() { 73 | } 74 | 75 | public SubscriberConf(int topicId) { 76 | this.topicId = topicId; 77 | } 78 | 79 | public int getTopicId() { 80 | return topicId; 81 | } 82 | 83 | public SubscriberConf topicId(int topicId) { 84 | this.topicId = topicId; 85 | return this; 86 | } 87 | 88 | public int getReceiveBufferPackets() { 89 | return receiveBufferPackets; 90 | } 91 | 92 | public SubscriberConf receiveBufferPackets(int receiveBufferPackets) { 93 | this.receiveBufferPackets = receiveBufferPackets; 94 | return this; 95 | } 96 | 97 | public long getMaxDelayRetransMS() { 98 | return maxDelayRetransMS; 99 | } 100 | 101 | public SubscriberConf maxDelayRetransMS(long maxDelyRetransMS) { 102 | this.maxDelayRetransMS = maxDelyRetransMS; 103 | return this; 104 | } 105 | 106 | public long getMaxDelayNextRetransMS() { 107 | return maxDelayNextRetransMS; 108 | } 109 | 110 | public SubscriberConf maxDelayNextRetransMS(long maxDelayNextRetransMS) { 111 | this.maxDelayNextRetransMS = maxDelayNextRetransMS; 112 | return this; 113 | } 114 | 115 | public long getSenderHBTimeout() { 116 | return senderHBTimeout; 117 | } 118 | 119 | public SubscriberConf setSenderHBTimeout(long senderHBTimeout) { 120 | this.senderHBTimeout = senderHBTimeout; 121 | return this; 122 | } 123 | 124 | public boolean isUnreliable() { 125 | return unreliable; 126 | } 127 | 128 | public SubscriberConf unreliable(boolean unreliable) { 129 | this.unreliable = unreliable; 130 | return this; 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /attic/bigtest/TableDataQuery.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest; 2 | 3 | import de.ruedigermoeller.fastcast.bigtest.services.HTHost; 4 | import de.ruedigermoeller.fastcast.bigtest.services.HTListener; 5 | import org.nustaq.fastcast.api.FCFutureResultHandler; 6 | import org.nustaq.fastcast.api.FCRemoting; 7 | import org.nustaq.fastcast.api.FastCast; 8 | 9 | import java.io.IOException; 10 | import java.util.Iterator; 11 | import java.util.Objects; 12 | import java.util.concurrent.ConcurrentHashMap; 13 | import java.util.concurrent.Semaphore; 14 | 15 | /** 16 | * Created with IntelliJ IDEA. 17 | * User: ruedi 18 | * Date: 9/17/13 19 | * Time: 7:05 PM 20 | * To change this template use File | Settings | File Templates. 21 | */ 22 | public class TableDataQuery { 23 | HTHost remoteHTHost; 24 | HTListener localHTListener; 25 | 26 | 27 | public TableDataQuery() { 28 | } 29 | 30 | public void start() { 31 | FCRemoting rem = FastCast.getFastCast(); 32 | try { 33 | rem.joinCluster("shared/bigtest.yaml", "TQuery", null); 34 | } catch (IOException e) { 35 | e.printStackTrace(); 36 | System.exit(0); 37 | } 38 | 39 | // just talk remotely to those (no local instance) 40 | remoteHTHost = (HTHost) rem.startSending("hthost"); 41 | localHTListener = (HTListener) rem.getService("htlisten"); 42 | localHTListener.setListener(new HTListener.DataListener() { 43 | @Override 44 | public void changeReceived(int id, Object prev, Object newVal) { 45 | lastRec = System.currentTimeMillis();; 46 | } 47 | }); 48 | rem.startReceiving("htlisten"); 49 | 50 | System.out.println("started "+rem.getNodeId()); 51 | new Thread("waitForTest") { 52 | public void run() { 53 | while( true ) { 54 | long now = System.currentTimeMillis(); 55 | if ( lastRec != 0 && now-lastRec > 5000 ) { 56 | test.release(); 57 | } 58 | try { 59 | Thread.sleep(1000); 60 | } catch (InterruptedException e) { 61 | e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. 62 | } 63 | } 64 | } 65 | }.start(); 66 | processingLoop(); 67 | } 68 | 69 | int responseCount; 70 | long lastRec = 0; 71 | Semaphore test = new Semaphore(0); 72 | String strSed[] = { "one", "two", "three", "four" }; 73 | private void processingLoop() { 74 | //while( true ) 75 | { 76 | try { 77 | responseCount = 0; 78 | final long tim = System.currentTimeMillis(); 79 | test.acquire(); 80 | System.out.println("startInq"); 81 | remoteHTHost.getTableData(new FCFutureResultHandler() { 82 | long lastRec = 0; 83 | ConcurrentHashMap res = new ConcurrentHashMap(); 84 | 85 | @Override 86 | public void timeoutReached() { 87 | System.out.println("finshed "+responseCount+" in "+(lastRec-tim)+" ms. res size "+res.size()); 88 | ConcurrentHashMap mirror = localHTListener.getMirror(); 89 | System.out.println("mirror/res siz "+mirror.size()+" "+res.size()); 90 | for (Iterator iterator = mirror.keySet().iterator(); iterator.hasNext(); ) { 91 | Object next = iterator.next(); 92 | if ( !Objects.deepEquals(mirror.get(next),res.get(next)) ) { 93 | System.out.println("1st error at "+next); 94 | System.out.println(" "+mirror.get(next)+" "+res.get(next)); 95 | break; 96 | } 97 | } 98 | System.out.println("comparision done"); 99 | } 100 | 101 | @Override 102 | public void resultReceived(Object obj, String sender) { 103 | responseCount++; 104 | lastRec = System.currentTimeMillis(); 105 | Object val[] = (Object[]) obj; 106 | res.put(val[0],val[1]); 107 | extendTimeout(); 108 | } 109 | }); 110 | 111 | } catch (InterruptedException e) { 112 | 113 | } 114 | } 115 | } 116 | 117 | public static void main( String arg[] ) throws IOException { 118 | TableDataQuery clusterClientNode = new TableDataQuery(); 119 | clusterClientNode.start(); 120 | } 121 | 122 | } -------------------------------------------------------------------------------- /attic/bigtest/DataMutatingClient.java: -------------------------------------------------------------------------------- 1 | package de.ruedigermoeller.fastcast.bigtest; 2 | 3 | /** 4 | * Created with IntelliJ IDEA. 5 | * User: ruedi 6 | * Date: 9/16/13 7 | * Time: 9:21 PM 8 | * To change this template use File | Settings | File Templates. 9 | */ 10 | 11 | import de.ruedigermoeller.fastcast.bigtest.services.HTHost; 12 | import de.ruedigermoeller.fastcast.bigtest.services.HTListener; 13 | import org.nustaq.fastcast.api.FCRemoting; 14 | import org.nustaq.fastcast.api.FastCast; 15 | 16 | import java.io.IOException; 17 | import java.util.concurrent.atomic.AtomicBoolean; 18 | 19 | /** 20 | * directly sends to HTHost and receives copy. not communication to ReqServer 21 | */ 22 | public class DataMutatingClient implements HTListener.DataListener { 23 | HTHost remoteHTHost; 24 | HTListener localHTListener; 25 | 26 | 27 | public DataMutatingClient() { 28 | } 29 | 30 | public void start() { 31 | FCRemoting rem = FastCast.getFastCast(); 32 | try { 33 | rem.joinCluster("shared/bigtest.yaml", "MClient", null); 34 | } catch (IOException e) { 35 | e.printStackTrace(); 36 | System.exit(0); 37 | } 38 | 39 | // just talk remotely to those (no local instance) 40 | remoteHTHost = (HTHost) rem.startSending("hthost"); 41 | 42 | // install htlisten in this node 43 | rem.startReceiving("htlisten"); 44 | localHTListener = (HTListener) rem.getService("htlisten"); 45 | localHTListener.setListener(this); 46 | 47 | // let data host nodes assign modulo (~sharding) 48 | remoteHTHost.syncHostNumbers(); 49 | 50 | System.out.println("started "+rem.getNodeId()); 51 | 52 | try { 53 | Thread.sleep(3000); 54 | } catch (InterruptedException e) { 55 | e.printStackTrace(); 56 | } 57 | processingLoop(); 58 | } 59 | 60 | int requestCount; long tim = System.currentTimeMillis(); int responseCount; 61 | byte toSend[] = new byte[100]; 62 | String strSed[] = { "one", "two", "three", "four" }; 63 | boolean bool = true; 64 | private void processingLoop() { 65 | while( bool ) { 66 | final int random = (int) (Math.random() * 1000000); 67 | requestCount++; 68 | remoteHTHost.putData(random,strSed); 69 | } 70 | int maxSend = 1000000; 71 | for (int random = 0; random < maxSend; random++) { 72 | requestCount++; 73 | remoteHTHost.putData(random,strSed); 74 | // try { 75 | // Thread.sleep(1); 76 | // } catch (InterruptedException e) { 77 | // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. 78 | // } 79 | } 80 | try { 81 | Thread.sleep(1000); 82 | } catch (InterruptedException e) { 83 | e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. 84 | } 85 | System.out.println("***** loop finished"); 86 | remoteHTHost.printTableSize(); 87 | int count = 0; 88 | while( localHTListener.getMirror().size() < maxSend && count < 10) { 89 | System.out.println(" mirror size "+count+" s lag " + localHTListener.getMirror().size()); 90 | try { 91 | Thread.sleep(1000); 92 | } catch (InterruptedException e) { 93 | e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. 94 | } 95 | count++; 96 | } 97 | int errCount = 0; 98 | for (int random = 0; errCount < 100 && random < maxSend; random++) { 99 | if ( localHTListener.getMirror().get(random) == null ) { 100 | System.out.println("Error at "+random); 101 | errCount++; 102 | } 103 | } 104 | } 105 | 106 | AtomicBoolean lock = new AtomicBoolean(false); 107 | @Override 108 | public void changeReceived(int id, Object prev, Object newVal) { 109 | responseCount++; 110 | if (lock.compareAndSet(false,true)) { 111 | long now = System.currentTimeMillis(); 112 | if ( now - tim > 1000 ) { 113 | System.out.println(""+Thread.currentThread().getName()+" resp "+responseCount+" req:"+requestCount+ " in "+(now-tim)+" millis"); 114 | // System.out.println("mirror size " + localHTListener.getMirror().size()); 115 | requestCount = responseCount = 0; 116 | tim = now; 117 | } 118 | lock.set(false); 119 | } 120 | } 121 | 122 | 123 | public static void main( String arg[] ) throws IOException { 124 | DataMutatingClient clusterClientNode = new DataMutatingClient(); 125 | clusterClientNode.start(); 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/ReceiveBufferDispatcher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2014, Ruediger Moeller. All rights reserved. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | * 19 | * Date: 03.01.14 20 | * Time: 21:19 21 | * To change this template use File | Settings | File Templates. 22 | */ 23 | package org.nustaq.fastcast.impl; 24 | 25 | import org.nustaq.fastcast.api.FCSubscriber; 26 | import org.nustaq.fastcast.util.FCLog; 27 | import org.nustaq.offheap.structs.structtypes.StructString; 28 | 29 | import java.util.ArrayList; 30 | import java.util.Date; 31 | import java.util.Iterator; 32 | import java.util.List; 33 | import java.util.concurrent.ConcurrentHashMap; 34 | 35 | /* 36 | * Created with IntelliJ IDEA. 37 | * User: ruedi 38 | * Date: 14.08.13 39 | * Time: 00:42 40 | * To change this template use File | Settings | File Templates. 41 | */ 42 | 43 | /** 44 | * multiplexes messages of different publishers to their 45 | * associated PacketReceiveBuffer 46 | */ 47 | public class ReceiveBufferDispatcher { 48 | 49 | ConcurrentHashMap bufferMap = new ConcurrentHashMap(); 50 | 51 | int packetSize; 52 | String nodeId; 53 | int historySize; 54 | int topic; 55 | FCSubscriber receiver; 56 | Topic topicEntry; 57 | 58 | public ReceiveBufferDispatcher(int packetSize, String nodeId, Topic entry, FCSubscriber rec) { 59 | receiver = rec; 60 | this.packetSize = packetSize; 61 | this.nodeId = nodeId; 62 | this.historySize = entry.getSubscriberConf().getReceiveBufferPackets(); 63 | this.topic = entry.getTopicId(); 64 | topicEntry = entry; 65 | } 66 | 67 | public Topic getTopicEntry() { 68 | return topicEntry; 69 | } 70 | 71 | public PacketReceiveBuffer getBuffer(StructString sender) { 72 | PacketReceiveBuffer receiveBuffer = bufferMap.get(sender); 73 | if ( receiveBuffer == null ) { 74 | int hSize = historySize; 75 | if ( ((long)hSize*packetSize) > Integer.MAX_VALUE-2*packetSize ) { 76 | final int newHist = (Integer.MAX_VALUE - 2 * packetSize) / packetSize; 77 | topicEntry.getSubscriberConf().receiveBufferPackets(newHist); 78 | FCLog.get().warn("int overflow, degrading history size from "+hSize+" to "+newHist); 79 | historySize = newHist; 80 | } 81 | receiveBuffer = new PacketReceiveBuffer(packetSize,nodeId,historySize,sender.toString(), topicEntry, receiver); 82 | bufferMap.put((StructString) sender.createCopy(),receiveBuffer); 83 | } 84 | return receiveBuffer; 85 | } 86 | 87 | /** 88 | * if a sender stops sending, remove from map to free memory 89 | * @param senderName 90 | */ 91 | public void cleanup(String senderName) { 92 | StructString struct = new StructString(senderName); 93 | PacketReceiveBuffer packetReceiveBuffer = bufferMap.get(struct); 94 | bufferMap.remove(struct); 95 | if ( packetReceiveBuffer != null ) { 96 | FCSubscriber subscriber = packetReceiveBuffer.getTopicEntry().getSubscriber(); 97 | if ( subscriber != null ) { 98 | subscriber.senderTerminated(senderName); 99 | } else { 100 | FCLog.get().severe("no subscriber for stopped sender", null); 101 | } 102 | packetReceiveBuffer.terminate(); 103 | } 104 | else { 105 | FCLog.get().warn("cannot find packetReceiver to terminate"); 106 | } 107 | } 108 | 109 | public void cleanupTopic() { 110 | ArrayList keys = new ArrayList(bufferMap.keySet()); 111 | for (int i = 0; i < keys.size(); i++) { 112 | StructString o = keys.get(i); 113 | cleanup(o.toString()); 114 | } 115 | } 116 | 117 | public void getTimedOutSenders(long now, long timeout, List res) { 118 | for (Iterator iterator = bufferMap.values().iterator(); iterator.hasNext(); ) { 119 | PacketReceiveBuffer next = iterator.next(); 120 | if ( now - next.getLastHBMillis() > timeout ) { 121 | System.out.println("timeout "+new Date(now)+" last:"+new Date(next.getLastHBMillis())); 122 | res.add(next.getReceivesFrom()); 123 | } 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/async/AsyncLatPublisher.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.latency.async; 2 | 3 | import org.HdrHistogram.Histogram; 4 | import org.nustaq.fastcast.api.FastCast; 5 | import org.nustaq.fastcast.api.util.ObjectPublisher; 6 | import org.nustaq.fastcast.api.util.ObjectSubscriber; 7 | import org.nustaq.fastcast.util.RateMeasure; 8 | 9 | import java.util.concurrent.Executor; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.TimeUnit; 12 | 13 | /** 14 | * Created by ruedi on 23/01/15. 15 | * 16 | * This test measures asynchronous latency using multithreaded receiver processing. 17 | * 18 | * - Messages are sent from A to B 19 | * - B receives a message decodes it using serialization and puts it to a dedicated processing thread (see ObjectSubscriber class) 20 | * - Inside the processing thread, the same message is sent back from B to A 21 | * - A receives the message back, decodes it and computes the time in nanos 22 | * 23 | * So the following actions are measured: 24 | * - send message via network 25 | * - receive message from network 26 | * - decode message 27 | * - enqueue message to another thread 28 | * - [no processing] encode message again 29 | * - send message 30 | * - receive message 31 | * - decode message 32 | * 33 | * Note that this test uses fast-serialization for en/decoding. Using fst-structs will probably yield significant 34 | * better results (especially reduce GC related outliers) 35 | * 36 | * note: in order to test this on localhost, reduce the localhost mtu to 1550 bytes, else latency might get very high 37 | * 38 | * $ sudo ifconfig lo mtu 1550 39 | * 40 | * yields 25 microseconds rtt at 30k events per second 41 | * 42 | * note 1: pps setting determines the point batching starts. On a decent system/OS a pps of 50_000 is not a problem, 43 | * however many systems have built in limits (e.g. some windows cap at 25k on localhost). As batching trades latency for throughput 44 | * the higher pps, the lower the latency but also troughput 45 | * 46 | */ 47 | public class AsyncLatPublisher { 48 | 49 | public static final String CFG_FILE_PATH = "src/main/java/org/nustaq/fastcast/examples/latency/async/fc.kson"; 50 | 51 | FastCast fastCast; 52 | ObjectPublisher pub; 53 | Histogram hi = new Histogram(TimeUnit.SECONDS.toNanos(2),3); 54 | Executor dumper = Executors.newCachedThreadPool(); 55 | 56 | public void initFastCast() throws Exception { 57 | fastCast = FastCast.getFastCast(); 58 | fastCast.setNodeId("PUB"); 59 | fastCast.loadConfig(CFG_FILE_PATH); 60 | 61 | pub = new ObjectPublisher( 62 | fastCast.onTransport("default").publish("stream"), 63 | AsyncLatMessage.class 64 | ); 65 | 66 | fastCast.onTransport("back").subscribe( "back", 67 | new ObjectSubscriber(false,AsyncLatMessage.class) { 68 | @Override 69 | protected void objectReceived(String s, long l, Object o) { 70 | if ( "END".equals(o) ) { 71 | final Histogram oldHi = hi; 72 | hi = new Histogram(TimeUnit.SECONDS.toNanos(2),3); 73 | // no lambdas to stay 1.7 compatible 74 | // move printing out of the receiving thread 75 | dumper.execute(new Runnable() { 76 | @Override 77 | public void run() { 78 | oldHi.outputPercentileDistribution(System.out,1000.0); 79 | } 80 | }); 81 | // hi.reset(); 82 | return; 83 | } 84 | final long value = System.nanoTime() - ((AsyncLatMessage) o).getSendTimeStampNanos(); 85 | if ( value < 1_000_000_000 ) 86 | hi.recordValue(value); 87 | } 88 | 89 | @Override 90 | public boolean dropped() { 91 | System.exit(-2); 92 | return false; 93 | } 94 | }); 95 | 96 | } 97 | 98 | public void run( int pauseNanos, int numEvents ) throws Throwable { 99 | RateMeasure report = new RateMeasure("send rate"); 100 | AsyncLatMessage event = new AsyncLatMessage(System.nanoTime(), 0, 0+1, 10, 110); 101 | // int msgCount = 0; 102 | for ( int i = 0; i < numEvents; i++ ) { 103 | double bidPrc = Math.random() * 10; 104 | event.setBidPrc(bidPrc); 105 | event.setAskPrc(bidPrc+1); 106 | event.setSendTimeStampNanos(System.nanoTime()); 107 | pub.sendObject( null, event, true ); 108 | report.count(); 109 | long time = System.nanoTime(); 110 | while( System.nanoTime() - time < pauseNanos ) { 111 | // spin 112 | } 113 | // msgCount++; 114 | // if ( (msgCount%10000) == 0 ) { 115 | // System.out.println("count "+msgCount); 116 | // } 117 | } 118 | pub.sendObject(null,"END", true); 119 | } 120 | 121 | public static void main(String arg[]) throws Throwable { 122 | AsyncLatPublisher pub = new AsyncLatPublisher(); 123 | 124 | pub.initFastCast(); 125 | while (true) 126 | pub.run( 2_000, 1_000_000 ); // 93_000 = 10k, 27_000 = 30k, 10_500 = 70k, 4_900 = 140k 127 | 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /examples/src/main/java/org/nustaq/fastcast/examples/latency/async/allocfree/AsyncPubStruct.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.examples.latency.async.allocfree; 2 | 3 | import org.HdrHistogram.Histogram; 4 | import org.nustaq.fastcast.api.FCPublisher; 5 | import org.nustaq.fastcast.api.FCSubscriber; 6 | import org.nustaq.fastcast.api.FastCast; 7 | import org.nustaq.fastcast.api.util.ObjectPublisher; 8 | import org.nustaq.fastcast.api.util.ObjectSubscriber; 9 | import org.nustaq.fastcast.examples.latency.async.AsyncLatMessage; 10 | import org.nustaq.fastcast.util.RateMeasure; 11 | import org.nustaq.offheap.bytez.Bytez; 12 | import org.nustaq.offheap.bytez.onheap.HeapBytez; 13 | import org.nustaq.offheap.structs.FSTStruct; 14 | import org.nustaq.offheap.structs.structtypes.StructString; 15 | import org.nustaq.offheap.structs.unsafeimpl.FSTStructFactory; 16 | 17 | import java.util.concurrent.Executor; 18 | import java.util.concurrent.Executors; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | /** 22 | * Created by ruedi on 26/01/15. 23 | * same as async test but using structs instead of serialization and some moderate memory reuse to reduce 24 | * allocation rate 25 | */ 26 | public class AsyncPubStruct { 27 | 28 | public static final String CFG_FILE_PATH = "src/main/java/org/nustaq/fastcast/examples/latency/async/fc.kson"; 29 | 30 | FastCast fastCast; 31 | FCPublisher pub; 32 | Histogram hi = new Histogram(TimeUnit.SECONDS.toNanos(2),3); 33 | Executor dumper = Executors.newCachedThreadPool(); 34 | 35 | public void initFastCast() throws Exception { 36 | fastCast = FastCast.getFastCast(); 37 | fastCast.setNodeId("PUB"); 38 | fastCast.loadConfig(CFG_FILE_PATH); 39 | 40 | pub = fastCast.onTransport("default").publish("stream"); 41 | 42 | // pointer to message 43 | final FSTStruct msg = FSTStructFactory.getInstance().createEmptyStructPointer(FSTStruct.class); 44 | 45 | fastCast.onTransport("back").subscribe( "back", new FCSubscriber() { 46 | @Override 47 | public void messageReceived(String sender, long sequence, Bytez b, long off, int len) { 48 | // as structs decode literally in zero time, we can decode inside receiver thread 49 | msg.baseOn(b, (int) off); 50 | Class type = msg.getPointedClass(); 51 | if ( type == StructString.class ) { 52 | // sequence end, print histogram 53 | final Histogram oldHi = hi; 54 | hi = new Histogram(TimeUnit.SECONDS.toNanos(2),3); 55 | // no lambdas to stay 1.7 compatible 56 | // move printing out of the receiving thread 57 | dumper.execute(new Runnable() { 58 | @Override 59 | public void run() { 60 | oldHi.outputPercentileDistribution(System.out,1000.0); 61 | } 62 | }); 63 | } else { // a regular message, record latency 64 | AsyncLatMessageStruct mdata = msg.cast(); 65 | long value = System.nanoTime() - mdata.getSendTimeStampNanos(); 66 | if ( value < 1_000_000_000 ) { // avoid AIOB during init + JITTING 67 | hi.recordValue(value); 68 | } 69 | // a real app would need to copy the message (recycle byte arrays/objects !) and 70 | // run msg processing in an executor to get out of the receiving thread. 71 | // mdata gets invalid on finish of this method 72 | } 73 | } 74 | 75 | @Override 76 | public boolean dropped() { 77 | System.out.println("DROPPED"); 78 | System.exit(-1); 79 | return false; 80 | } 81 | 82 | @Override 83 | public void senderTerminated(String senderNodeId) { 84 | 85 | } 86 | 87 | @Override 88 | public void senderBootstrapped(String receivesFrom, long seqNo) { 89 | 90 | } 91 | }); 92 | } 93 | 94 | public void run( int pauseNanos, int numEvents ) throws Throwable { 95 | RateMeasure report = new RateMeasure("send rate"); 96 | // alloc + init a single instance which is reused then. if no allocator is used, default is OnHeap Alloc (struct is byte[] backed) 97 | AsyncLatMessageStruct event = (AsyncLatMessageStruct) new AsyncLatMessageStruct(System.nanoTime(), 0, 0+1, 10, 110).toOffHeap(); 98 | byte underlying[] = event.getBase().asByteArray(); 99 | byte[] end = new StructString("END").toOffHeap().getBase().asByteArray(); 100 | for ( int i = 0; i < numEvents; i++ ) { 101 | double bidPrc = Math.random() * 10; 102 | event.setBidPrc(bidPrc); 103 | event.setAskPrc(bidPrc+1); 104 | event.setSendTimeStampNanos(System.nanoTime()); 105 | while( ! pub.offer(null, underlying, 0, underlying.length, true) ) { 106 | // spin 107 | } 108 | report.count(); 109 | long time = System.nanoTime(); 110 | while( System.nanoTime() - time < pauseNanos ) { 111 | // spin 112 | } 113 | } 114 | while( ! pub.offer(null, end, 0, end.length, true) ) { 115 | // spin 116 | } 117 | } 118 | 119 | public static void main(String arg[]) throws Throwable { 120 | AsyncPubStruct pub = new AsyncPubStruct(); 121 | 122 | pub.initFastCast(); 123 | while (true) 124 | pub.run( 1000, 5_000_000 ); // 93_000 = 10k, 27_000 = 30k, 10_500 = 70k, 4_900 = 140k 125 | 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /src/main/java/org/nustaq/fastcast/impl/BatchingController.java: -------------------------------------------------------------------------------- 1 | package org.nustaq.fastcast.impl; 2 | 3 | import org.nustaq.fastcast.util.FCLog; 4 | import org.nustaq.fastcast.util.RateMeasure; 5 | 6 | /** 7 | * Created by ruedi on 24.01.2015. 8 | */ 9 | public class BatchingController { 10 | 11 | public static final int SIZE_OBSERVATION_WINDOW_TICKS = 4; 12 | 13 | public static enum Action { 14 | NONE, 15 | BATCH, 16 | BLOCK 17 | } 18 | 19 | public static final int RATE_PRECISION = 20; 20 | volatile static long MinResolution = 0; 21 | 22 | long lastCheckTS; 23 | 24 | int packetCounter; 25 | 26 | long nanosPerTick; 27 | long ticksPerSec; 28 | int packetsPerTick; 29 | 30 | int ratePerSecond; // unused, just memoized 31 | 32 | public BatchingController(int ratePerSecond) { 33 | this.ratePerSecond = ratePerSecond; 34 | if ( MinResolution == 0 ) { 35 | calibrate(); 36 | } 37 | nanosPerTick = MinResolution; 38 | do { 39 | ticksPerSec = 1_000_000_000L / nanosPerTick; 40 | packetsPerTick = (int) (ratePerSecond / ticksPerSec); 41 | if (packetsPerTick < RATE_PRECISION) { 42 | nanosPerTick *=2; 43 | } 44 | } while ( packetsPerTick < RATE_PRECISION ); 45 | FCLog.get().info("rate limiting in window "+4*nanosPerTick+" ns ("+(4*nanosPerTick)/1000+" micros) ("+(4*nanosPerTick)/1000/1000+" millis)"); 46 | } 47 | 48 | public int getRatePerSecond() { 49 | return ratePerSecond; 50 | } 51 | 52 | @Override 53 | public String toString() { 54 | return "BatchingController{" + 55 | "nanosPerTick=" + nanosPerTick + 56 | ", ticksPerSec=" + ticksPerSec + 57 | ", packetsPerTick=" + packetsPerTick + 58 | ", actualRate=" + packetsPerTick*ticksPerSec + 59 | '}'; 60 | } 61 | 62 | private void calibrate() { 63 | long sum = 0; 64 | for ( int n = 0; n < 100; n++ ) { 65 | long nanos = System.nanoTime(); 66 | while (System.nanoTime() == nanos) { 67 | // spin 68 | } 69 | long diff = System.nanoTime() - nanos; 70 | // System.out.println(diff); 71 | sum += diff; 72 | } 73 | MinResolution = Math.max(1, sum / 100); 74 | System.out.println("timer resolution:"+MinResolution+" ns"); 75 | } 76 | 77 | public int getElapsedTicks() { 78 | final long now = System.nanoTime(); 79 | if (lastCheckTS == 0) { 80 | lastCheckTS = now; 81 | return 0; 82 | } 83 | if (now - lastCheckTS > SIZE_OBSERVATION_WINDOW_TICKS *nanosPerTick ) { 84 | long elapsedTicks = (now-lastCheckTS) / nanosPerTick; 85 | lastCheckTS += nanosPerTick; 86 | packetCounter -= packetsPerTick; 87 | return 0; 88 | } 89 | return (int) ((now -lastCheckTS)/ nanosPerTick); 90 | } 91 | 92 | public int getAllowedPackets() { 93 | return getElapsedTicks()*packetsPerTick+1; 94 | } 95 | 96 | public void countPacket() { 97 | packetCounter++; 98 | } 99 | 100 | public Action getAction() { 101 | final int allowedPackets = getAllowedPackets(); 102 | if ( packetCounter < allowedPackets ) { 103 | return Action.NONE; 104 | } else if ( packetCounter < allowedPackets + packetsPerTick*2 ) 105 | return Action.BATCH; 106 | return Action.BLOCK; 107 | } 108 | 109 | // 110 | // public void block() { 111 | // while ( packetCounter >= maxRatePerInterval) { 112 | // while( (System.nanoTime() - lastCheck) < interValNanos ) { 113 | // // spin 114 | // } 115 | // lastCheck = System.nanoTime(); 116 | // packetCounter -= maxRatePerInterval; 117 | // } 118 | // } 119 | 120 | public static void main( String arg[] ) { 121 | 122 | // System.out.println(new BatchingController(100)); 123 | // System.out.println(new BatchingController(200)); 124 | // System.out.println(new BatchingController(1000)); 125 | // System.out.println(new BatchingController(2000)); 126 | // System.out.println(new BatchingController(5000)); 127 | // System.out.println(new BatchingController(10000)); 128 | // System.out.println(new BatchingController(15000)); 129 | // System.out.println(new BatchingController(20000)); 130 | // System.out.println(new BatchingController(30000)); 131 | // System.out.println(new BatchingController(50000)); 132 | // System.out.println(new BatchingController(70000)); 133 | // System.out.println(new BatchingController(100000)); 134 | // System.out.println(new BatchingController(500000)); 135 | 136 | BatchingController limiter = new BatchingController(10000); 137 | RateMeasure m = new RateMeasure("msg rate"); 138 | RateMeasure pm = new RateMeasure("packet rate"); 139 | int msgPerPackAssumption = 4; 140 | int batchCount = 0; 141 | while( true ) { 142 | // LockSupport.parkNanos(1_000); 143 | // Sleeper.spinMicros(300); 144 | Action a = limiter.getAction(); 145 | if ( a == Action.NONE ) { 146 | limiter.countPacket(); 147 | pm.count(); 148 | m.count(); 149 | } else if ( a == Action.BATCH ) { 150 | if (batchCount == msgPerPackAssumption ) { 151 | limiter.countPacket(); 152 | pm.count(); 153 | batchCount = 0; 154 | } 155 | batchCount++; 156 | m.count(); 157 | } else /*block*/ { 158 | int x = 0; 159 | } 160 | } 161 | } 162 | } 163 | --------------------------------------------------------------------------------