├── .gitignore ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── assembly.xml ├── index.html ├── lib ├── kryo-1.01.jar ├── minlog-1.2.jar ├── reflectasm-0.8.jar └── zookeeper-3.1.1.jar ├── pom.xml ├── scripts ├── generate_load.sh ├── run_adapter.sh ├── run_client_adapter.sh └── s4_start.sh └── src ├── main ├── java │ └── io │ │ └── s4 │ │ ├── MainApp.java │ │ ├── adapter │ │ └── Adapter.java │ │ ├── client │ │ ├── Adapter.java │ │ ├── ClientConnection.java │ │ ├── ClientReadMode.java │ │ ├── ClientStub.java │ │ ├── ClientWriteMode.java │ │ ├── GenericJsonClientStub.java │ │ ├── Handshake.java │ │ ├── IOChannel.java │ │ ├── InputStub.java │ │ ├── OutputStub.java │ │ └── util │ │ │ └── ObjectBuilder.java │ │ ├── collector │ │ ├── Event.java │ │ ├── EventListener.java │ │ ├── EventRecord.java │ │ └── EventWrapper.java │ │ ├── dispatcher │ │ ├── Dispatcher.java │ │ ├── EventDispatcher.java │ │ ├── MultiDispatcher.java │ │ ├── StreamExcludingDispatcher.java │ │ ├── StreamSelectingDispatcher.java │ │ ├── partitioner │ │ │ ├── BroadcastPartitioner.java │ │ │ ├── CompoundKeyInfo.java │ │ │ ├── DefaultHasher.java │ │ │ ├── DefaultPartitioner.java │ │ │ ├── DummyPartitioner.java │ │ │ ├── HashAlgorithm.java │ │ │ ├── Hasher.java │ │ │ ├── KeyInfo.java │ │ │ ├── Partitioner.java │ │ │ ├── RoundRobinPartitioner.java │ │ │ ├── TestDefaultPartitioner.java │ │ │ └── VariableKeyPartitioner.java │ │ └── transformer │ │ │ └── Transformer.java │ │ ├── emitter │ │ ├── CommLayerEmitter.java │ │ └── EventEmitter.java │ │ ├── listener │ │ ├── CommLayerListener.java │ │ ├── EventHandler.java │ │ ├── EventListener.java │ │ └── EventProducer.java │ │ ├── logger │ │ ├── Log4jMonitor.java │ │ ├── Monitor.java │ │ └── TraceMessage.java │ │ ├── message │ │ ├── PrototypeRequest.java │ │ ├── Request.java │ │ ├── Response.java │ │ └── SinglePERequest.java │ │ ├── persist │ │ ├── ConMapPersister.java │ │ ├── DumpingPersister.java │ │ ├── HashMapPersister.java │ │ └── Persister.java │ │ ├── processor │ │ ├── AbstractPE.java │ │ ├── AbstractWindowingPE.java │ │ ├── AsynchronousEventProcessor.java │ │ ├── ControlEventProcessor.java │ │ ├── EventAdvice.java │ │ ├── JoinPE.java │ │ ├── OutputFormatter.java │ │ ├── OverloadDispatcher.java │ │ ├── OverloadDispatcherGenerator.java │ │ ├── OverloadDispatcherSlot.java │ │ ├── PEContainer.java │ │ ├── PrintEventPE.java │ │ ├── ProcessingElement.java │ │ ├── PrototypeWrapper.java │ │ ├── ReroutePE.java │ │ └── SimpleCountingPE.java │ │ ├── schema │ │ ├── AvroSchemaManager.java │ │ ├── Schema.java │ │ ├── SchemaContainer.java │ │ └── SchemaManager.java │ │ ├── serialize │ │ ├── AvroSerDeser.java │ │ ├── AvroUtils.java │ │ ├── KryoSerDeser.java │ │ └── SerializerDeserializer.java │ │ ├── test │ │ ├── TestPersisterEventClock.java │ │ └── TestPersisterWallClock.java │ │ └── util │ │ ├── AvroSchemaSupplementer.java │ │ ├── ByteArrayIOChannel.java │ │ ├── Cloner.java │ │ ├── ClonerGenerator.java │ │ ├── DoubleOutputFormatter.java │ │ ├── GsonUtil.java │ │ ├── KeyUtil.java │ │ ├── LoadGenerator.java │ │ ├── MethodInvoker.java │ │ ├── MetricsName.java │ │ ├── MiscConstants.java │ │ ├── NumberUtils.java │ │ ├── PreprodLogger.java │ │ ├── ReverseDoubleOutputFormatter.java │ │ ├── ReverseIntegerOutputFormatter.java │ │ ├── S4Util.java │ │ ├── SlotUtils.java │ │ ├── ToStringOutputFormatter.java │ │ ├── Watcher.java │ │ └── clock │ │ ├── Clock.java │ │ ├── ClockStreamsLoader.java │ │ ├── DrivenClock.java │ │ ├── EventClock.java │ │ ├── TimerRequest.java │ │ └── WallClock.java └── resources │ ├── adapter_conf.xml │ ├── client_adapter_conf.xml │ ├── client_stub_conf.xml │ ├── clusters_example.xml │ ├── event_clock.xml │ ├── log4j.xml │ ├── s4_core.properties_header_redbutton │ ├── s4_core.properties_header_typical │ ├── s4_core_conf_typical.xml │ └── wall_clock.xml └── test ├── java └── io │ └── s4 │ └── processor │ ├── MockPE.java │ └── TestPrototypeWrapper.java └── resources └── log4j.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # Eclipse meta-information 2 | /.project 3 | /.classpath 4 | /.settings 5 | 6 | # Build directory 7 | /target/ 8 | !/target/site/apidocs/ 9 | !/target/s4_core*.tar.gz 10 | 11 | # SVN 12 | .svn 13 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | ======================================================================= 2 | NOTICE file for use with, and corresponding to Section 4 of, 3 | the Apache License, Version 2.0, in this case for the S4 project. 4 | ========================================================================= 5 | 6 | This product includes software developed by 7 | Yahoo! Inc. (www.yahoo.com) 8 | Copyright (c) 2010 Yahoo! Inc. All rights reserved. 9 | 10 | This product includes software developed by 11 | The Apache Software Foundation (http://www.apache.org/). 12 | 13 | This product includes software (Jackson) developed by 14 | The Codehaus Foundation (http://www.codehaus.org/). 15 | 16 | Kryo binaries, distributed in lib/kryo-1.01.jar, developed by 17 | Nathan Sweet 18 | Copyright (c) 2008, Nathan Sweet. 19 | All rights reserved. 20 | 21 | json.org binaries 22 | Copyright (c) 2002 JSON.org 23 | Permission is hereby granted, free of charge, to any person 24 | obtaining a copy of this software and associated documentation 25 | files (the "Software"), to deal in the Software without 26 | restriction, including without limitation the rights to use, 27 | copy, modify, merge, publish, distribute, sublicense, and/or 28 | sell copies of the Software, and to permit persons to whom the 29 | Software is furnished to do so, subject to the following conditions: 30 | - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 31 | - The Software shall be used for Good, not Evil. 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | S4 Core Classes 2 | =============== 3 | 4 | Introduction 5 | ------------ 6 | This is a component of the S4 streaming system. For more information, see [s4.io](http://s4.io) 7 | 8 | Requirements 9 | ------------ 10 | 11 | * Linux 12 | * Java 1.6 13 | * Maven 14 | * S4 Communication Layer 15 | 16 | Build Instructions 17 | ------------------ 18 | 19 | 1. First build and install the comm package in your Maven repository. 20 | 21 | 2. Kryo, Reflectasm, and minlog must be installed to your local Maven repository manually. 22 | The jars are present in lib/ within this project. To install, run the following commands: 23 | 24 | mvn install:install-file -DgroupId=com.esotericsoftware -DartifactId=kryo -Dversion=1.01 -Dpackaging=jar -Dfile=lib/kryo-1.01.jar 25 | mvn install:install-file -DgroupId=com.esotericsoftware -DartifactId=reflectasm -Dversion=0.8 -Dpackaging=jar -Dfile=lib/reflectasm-0.8.jar 26 | mvn install:install-file -DgroupId=com.esotericsoftware -DartifactId=minlog -Dversion=1.2 -Dpackaging=jar -Dfile=lib/minlog-1.2.jar 27 | 28 | 3. Build and install using Maven 29 | 30 | mvn assembly:assembly install 31 | 32 | -------------------------------------------------------------------------------- /assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | tar.gz 6 | 7 | false 8 | 9 | 10 | false 11 | s4_core/lib 12 | runtime 13 | false 14 | 15 | 16 | 17 | 18 | ${project.basedir}/NOTICE.txt 19 | 20 | NOTICE.txt 21 | 22 | 23 | ${project.basedir}/src/main/resources/s4_core_conf_typical.xml 24 | s4_core/conf/typical 25 | s4_core_conf.xml 26 | 27 | 28 | ${project.basedir}/src/main/resources/wall_clock.xml 29 | s4_core/conf/typical 30 | wall_clock.xml 31 | 32 | 33 | ${project.basedir}/src/main/resources/event_clock.xml 34 | s4_core/conf/typical 35 | event_clock.xml 36 | 37 | 38 | ${project.basedir}/src/main/resources/s4_core.properties_header_typical 39 | s4_core/conf/typical 40 | s4_core.properties_header 41 | 42 | 43 | ${project.basedir}/src/main/resources/s4_core_conf_typical.xml 44 | s4_core/conf/redbutton 45 | s4_core_conf.xml 46 | 47 | 48 | ${project.basedir}/src/main/resources/wall_clock.xml 49 | s4_core/conf/redbutton 50 | wall_clock.xml 51 | 52 | 53 | ${project.basedir}/src/main/resources/event_clock.xml 54 | s4_core/conf/redbutton 55 | event_clock.xml 56 | 57 | 58 | ${project.basedir}/src/main/resources/s4_core.properties_header_redbutton 59 | s4_core/conf/redbutton 60 | s4_core.properties_header 61 | 62 | 63 | ${project.basedir}/src/main/resources/clusters_example.xml 64 | s4_core/conf/redbutton 65 | clusters.xml 66 | 67 | 68 | 69 | 70 | ${project.basedir}/target 71 | s4_core/lib 72 | 73 | *.jar 74 | 75 | 76 | 77 | ${project.basedir}/src/main/resources 78 | s4_core/conf/typical 79 | 80 | adapter_conf.xml 81 | log4j.xml 82 | 83 | 84 | 85 | ${project.basedir}/src/main/resources 86 | s4_core/conf/redbutton 87 | 88 | adapter_conf.xml 89 | client_adapter_conf.xml 90 | client_stub_conf.xml 91 | log4j.xml 92 | 93 | 94 | 95 | ${project.basedir}/scripts 96 | 0755 97 | bin 98 | 99 | s4_start.sh 100 | generate_load.sh 101 | run_adapter.sh 102 | run_client_adapter.sh 103 | 104 | 105 | 106 | ${project.basedir} 107 | s4_core/lock 108 | 109 | **/* 110 | 111 | 112 | 113 | ${project.basedir} 114 | s4_core/log 115 | 116 | **/* 117 | 118 | 119 | 120 | ${project.basedir}/ 121 | s4_apps 122 | 123 | **/* 124 | 125 | 126 | 127 | ${project.basedir} 128 | s4_exts 129 | 130 | **/* 131 | 132 | 133 | 134 | 135 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | S4 Core Classes 4 | 5 | 6 | Javadocs 7 | 8 | 9 | -------------------------------------------------------------------------------- /lib/kryo-1.01.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/s4/core/be285416364777531034950794ed545eb223d1ac/lib/kryo-1.01.jar -------------------------------------------------------------------------------- /lib/minlog-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/s4/core/be285416364777531034950794ed545eb223d1ac/lib/minlog-1.2.jar -------------------------------------------------------------------------------- /lib/reflectasm-0.8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/s4/core/be285416364777531034950794ed545eb223d1ac/lib/reflectasm-0.8.jar -------------------------------------------------------------------------------- /lib/zookeeper-3.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/s4/core/be285416364777531034950794ed545eb223d1ac/lib/zookeeper-3.1.1.jar -------------------------------------------------------------------------------- /scripts/generate_load.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | osx=false 4 | case "`uname`" in 5 | Darwin*) osx=true;; 6 | esac 7 | 8 | if $osx; then 9 | READLINK="stat" 10 | else 11 | READLINK="readlink" 12 | fi 13 | 14 | #--------------------------------------------- 15 | # USAGE and read arguments 16 | #--------------------------------------------- 17 | 18 | if [ "$1" == "-h" ]; then 19 | echo "Usage: $0" >&2 20 | echo " -c s4 core home" >&2 21 | echo " -a sender cluster name" >&2 22 | echo " -g listener cluster name (same as -a if not specified)" >&2 23 | echo " -z cluster manager address (hostname:port)" >&2 24 | echo " -r emit rate" >&2 25 | echo " -d rate display interval" >&2 26 | echo " -s comma delimited list of schema files" >&2 27 | echo " -u path to user defined event classes" >&2 28 | echo " -x redbutton mode (explicit cluster management)" >&2 29 | echo " -h help" >&2 30 | exit 1 31 | fi 32 | 33 | BASE_DIR=`dirname $($READLINK -f $0)` 34 | CORE_HOME=`$READLINK -f ${BASE_DIR}/../s4_core` 35 | CP_SEP=":" 36 | REDBUTTON_MODE="false" 37 | 38 | while getopts ":c:z:a:g:r:d:l:u:x" opt; 39 | do case "$opt" in 40 | c) CORE_HOME=$OPTARG;; 41 | z) CLUSTER_MANAGER=$OPTARG;; 42 | a) SENDER_CLUSTER_NAME=$OPTARG;; 43 | g) LISTENER_CLUSTER_NAME=$OPTARG;; 44 | r) RATE=$OPTARG;; 45 | d) DISPLAY_INTERVAL=$OPTARG;; 46 | l) LOCK_DIR=$OPTARG;; 47 | u) USER_CLASS_PATH=$OPTARG;; 48 | x) REDBUTTON_MODE="true";; 49 | \?) 50 | echo "Invalid option: -$OPTARG" >&2 51 | exit 1 52 | ;; 53 | :) 54 | echo "Option -$OPTARG requires an argument." >&2 55 | exit 1 56 | ;; 57 | esac 58 | done 59 | shift $(($OPTIND-1)) 60 | 61 | INPUT_FILE=$1 62 | 63 | if [ "x$CLUSTER_MANAGER" == "x" ] ; then 64 | CLUSTER_MANAGER="localhost:2181" 65 | fi 66 | 67 | if [ "x$SENDER_CLUSTER_NAME" == "x" ] ; then 68 | SENDER_CLUSTER_NAME="s4" 69 | fi 70 | 71 | if [ "x$LISTENER_CLUSTER_NAME" == "x" ] ; then 72 | LISTENER_CLUSTER_NAME=$SENDER_CLUSTER_NAME 73 | fi 74 | 75 | if [ "x$RATE" == "x" ] ; then 76 | RATE=80 77 | fi 78 | 79 | if [ "x$DISPLAY_INTERVAL" == "x" ] ; then 80 | DISPLAY_INTERVAL=15 81 | fi 82 | 83 | if [ "x$SCHEMA_FILE_LIST" != "x" ] ; then 84 | SCHEMA_FILE_LIST="${SCHEMA_FILE_LIST}," 85 | fi 86 | SCHEMA_FILE_LIST="${SCHEMA_FILE_LIST}${CORE_HOME}/conf/typical/schemas/EventWrapper_schema.js" 87 | 88 | if [ "x$LOCK_DIR" == "x" ] ; then 89 | LOCK_DIR="${CORE_HOME}/lock" 90 | fi 91 | 92 | echo "Cluster manager ${CLUSTER_MANAGER}" 93 | echo "Sender cluster name ${SENDER_CLUSTER_NAME}" 94 | echo "Listener cluster name ${LISTENER_CLUSTER_NAME}" 95 | echo "Rate ${RATE}" 96 | echo "Display interval ${DISPLAY_INTERVAL}" 97 | echo "Schema list ${SCHEMA_FILE_LIST}" 98 | 99 | JAVA_LOC="" 100 | if [ "x$JAVA_HOME" != "x" ] ; then 101 | JAVA_LOC=${JAVA_HOME}"/bin/" 102 | fi 103 | 104 | JAVA_OPTS="" 105 | if [ "x$LOCK_DIR" != "x" ] ; then 106 | JAVA_OPTS="$JAVA_OPTS -Dlock_dir=$LOCK_DIR " 107 | fi 108 | 109 | echo "java location is ${JAVA_LOC}" 110 | echo -n "JAVA VERSION=" 111 | echo `${JAVA_LOC}java -version` 112 | #--------------------------------------------- 113 | #ADDING CORE JARS TO CLASSPATH 114 | #--------------------------------------------- 115 | 116 | CLASSPATH=`find $CORE_HOME -name "*.jar" | awk '{p=$0"'$CP_SEP'"p;} END {print p}'` 117 | if [ $REDBUTTON_MODE == "true" ] ; then 118 | CLASSPATH=${CLASSPATH}${CP_SEP}${CORE_HOME}/conf/redbutton 119 | JAVA_OPTS="$JAVA_OPTS -Dcommlayer.mode=static" 120 | fi 121 | 122 | if [ "x$USER_CLASS_PATH" != "x" ] ; then 123 | CLASSPATH=${CLASSPATH}${CP_SEP}${USER_CLASS_PATH} 124 | fi 125 | 126 | 127 | CMD="${JAVA_LOC}java $JAVA_OPTS -classpath $CLASSPATH io.s4.util.LoadGenerator -a ${SENDER_CLUSTER_NAME} -g ${LISTENER_CLUSTER_NAME} -z ${CLUSTER_MANAGER} -r${RATE} -d ${DISPLAY_INTERVAL} $INPUT_FILE" 128 | echo "Running ${CMD}" 129 | $CMD 130 | -------------------------------------------------------------------------------- /scripts/run_adapter.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | get_property() 4 | { 5 | val=`sed '/^\#/d' ${CONF_LOC}/s4_core.properties_header | grep $1 | tail -n 1 | sed 's/^[^=]*=//;s/^[[:space:]]*//;s/[[:space:]]*$//'` 6 | echo "$val" 7 | } 8 | 9 | osx=false 10 | case "`uname`" in 11 | Darwin*) osx=true;; 12 | esac 13 | 14 | if $osx; then 15 | READLINK="stat" 16 | else 17 | READLINK="readlink" 18 | fi 19 | 20 | #--------------------------------------------- 21 | # USAGE and read arguments 22 | #--------------------------------------------- 23 | 24 | if [ "$1" == "-h" ]; then 25 | echo "Usage: $0" >&2 26 | echo " -c s4 core home" >&2 27 | echo " -a sender cluster name" >&2 28 | echo " -g listener cluster name (same as -a if not specified)" >&2 29 | echo " -z cluster manager address (hostname:port)" >&2 30 | echo " -u path to user defined data adapter classes" >&2 31 | echo " -x redbutton mode (explicit cluster management)" >&2 32 | echo " -d legacy data adapter conf file" >&2 33 | echo " -h help" >&2 34 | exit 1 35 | fi 36 | 37 | BASE_DIR=`dirname $($READLINK -f $0)` 38 | CORE_HOME=`$READLINK -f ${BASE_DIR}/../s4_core` 39 | APPS_HOME=`$READLINK -f ${BASE_DIR}/../s4_apps` 40 | CP_SEP=":" 41 | REDBUTTON_MODE="false" 42 | 43 | while getopts ":c:z:a:l:u:d:x" opt; 44 | do case "$opt" in 45 | c) CORE_HOME=$OPTARG;; 46 | a) APPS_HOME=$OPTARG;; 47 | z) CLUSTER_MANAGER=$OPTARG;; 48 | a) SENDER_CLUSTER_NAME=$OPTARG;; 49 | g) LISTENER_CLUSTER_NAME=$OPTARG;; 50 | l) LOCK_DIR=$OPTARG;; 51 | u) USER_CLASS_PATH=$OPTARG;; 52 | d) DATA_ADAPTER_CONF=$OPTARG;; 53 | x) REDBUTTON_MODE="true";; 54 | \?) 55 | echo "Invalid option: -$OPTARG" >&2 56 | exit 1 57 | ;; 58 | :) 59 | echo "Option -$OPTARG requires an argument." >&2 60 | exit 1 61 | ;; 62 | esac 63 | done 64 | shift $(($OPTIND-1)) 65 | 66 | CONF_TYPE=$1 67 | if [ "x$CONF_TYPE" == "x" ] ; then 68 | CONF_TYPE="redbutton" 69 | fi 70 | 71 | CONF_FILE=${CORE_HOME}"/conf/"${CONF_TYPE}"/adapter_conf.xml" 72 | CONF_LOC=`dirname $CONF_FILE` 73 | LOG_LOC="${CORE_HOME}/logs" 74 | COMMLAYER_MODE=$(get_property "commlayer_mode") 75 | 76 | if [ "x$CLUSTER_MANAGER" == "x" ] ; then 77 | CLUSTER_MANAGER="localhost:2181" 78 | fi 79 | 80 | if [ "x$SENDER_CLUSTER_NAME" == "x" ] ; then 81 | SENDER_CLUSTER_NAME="s4" 82 | fi 83 | 84 | if [ "x$LISTENER_CLUSTER_NAME" == "x" ] ; then 85 | LISTENER_CLUSTER_NAME=$SENDER_CLUSTER_NAME 86 | fi 87 | 88 | if [ "x$LOCK_DIR" == "x" ] ; then 89 | LOCK_DIR="${CORE_HOME}/lock" 90 | fi 91 | 92 | if [ "x$DATA_ADAPTER_CONF" == "x" ] ; then 93 | echo "No data adapter configuration specified" 94 | exit 1 95 | fi 96 | 97 | echo "Cluster manager ${CLUSTER_MANAGER}" 98 | echo "Sender cluster name ${SENDER_CLUSTER_NAME}" 99 | echo "Listener cluster name ${LISTENER_CLUSTER_NAME}" 100 | 101 | JAVA_LOC="" 102 | if [ "x$JAVA_HOME" != "x" ] ; then 103 | JAVA_LOC=${JAVA_HOME}"/bin/" 104 | fi 105 | 106 | JAVA_OPTS="" 107 | if [ "x$LOCK_DIR" != "x" ] ; then 108 | JAVA_OPTS="$JAVA_OPTS -Dlock_dir=$LOCK_DIR " 109 | fi 110 | 111 | JAVA_OPTS="$JAVA_OPTS -Dlog_loc=${LOG_LOC} " 112 | 113 | echo "java location is ${JAVA_LOC}" 114 | echo -n "JAVA VERSION=" 115 | echo `${JAVA_LOC}java -version` 116 | #--------------------------------------------- 117 | #ADDING CORE JARS TO CLASSPATH 118 | #--------------------------------------------- 119 | 120 | CLASSPATH=`find $CORE_HOME -name "*.jar" | awk '{p=$0"'$CP_SEP'"p;} END {print p}'` 121 | CLASSPATH=$CLASSPATH$CP_SEP`find $APPS_HOME -name "*.jar" | awk '{p=$0"'$CP_SEP'"p;} END {print p}'` 122 | JAVA_OPTS="$JAVA_OPTS -Dzk_session_timeout=5000" 123 | 124 | CLASSPATH=${CLASSPATH}${CP_SEP}${CONF_LOC} 125 | if [ $REDBUTTON_MODE == "true" ] ; then 126 | JAVA_OPTS="$JAVA_OPTS -Dcommlayer.mode=static" 127 | else 128 | JAVA_OPTS="$JAVA_OPTS -Dcommlayer.mode=${COMMLAYER_MODE}" 129 | fi 130 | 131 | if [ "x$USER_CLASS_PATH" != "x" ] ; then 132 | CLASSPATH=${CLASSPATH}${CP_SEP}${USER_CLASS_PATH} 133 | fi 134 | 135 | MKTEMP_ARGS="" 136 | 137 | if $osx ; then 138 | MKTEMP_ARGS="tmpXXXX" 139 | fi 140 | 141 | TMP1=`mktemp -d $MKTEMP_ARGS` 142 | echo "Temp is $TMP1" 143 | echo "appName=${SENDER_CLUSTER_NAME}" > $TMP1/adapter.properties 144 | echo "listenerAppName=${LISTENER_CLUSTER_NAME}" >> $TMP1/adapter.properties 145 | cat $TMP1/adapter.properties 146 | 147 | CLASSPATH=${CLASSPATH}${CP_SEP}${TMP1} 148 | 149 | CMD="${JAVA_LOC}java $JAVA_OPTS -Dlog4j.configuration=file:${CONF_LOC}/log4j.xml -classpath $CLASSPATH io.s4.adapter.Adapter -t ${CONF_TYPE} -c ${CORE_HOME} -d ${DATA_ADAPTER_CONF}" 150 | echo "Running ${CMD}" 151 | $CMD 152 | -------------------------------------------------------------------------------- /scripts/run_client_adapter.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | get_property() 4 | { 5 | val=`sed '/^\#/d' ${CONF_LOC}/s4_core.properties_header | grep $1 | tail -n 1 | sed 's/^[^=]*=//;s/^[[:space:]]*//;s/[[:space:]]*$//'` 6 | echo "$val" 7 | } 8 | 9 | osx=false 10 | case "`uname`" in 11 | Darwin*) osx=true;; 12 | esac 13 | 14 | if $osx; then 15 | READLINK="stat" 16 | else 17 | READLINK="readlink" 18 | fi 19 | 20 | #--------------------------------------------- 21 | # USAGE and read arguments 22 | #--------------------------------------------- 23 | 24 | if [ "$1" == "-h" ]; then 25 | echo "Usage: $0" >&2 26 | echo " -c s4 core home" >&2 27 | echo " -a s4 apps home" >&2 28 | echo " -s sender cluster name" >&2 29 | echo " -g listener cluster name (same as -s if not specified)" >&2 30 | echo " -z cluster manager address (hostname:port)" >&2 31 | echo " -u path to user defined data adapter classes" >&2 32 | echo " -x redbutton mode (explicit cluster management)" >&2 33 | echo " -d client stub conf file" >&2 34 | echo " -h help" >&2 35 | exit 1 36 | fi 37 | 38 | BASE_DIR=`dirname $($READLINK -f $0)` 39 | CORE_HOME=`$READLINK -f ${BASE_DIR}/../s4_core` 40 | APPS_HOME=`$READLINK -f ${BASE_DIR}/../s4_apps` 41 | CP_SEP=":" 42 | REDBUTTON_MODE="false" 43 | 44 | while getopts ":c:z:a:s:g:l:u:d:x" opt; 45 | do case "$opt" in 46 | c) CORE_HOME=$OPTARG;; 47 | a) APPS_HOME=$OPTARG;; 48 | z) CLUSTER_MANAGER=$OPTARG;; 49 | s) SENDER_CLUSTER_NAME=$OPTARG;; 50 | g) LISTENER_CLUSTER_NAME=$OPTARG;; 51 | l) LOCK_DIR=$OPTARG;; 52 | u) USER_CLASS_PATH=$OPTARG;; 53 | d) DATA_ADAPTER_CONF=$OPTARG;; 54 | x) REDBUTTON_MODE="true";; 55 | \?) 56 | echo "Invalid option: -$OPTARG" >&2 57 | exit 1 58 | ;; 59 | :) 60 | echo "Option -$OPTARG requires an argument." >&2 61 | exit 1 62 | ;; 63 | esac 64 | done 65 | shift $(($OPTIND-1)) 66 | 67 | CONF_TYPE=$1 68 | if [ "x$CONF_TYPE" == "x" ] ; then 69 | CONF_TYPE="redbutton" 70 | fi 71 | 72 | CONF_FILE=${CORE_HOME}"/conf/"${CONF_TYPE}"/adapter_conf.xml" 73 | CONF_LOC=`dirname $CONF_FILE` 74 | LOG_LOC="${CORE_HOME}/logs" 75 | COMMLAYER_MODE=$(get_property "commlayer_mode") 76 | 77 | if [ "x$CLUSTER_MANAGER" == "x" ] ; then 78 | CLUSTER_MANAGER="localhost:2181" 79 | fi 80 | 81 | if [ "x$SENDER_CLUSTER_NAME" == "x" ] ; then 82 | SENDER_CLUSTER_NAME="s4" 83 | fi 84 | 85 | if [ "x$LISTENER_CLUSTER_NAME" == "x" ] ; then 86 | LISTENER_CLUSTER_NAME=$SENDER_CLUSTER_NAME 87 | fi 88 | 89 | if [ "x$LOCK_DIR" == "x" ] ; then 90 | LOCK_DIR="${CORE_HOME}/lock" 91 | fi 92 | 93 | if [ "x$DATA_ADAPTER_CONF" == "x" ] ; then 94 | echo "No data adapter configuration specified" 95 | exit 1 96 | fi 97 | 98 | echo "Cluster manager ${CLUSTER_MANAGER}" 99 | echo "Sender cluster name ${SENDER_CLUSTER_NAME}" 100 | echo "Listener cluster name ${LISTENER_CLUSTER_NAME}" 101 | 102 | JAVA_LOC="" 103 | if [ "x$JAVA_HOME" != "x" ] ; then 104 | JAVA_LOC=${JAVA_HOME}"/bin/" 105 | fi 106 | 107 | JAVA_OPTS="" 108 | if [ "x$LOCK_DIR" != "x" ] ; then 109 | JAVA_OPTS="$JAVA_OPTS -Dlock_dir=$LOCK_DIR " 110 | fi 111 | 112 | JAVA_OPTS="$JAVA_OPTS -Dlog_loc=${LOG_LOC} " 113 | 114 | echo "java location is ${JAVA_LOC}" 115 | echo -n "JAVA VERSION=" 116 | echo `${JAVA_LOC}java -version` 117 | #--------------------------------------------- 118 | #ADDING CORE JARS TO CLASSPATH 119 | #--------------------------------------------- 120 | 121 | CLASSPATH=`find $CORE_HOME -name "*.jar" | awk '{p=$0"'$CP_SEP'"p;} END {print p}'` 122 | CLASSPATH=$CLASSPATH$CP_SEP`find $APPS_HOME -name "*.jar" | awk '{p=$0"'$CP_SEP'"p;} END {print p}'` 123 | JAVA_OPTS="$JAVA_OPTS -Dzk_session_timeout=5000" 124 | 125 | CLASSPATH=${CLASSPATH}${CP_SEP}${CONF_LOC} 126 | if [ $REDBUTTON_MODE == "true" ] ; then 127 | JAVA_OPTS="$JAVA_OPTS -Dcommlayer.mode=static" 128 | else 129 | JAVA_OPTS="$JAVA_OPTS -Dcommlayer.mode=${COMMLAYER_MODE}" 130 | fi 131 | 132 | if [ "x$USER_CLASS_PATH" != "x" ] ; then 133 | CLASSPATH=${CLASSPATH}${CP_SEP}${USER_CLASS_PATH} 134 | fi 135 | 136 | MKTEMP_ARGS="" 137 | 138 | if $osx ; then 139 | MKTEMP_ARGS="tmpXXXX" 140 | fi 141 | 142 | TMP1=`mktemp -d $MKTEMP_ARGS` 143 | echo "Temp is $TMP1" 144 | echo "appName=${SENDER_CLUSTER_NAME}" > $TMP1/adapter.properties 145 | echo "listenerAppName=${LISTENER_CLUSTER_NAME}" >> $TMP1/adapter.properties 146 | echo "listener_max_queue_size=8000" >> $TMP1/adapter.properties 147 | cat $TMP1/adapter.properties 148 | 149 | CLASSPATH=${CLASSPATH}${CP_SEP}${TMP1} 150 | 151 | CMD="${JAVA_LOC}java $JAVA_OPTS -Dlog4j.configuration=file:${CONF_LOC}/log4j.xml -classpath $CLASSPATH io.s4.client.Adapter -t ${CONF_TYPE} -c ${CORE_HOME} -d ${DATA_ADAPTER_CONF}" 152 | echo "Running ${CMD}" 153 | $CMD 154 | -------------------------------------------------------------------------------- /src/main/java/io/s4/client/ClientConnection.java: -------------------------------------------------------------------------------- 1 | package io.s4.client; 2 | 3 | import io.s4.collector.EventWrapper; 4 | import io.s4.message.Request; 5 | 6 | import java.io.IOException; 7 | import java.net.Socket; 8 | import java.util.HashSet; 9 | import java.util.List; 10 | import java.util.UUID; 11 | 12 | /** 13 | * Connection to a client. A Stub has a collection of connections. 14 | */ 15 | public class ClientConnection { 16 | /** 17 | * 18 | */ 19 | private final ClientStub clientStub; 20 | 21 | /** 22 | * TCP/IP socket used to communicate with client. 23 | */ 24 | private final Socket socket; 25 | 26 | public final IOChannel io; 27 | 28 | public final ClientReadMode clientReadMode; 29 | public final ClientWriteMode clientWriteMode; 30 | 31 | /** 32 | * GUID of client. 33 | */ 34 | public final UUID uuid; 35 | 36 | public ClientConnection(ClientStub clientStub, Socket socket, UUID uuid, 37 | ClientReadMode clientReadMode, ClientWriteMode clientWriteMode) 38 | throws IOException { 39 | this.clientStub = clientStub; 40 | this.uuid = uuid; 41 | this.socket = socket; 42 | this.io = this.clientStub.createIOChannel(socket); 43 | this.clientReadMode = clientReadMode; 44 | this.clientWriteMode = clientWriteMode; 45 | } 46 | 47 | public boolean good() { 48 | return socket.isConnected(); 49 | } 50 | 51 | public void close() { 52 | synchronized (this.clientStub.clients) { 53 | ClientStub.logger.info("closing connection to client " + uuid); 54 | this.clientStub.clients.remove(this.uuid); 55 | } 56 | 57 | try { 58 | socket.close(); 59 | } catch (IOException e) { 60 | ClientStub.logger.error("problem closing client connection to client " 61 | + uuid, 62 | e); 63 | } 64 | } 65 | 66 | private HashSet includeStreams = new HashSet(); 67 | private HashSet excludeStreams = new HashSet(); 68 | 69 | public void includeStreams(List streams) { 70 | includeStreams.addAll(streams); 71 | } 72 | 73 | public void excludeStreams(List streams) { 74 | excludeStreams.addAll(streams); 75 | } 76 | 77 | /** 78 | * Stream is accepted if and only if: 79 | * 80 | * (A) readMode is Select and: 1. indifferent to stream inclusion OR stream 81 | * is included AND 2. indifferent to stream exclusion OR stream is not 82 | * excluded. 83 | * 84 | * OR (B) readMode is All 85 | * 86 | * @return true if and only if stream is accepted 87 | */ 88 | public boolean streamAccepted(String s) { 89 | switch (clientReadMode) { 90 | case None: 91 | case Private: 92 | return false; 93 | 94 | case Select: 95 | return (includeStreams.isEmpty() || includeStreams.contains(s)) 96 | && (excludeStreams.isEmpty() || !excludeStreams.contains(s)); 97 | 98 | case All: 99 | return true; 100 | 101 | default: 102 | return false; 103 | } 104 | } 105 | 106 | private Thread receiverThread = null; 107 | 108 | public void start() { 109 | if (clientWriteMode == ClientWriteMode.Enabled) 110 | (receiverThread = new Thread(receiver)).start(); 111 | } 112 | 113 | public void stop() { 114 | if (receiverThread != null) { 115 | receiverThread.interrupt(); 116 | receiverThread = null; 117 | } 118 | } 119 | 120 | public final Runnable receiver = new Runnable() { 121 | public void run() { 122 | try { 123 | while (good()) { 124 | byte[] b = io.recv(); 125 | 126 | // null, empty => goodbye 127 | if (b == null || b.length == 0) { 128 | ClientStub.logger.info("client session ended " + uuid); 129 | break; 130 | } 131 | 132 | EventWrapper ew = ClientConnection.this.clientStub.eventWrapperFromBytes(b); 133 | if (ew == null) 134 | continue; 135 | 136 | Object event = ew.getEvent(); 137 | if (event instanceof Request) { 138 | decorateRequest((Request) event); 139 | ClientStub.logger.info("Decorated client request: " 140 | + ew.toString()); 141 | } 142 | 143 | ClientConnection.this.clientStub.injectEvent(ew); 144 | } 145 | } catch (IOException e) { 146 | ClientStub.logger.info("error while reading from client " 147 | + uuid, e); 148 | 149 | } finally { 150 | close(); 151 | } 152 | } 153 | 154 | private void decorateRequest(Request r) { 155 | // add UUID of client into request. 156 | Request.RInfo info = r.getRInfo(); 157 | 158 | if (info != null && info instanceof Request.ClientRInfo) 159 | ((Request.ClientRInfo) info).setRequesterUUID(ClientConnection.this.uuid); 160 | } 161 | }; 162 | 163 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/client/ClientReadMode.java: -------------------------------------------------------------------------------- 1 | package io.s4.client; 2 | 3 | /** 4 | * Client mode. 5 | */ 6 | public enum ClientReadMode { 7 | None(false, false), Private(true, false), Select(true, true), All(true, true); 8 | 9 | private final boolean priv; 10 | private final boolean pub; 11 | 12 | ClientReadMode(boolean priv, boolean pub) { 13 | this.priv = priv; 14 | this.pub = pub; 15 | } 16 | 17 | public boolean takePublic() { 18 | return pub; 19 | } 20 | 21 | public boolean takePrivate() { 22 | return priv; 23 | } 24 | 25 | public static ClientReadMode fromString(String s) { 26 | if (s.equalsIgnoreCase("none")) 27 | return None; 28 | else if (s.equalsIgnoreCase("private")) 29 | return Private; 30 | else if (s.equalsIgnoreCase("select")) 31 | return Select; 32 | else if (s.equalsIgnoreCase("all")) 33 | return All; 34 | else 35 | return null; 36 | } 37 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/client/ClientWriteMode.java: -------------------------------------------------------------------------------- 1 | package io.s4.client; 2 | 3 | /** 4 | * Client's write mode. 5 | */ 6 | public enum ClientWriteMode { 7 | Enabled, Disabled; 8 | 9 | public static ClientWriteMode fromString(String s) { 10 | if (s.equalsIgnoreCase("enabled")) 11 | return Enabled; 12 | else if (s.equalsIgnoreCase("disabled")) 13 | return Disabled; 14 | else 15 | return null; 16 | } 17 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/client/GenericJsonClientStub.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.client; 17 | 18 | import io.s4.client.util.ObjectBuilder; 19 | import io.s4.collector.EventWrapper; 20 | import io.s4.util.GsonUtil; 21 | 22 | import java.nio.charset.Charset; 23 | 24 | import org.json.JSONArray; 25 | import org.json.JSONException; 26 | import org.json.JSONObject; 27 | 28 | public class GenericJsonClientStub extends ClientStub { 29 | 30 | // private static final ObjectBuilder builder = new ObjectBuilder(); 31 | // private static final Gson builder = new Gson(); 32 | 33 | private static final Info protocolInfo = new Info("generic-json", 1, 0); 34 | 35 | @Override 36 | public Info getProtocolInfo() { 37 | return protocolInfo; 38 | } 39 | 40 | @Override 41 | public EventWrapper eventWrapperFromBytes(byte[] v) { 42 | try { 43 | // interpret v as a JSON string 44 | String s = new String(v, Charset.forName("UTF8")); 45 | JSONObject json = new JSONObject(s); 46 | 47 | String streamName = json.getString("stream"); 48 | String className = json.getString("class"); 49 | 50 | Class clazz; 51 | try { 52 | clazz = Class.forName(className); 53 | } catch (ClassNotFoundException e) { 54 | throw new ObjectBuilder.Exception("bad class name for json-encoded object: " 55 | + className, 56 | e); 57 | } 58 | 59 | String[] keyNames = null; 60 | JSONArray keyArray = json.optJSONArray("keys"); 61 | if (keyArray != null) { 62 | keyNames = new String[keyArray.length()]; 63 | for (int i = 0; i < keyNames.length; ++i) { 64 | keyNames[i] = keyArray.optString(i); 65 | } 66 | } 67 | 68 | String jevent = json.getString("object"); 69 | 70 | Object obj = GsonUtil.get().fromJson(jevent, clazz); 71 | 72 | return new EventWrapper(streamName, keyNames, obj); 73 | 74 | } catch (JSONException e) { 75 | logger.error("problem with event JSON", e); 76 | } catch (ObjectBuilder.Exception e) { 77 | logger.error("failed to build object from JSON", e); 78 | } 79 | 80 | return null; 81 | } 82 | 83 | @Override 84 | public byte[] bytesFromEventWrapper(EventWrapper ew) { 85 | JSONObject jevent = new JSONObject(); 86 | 87 | Object obj = ew.getEvent(); 88 | 89 | try { 90 | jevent.put("stream", ew.getStreamName()); 91 | jevent.put("class", obj.getClass().getName()); 92 | jevent.put("object", GsonUtil.get().toJson(obj)); 93 | 94 | return jevent.toString().getBytes(Charset.forName("UTF8")); 95 | 96 | } catch (JSONException e) { 97 | logger.error("exception while converting event wrapper to bytes.", 98 | e); 99 | return null; 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/io/s4/client/IOChannel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.client; 17 | 18 | import java.io.IOException; 19 | 20 | public interface IOChannel { 21 | byte[] recv() throws IOException; 22 | 23 | void send(byte[] v) throws IOException; 24 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/client/InputStub.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.client; 17 | 18 | import io.s4.listener.EventProducer; 19 | 20 | public interface InputStub extends EventProducer { 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/io/s4/client/OutputStub.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.client; 17 | 18 | import io.s4.processor.AsynchronousEventProcessor; 19 | 20 | import java.util.List; 21 | 22 | public interface OutputStub extends AsynchronousEventProcessor { 23 | 24 | List getAcceptedStreams(); 25 | 26 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/collector/Event.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.collector; 17 | 18 | import io.s4.dispatcher.Dispatcher; 19 | import io.s4.dispatcher.partitioner.CompoundKeyInfo; 20 | import io.s4.dispatcher.partitioner.KeyInfo; 21 | import io.s4.dispatcher.partitioner.KeyInfo.KeyPathElement; 22 | import io.s4.dispatcher.partitioner.KeyInfo.KeyPathElementIndex; 23 | import io.s4.dispatcher.partitioner.KeyInfo.KeyPathElementName; 24 | 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | public class Event extends EventRecord { 30 | private String eventName; 31 | private long timestamp; 32 | private List compoundKeys = new ArrayList(); 33 | private boolean debug = false; 34 | public static final String EVENT_NAME_KEY = "S4__eventName"; 35 | public static final String TIMESTAMP_KEY = "S4__timestamp"; 36 | 37 | public void setDebug(boolean debug) { 38 | this.debug = debug; 39 | } 40 | 41 | public Event(Map passedEventData) { 42 | super(passedEventData); 43 | 44 | eventName = this.get(EVENT_NAME_KEY, "unknown"); 45 | timestamp = this.get(TIMESTAMP_KEY, -1L); 46 | 47 | List plainCompoundKeyList = null; 48 | if ((plainCompoundKeyList = get(Dispatcher.PARTITION_INFO_KEY, 49 | EMPTY_LIST)) != EMPTY_LIST) { 50 | for (EventRecord plainCompoundKeyInfo : plainCompoundKeyList) { 51 | CompoundKeyInfo compoundKeyInfo = new CompoundKeyInfo(); 52 | compoundKeyInfo.setCompoundValue(plainCompoundKeyInfo.get("compoundValue", 53 | (String) null)); 54 | compoundKeyInfo.setCompoundKey(plainCompoundKeyInfo.get("compoundKey", 55 | (String) null)); 56 | compoundKeys.add(compoundKeyInfo); 57 | for (EventRecord plainKeyInfo : plainCompoundKeyInfo.get("keyInfoList", 58 | EMPTY_LIST)) { 59 | KeyInfo keyInfo = new KeyInfo(); 60 | for (EventRecord plainKeyPathElement : plainKeyInfo.get("keyPathElementList", 61 | EMPTY_LIST)) { 62 | String keyName = plainKeyPathElement.get("keyName", 63 | (String) null); 64 | Integer index = plainKeyPathElement.get("index", 65 | (Integer) null); 66 | 67 | if (keyName != null) { 68 | keyInfo.addElementToPath(keyName); 69 | } else if (index != null) { 70 | keyInfo.addElementToPath(index); 71 | } 72 | } 73 | compoundKeyInfo.addKeyInfo(keyInfo); 74 | } 75 | } 76 | } 77 | if (debug) { 78 | for (CompoundKeyInfo compoundKeyInfo : compoundKeys) { 79 | System.out.println("CompoundKey: " 80 | + compoundKeyInfo.getCompoundValue()); 81 | for (KeyInfo keyInfo : compoundKeyInfo.getKeyInfoList()) { 82 | String keyPath = ""; 83 | for (KeyPathElement keyPathElement : keyInfo.getKeyPath()) { 84 | if (keyPathElement instanceof KeyPathElementIndex) { 85 | keyPath += "[" 86 | + ((KeyPathElementIndex) keyPathElement).getIndex() 87 | + "]"; 88 | } else { 89 | if (keyPath.length() > 0) { 90 | keyPath += "/"; 91 | } 92 | keyPath += ((KeyPathElementName) keyPathElement).getKeyName(); 93 | } 94 | } 95 | System.out.println(" " + keyPath); 96 | } 97 | } 98 | } 99 | } 100 | 101 | public List getCompoundKeys() { 102 | return compoundKeys; 103 | } 104 | 105 | public String getEventName() { 106 | return eventName; 107 | } 108 | 109 | public long getTimeStamp() { 110 | return timestamp; 111 | } 112 | 113 | public List> getCompoundKeyList() { 114 | return get(Dispatcher.PARTITION_INFO_KEY, 115 | new ArrayList>()); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/io/s4/collector/EventListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.collector; 17 | 18 | import static io.s4.util.MetricsName.S4_CORE_METRICS; 19 | import static io.s4.util.MetricsName.S4_EVENT_METRICS; 20 | import static io.s4.util.MetricsName.generic_listener_msg_in_ct; 21 | import io.s4.listener.EventHandler; 22 | import io.s4.logger.Monitor; 23 | import io.s4.processor.AsynchronousEventProcessor; 24 | import io.s4.processor.PEContainer; 25 | 26 | import org.apache.log4j.Logger; 27 | 28 | public class EventListener implements EventHandler { 29 | private static Logger logger = Logger.getLogger(EventListener.class); 30 | private int eventCount = 0; 31 | private AsynchronousEventProcessor eventProcessor; 32 | private io.s4.listener.EventListener rawListener; 33 | private Monitor monitor; 34 | 35 | public void setMonitor(Monitor monitor) { 36 | this.monitor = monitor; 37 | } 38 | 39 | public void setPeContainer(PEContainer peContainer) { 40 | this.eventProcessor = peContainer; 41 | } 42 | 43 | public void setEventProcessor(AsynchronousEventProcessor eventProcessor) { 44 | this.eventProcessor = eventProcessor; 45 | } 46 | 47 | public void setRawListener(io.s4.listener.EventListener rawListener) { 48 | this.rawListener = rawListener; 49 | } 50 | 51 | public io.s4.listener.EventListener getRawListener() { 52 | return this.rawListener; 53 | } 54 | 55 | public int getEventCount() { 56 | return eventCount; 57 | } 58 | 59 | public EventListener() { 60 | 61 | } 62 | 63 | public void init() { 64 | rawListener.addHandler(this); 65 | } 66 | 67 | public void processEvent(EventWrapper eventWrapper) { 68 | try { 69 | synchronized (this) { 70 | eventCount++; 71 | } 72 | if (logger.isDebugEnabled()) { 73 | logger.debug("STEP 3 (EventListener): peContainer.addEvent - " 74 | + eventWrapper.getEvent().toString()); 75 | } 76 | eventProcessor.queueWork(eventWrapper); 77 | 78 | if (monitor != null) { 79 | monitor.increment(generic_listener_msg_in_ct.toString(), 80 | 1, 81 | S4_EVENT_METRICS.toString(), 82 | "et", 83 | eventWrapper.getStreamName()); 84 | monitor.increment(generic_listener_msg_in_ct.toString(), 85 | 1, 86 | S4_CORE_METRICS.toString()); 87 | } 88 | } catch (Exception e) { 89 | logger.error("Exception in processEvent on thread " 90 | + Thread.currentThread().getId(), e); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/io/s4/collector/EventWrapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.collector; 17 | 18 | import io.s4.dispatcher.partitioner.CompoundKeyInfo; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | import java.util.StringTokenizer; 23 | 24 | public class EventWrapper { 25 | private List compoundKeys = null; 26 | private List> compoundKeyNames = null; 27 | private Object event; 28 | private String streamName; 29 | 30 | public List getCompoundKeys() { 31 | return compoundKeys; 32 | } 33 | 34 | public Object getEvent() { 35 | return event; 36 | } 37 | 38 | public String getStreamName() { 39 | return streamName; 40 | } 41 | 42 | public List> getCompoundKeyNames() { 43 | return compoundKeyNames; 44 | } 45 | 46 | public EventWrapper() { 47 | compoundKeys = new ArrayList(); 48 | } 49 | 50 | public EventWrapper(String streamName, Object event, 51 | List compoundKeys) { 52 | this.streamName = streamName; 53 | this.event = event; 54 | this.compoundKeys = compoundKeys; 55 | } 56 | 57 | public EventWrapper(String streamName, String[] compoundKeyStrings, 58 | Object event) { 59 | this.streamName = streamName; 60 | this.event = event; 61 | 62 | if (compoundKeyStrings != null) { 63 | this.compoundKeyNames = new ArrayList>(compoundKeyStrings.length); 64 | 65 | for (String keyAsString : compoundKeyStrings) { 66 | List keyNameElements = new ArrayList(); 67 | StringTokenizer st = new StringTokenizer(keyAsString, "/"); 68 | while (st.hasMoreTokens()) { 69 | keyNameElements.add(st.nextToken()); 70 | } 71 | compoundKeyNames.add(keyNameElements); 72 | } 73 | } 74 | } 75 | 76 | public String toString() { 77 | return "stream:" + getStreamName() + " keys:" + getCompoundKeys() 78 | + " keyNames:" + getCompoundKeyNames() + " event:" + getEvent(); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/EventDispatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher; 17 | 18 | import java.util.List; 19 | 20 | public interface EventDispatcher { 21 | 22 | /** 23 | * Dispatch event using stream name. Partitioners may be used to partition 24 | * the event, possibly based on a pre-determined set of fixed named keys. 25 | * 26 | * @param streamName 27 | * name of stream to dispatch on 28 | * @param event 29 | * object to dispatch 30 | */ 31 | void dispatchEvent(String streamName, Object event); 32 | 33 | /** 34 | * Dispatch event using a stream name and using a set of named keys. 35 | * VariableKeyPartitioners may be used to partition the event. 36 | * 37 | * @param streamName 38 | * name of stream to dispatch on 39 | * @param compoundKeyNames 40 | * keys to use for dispatching 41 | * @param event 42 | * object to dispatch 43 | */ 44 | void dispatchEvent(String streamName, List> compoundKeyNames, 45 | Object event); 46 | 47 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/MultiDispatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * Dispatcher that sends events through multiple abstract dispatchers. 22 | */ 23 | public class MultiDispatcher implements EventDispatcher { 24 | 25 | private EventDispatcher[] dispatchers = null; 26 | 27 | public void setDispatchers(EventDispatcher[] dispatchers) { 28 | this.dispatchers = dispatchers; 29 | } 30 | 31 | @Override 32 | public void dispatchEvent(String streamName, Object event) { 33 | if (dispatchers != null) { 34 | for (EventDispatcher dispatcher : dispatchers) { 35 | dispatcher.dispatchEvent(streamName, event); 36 | } 37 | } 38 | } 39 | 40 | @Override 41 | public void dispatchEvent(String streamName, 42 | List> compoundKeyNames, Object event) { 43 | if (dispatchers != null) { 44 | for (EventDispatcher dispatcher : dispatchers) { 45 | dispatcher.dispatchEvent(streamName, compoundKeyNames, event); 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/StreamExcludingDispatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher; 17 | 18 | import java.util.Arrays; 19 | import java.util.HashSet; 20 | import java.util.List; 21 | 22 | public class StreamExcludingDispatcher implements EventDispatcher { 23 | 24 | private EventDispatcher dispatcher = null; 25 | 26 | public void setDispatcher(EventDispatcher dispatcher) { 27 | this.dispatcher = dispatcher; 28 | } 29 | 30 | private HashSet streams = null; 31 | 32 | public void setStreams(String[] streams) { 33 | this.streams = new HashSet(Arrays.asList(streams)); 34 | } 35 | 36 | @Override 37 | public void dispatchEvent(String streamName, Object event) { 38 | if (dispatcher != null 39 | && (streams == null || !streams.contains(streamName))) { 40 | dispatcher.dispatchEvent(streamName, event); 41 | } 42 | } 43 | 44 | @Override 45 | public void dispatchEvent(String streamName, 46 | List> compoundKeyNames, Object event) { 47 | if (dispatcher != null 48 | && (streams == null || !streams.contains(streamName))) { 49 | dispatcher.dispatchEvent(streamName, compoundKeyNames, event); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/StreamSelectingDispatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher; 17 | 18 | import java.util.Arrays; 19 | import java.util.HashSet; 20 | import java.util.List; 21 | 22 | public class StreamSelectingDispatcher implements EventDispatcher { 23 | 24 | private EventDispatcher dispatcher = null; 25 | 26 | public void setDispatcher(EventDispatcher dispatcher) { 27 | this.dispatcher = dispatcher; 28 | } 29 | 30 | private HashSet streams = null; 31 | 32 | public void setStreams(String[] streams) { 33 | this.streams = new HashSet(Arrays.asList(streams)); 34 | } 35 | 36 | @Override 37 | public void dispatchEvent(String streamName, Object event) { 38 | if (dispatcher != null && streams != null 39 | && streams.contains(streamName)) { 40 | dispatcher.dispatchEvent(streamName, event); 41 | } 42 | } 43 | 44 | @Override 45 | public void dispatchEvent(String streamName, 46 | List> compoundKeyNames, Object event) { 47 | if (dispatcher != null && streams != null 48 | && streams.contains(streamName)) { 49 | dispatcher.dispatchEvent(streamName, compoundKeyNames, event); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/BroadcastPartitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | /** 22 | * Partition events to be sent to all parts. This can be used in conjunction 23 | * with a Dispatcher to broadcast events. 24 | */ 25 | public class BroadcastPartitioner implements Partitioner { 26 | @Override 27 | public List partition(String streamName, Object event, 28 | int partitionCount) { 29 | 30 | List partitionInfoList = new ArrayList(); 31 | 32 | // assign to all partitions 33 | for (int i = 0; i < partitionCount; ++i) { 34 | CompoundKeyInfo partitionInfo = new CompoundKeyInfo(); 35 | 36 | partitionInfo.setPartitionId(i); 37 | 38 | partitionInfoList.add(partitionInfo); 39 | } 40 | 41 | return partitionInfoList; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/CompoundKeyInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import java.io.Serializable; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | public class CompoundKeyInfo implements Serializable { 23 | List keyInfoList = new ArrayList(); 24 | int partitionId = -1; 25 | String compoundValue; 26 | String compoundKey; 27 | 28 | public CompoundKeyInfo() { 29 | } 30 | 31 | public void addKeyInfo(KeyInfo keyInfo) { 32 | keyInfoList.add(keyInfo); 33 | } 34 | 35 | public void setPartitionId(int partitionId) { 36 | this.partitionId = partitionId; 37 | } 38 | 39 | public void setCompoundKey(String compoundKey) { 40 | this.compoundKey = compoundKey; 41 | } 42 | 43 | public void setCompoundValue(String compoundValue) { 44 | this.compoundValue = compoundValue; 45 | } 46 | 47 | public List getKeyInfoList() { 48 | return keyInfoList; 49 | } 50 | 51 | public int getPartitionId() { 52 | return partitionId; 53 | } 54 | 55 | public String getCompoundKey() { 56 | if (compoundKey == null) { 57 | StringBuffer compoundKeyBuffer = new StringBuffer(); 58 | for (KeyInfo keyInfo : this.getKeyInfoList()) { 59 | if (compoundKeyBuffer.length() > 0) { 60 | compoundKeyBuffer.append(","); 61 | } 62 | compoundKeyBuffer.append(keyInfo.toString()); 63 | } 64 | compoundKey = compoundKeyBuffer.toString(); 65 | } 66 | return compoundKey; 67 | } 68 | 69 | public String getCompoundValue() { 70 | return this.compoundValue; 71 | } 72 | 73 | public String toString() { 74 | return "{" + getCompoundKey() + " = " + getCompoundValue() + "}:" 75 | + getPartitionId(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/DefaultHasher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | public class DefaultHasher implements Hasher { 19 | 20 | HashAlgorithm hashAlgorithm = HashAlgorithm.FNV1_64_HASH; 21 | 22 | @Override 23 | public long hash(Object hashKey) { 24 | return hashAlgorithm.hash(String.valueOf(hashKey)); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/DummyPartitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | public class DummyPartitioner implements Partitioner { 22 | 23 | @Override 24 | public List partition(String streamName, Object event, int partitionCount) { 25 | CompoundKeyInfo partitionInfo = new CompoundKeyInfo(); 26 | partitionInfo.setPartitionId(0); 27 | List partitionInfoList = new ArrayList(); 28 | partitionInfoList.add(partitionInfo); 29 | 30 | return partitionInfoList; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/HashAlgorithm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import io.s4.util.KeyUtil; 19 | 20 | import java.security.MessageDigest; 21 | import java.security.NoSuchAlgorithmException; 22 | import java.util.zip.CRC32; 23 | 24 | /** from 25 | * http://github.com/dustin/java-memcached-client/blob/master/src/main/java/net/spy/memcached/HashAlgorithm.java 26 | */ 27 | 28 | /** 29 | * Known hashing algorithms for locating a server for a key. Note that all hash 30 | * algorithms return 64-bits of hash, but only the lower 32-bits are 31 | * significant. This allows a positive 32-bit number to be returned for all 32 | * cases. 33 | */ 34 | public enum HashAlgorithm { 35 | 36 | /** 37 | * Native hash (String.hashCode()). 38 | */ 39 | NATIVE_HASH, 40 | /** 41 | * CRC32_HASH as used by the perl API. This will be more consistent both 42 | * across multiple API users as well as java versions, but is mostly likely 43 | * significantly slower. 44 | */ 45 | CRC32_HASH, 46 | /** 47 | * FNV hashes are designed to be fast while maintaining a low collision 48 | * rate. The FNV speed allows one to quickly hash lots of data while 49 | * maintaining a reasonable collision rate. 50 | * 51 | * @see fnv 52 | * comparisons 53 | * @see fnv at 54 | * wikipedia 55 | */ 56 | FNV1_64_HASH, 57 | /** 58 | * Variation of FNV. 59 | */ 60 | FNV1A_64_HASH, 61 | /** 62 | * 32-bit FNV1. 63 | */ 64 | FNV1_32_HASH, 65 | /** 66 | * 32-bit FNV1a. 67 | */ 68 | FNV1A_32_HASH, 69 | /** 70 | * MD5-based hash algorithm used by ketama. 71 | */ 72 | KETAMA_HASH; 73 | 74 | private static final long FNV_64_INIT = 0xcbf29ce484222325L; 75 | private static final long FNV_64_PRIME = 0x100000001b3L; 76 | 77 | private static final long FNV_32_INIT = 2166136261L; 78 | private static final long FNV_32_PRIME = 16777619; 79 | 80 | /** 81 | * Compute the hash for the given key. 82 | * 83 | * @return a positive integer hash 84 | */ 85 | public long hash(final String k) { 86 | long rv = 0; 87 | switch (this) { 88 | case NATIVE_HASH: 89 | rv = k.hashCode(); 90 | break; 91 | case CRC32_HASH: 92 | // return (crc32(shift) >> 16) & 0x7fff; 93 | CRC32 crc32 = new CRC32(); 94 | crc32.update(KeyUtil.getKeyBytes(k)); 95 | rv = (crc32.getValue() >> 16) & 0x7fff; 96 | break; 97 | case FNV1_64_HASH: { 98 | // Thanks to pierre@demartines.com for the pointer 99 | rv = FNV_64_INIT; 100 | int len = k.length(); 101 | for (int i = 0; i < len; i++) { 102 | rv *= FNV_64_PRIME; 103 | rv ^= k.charAt(i); 104 | } 105 | } 106 | break; 107 | case FNV1A_64_HASH: { 108 | rv = FNV_64_INIT; 109 | int len = k.length(); 110 | for (int i = 0; i < len; i++) { 111 | rv ^= k.charAt(i); 112 | rv *= FNV_64_PRIME; 113 | } 114 | } 115 | break; 116 | case FNV1_32_HASH: { 117 | rv = FNV_32_INIT; 118 | int len = k.length(); 119 | for (int i = 0; i < len; i++) { 120 | rv *= FNV_32_PRIME; 121 | rv ^= k.charAt(i); 122 | } 123 | } 124 | break; 125 | case FNV1A_32_HASH: { 126 | rv = FNV_32_INIT; 127 | int len = k.length(); 128 | for (int i = 0; i < len; i++) { 129 | rv ^= k.charAt(i); 130 | rv *= FNV_32_PRIME; 131 | } 132 | } 133 | break; 134 | case KETAMA_HASH: 135 | byte[] bKey = computeMd5(k); 136 | rv = ((long) (bKey[3] & 0xFF) << 24) 137 | | ((long) (bKey[2] & 0xFF) << 16) 138 | | ((long) (bKey[1] & 0xFF) << 8) | (bKey[0] & 0xFF); 139 | break; 140 | default: 141 | assert false; 142 | } 143 | return rv & 0xffffffffL; /* Truncate to 32-bits */ 144 | } 145 | 146 | /** 147 | * Get the md5 of the given key. 148 | */ 149 | public static byte[] computeMd5(String k) { 150 | MessageDigest md5; 151 | try { 152 | md5 = MessageDigest.getInstance("MD5"); 153 | } catch (NoSuchAlgorithmException e) { 154 | throw new RuntimeException("MD5 not supported", e); 155 | } 156 | md5.reset(); 157 | md5.update(KeyUtil.getKeyBytes(k)); 158 | return md5.digest(); 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/Hasher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | public interface Hasher { 19 | public long hash(Object hashKey); 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/KeyInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import java.io.Serializable; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | public class KeyInfo implements Serializable { 23 | List keyPath = new ArrayList(); 24 | String value; 25 | 26 | public void setValue(String value) { 27 | this.value = value; 28 | } 29 | 30 | public String getValue() { 31 | return this.value; 32 | } 33 | 34 | public void addElementToPath(String keyName) { 35 | keyPath.add(new KeyPathElementName(keyName)); 36 | } 37 | 38 | public void addElementToPath(int index) { 39 | keyPath.add(new KeyPathElementIndex(index)); 40 | } 41 | 42 | private void addElementToPath(KeyPathElement keyPathElement) { 43 | keyPath.add(keyPathElement); 44 | } 45 | 46 | public List getKeyPath() { 47 | return keyPath; 48 | } 49 | 50 | public static class KeyPathElement implements Serializable { 51 | public enum PathElementType { 52 | KEY_NAME, INDEX 53 | } 54 | 55 | PathElementType pathElementType; 56 | 57 | public PathElementType getPathElementType() { 58 | return pathElementType; 59 | } 60 | 61 | } 62 | 63 | public static class KeyPathElementName extends KeyPathElement { 64 | String keyName; 65 | 66 | public KeyPathElementName() { 67 | 68 | } 69 | 70 | public KeyPathElementName(String keyName) { 71 | pathElementType = PathElementType.KEY_NAME; 72 | this.keyName = keyName; 73 | } 74 | 75 | public String getKeyName() { 76 | return keyName; 77 | } 78 | } 79 | 80 | public static class KeyPathElementIndex extends KeyPathElement { 81 | int index; 82 | 83 | public KeyPathElementIndex() { 84 | 85 | } 86 | 87 | public KeyPathElementIndex(int index) { 88 | pathElementType = PathElementType.INDEX; 89 | this.index = index; 90 | } 91 | 92 | public int getIndex() { 93 | return index; 94 | } 95 | } 96 | 97 | public KeyInfo copy() { 98 | KeyInfo newKeyInfo = new KeyInfo(); 99 | for (KeyPathElement element : keyPath) { 100 | newKeyInfo.addElementToPath(element); 101 | } 102 | return newKeyInfo; 103 | } 104 | 105 | public String toString() { 106 | return toString(false); 107 | } 108 | 109 | public String toString(boolean showFull) { 110 | StringBuffer sb = new StringBuffer(); 111 | for (KeyPathElement element : keyPath) { 112 | if (element.getPathElementType() == KeyPathElement.PathElementType.KEY_NAME) { 113 | if (sb.length() > 0) { 114 | sb.append("/"); 115 | } 116 | sb.append(((KeyPathElementName) element).getKeyName()); 117 | } else if (showFull) { 118 | sb.append("[") 119 | .append(((KeyPathElementIndex) element).getIndex()) 120 | .append("]"); 121 | } 122 | } 123 | return sb.toString(); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/Partitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import java.util.List; 19 | 20 | public interface Partitioner { 21 | public List partition(String streamName, Object event, int partitionCount); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/RoundRobinPartitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | public class RoundRobinPartitioner implements Partitioner { 22 | private int counter = 0; 23 | 24 | @Override 25 | public List partition(String streamName, Object event, int partitionCount) { 26 | 27 | CompoundKeyInfo partitionInfo = new CompoundKeyInfo(); 28 | int partitionId = 0; 29 | 30 | synchronized (this) { 31 | counter++; 32 | if (counter < 0) { 33 | counter = 0; 34 | } 35 | partitionId = counter % partitionCount; 36 | } 37 | 38 | partitionInfo.setPartitionId(partitionId); 39 | List partitionInfoList = new ArrayList(); 40 | partitionInfoList.add(partitionInfo); 41 | 42 | return partitionInfoList; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/TestDefaultPartitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | public class TestDefaultPartitioner { 22 | public static void main(String[] args) { 23 | DefaultPartitioner dp1 = new DefaultPartitioner(); 24 | DefaultPartitioner dp2 = new DefaultPartitioner(); 25 | dp1.setDebug(true); 26 | dp1.setHashKey(new String[] { "list1/val1", "list1/val2", "query" }); 27 | dp1.setHasher(new DefaultHasher()); 28 | 29 | dp2.setDebug(true); 30 | dp2.setHashKey(new String[] { "user" }); 31 | dp2.setHasher(new DefaultHasher()); 32 | 33 | TopLevel tl1 = new TopLevel(); 34 | tl1.setQuery("Hello there"); 35 | tl1.setUser("spitzer"); 36 | 37 | for (int i = 0; i < 4; i++) { 38 | Nested n = new Nested(); 39 | n.setVal1(i + 77); 40 | n.setVal2(i / 10.7); 41 | tl1.addNested(n); 42 | } 43 | 44 | dp1.partition("test", tl1, 4); 45 | dp2.partition("test", tl1, 4); 46 | 47 | } 48 | 49 | static class TopLevel { 50 | private String query; 51 | private List list1 = new ArrayList(); 52 | private String user; 53 | 54 | public String getQuery() { 55 | return query; 56 | } 57 | 58 | public void setQuery(String query) { 59 | this.query = query; 60 | } 61 | 62 | public List getList1() { 63 | return list1; 64 | } 65 | 66 | public void setList1(List list1) { 67 | this.list1 = list1; 68 | } 69 | 70 | public String getUser() { 71 | return user; 72 | } 73 | 74 | public void setUser(String user) { 75 | this.user = user; 76 | } 77 | 78 | public void addNested(Nested nested) { 79 | list1.add(nested); 80 | } 81 | } 82 | 83 | static class Nested { 84 | long val1; 85 | double val2; 86 | 87 | public Nested() { 88 | 89 | } 90 | 91 | public long getVal1() { 92 | return val1; 93 | } 94 | 95 | public void setVal1(long val1) { 96 | this.val1 = val1; 97 | } 98 | 99 | public double getVal2() { 100 | return val2; 101 | } 102 | 103 | public void setVal2(double val2) { 104 | this.val2 = val2; 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/partitioner/VariableKeyPartitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.partitioner; 17 | 18 | import java.util.List; 19 | 20 | public interface VariableKeyPartitioner { 21 | public List partition(String streamName, 22 | List> keyNames, 23 | Object event, int partitionCount); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/io/s4/dispatcher/transformer/Transformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.dispatcher.transformer; 17 | 18 | 19 | public interface Transformer { 20 | public Object transform(Object event); 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/io/s4/emitter/EventEmitter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.emitter; 17 | 18 | import io.s4.collector.EventWrapper; 19 | 20 | public interface EventEmitter { 21 | public void emit(int partitionId, EventWrapper eventWrapper); 22 | 23 | public int getNodeCount(); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/io/s4/listener/EventHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.listener; 17 | 18 | import io.s4.collector.EventWrapper; 19 | 20 | public interface EventHandler { 21 | void processEvent(EventWrapper eventWrapper); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/s4/listener/EventListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.listener; 17 | 18 | public interface EventListener extends EventProducer { 19 | 20 | int getId(); 21 | 22 | String getAppName(); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/io/s4/listener/EventProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.listener; 17 | 18 | public interface EventProducer { 19 | 20 | void addHandler(EventHandler handler); 21 | 22 | boolean removeHandler(EventHandler handler); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/io/s4/logger/Log4jMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.logger; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | import java.util.Timer; 21 | import java.util.TimerTask; 22 | import java.util.concurrent.ConcurrentHashMap; 23 | 24 | public class Log4jMonitor extends TimerTask implements Monitor { 25 | Map metricMap = new ConcurrentHashMap(); 26 | private String loggerName = "s4"; 27 | private int flushInterval = 600; // default is every 10 minutes 28 | 29 | private Timer timer = new Timer(); 30 | private Map defaultMap = new HashMap(); 31 | 32 | public void setLoggerName(String loggerName) { 33 | this.loggerName = loggerName; 34 | } 35 | 36 | public void setFlushInterval(int flushInterval) { 37 | this.flushInterval = flushInterval; 38 | } 39 | 40 | public void init() { 41 | if (flushInterval > 0) { 42 | timer.scheduleAtFixedRate(this, 43 | flushInterval * 1000, 44 | flushInterval * 1000); 45 | } 46 | } 47 | 48 | // TODO: this will be removed after changing above functions 49 | public void set(String metricName, int value) { 50 | metricMap.put(metricName, value); 51 | } 52 | 53 | public void flushStats() { 54 | org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(loggerName); 55 | for (String key : metricMap.keySet()) { 56 | String message = key + " = " + metricMap.get(key); 57 | logger.info(message); 58 | metricMap.remove(key); 59 | } 60 | if (defaultMap != null) { 61 | for (String key : defaultMap.keySet()) { 62 | // TODO: need to be changed 63 | set(key, defaultMap.get(key)); 64 | } 65 | } 66 | } 67 | 68 | public void run() { 69 | flushStats(); 70 | } 71 | 72 | @Override 73 | public void increment(String metricName, int increment) { 74 | Integer currValue = metricMap.get(metricName); 75 | if (currValue == null) { 76 | currValue = 0; 77 | } 78 | currValue += increment; 79 | metricMap.put(metricName, currValue); 80 | } 81 | 82 | @Override 83 | public void setDefaultValue(String key, int val) { 84 | // TODO Auto-generated method stub 85 | 86 | } 87 | 88 | @Override 89 | public void increment(String metricName, int increment, String metricEventName, String... furtherDistinctions) { 90 | increment(buildMetricName(metricName, 91 | metricEventName, 92 | furtherDistinctions), 93 | increment); 94 | 95 | } 96 | 97 | @Override 98 | public void set(String metricName, int value, String metricEventName, String... furtherDistinctions) { 99 | metricMap.put(buildMetricName(metricName, 100 | metricEventName, 101 | furtherDistinctions), 102 | value); 103 | } 104 | 105 | private String buildMetricName(String metricName, String metricEventName, String[] furtherDistinctions) { 106 | StringBuffer sb = new StringBuffer(metricEventName); 107 | sb.append(":"); 108 | sb.append(metricName); 109 | if (furtherDistinctions != null) { 110 | for (String furtherDistinction : furtherDistinctions) { 111 | sb.append(":"); 112 | sb.append(furtherDistinction); 113 | } 114 | } 115 | return sb.toString().intern(); 116 | 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/io/s4/logger/Monitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.logger; 17 | 18 | public interface Monitor { 19 | public void increment(String metricName, int increment, String metricEventName, String... aggKeys); 20 | 21 | public void increment(String metricName, int increment); 22 | 23 | public void set(String metricName, int value, String metricEventName, String... aggKeys); 24 | 25 | public void set(String metricName, int value); 26 | 27 | public void flushStats(); 28 | 29 | public void setDefaultValue(String key, int val); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/io/s4/logger/TraceMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.logger; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | public class TraceMessage { 22 | private long traceId; 23 | private Map propertyMap = new HashMap(); 24 | 25 | public void setTraceId(long traceId) { 26 | this.traceId = traceId; 27 | } 28 | 29 | public long getTraceId() { 30 | return this.traceId; 31 | } 32 | 33 | public void setProperty(String name, String value) { 34 | this.propertyMap.put(name, value); 35 | } 36 | 37 | public String toString() { 38 | return this.traceId + "; " + this.propertyMap.toString(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/io/s4/message/PrototypeRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.message; 17 | 18 | import io.s4.dispatcher.partitioner.CompoundKeyInfo; 19 | import io.s4.dispatcher.partitioner.Hasher; 20 | import io.s4.processor.PrototypeWrapper; 21 | import io.s4.util.MethodInvoker; 22 | 23 | import java.util.ArrayList; 24 | import java.util.Collections; 25 | import java.util.HashMap; 26 | import java.util.List; 27 | 28 | /** 29 | * A request for a value from the prototype of PEs. 30 | */ 31 | public class PrototypeRequest extends Request { 32 | 33 | private final List query; 34 | 35 | public PrototypeRequest(List query, RInfo info) { 36 | this.query = query; 37 | this.rinfo = info; 38 | } 39 | 40 | public PrototypeRequest(List query) { 41 | this.query = query; 42 | this.rinfo = null; 43 | } 44 | 45 | public PrototypeRequest() { 46 | this.query = Collections. emptyList(); 47 | this.rinfo = null; 48 | } 49 | 50 | public String toString() { 51 | return "PROTOTYPE: query=[" + query + "] info=[" + rinfo + "]"; 52 | } 53 | 54 | /** 55 | * List of queries to execute. 56 | * 57 | * @return list of queries 58 | */ 59 | public List getQuery() { 60 | return query; 61 | } 62 | 63 | /** 64 | * Evaluate Request on a particular PE Prototype. 65 | * 66 | * @param pe 67 | * prototype 68 | * @return Response object. 69 | */ 70 | public Response evaluate(PrototypeWrapper pw) { 71 | 72 | HashMap results = new HashMap(); 73 | HashMap exceptions = new HashMap(); 74 | 75 | for (String q : query) { 76 | if (q.startsWith("$")) { 77 | // requests for getters should be of the form $fieldA. 78 | // Responds with pe.getFieldA() 79 | try { 80 | Object res = MethodInvoker.invokeGetter(pw, q.substring(1)); 81 | results.put(q, res); 82 | 83 | } catch (Exception e) { 84 | exceptions.put(q, e.toString()); 85 | } 86 | 87 | } else if (q.equalsIgnoreCase("count")) { 88 | // Some aggregate operators 89 | results.put(q, pw.getPECount()); 90 | 91 | } else { 92 | exceptions.put(q, "Query Parse Error"); 93 | } 94 | } 95 | 96 | return new Response(results, exceptions, this); 97 | } 98 | 99 | public List partition(Hasher h, String delim, int partCount) { 100 | // send to all partitions 101 | List partitionInfoList = new ArrayList(); 102 | 103 | for (int i = 0; i < partCount; ++i) { 104 | CompoundKeyInfo partitionInfo = new CompoundKeyInfo(); 105 | partitionInfo.setPartitionId(i); 106 | partitionInfoList.add(partitionInfo); 107 | } 108 | 109 | return partitionInfoList; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/io/s4/message/Request.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.message; 17 | 18 | import io.s4.dispatcher.partitioner.CompoundKeyInfo; 19 | import io.s4.dispatcher.partitioner.Hasher; 20 | import io.s4.util.GsonUtil; 21 | 22 | import java.lang.reflect.Type; 23 | import java.util.List; 24 | import java.util.UUID; 25 | 26 | import com.google.gson.InstanceCreator; 27 | 28 | abstract public class Request { 29 | 30 | protected RInfo rinfo = nullRInfo; 31 | 32 | public final static RInfo nullRInfo = new NullRInfo(); 33 | 34 | /** 35 | * Requester/Return information 36 | */ 37 | abstract public static class RInfo { 38 | 39 | private long id = 0; 40 | 41 | /** 42 | * Identity of request. This is typically specified by the requester. 43 | */ 44 | public long getId() { 45 | return id; 46 | } 47 | 48 | public void setId(int id) { 49 | this.id = id; 50 | } 51 | 52 | private String stream; 53 | 54 | /** 55 | * Stream name on which response should be sent. 56 | * 57 | * @return stream name. 58 | */ 59 | public String getStream() { 60 | return stream; 61 | } 62 | 63 | public void setStream(String stream) { 64 | this.stream = stream; 65 | } 66 | 67 | private int partition; 68 | 69 | /** 70 | * Partition Id from which this request originated. This may be used to 71 | * return a response to the same partition. 72 | * 73 | * @return 74 | */ 75 | public int getPartition() { 76 | return partition; 77 | } 78 | 79 | public void setPartition(int partition) { 80 | this.partition = partition; 81 | } 82 | 83 | // Tell Gson how to instantiate one of these: create a ClientRInfo 84 | static { 85 | InstanceCreator creator = new InstanceCreator() { 86 | public io.s4.message.Request.RInfo createInstance(Type type) { 87 | return new io.s4.message.Request.ClientRInfo(); 88 | } 89 | }; 90 | 91 | GsonUtil.registerTypeAdapter(RInfo.class, creator); 92 | } 93 | 94 | } 95 | 96 | public static class ClientRInfo extends RInfo { 97 | private UUID requesterUUID = null; 98 | 99 | /** 100 | * Identity of requesting client. This is used to send the response back 101 | * to the client. 102 | * 103 | * @return UUID of the client from which the request originated. 104 | */ 105 | public UUID getRequesterUUID() { 106 | return requesterUUID; 107 | } 108 | 109 | public void setRequesterUUID(UUID requesterUUID) { 110 | this.requesterUUID = requesterUUID; 111 | } 112 | 113 | public String toString() { 114 | return "(id:" + getId() + " requester:" + getRequesterUUID() 115 | + " partition:" + getPartition() + " stream:" + getStream() 116 | + ")"; 117 | } 118 | } 119 | 120 | public static class PERInfo extends RInfo { 121 | private String requesterKey = null; 122 | 123 | /** 124 | * Identity of requesting PE. This is used to route the response back to 125 | * the originating PE. 126 | * 127 | * @return key value of the PE from which the request originated. 128 | */ 129 | public String getRequesterKey() { 130 | return requesterKey; 131 | } 132 | 133 | public void setRequesterKey(String requesterKey) { 134 | this.requesterKey = requesterKey; 135 | } 136 | 137 | public String toString() { 138 | return "(id:" + getId() + " requester:" + getRequesterKey() 139 | + " partition:" + getPartition() + " stream:" + getStream() 140 | + ")"; 141 | } 142 | } 143 | 144 | public static class NullRInfo extends RInfo { 145 | public NullRInfo() { 146 | super.stream = "@null"; 147 | super.partition = -1; 148 | } 149 | } 150 | 151 | /** 152 | * Query metainformation. 153 | * 154 | * @return Info representing origin of request. 155 | */ 156 | public RInfo getRInfo() { 157 | return rinfo; 158 | } 159 | 160 | /** 161 | * Query metainformation. 162 | */ 163 | public void setRInfo(RInfo rinfo) { 164 | this.rinfo = rinfo; 165 | } 166 | 167 | /** 168 | * Partition itself. This is used by the default partitioner. 169 | * 170 | * @param h 171 | * hasher 172 | * @param delim 173 | * delimiter used to concatenate compound key values 174 | * @param partCount 175 | * number of partitions 176 | * @return list of compound keys: one event may have to be sent to multiple 177 | * nodes. 178 | */ 179 | abstract public List partition(Hasher h, String delim, 180 | int partCount); 181 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/message/Response.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.message; 17 | 18 | import io.s4.dispatcher.partitioner.CompoundKeyInfo; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | public class Response { 25 | 26 | private Map result; 27 | 28 | private Map exception; 29 | 30 | private Request request; 31 | 32 | public Response(Map result, Request request) { 33 | this.result = result; 34 | this.request = request; 35 | } 36 | 37 | public Response(Map result, Map exception, 38 | Request request) { 39 | this.result = result; 40 | this.exception = exception; 41 | this.request = request; 42 | } 43 | 44 | public Response() { 45 | result = null; 46 | exception = null; 47 | request = null; 48 | } 49 | 50 | /** 51 | * Result of a request. 52 | * 53 | * @return map from query strings o corresponding values. 54 | */ 55 | public Map getResult() { 56 | return result; 57 | } 58 | 59 | public Map getException() { 60 | return exception; 61 | } 62 | 63 | public Request getRequest() { 64 | return request; 65 | } 66 | 67 | public Request.RInfo getRInfo() { 68 | return (request != null ? request.getRInfo() : null); 69 | } 70 | 71 | public String toString() { 72 | return "[" + result + "] (" + request + ")"; 73 | } 74 | 75 | public List partition(int partCount) { 76 | // partition id is available from the request info object 77 | 78 | int p = this.getRInfo().getPartition(); 79 | List partitionInfoList = null; 80 | 81 | if (p >= 0 && p < partCount) { 82 | CompoundKeyInfo partitionInfo = new CompoundKeyInfo(); 83 | partitionInfo.setPartitionId(p); 84 | 85 | partitionInfoList = new ArrayList(); 86 | partitionInfoList.add(partitionInfo); 87 | } 88 | 89 | return partitionInfoList; 90 | } 91 | 92 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/message/SinglePERequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.message; 17 | 18 | import io.s4.dispatcher.partitioner.CompoundKeyInfo; 19 | import io.s4.dispatcher.partitioner.Hasher; 20 | import io.s4.dispatcher.partitioner.KeyInfo; 21 | import io.s4.processor.ProcessingElement; 22 | import io.s4.util.MethodInvoker; 23 | 24 | import java.util.ArrayList; 25 | import java.util.Collections; 26 | import java.util.HashMap; 27 | import java.util.List; 28 | 29 | import org.springframework.util.StringUtils; 30 | 31 | /** 32 | * A request for a value from a particular PE. 33 | */ 34 | public class SinglePERequest extends Request { 35 | 36 | private final List target; 37 | 38 | private final List query; 39 | 40 | public SinglePERequest(List target, List query, RInfo info) { 41 | this.target = target; 42 | this.query = query; 43 | this.rinfo = info; 44 | } 45 | 46 | public SinglePERequest(List target, List query) { 47 | this.target = target; 48 | this.query = query; 49 | this.rinfo = null; 50 | } 51 | 52 | public SinglePERequest() { 53 | this.target = Collections. emptyList(); 54 | this.query = Collections. emptyList(); 55 | this.rinfo = null; 56 | } 57 | 58 | public String toString() { 59 | return "target:" + target + " query:" + query + " info:" + rinfo; 60 | } 61 | 62 | /** 63 | * Fields used to target a particular PE. 64 | * 65 | * @return list of targeting values. Order matters. 66 | */ 67 | public List getTarget() { 68 | return target; 69 | } 70 | 71 | /** 72 | * List of field names that have to be read form target PE. 73 | * 74 | * @return list of field name strings. 75 | */ 76 | public List getQuery() { 77 | return query; 78 | } 79 | 80 | /** 81 | * Evaluate Request on a particular PE. 82 | * 83 | * @param pe 84 | * @return Response object. 85 | */ 86 | public Response evaluate(ProcessingElement pe) { 87 | 88 | HashMap results = new HashMap(); 89 | HashMap exceptions = new HashMap(); 90 | 91 | for (String q : query) { 92 | // requests for getters should be of the form $field. Responds with 93 | // pe.getField() 94 | if (q.startsWith("$")) { 95 | try { 96 | Object res = MethodInvoker.invokeGetter(pe, q.substring(1)); 97 | results.put(q, res); 98 | 99 | } catch (Exception e) { 100 | exceptions.put(q, e.toString()); 101 | } 102 | } 103 | } 104 | 105 | return new Response(results, exceptions, this); 106 | } 107 | 108 | public List partition(Hasher h, String delim, int partCount) { 109 | List valueList = this.getTarget(); 110 | if (valueList == null) 111 | return null; 112 | 113 | // First, build the key 114 | KeyInfo keyInfo = new KeyInfo(); 115 | // special kay name to denote request 116 | keyInfo.addElementToPath("#req"); 117 | 118 | // for value, concatenate list of values from Request's target field. 119 | String stringValue = StringUtils.collectionToDelimitedString(valueList, 120 | delim); 121 | keyInfo.setValue(stringValue); 122 | 123 | // partition id is derived form string value, as usual 124 | int partitionId = (int) (h.hash(stringValue) % partCount); 125 | 126 | CompoundKeyInfo partitionInfo = new CompoundKeyInfo(); 127 | partitionInfo.addKeyInfo(keyInfo); 128 | partitionInfo.setCompoundValue(stringValue); 129 | partitionInfo.setPartitionId(partitionId); 130 | 131 | List partitionInfoList = new ArrayList(); 132 | partitionInfoList.add(partitionInfo); 133 | 134 | return partitionInfoList; 135 | } 136 | 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/io/s4/persist/ConMapPersister.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.persist; 17 | 18 | import io.s4.util.clock.Clock; 19 | 20 | import java.util.Enumeration; 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | import java.util.Set; 24 | import java.util.concurrent.ConcurrentHashMap; 25 | import java.util.concurrent.atomic.AtomicInteger; 26 | 27 | import org.apache.log4j.Logger; 28 | 29 | public class ConMapPersister implements Persister { 30 | private AtomicInteger persistCount = new AtomicInteger(0); 31 | private boolean selfClean = false; 32 | private int cleanWaitTime = 40; // 20 seconds by default 33 | private String loggerName = "s4"; 34 | ConcurrentHashMap cache; 35 | Clock s4Clock; 36 | 37 | private int startCapacity = 5000; 38 | 39 | public void setStartCapacity(int startCapacity) { 40 | this.startCapacity = startCapacity; 41 | } 42 | 43 | public int getStartCapacity() { 44 | return startCapacity; 45 | } 46 | 47 | public void setSelfClean(boolean selfClean) { 48 | this.selfClean = selfClean; 49 | } 50 | 51 | public void setCleanWaitTime(int cleanWaitTime) { 52 | this.cleanWaitTime = cleanWaitTime; 53 | } 54 | 55 | public void setLoggerName(String loggerName) { 56 | this.loggerName = loggerName; 57 | } 58 | 59 | public ConMapPersister(Clock s4Clock) { 60 | this.s4Clock = s4Clock; 61 | } 62 | 63 | public void setS4Clock(Clock s4Clock) { 64 | this.s4Clock = s4Clock; 65 | } 66 | 67 | public ConMapPersister() { 68 | } 69 | 70 | public void init() { 71 | cache = new ConcurrentHashMap(this.getStartCapacity()); 72 | 73 | if (selfClean) { 74 | Runnable r = new Runnable() { 75 | public void run() { 76 | while (!Thread.interrupted()) { 77 | int cleanCount = ConMapPersister.this.cleanOutGarbage(); 78 | Logger.getLogger(loggerName).info("Cleaned out " 79 | + cleanCount + " entries; Persister has " 80 | + cache.size() + " entries"); 81 | try { 82 | Thread.sleep(cleanWaitTime * 1000); 83 | } catch (InterruptedException ie) { 84 | Thread.currentThread().interrupt(); 85 | } 86 | } 87 | } 88 | }; 89 | Thread t = new Thread(r); 90 | t.start(); 91 | t.setPriority(Thread.MIN_PRIORITY); 92 | } 93 | } 94 | 95 | public int getQueueSize() { 96 | return 0; 97 | } 98 | 99 | public int getPersistCount() { 100 | return persistCount.get(); 101 | } 102 | 103 | public int getCacheEntryCount() { 104 | return cache.size(); 105 | } 106 | 107 | public void setAsynch(String key, Object value, int period) { 108 | // there really is no asynch for the local cache 109 | set(key, value, period); 110 | } 111 | 112 | public void set(String key, Object value, int period) { 113 | persistCount.getAndIncrement(); 114 | CacheEntry ce = new CacheEntry(); 115 | ce.value = value; 116 | ce.period = period; 117 | ce.addTime = s4Clock.getCurrentTime(); 118 | cache.put(key, ce); 119 | } 120 | 121 | public Object get(String key) { 122 | CacheEntry ce = cache.get(key); 123 | if (ce == null) { 124 | return null; 125 | } 126 | 127 | if (ce.isExpired()) { 128 | return null; 129 | } 130 | 131 | return ce.value; 132 | } 133 | 134 | public Map getBulk(String[] keys) { 135 | HashMap map = new HashMap(); 136 | for (String key : keys) { 137 | Object value = get(key); 138 | if (value != null) { 139 | map.put(key, value); 140 | } 141 | } 142 | return map; 143 | } 144 | 145 | public Object getObject(String key) { 146 | return get(key); 147 | } 148 | 149 | public Map getBulkObjects(String[] keys) { 150 | return getBulk(keys); 151 | } 152 | 153 | public void remove(String key) { 154 | cache.remove(key); 155 | } 156 | 157 | public int cleanOutGarbage() { 158 | int count = 0; 159 | for (Enumeration en = cache.keys(); en.hasMoreElements();) { 160 | String key = (String) en.nextElement(); 161 | CacheEntry ce = cache.get(key); 162 | if (ce != null && ce.isExpired()) { 163 | count++; 164 | cache.remove(key); 165 | } 166 | } 167 | return count; 168 | } 169 | 170 | public Set keySet() { 171 | return cache.keySet(); 172 | } 173 | 174 | public class CacheEntry { 175 | Object value; 176 | long addTime; 177 | int period; 178 | 179 | public boolean isExpired() { 180 | if (period > 0) { 181 | if ((addTime + (1000 * (long) period)) <= s4Clock.getCurrentTime()) { 182 | return true; 183 | } 184 | } 185 | return false; 186 | } 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/main/java/io/s4/persist/DumpingPersister.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.persist; 17 | 18 | import io.s4.processor.OutputFormatter; 19 | import io.s4.util.clock.Clock; 20 | 21 | import java.io.BufferedWriter; 22 | import java.io.File; 23 | import java.io.FileOutputStream; 24 | import java.io.IOException; 25 | import java.io.OutputStreamWriter; 26 | import java.util.HashSet; 27 | import java.util.Map; 28 | import java.util.Set; 29 | import java.util.UUID; 30 | import java.util.regex.Matcher; 31 | import java.util.regex.Pattern; 32 | 33 | import org.apache.log4j.Logger; 34 | 35 | public class DumpingPersister extends ConMapPersister implements Runnable { 36 | 37 | public DumpingPersister() { 38 | } 39 | 40 | public DumpingPersister(Clock s4Clock) { 41 | super(s4Clock); 42 | // TODO Auto-generated constructor stub 43 | } 44 | 45 | private String dumpFilePrefix; 46 | private Map regexFormatter; 47 | private Pattern[] patterns; 48 | private OutputFormatter[] formatters; 49 | private long outputTimeBoundary; 50 | 51 | public void setDumpFilePrefix(String dumpFilePrefix) { 52 | this.dumpFilePrefix = dumpFilePrefix; 53 | } 54 | 55 | public void setRegexFormatter(Map regexFormatter) { 56 | this.regexFormatter = regexFormatter; 57 | } 58 | 59 | public void setOutputTimeBoundary(long outputTimeBoundary) { 60 | this.outputTimeBoundary = outputTimeBoundary; 61 | } 62 | 63 | public void init() { 64 | super.init(); 65 | 66 | Set regexes = regexFormatter.keySet(); 67 | patterns = new Pattern[regexes.size()]; 68 | formatters = new OutputFormatter[regexes.size()]; 69 | 70 | int i = 0; 71 | for (String regex : regexes) { 72 | patterns[i] = Pattern.compile(regex); 73 | formatters[i] = regexFormatter.get(regex); 74 | i++; 75 | } 76 | 77 | Thread t = new Thread(this); 78 | t.start(); 79 | } 80 | 81 | public void run() { 82 | long boundaryInMillis = outputTimeBoundary * 1000; 83 | long currentTime = System.currentTimeMillis(); 84 | while (!Thread.interrupted()) { 85 | long currentBoundary = (currentTime / boundaryInMillis) 86 | * boundaryInMillis; 87 | long interval = ((currentBoundary + boundaryInMillis) - System.currentTimeMillis()); 88 | if (interval > 0) { 89 | try { 90 | Thread.sleep(interval); 91 | } catch (InterruptedException ie) { 92 | Thread.currentThread().interrupt(); 93 | } 94 | } 95 | 96 | currentTime = System.currentTimeMillis(); 97 | 98 | try { 99 | output(); 100 | } catch (Exception e) { 101 | Logger.getLogger("s4").error("Exception dumping persister", e); 102 | } 103 | } 104 | } 105 | 106 | public void output() { 107 | File file = new File(dumpFilePrefix + UUID.randomUUID().toString()); 108 | Logger.getLogger("s4").info("Dumping to " + file); 109 | FileOutputStream fos = null; 110 | OutputStreamWriter osw = null; 111 | BufferedWriter dumpWriter = null; 112 | 113 | try { 114 | fos = new FileOutputStream(file); 115 | osw = new OutputStreamWriter(fos); 116 | dumpWriter = new BufferedWriter(osw); 117 | } catch (IOException ioe) { 118 | throw new RuntimeException(ioe); 119 | } 120 | 121 | int savedPriority = Thread.currentThread().getPriority(); 122 | try { 123 | 124 | Thread.currentThread().setPriority(Thread.MIN_PRIORITY); 125 | Set keys = new HashSet(); 126 | for (String key : this.keySet()) { 127 | keys.add(key); 128 | } 129 | 130 | for (String key : keys) { 131 | Object value = this.get(key); 132 | if (value == null) { 133 | continue; 134 | } 135 | for (int patternId = 0; patternId < patterns.length; patternId++) { 136 | Matcher m = patterns[patternId].matcher(key); 137 | if (m.matches()) { 138 | Object formattedValue = formatters[patternId].format(value); 139 | dumpWriter.write(key + " = " + formattedValue + "\n"); 140 | } 141 | } 142 | 143 | } 144 | } catch (Exception e) { 145 | throw new RuntimeException(e); 146 | } finally { 147 | Thread.currentThread().setPriority(savedPriority); 148 | try { 149 | dumpWriter.close(); 150 | } catch (Exception e) { 151 | } 152 | try { 153 | osw.close(); 154 | } catch (Exception e) { 155 | } 156 | try { 157 | fos.close(); 158 | } catch (Exception e) { 159 | } 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/AsynchronousEventProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | import io.s4.collector.EventWrapper; 19 | 20 | public interface AsynchronousEventProcessor { 21 | 22 | void queueWork(EventWrapper eventWrapper); 23 | 24 | // This will always be called by a different thread than the one executing 25 | // run() 26 | int getQueueSize(); 27 | 28 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/ControlEventProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | import io.s4.collector.EventWrapper; 19 | import io.s4.dispatcher.EventDispatcher; 20 | import io.s4.dispatcher.partitioner.CompoundKeyInfo; 21 | import io.s4.message.PrototypeRequest; 22 | import io.s4.message.Response; 23 | import io.s4.message.SinglePERequest; 24 | 25 | import java.util.List; 26 | 27 | /** 28 | * Processes control events. 29 | */ 30 | public class ControlEventProcessor { 31 | 32 | private EventDispatcher dispatcher; 33 | 34 | public void setDispatcher(EventDispatcher dispatcher) { 35 | this.dispatcher = dispatcher; 36 | } 37 | 38 | public void process(EventWrapper e, PrototypeWrapper p) { 39 | String en = e.getStreamName(); // e.g. "#joinPe01" 40 | String pn = p.getId(); // e.g. "JoinPE01" 41 | 42 | // stream name has to match PE's ID (modulo case). 43 | // e.g. "#joinPe01" will match "JoinPE01" 44 | if (!en.regionMatches(true, 1, pn, 0, pn.length())) 45 | return; 46 | 47 | execute(e, p); 48 | } 49 | 50 | protected void execute(EventWrapper e, PrototypeWrapper p) { 51 | List keyInfoList = e.getCompoundKeys(); 52 | Object event = e.getEvent(); 53 | 54 | if (event instanceof SinglePERequest) { 55 | // Handle Requests to individual PEs 56 | if (keyInfoList.isEmpty()) 57 | return; 58 | 59 | CompoundKeyInfo keyInfo = keyInfoList.get(0); 60 | 61 | String keyVal = keyInfo.getCompoundValue(); 62 | 63 | ProcessingElement pe = p.lookupPE(keyVal); 64 | 65 | Response response = ((SinglePERequest) event).evaluate(pe); 66 | String stream = response.getRInfo().getStream(); 67 | 68 | dispatcher.dispatchEvent(stream, response); 69 | 70 | } else if (event instanceof PrototypeRequest) { 71 | // Or handle aggregate requests to Prototypes. 72 | Response response = ((PrototypeRequest) event).evaluate(p); 73 | String stream = response.getRInfo().getStream(); 74 | 75 | dispatcher.dispatchEvent(stream, response); 76 | } 77 | 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/EventAdvice.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | public class EventAdvice { 19 | String eventName; 20 | String key; 21 | 22 | public EventAdvice(String eventName, String key) { 23 | this.eventName = eventName; 24 | this.key = key; 25 | } 26 | 27 | public String toString() { 28 | return eventName + ":{" + key + "}"; 29 | } 30 | 31 | public String getKey() { 32 | return key; 33 | } 34 | 35 | public String getEventName() { 36 | return eventName; 37 | } 38 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/OutputFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | public interface OutputFormatter { 19 | public Object format(Object outputValue); 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/OverloadDispatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | public interface OverloadDispatcher { 19 | public void dispatch(Object pe, Object event); 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/OverloadDispatcherSlot.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | public interface OverloadDispatcherSlot { 19 | public void dispatch(Object slot, Object event, long slotTime, AbstractWindowingPE pe); 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/PrintEventPE.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | import org.apache.log4j.Logger; 19 | 20 | public class PrintEventPE extends AbstractPE { 21 | 22 | private String id = "PrintEventPE"; 23 | 24 | public String getId() { 25 | return id; 26 | } 27 | 28 | public void setId(String id) { 29 | this.id = id; 30 | } 31 | 32 | @Override 33 | public void output() { 34 | // TODO Auto-generated method stub 35 | 36 | } 37 | 38 | public void processEvent(Object event) { 39 | Logger.getLogger("s4").info(event); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/ProcessingElement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | import io.s4.dispatcher.partitioner.CompoundKeyInfo; 19 | 20 | import java.util.List; 21 | 22 | public interface ProcessingElement extends Cloneable { 23 | void execute(String streamName, CompoundKeyInfo compoundKeyInfo, Object event); 24 | 25 | List advise(); 26 | 27 | public int getTtl(); 28 | 29 | public Object clone(); 30 | 31 | public String getId(); 32 | 33 | public String getInitMethod(); 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/PrototypeWrapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | import io.s4.persist.ConMapPersister; 19 | import io.s4.persist.Persister; 20 | import io.s4.util.clock.Clock; 21 | 22 | import java.lang.reflect.Method; 23 | import java.util.List; 24 | 25 | import org.apache.log4j.Logger; 26 | 27 | public class PrototypeWrapper { 28 | 29 | private static Logger logger = Logger.getLogger(PrototypeWrapper.class); 30 | private ProcessingElement prototype; 31 | Persister lookupTable; 32 | 33 | public String getId() { 34 | return prototype.getId(); 35 | } 36 | 37 | public PrototypeWrapper(ProcessingElement prototype, Clock s4Clock) { 38 | this.prototype = prototype; 39 | lookupTable = new ConMapPersister(s4Clock); 40 | System.out.println("Using ConMapPersister .."); 41 | // this bit of reflection is not a performance issue because it is only 42 | // invoked at configuration time 43 | try { 44 | ((ConMapPersister) lookupTable).setSelfClean(true); 45 | ((ConMapPersister) lookupTable).init(); 46 | // set the persister in prototype 47 | Method method = prototype.getClass().getMethod("setLookupTable", 48 | Persister.class); 49 | method.invoke(prototype, lookupTable); 50 | } catch (NoSuchMethodException e) { 51 | // this is expected 52 | } catch (Exception e) { 53 | // this is not expected 54 | Logger.getLogger("s4") 55 | .error("Exception invoking setLookupTable on prototype", e); 56 | } 57 | } 58 | 59 | /** 60 | * Find PE corresponding to keyValue. If no such PE exists, then a new one 61 | * is created by cloning the prototype and this is returned. As a 62 | * side-effect, the last update time for the PE in the lookup table is 63 | * modified. 64 | * 65 | * @param keyValue 66 | * key value 67 | * @return PE corresponding to keyValue. 68 | */ 69 | public ProcessingElement getPE(String keyValue) { 70 | ProcessingElement pe = null; 71 | try { 72 | pe = (ProcessingElement) lookupTable.get(keyValue); 73 | if (pe == null) { 74 | pe = (ProcessingElement) prototype.clone(); 75 | //invoke the initialization method if it has been specified 76 | if (pe.getInitMethod() != null) { 77 | Method initMethod = pe.getClass().getMethod(pe.getInitMethod(), new Class[0]); 78 | initMethod.invoke(pe, (new Object[0])); 79 | } 80 | 81 | } 82 | // update the last update time on the entry 83 | lookupTable.set(keyValue, pe, prototype.getTtl()); 84 | 85 | } catch (Exception e) { 86 | logger.error("exception when looking up pe for key:" + keyValue, e); 87 | } 88 | 89 | return pe; 90 | } 91 | 92 | /** 93 | * Find PE corresponding to keyValue. If no such PE exists, then null is 94 | * returned. Note: the last update time is not modified in the lookup table. 95 | * 96 | * @param keyValue 97 | * key value 98 | * @return PE corresponding to keyValue, if such a PE exists. Null 99 | * otherwise. 100 | */ 101 | public ProcessingElement lookupPE(String keyValue) { 102 | ProcessingElement pe = null; 103 | 104 | try { 105 | pe = (ProcessingElement) lookupTable.get(keyValue); 106 | 107 | } catch (Exception e) { 108 | logger.error("exception when looking up pe for key:" + keyValue, e); 109 | } 110 | 111 | return pe; 112 | } 113 | 114 | public int getPECount() { 115 | return lookupTable.keySet().size(); 116 | } 117 | 118 | public List advise() { 119 | return prototype.advise(); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/ReroutePE.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | import io.s4.dispatcher.EventDispatcher; 19 | import io.s4.dispatcher.transformer.Transformer; 20 | import io.s4.util.Cloner; 21 | import io.s4.util.ClonerGenerator; 22 | 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | import org.apache.log4j.Logger; 27 | 28 | public class ReroutePE extends AbstractPE { 29 | private static Logger logger = Logger.getLogger(ReroutePE.class); 30 | private EventDispatcher dispatcher; 31 | private Transformer[] transformers = new Transformer[0]; 32 | // private List keys; 33 | private String id = "ReroutePE"; 34 | private String outputStreamName; 35 | 36 | public String getId() { 37 | return id; 38 | } 39 | 40 | public void setId(String id) { 41 | this.id = id; 42 | } 43 | 44 | public void setDispatcher(EventDispatcher dispatcher) { 45 | this.dispatcher = dispatcher; 46 | } 47 | 48 | public void setTransformers(Transformer[] transformers) { 49 | this.transformers = transformers; 50 | } 51 | 52 | // public void setEventNames(String[] eventNames) { 53 | // keys = new ArrayList(); 54 | // for (String eventName : eventNames) { 55 | // keys.add(new EventAdvice(eventName, "*")); 56 | // } 57 | // } 58 | 59 | // @Override 60 | // public List advise() { 61 | // return keys; 62 | // } 63 | 64 | public void setOutputStreamName(String outputStreamName) { 65 | this.outputStreamName = outputStreamName; 66 | } 67 | 68 | private Map clonerMap = new HashMap(); 69 | 70 | public void processEvent(Object event) { 71 | Object newEvent = event; 72 | if (transformers != null && transformers.length > 0) { 73 | Cloner cloner = clonerMap.get(event.getClass().getName()); 74 | if (cloner == null) { 75 | ClonerGenerator cg = new ClonerGenerator(); 76 | // generate byte code that knows how to call the clone method on 77 | // this event 78 | Class clonerClass = cg.generate(event.getClass()); 79 | try { 80 | cloner = (Cloner) clonerClass.newInstance(); 81 | clonerMap.put(event.getClass().getName(), cloner); 82 | } catch (InstantiationException ie) { 83 | Logger.getLogger(this.getClass()).error(ie); 84 | throw new RuntimeException(ie); 85 | } catch (IllegalAccessException ias) { 86 | Logger.getLogger(this.getClass()).error(ias); 87 | throw new RuntimeException(ias); 88 | } 89 | } 90 | newEvent = cloner.clone(event); 91 | } 92 | 93 | for (Transformer transformer : transformers) { 94 | newEvent = transformer.transform(newEvent); 95 | if (newEvent == null) { 96 | return; 97 | } 98 | } 99 | 100 | dispatcher.dispatchEvent(outputStreamName, newEvent); 101 | } 102 | 103 | public int getTtl() { 104 | return -1; // never die 105 | } 106 | 107 | @Override 108 | public void output() { 109 | // TODO Auto-generated method stub 110 | 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/io/s4/processor/SimpleCountingPE.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.processor; 17 | 18 | import io.s4.persist.Persister; 19 | 20 | import java.util.List; 21 | 22 | public class SimpleCountingPE extends AbstractPE { 23 | private boolean clearOnOutput; 24 | private OutputFormatter outputFormatter; 25 | private Persister persister; 26 | private int persistTime; 27 | private String keyPrefix = "s4:counter"; 28 | private boolean dirty = false; 29 | private String id = "SimpleCountingPE"; 30 | 31 | public String getId() { 32 | return id; 33 | } 34 | 35 | public void setId(String id) { 36 | this.id = id; 37 | } 38 | 39 | public void setClearOnOutput(boolean clearOnOutput) { 40 | this.clearOnOutput = clearOnOutput; 41 | } 42 | 43 | public void setOutputFormatter(OutputFormatter outputFormatter) { 44 | this.outputFormatter = outputFormatter; 45 | } 46 | 47 | public void setPersister(Persister persister) { 48 | this.persister = persister; 49 | } 50 | 51 | public void setPersistTime(int persistTime) { 52 | this.persistTime = persistTime; 53 | } 54 | 55 | public void setKeyPrefix(String keyPrefix) { 56 | this.keyPrefix = keyPrefix; 57 | } 58 | 59 | private long counter = 0; 60 | 61 | @Override 62 | public void output() { 63 | Object outputValue = null; 64 | synchronized (this) { 65 | if (!dirty) { 66 | return; 67 | } 68 | outputValue = new Long(counter); 69 | if (clearOnOutput) { 70 | counter = 0; 71 | } 72 | dirty = false; 73 | } 74 | 75 | List simpleKeyValues = this.getKeyValue(); 76 | StringBuffer keyBuffer = new StringBuffer(keyPrefix); 77 | for (Object simpleKeyValue : simpleKeyValues) { 78 | keyBuffer.append(":"); 79 | keyBuffer.append(String.valueOf(simpleKeyValue)); 80 | } 81 | 82 | if (outputFormatter != null) { 83 | outputValue = outputFormatter.format(outputValue); 84 | } 85 | 86 | try { 87 | persister.set(keyBuffer.toString(), outputValue, persistTime); 88 | } catch (InterruptedException ie) { 89 | Thread.currentThread().interrupt(); 90 | } 91 | 92 | } 93 | 94 | public void processEvent(Object event) { 95 | synchronized (this) { 96 | counter++; 97 | dirty = true; 98 | } 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/io/s4/schema/AvroSchemaManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.schema; 17 | 18 | import java.util.Map; 19 | import java.util.concurrent.ConcurrentHashMap; 20 | 21 | import org.apache.avro.Schema; 22 | import org.apache.log4j.Logger; 23 | 24 | public class AvroSchemaManager extends SchemaManager { 25 | Map schemaMap = new ConcurrentHashMap(); 26 | 27 | public void init() { 28 | super.init(); 29 | for (String schemaName : schemaStringMap.keySet()) { 30 | String schemaString = schemaStringMap.get(schemaName); 31 | schemaMap.put(schemaName, Schema.parse(schemaString)); 32 | if (Logger.getLogger("s4").isDebugEnabled()) { 33 | Logger.getLogger("s4").debug("Putting schema " + schemaName 34 | + ": " + schemaMap.get(schemaName)); 35 | } 36 | } 37 | 38 | } 39 | 40 | public void addSchemaFromFile(String schemaName, String schemaFilename) { 41 | super.addSchemaFromFile(schemaName, schemaFilename); 42 | String schemaString = schemaStringMap.get(schemaName); 43 | schemaMap.put(schemaName, Schema.parse(schemaString)); 44 | } 45 | 46 | public void addSchema(String schemaName, String schemaString) { 47 | super.addSchema(schemaName, schemaString); 48 | schemaMap.put(schemaName, Schema.parse(schemaString)); 49 | } 50 | 51 | public Schema getCompiledSchema(String schemaName) { 52 | return schemaMap.get(schemaName); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/io/s4/schema/SchemaContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.schema; 17 | 18 | import java.util.Map; 19 | import java.util.concurrent.ConcurrentHashMap; 20 | 21 | public class SchemaContainer { 22 | private Map schemaMap = new ConcurrentHashMap(); 23 | 24 | public Schema getSchema(Class clazz) { 25 | String schemaKey = clazz.getClassLoader().toString() + ":" 26 | + clazz.getName(); 27 | 28 | Schema schema = schemaMap.get(schemaKey); 29 | if (schema == null) { 30 | schema = new Schema(clazz); 31 | schemaMap.put(schemaKey, schema); 32 | } 33 | 34 | return schema; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/s4/schema/SchemaManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.schema; 17 | 18 | import java.io.BufferedReader; 19 | import java.io.File; 20 | import java.io.FileReader; 21 | import java.io.IOException; 22 | import java.util.Map; 23 | import java.util.concurrent.ConcurrentHashMap; 24 | 25 | import org.apache.log4j.Logger; 26 | 27 | public class SchemaManager { 28 | protected Map schemaStringMap = new ConcurrentHashMap(); 29 | private Map schemaFilenameMap; 30 | 31 | public void init() { 32 | if (schemaFilenameMap == null) { 33 | return; 34 | } 35 | 36 | for (String schemaName : schemaFilenameMap.keySet()) { 37 | String schemaFilename = schemaFilenameMap.get(schemaName); 38 | addSchemaFromFile(schemaName, schemaFilename); 39 | } 40 | } 41 | 42 | public void setSchemas(Map schemaFilenameMap) { 43 | this.schemaFilenameMap = schemaFilenameMap; 44 | } 45 | 46 | public String getSchemaString(String schemaName) { 47 | return schemaStringMap.get(schemaName); 48 | } 49 | 50 | public void addSchemaFromFile(String schemaName, String schemaFilename) { 51 | BufferedReader br = null; 52 | FileReader fr = null; 53 | try { 54 | File schemaFile = new File(schemaFilename); 55 | if (schemaFile.exists()) { 56 | fr = new FileReader(schemaFilename); 57 | br = new BufferedReader(fr); 58 | StringBuffer schemaBuffer = new StringBuffer(); 59 | String line = null; 60 | while ((line = br.readLine()) != null) { 61 | schemaBuffer.append(line); 62 | } 63 | schemaStringMap.put(schemaName, schemaBuffer.toString()); 64 | } else { 65 | Logger.getLogger("s4").error("Missing schema file:" 66 | + schemaFilename + " for type:" + schemaName); 67 | } 68 | } catch (IOException ioe) { 69 | Logger.getLogger("s4").error("Exception reading schema file " 70 | + schemaFilename, 71 | ioe); 72 | } finally { 73 | if (br != null) { 74 | try { 75 | br.close(); 76 | } catch (Exception e) { 77 | } 78 | } 79 | if (fr != null) { 80 | try { 81 | fr.close(); 82 | } catch (Exception e) { 83 | } 84 | 85 | } 86 | } 87 | } 88 | 89 | public void addSchema(String schemaName, String schema) { 90 | schemaStringMap.put(schemaName, schema); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/io/s4/serialize/AvroUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.serialize; 17 | 18 | import java.io.ByteArrayInputStream; 19 | import java.io.ByteArrayOutputStream; 20 | 21 | import org.apache.avro.Schema; 22 | import org.apache.avro.generic.GenericDatumReader; 23 | import org.apache.avro.generic.GenericDatumWriter; 24 | import org.apache.avro.generic.GenericRecord; 25 | import org.apache.avro.io.BinaryDecoder; 26 | import org.apache.avro.io.BinaryEncoder; 27 | 28 | public class AvroUtils { 29 | public static GenericRecord deserialize(Schema schema, byte[] array) 30 | throws Exception { 31 | GenericDatumReader serveReader = new GenericDatumReader(schema); 32 | return serveReader.read(null, 33 | new BinaryDecoder(new ByteArrayInputStream(array))); 34 | } 35 | 36 | public static byte[] serialize(Schema schema, GenericRecord content) 37 | throws Exception { 38 | GenericDatumWriter serveWriter = new GenericDatumWriter(schema); 39 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 40 | serveWriter.write(content, new BinaryEncoder(out)); 41 | return out.toByteArray(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/s4/serialize/KryoSerDeser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.serialize; 17 | 18 | import java.nio.ByteBuffer; 19 | 20 | import com.esotericsoftware.kryo.Kryo; 21 | import com.esotericsoftware.kryo.ObjectBuffer; 22 | import com.esotericsoftware.kryo.serialize.SimpleSerializer; 23 | 24 | public class KryoSerDeser implements SerializerDeserializer { 25 | 26 | private Kryo kryo = new Kryo(); 27 | 28 | private int initialBufferSize = 2048; 29 | private int maxBufferSize = 256*1024; 30 | 31 | public void setInitialBufferSize(int initialBufferSize) { 32 | this.initialBufferSize = initialBufferSize; 33 | } 34 | 35 | public void setMaxBufferSize(int maxBufferSize) { 36 | this.maxBufferSize = maxBufferSize; 37 | } 38 | 39 | public KryoSerDeser() { 40 | kryo.setRegistrationOptional(true); 41 | 42 | // UUIDs don't have a no-arg constructor. 43 | kryo.register(java.util.UUID.class, 44 | new SimpleSerializer() { 45 | @Override 46 | public java.util.UUID read(ByteBuffer buf) { 47 | return new java.util.UUID(buf.getLong(), 48 | buf.getLong()); 49 | } 50 | 51 | @Override 52 | public void write(ByteBuffer buf, java.util.UUID uuid) { 53 | buf.putLong(uuid.getMostSignificantBits()); 54 | buf.putLong(uuid.getLeastSignificantBits()); 55 | } 56 | 57 | }); 58 | } 59 | 60 | @Override 61 | public Object deserialize(byte[] rawMessage) { 62 | ObjectBuffer buffer = new ObjectBuffer(kryo, initialBufferSize, maxBufferSize); 63 | return buffer.readClassAndObject(rawMessage); 64 | } 65 | 66 | @Override 67 | public byte[] serialize(Object message) { 68 | ObjectBuffer buffer = new ObjectBuffer(kryo, initialBufferSize, maxBufferSize); 69 | return buffer.writeClassAndObject(message); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/io/s4/serialize/SerializerDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.serialize; 17 | 18 | public interface SerializerDeserializer { 19 | public byte[] serialize(Object message); 20 | 21 | public Object deserialize(byte[] rawMessage); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/s4/test/TestPersisterEventClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.test; 17 | 18 | import io.s4.persist.ConMapPersister; 19 | import io.s4.persist.HashMapPersister; 20 | import io.s4.persist.Persister; 21 | import io.s4.util.clock.EventClock; 22 | 23 | public class TestPersisterEventClock { 24 | 25 | static EventClock s4Clock; 26 | static Persister persister; 27 | 28 | public static void main(String[] args) { 29 | TestPersisterEventClock testPersisterClock = new TestPersisterEventClock(); 30 | s4Clock = new EventClock(); 31 | s4Clock.updateTime(69990000); 32 | TimeUpdaterThread timeUpdater = new TimeUpdaterThread(s4Clock); 33 | Thread timeUpdaterThread = new Thread(timeUpdater); 34 | timeUpdaterThread.start(); 35 | persister = new HashMapPersister(s4Clock); 36 | testPersisterClock.testPersister(persister); 37 | persister = new ConMapPersister(s4Clock); 38 | testPersisterClock.testPersister(persister); 39 | } 40 | 41 | public void testPersister(Persister persister) { 42 | HashMapPersister hp = null; 43 | ConMapPersister cp = null; 44 | if (persister instanceof HashMapPersister) { 45 | hp = (HashMapPersister) persister; 46 | hp.init(); 47 | 48 | hp.set("mykey1", "Test1", 40); 49 | hp.set("mykey2", "Test2", 48); 50 | hp.set("mykey3", "Test2", -1); 51 | 52 | try { 53 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 1); 54 | } catch (Exception e) { 55 | } 56 | 57 | System.out.println("mykey1: " + hp.get("mykey1")); 58 | System.out.println("mykey2: " + hp.get("mykey2")); 59 | System.out.println("mykey3: " + hp.get("mykey3")); 60 | 61 | System.out.println("Going to sleep..."); 62 | try { 63 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 41000); 64 | } catch (Exception e) { 65 | } 66 | System.out.println("Waking up"); 67 | 68 | System.out.println("mykey1: " + hp.get("mykey1")); 69 | System.out.println("mykey2: " + hp.get("mykey2")); 70 | System.out.println("mykey3: " + hp.get("mykey3")); 71 | 72 | System.out.println("Going to sleep..."); 73 | try { 74 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 10000); 75 | } catch (Exception e) { 76 | } 77 | System.out.println("Waking up"); 78 | 79 | System.out.println("mykey1: " + hp.get("mykey1")); 80 | System.out.println("mykey2: " + hp.get("mykey2")); 81 | System.out.println("mykey3: " + hp.get("mykey3")); 82 | System.out.println("cleanUp: " + hp.cleanOutGarbage()); 83 | 84 | } 85 | if (persister instanceof ConMapPersister) { 86 | cp = (ConMapPersister) persister; 87 | cp.init(); 88 | 89 | cp.set("mykey1", "Test1", 40); 90 | cp.set("mykey2", "Test2", 48); 91 | cp.set("mykey3", "Test2", -1); 92 | 93 | try { 94 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 1); 95 | } catch (Exception e) { 96 | } 97 | 98 | System.out.println("mykey1: " + cp.get("mykey1")); 99 | System.out.println("mykey2: " + cp.get("mykey2")); 100 | System.out.println("mykey3: " + cp.get("mykey3")); 101 | 102 | System.out.println("Going to sleep..."); 103 | try { 104 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 41000); 105 | } catch (Exception e) { 106 | } 107 | System.out.println("Waking up"); 108 | 109 | System.out.println("mykey1: " + cp.get("mykey1")); 110 | System.out.println("mykey2: " + cp.get("mykey2")); 111 | System.out.println("mykey3: " + cp.get("mykey3")); 112 | 113 | System.out.println("Going to sleep..."); 114 | try { 115 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 10000); 116 | } catch (Exception e) { 117 | } 118 | System.out.println("Waking up"); 119 | 120 | System.out.println("mykey1: " + cp.get("mykey1")); 121 | System.out.println("mykey2: " + cp.get("mykey2")); 122 | System.out.println("mykey3: " + cp.get("mykey3")); 123 | System.out.println("cleanUp: " + cp.cleanOutGarbage()); 124 | } 125 | } 126 | 127 | static class TimeUpdaterThread implements Runnable { 128 | 129 | EventClock s4Clock; 130 | 131 | public TimeUpdaterThread(EventClock s4Clock) { 132 | this.s4Clock = s4Clock; 133 | } 134 | 135 | @Override 136 | public void run() { 137 | // TODO Auto-generated method stub 138 | 139 | // tick the clock 140 | for (long currentTime = 69996400; currentTime <= 70099900; currentTime += 400) { 141 | System.out.println("Setting time to " + currentTime); 142 | s4Clock.updateTime(currentTime); 143 | try { 144 | Thread.sleep(400); 145 | } catch (InterruptedException ie) { 146 | } 147 | } 148 | } 149 | 150 | } 151 | 152 | } 153 | -------------------------------------------------------------------------------- /src/main/java/io/s4/test/TestPersisterWallClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.test; 17 | 18 | import io.s4.persist.ConMapPersister; 19 | import io.s4.persist.HashMapPersister; 20 | import io.s4.persist.Persister; 21 | import io.s4.util.clock.WallClock; 22 | 23 | public class TestPersisterWallClock { 24 | 25 | static WallClock s4Clock; 26 | static Persister persister; 27 | 28 | public static void main(String[] args) { 29 | TestPersisterWallClock testPersisterClock = new TestPersisterWallClock(); 30 | s4Clock = new WallClock(); 31 | persister = new HashMapPersister(s4Clock); 32 | testPersisterClock.testPersister(persister); 33 | persister = new ConMapPersister(s4Clock); 34 | testPersisterClock.testPersister(persister); 35 | } 36 | 37 | public void testPersister(Persister persister) { 38 | HashMapPersister hp = null; 39 | ConMapPersister cp = null; 40 | if (persister instanceof HashMapPersister) { 41 | hp = (HashMapPersister) persister; 42 | hp.init(); 43 | 44 | hp.set("mykey1", "Test1", 40); 45 | hp.set("mykey2", "Test2", 48); 46 | hp.set("mykey3", "Test2", -1); 47 | 48 | try { 49 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 1); 50 | } catch (Exception e) { 51 | } 52 | 53 | System.out.println("mykey1: " + hp.get("mykey1")); 54 | System.out.println("mykey2: " + hp.get("mykey2")); 55 | System.out.println("mykey3: " + hp.get("mykey3")); 56 | 57 | System.out.println("Going to sleep..."); 58 | try { 59 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 41000); 60 | } catch (Exception e) { 61 | } 62 | System.out.println("Waking up"); 63 | 64 | System.out.println("mykey1: " + hp.get("mykey1")); 65 | System.out.println("mykey2: " + hp.get("mykey2")); 66 | System.out.println("mykey3: " + hp.get("mykey3")); 67 | 68 | System.out.println("Going to sleep..."); 69 | try { 70 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 10000); 71 | } catch (Exception e) { 72 | } 73 | System.out.println("Waking up"); 74 | 75 | System.out.println("mykey1: " + hp.get("mykey1")); 76 | System.out.println("mykey2: " + hp.get("mykey2")); 77 | System.out.println("mykey3: " + hp.get("mykey3")); 78 | System.out.println("cleanUp: " + hp.cleanOutGarbage()); 79 | 80 | } 81 | if (persister instanceof ConMapPersister) { 82 | cp = (ConMapPersister) persister; 83 | cp.init(); 84 | 85 | cp.set("mykey1", "Test1", 40); 86 | cp.set("mykey2", "Test2", 48); 87 | cp.set("mykey3", "Test2", -1); 88 | 89 | try { 90 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 1); 91 | } catch (Exception e) { 92 | } 93 | 94 | System.out.println("mykey1: " + cp.get("mykey1")); 95 | System.out.println("mykey2: " + cp.get("mykey2")); 96 | System.out.println("mykey3: " + cp.get("mykey3")); 97 | 98 | System.out.println("Going to sleep..."); 99 | try { 100 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 41000); 101 | } catch (Exception e) { 102 | } 103 | System.out.println("Waking up"); 104 | 105 | System.out.println("mykey1: " + cp.get("mykey1")); 106 | System.out.println("mykey2: " + cp.get("mykey2")); 107 | System.out.println("mykey3: " + cp.get("mykey3")); 108 | 109 | System.out.println("Going to sleep..."); 110 | try { 111 | s4Clock.waitForTime(s4Clock.getCurrentTime() + 10000); 112 | } catch (Exception e) { 113 | } 114 | System.out.println("Waking up"); 115 | 116 | System.out.println("mykey1: " + cp.get("mykey1")); 117 | System.out.println("mykey2: " + cp.get("mykey2")); 118 | System.out.println("mykey3: " + cp.get("mykey3")); 119 | System.out.println("cleanUp: " + cp.cleanOutGarbage()); 120 | } 121 | } 122 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/util/ByteArrayIOChannel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import io.s4.client.IOChannel; 19 | 20 | import java.io.IOException; 21 | import java.io.InputStream; 22 | import java.io.OutputStream; 23 | import java.net.Socket; 24 | 25 | public class ByteArrayIOChannel implements IOChannel { 26 | private InputStream in; 27 | private OutputStream out; 28 | 29 | public ByteArrayIOChannel(Socket socket) throws IOException { 30 | in = socket.getInputStream(); 31 | out = socket.getOutputStream(); 32 | } 33 | 34 | private void readBytes(byte[] s, int n) throws IOException { 35 | int r = 0; // bytes read so far 36 | 37 | do { 38 | // keep reading bytes till the required "n" are read 39 | int p = in.read(s, r, (n - r)); 40 | 41 | if (p == -1) { 42 | throw new IOException("reached end of stream after reading " 43 | + r + " bytes. expected " + n + " bytes"); 44 | } 45 | 46 | r += p; 47 | 48 | } while (r < n); 49 | } 50 | 51 | public byte[] recv() throws IOException { 52 | // first read size of byte array. 53 | // unsigned int, big endian: 0A0B0C0D -> {0A, 0B, 0C, 0D} 54 | byte[] s = { 0, 0, 0, 0 }; 55 | readBytes(s, 4); 56 | 57 | // to allow full range of int, using long for size 58 | int size = (int) ( // NOTE: type cast not necessary for int 59 | (0xff & s[0]) << 24 | (0xff & s[1]) << 16 | (0xff & s[2]) << 8 | (0xff & s[3]) << 0); 60 | 61 | if (size == 0) 62 | return null; 63 | 64 | byte[] v = new byte[size]; 65 | 66 | // read the message 67 | readBytes(v, size); 68 | 69 | return v; 70 | } 71 | 72 | public void send(byte[] v) throws IOException { 73 | byte[] s = { 0, 0, 0, 0 }; 74 | int size = v.length; 75 | 76 | s[3] = (byte) (size & 0xff); 77 | size >>= 8; 78 | s[2] = (byte) (size & 0xff); 79 | size >>= 8; 80 | s[1] = (byte) (size & 0xff); 81 | size >>= 8; 82 | s[0] = (byte) (size & 0xff); 83 | 84 | out.write(s); 85 | out.write(v); 86 | out.flush(); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/Cloner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | public interface Cloner { 19 | public Object clone(Object o); 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/DoubleOutputFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import io.s4.processor.OutputFormatter; 19 | 20 | import org.apache.log4j.Logger; 21 | 22 | public class DoubleOutputFormatter implements OutputFormatter { 23 | 24 | @Override 25 | public Object format(Object outputValue) { 26 | double doubleResult = 0.0; 27 | 28 | if (outputValue instanceof Double) { 29 | return outputValue; 30 | } 31 | 32 | try { 33 | doubleResult = ((Number) outputValue).doubleValue(); // outputValue 34 | // better be 35 | // convertible!! 36 | } catch (Exception e) { 37 | Logger.getLogger("s4") 38 | .error("Exception converting value to double", e); 39 | return null; 40 | } 41 | return doubleResult; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/GsonUtil.java: -------------------------------------------------------------------------------- 1 | package io.s4.util; 2 | 3 | import java.lang.reflect.Type; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import com.google.gson.Gson; 8 | import com.google.gson.GsonBuilder; 9 | import com.google.gson.JsonElement; 10 | import com.google.gson.JsonObject; 11 | import com.google.gson.JsonSerializationContext; 12 | import com.google.gson.JsonSerializer; 13 | 14 | public class GsonUtil { 15 | 16 | /** 17 | * This is a workaround for a bug in Gson: 18 | * 19 | * http://code.google.com/p/google-gson/issues/detail?id=279 20 | * "Templated collections of collections do not serialize correctly" 21 | */ 22 | private final static JsonSerializer objectSerializer = new JsonSerializer() { 23 | public JsonElement serialize(Object src, Type typeOfSrc, 24 | JsonSerializationContext context) { 25 | 26 | if (src.getClass() != Object.class) { 27 | return context.serialize(src, src.getClass()); 28 | } 29 | 30 | return new JsonObject(); 31 | } 32 | }; 33 | 34 | private static HashMap typeAdapters = new HashMap(); 35 | 36 | private volatile static Gson gson = null; 37 | 38 | // build gson ASAP 39 | static { 40 | build(); 41 | } 42 | 43 | /** 44 | * Add a type adapter to the chain. 45 | * 46 | * @param type 47 | * @param typeAdapter 48 | */ 49 | public static synchronized void registerTypeAdapter(Type type, 50 | Object typeAdapter) { 51 | typeAdapters.put(type, typeAdapter); 52 | build(); 53 | } 54 | 55 | /** 56 | * Get a Gson instance. 57 | * 58 | * @return Gson instance with all the adapters registered. 59 | */ 60 | public static Gson get() { 61 | return gson; 62 | } 63 | 64 | private static synchronized void build() { 65 | GsonBuilder b = (new GsonBuilder()).registerTypeAdapter(Object.class, 66 | objectSerializer); 67 | 68 | for (Map.Entry e : typeAdapters.entrySet()) { 69 | b.registerTypeAdapter(e.getKey(), e.getValue()); 70 | } 71 | 72 | gson = b.create(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/KeyUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import java.io.UnsupportedEncodingException; 19 | import java.util.ArrayList; 20 | import java.util.Collection; 21 | 22 | /** 23 | * from http://github.com/dustin/java-memcached-client/blob/master/src/main/java/net/spy/memcached/KeyUtil.java 24 | */ 25 | /** 26 | * Utilities for processing key values. 27 | */ 28 | public class KeyUtil { 29 | 30 | /** 31 | * Get the bytes for a key. 32 | * 33 | * @param k 34 | * the key 35 | * @return the bytes 36 | */ 37 | public static byte[] getKeyBytes(String k) { 38 | try { 39 | return k.getBytes("UTF-8"); 40 | } catch (UnsupportedEncodingException e) { 41 | throw new RuntimeException(e); 42 | } 43 | } 44 | 45 | /** 46 | * Get the keys in byte form for all of the string keys. 47 | * 48 | * @param keys 49 | * a collection of keys 50 | * @return return a collection of the byte representations of keys 51 | */ 52 | public static Collection getKeyBytes(Collection keys) { 53 | Collection rv = new ArrayList(keys.size()); 54 | for (String s : keys) { 55 | rv.add(getKeyBytes(s)); 56 | } 57 | return rv; 58 | } 59 | } -------------------------------------------------------------------------------- /src/main/java/io/s4/util/MethodInvoker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import java.lang.reflect.InvocationTargetException; 19 | import java.lang.reflect.Method; 20 | import java.util.HashMap; 21 | import java.util.concurrent.ConcurrentHashMap; 22 | 23 | /** 24 | * Utility class to invoke a method on an arbitrary object, if such a method is 25 | * defined. 26 | */ 27 | public class MethodInvoker { 28 | 29 | /** 30 | * Find and invoke a getter on an object. A getter for parameter N is a 31 | * public method whose name equals "get" + N, ignoring case, and which takes 32 | * zero arguments. If no such method is found, an exception is thrown. 33 | * 34 | * @param obj 35 | * object on which getter id to be invoked 36 | * @param name 37 | * parameter name 38 | * @return value returned by the getter method, if such a method is found. 39 | * Null if the object is null. 40 | * @throws Exception 41 | * if no suitable getter is found, or if the getter method 42 | * throws an exception. The latter is wrapped in an 43 | * {@link InvocationTargetException} 44 | */ 45 | public static Object invokeGetter(Object obj, String name) throws Exception { 46 | if (obj != null) { 47 | Method getter = findGetter(obj.getClass(), name); 48 | 49 | if (getter != null) { 50 | return getter.invoke(obj); 51 | 52 | } else { 53 | throw new NoGetterException(obj.getClass(), name); 54 | } 55 | 56 | } else { 57 | throw new Exception("Null Target"); 58 | } 59 | } 60 | 61 | private static ConcurrentHashMap, HashMap> gettersMap = new ConcurrentHashMap, HashMap>(); 62 | 63 | private static Method findGetter(Class clazz, String name) { 64 | HashMap getters = gettersMap.get(clazz); 65 | 66 | if (getters == null) { 67 | HashMap newGetters = reflectGetters(clazz); 68 | 69 | getters = gettersMap.putIfAbsent(clazz, newGetters); 70 | 71 | if (getters == null) 72 | getters = newGetters; 73 | } 74 | 75 | return getters.get(name.toLowerCase()); 76 | } 77 | 78 | private static HashMap reflectGetters(Class clazz) { 79 | HashMap map = new HashMap(); 80 | 81 | for (Method m : clazz.getMethods()) { 82 | // the method we are interested in should be named get* and take no 83 | // arguments. 84 | String n = m.getName(); 85 | 86 | if (m.getParameterTypes().length == 0 && n.startsWith("get")) { 87 | String name = n.substring(3).toLowerCase(); 88 | 89 | map.put(name, m); 90 | } 91 | } 92 | 93 | return map; 94 | } 95 | 96 | private static Class[] getTypes(Object[] args) { 97 | Class[] aT = new Class[args.length]; 98 | for (int i = 0; i < args.length; ++i) { 99 | aT[i] = args[i].getClass(); 100 | } 101 | 102 | return aT; 103 | } 104 | 105 | public static class NoGetterException extends Exception { 106 | public NoGetterException(Class clazz, String name) { 107 | super("No Getter for attribute " + clazz.getName() + "." + name); 108 | } 109 | } 110 | 111 | public static class NoMethodException extends Exception { 112 | public NoMethodException(Class clazz, String name, Object[] args) { 113 | super("No method found " + clazz.getName() + "." + name + "(" 114 | + MethodInvoker.getTypes(args) + ")"); 115 | } 116 | } 117 | 118 | public static class NullTargetException extends Exception { 119 | public NullTargetException() { 120 | super(); 121 | } 122 | } 123 | 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/MetricsName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | public enum MetricsName { 19 | // metrics event name 20 | S4_APP_METRICS("S4::S4AppMetrics"), S4_EVENT_METRICS("S4::S4EventMetrics"), S4_CORE_METRICS( 21 | "S4::S4CoreMetrics"), 22 | 23 | // metrics name 24 | low_level_listener_msg_in_ct("lll_in"), low_level_listener_msg_drop_ct( 25 | "lll_dr"), low_level_listener_qsz("lll_qsz"), low_level_listener_badmsg_ct( 26 | "lll_bad"), // exception can't be caught 27 | generic_listener_msg_in_ct("gl_in"), pecontainer_ev_dq_ct("pec_dq"), pecontainer_ev_nq_ct( 28 | "pec_nq"), pecontainer_msg_drop_ct("pec_dr"), pecontainer_qsz( 29 | "pec_qsz"), pecontainer_qsz_w("pec_qsz_w"), pecontainer_ev_process_ct( 30 | "pec_pr"), pecontainer_pe_ct("pec_pe"), pecontainer_ev_err_ct( 31 | "pec_err"), // exception can't be caught 32 | pecontainer_exec_elapse_time("pec_exec_t"), low_level_emitter_msg_out_ct( 33 | "lle_out"), low_level_emitter_out_err_ct("lle_err"), low_level_emitter_qsz( 34 | "lle_qsz"), s4_core_exit_ct("s4_ex_ct"), s4_core_free_mem("s4_fmem"), pe_join_ev_ct( 35 | "pe_j_ct"), pe_error_count("pe_err"); 36 | 37 | private final String eventShortName; 38 | 39 | private MetricsName(String eventShortName) { 40 | this.eventShortName = eventShortName; 41 | } 42 | 43 | public String toString() { 44 | return eventShortName; 45 | } 46 | 47 | public static void main(String[] args) { 48 | System.out.println(generic_listener_msg_in_ct.toString()); 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/MiscConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | public class MiscConstants { 19 | public final static String EVENT_WRAPPER_SCHEMA_NAME = "EventWrapper"; 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/NumberUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import java.math.BigInteger; 19 | 20 | public class NumberUtils { 21 | 22 | private final static BigInteger B64 = BigInteger.ZERO.setBit(64); 23 | 24 | public static BigInteger getLongAsUnsignedBigInteger(long number) { 25 | if (number >= 0) 26 | return BigInteger.valueOf(number); 27 | return BigInteger.valueOf(number).add(B64); 28 | } 29 | 30 | public static String getUnsignedBigIntegerAsHex(BigInteger bi) { 31 | String hexString = bi.toString(16); 32 | if (hexString.length() < 16) { 33 | String zeroes = "000000000000000"; 34 | hexString = zeroes.substring(0, 16 - hexString.length()) 35 | + hexString; 36 | } 37 | return hexString.toUpperCase(); 38 | } 39 | 40 | public static void main(String args[]) { 41 | BigInteger bi = getLongAsUnsignedBigInteger(0x00f1200000004561L); 42 | System.out.println(getUnsignedBigIntegerAsHex(bi)); 43 | 44 | bi = getLongAsUnsignedBigInteger(0x80f12000dd004561L); 45 | System.out.println(getUnsignedBigIntegerAsHex(bi)); 46 | 47 | bi = getLongAsUnsignedBigInteger(0x1161L); 48 | System.out.println(getUnsignedBigIntegerAsHex(bi)); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/PreprodLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import java.io.File; 19 | import java.io.FileOutputStream; 20 | import java.io.IOException; 21 | import java.text.DecimalFormat; 22 | 23 | import org.apache.log4j.Logger; 24 | 25 | public class PreprodLogger { 26 | 27 | String filenamePrefix; 28 | File file; 29 | FileOutputStream fos; 30 | 31 | private DecimalFormat formatter = new DecimalFormat("0000"); 32 | 33 | public void setFilenamePrefix(String filenamePrefix) { 34 | this.filenamePrefix = filenamePrefix; 35 | } 36 | 37 | public PreprodLogger() { 38 | } 39 | 40 | public void openNewFile() { 41 | for (int count = 1; true; count++) { 42 | String countString = formatter.format(count); 43 | String filename = filenamePrefix + "." + countString + ".txt"; 44 | file = new File(filename); 45 | if (!file.exists()) { 46 | break; 47 | } 48 | } 49 | 50 | try { 51 | fos = new FileOutputStream(file); 52 | } catch (IOException ioe) { 53 | Logger.getLogger("s4") 54 | .error("Some sort of exception opening event logging file", 55 | ioe); 56 | return; 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/ReverseDoubleOutputFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import io.s4.processor.OutputFormatter; 19 | 20 | public class ReverseDoubleOutputFormatter implements OutputFormatter { 21 | 22 | @Override 23 | public Object format(Object outputValue) { 24 | Double doubleObject = (Double) outputValue; 25 | return String.valueOf(doubleObject); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/ReverseIntegerOutputFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import io.s4.processor.OutputFormatter; 19 | 20 | public class ReverseIntegerOutputFormatter implements OutputFormatter { 21 | 22 | @Override 23 | public Object format(Object outputValue) { 24 | Integer integerObject = (Integer) outputValue; 25 | return String.valueOf(integerObject); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/S4Util.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | public class S4Util { 19 | public static long getPID() { 20 | String processName = java.lang.management.ManagementFactory.getRuntimeMXBean() 21 | .getName(); 22 | return Long.parseLong(processName.split("@")[0]); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/SlotUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | 19 | public class SlotUtils { 20 | 21 | private int slotSize; // slot size in seconds 22 | 23 | public SlotUtils(int slotSize) { 24 | this.slotSize = slotSize; 25 | } 26 | 27 | public void setSize(int slotSize) { 28 | this.slotSize = slotSize; 29 | } 30 | 31 | public Long getSlotAtTime(long time) { 32 | return slotSize * (long) Math.floor(time / slotSize); 33 | } 34 | 35 | public Long getCurrentSlot() { 36 | long currTimeStamp = System.currentTimeMillis() / 1000; // convert to 37 | // seconds 38 | Long slotTimeStamp = getSlotAtTime(currTimeStamp); 39 | return slotTimeStamp; 40 | } 41 | 42 | public Long getSlot(int index, long currTimeStamp) { 43 | Long slotTimeStamp = getSlotAtTime(currTimeStamp + index * slotSize); 44 | return slotTimeStamp; 45 | } 46 | 47 | public boolean isOutsideWindow(Long slot, int windowSize, long time) { 48 | boolean outside = false; 49 | long windowBoundary = getSlotAtTime(time) - windowSize; 50 | if (slot.longValue() < windowBoundary) { 51 | outside = true; 52 | } 53 | return outside; 54 | } 55 | 56 | public static void main(String[] args) { 57 | SlotUtils s = new SlotUtils(300); 58 | System.out.printf("%d\n", s.getCurrentSlot()); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/ToStringOutputFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.util; 17 | 18 | import io.s4.processor.OutputFormatter; 19 | 20 | public class ToStringOutputFormatter implements OutputFormatter { 21 | 22 | @Override 23 | public Object format(Object outputValue) { 24 | return String.valueOf(outputValue); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/clock/Clock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.util.clock; 18 | 19 | public interface Clock { 20 | 21 | public long waitForTime(long targetTime); 22 | 23 | public long getCurrentTime(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/clock/ClockStreamsLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.util.clock; 18 | 19 | 20 | import java.util.HashMap; 21 | 22 | import org.apache.log4j.Logger; 23 | 24 | public class ClockStreamsLoader { 25 | 26 | private static Logger logger = Logger.getLogger(ClockStreamsLoader.class); 27 | Clock s4Clock; 28 | HashMap streamFieldMap; 29 | 30 | public void setStreamFieldMap(HashMap streamFieldMap) { 31 | this.streamFieldMap = streamFieldMap; 32 | } 33 | 34 | public void setS4Clock(Clock s4Clock) { 35 | this.s4Clock = s4Clock; 36 | } 37 | 38 | public void addStreams() { 39 | if (s4Clock instanceof EventClock) { 40 | EventClock eventClock = (EventClock) s4Clock; 41 | for (String streamName : streamFieldMap.keySet()) { 42 | eventClock.addEventClockStream(streamName, 43 | streamFieldMap.get(streamName)); 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/clock/DrivenClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.util.clock; 18 | 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | import java.util.Map.Entry; 23 | import java.util.NavigableMap; 24 | import java.util.TreeMap; 25 | 26 | public class DrivenClock implements Clock { 27 | 28 | private volatile long currentTime; 29 | private NavigableMap> timerRequests = new TreeMap>(); 30 | 31 | public void updateTime(long newCurrentTime) { 32 | if (newCurrentTime < currentTime) { 33 | return; 34 | } 35 | List relevantRequests = null; 36 | synchronized (timerRequests) { 37 | currentTime = newCurrentTime; 38 | while (true) { 39 | // inspect the top of the timer request list and see if any 40 | // request 41 | // is 42 | // satisfied by the new current time 43 | Entry> entry = timerRequests 44 | .firstEntry(); 45 | if (entry == null || entry.getKey() > newCurrentTime) { 46 | break; 47 | } 48 | relevantRequests = timerRequests.remove(entry.getKey()); 49 | } 50 | if (relevantRequests != null) { 51 | for (TimerRequest timerRequest : relevantRequests) { 52 | timerRequest.wakeUp(newCurrentTime); 53 | } 54 | } 55 | } 56 | } 57 | 58 | public long waitForTime(long targetTime) { 59 | TimerRequest timerRequest = null; 60 | synchronized (timerRequests) { 61 | if (targetTime <= currentTime) { 62 | return currentTime; 63 | } 64 | timerRequest = new TimerRequest(targetTime); 65 | List requestsForTargetTime = timerRequests.get(targetTime); 66 | if (requestsForTargetTime == null) { 67 | requestsForTargetTime = new ArrayList(); 68 | timerRequests.put(targetTime, requestsForTargetTime); 69 | } 70 | requestsForTargetTime.add(timerRequest); 71 | } 72 | return timerRequest.waitForTargetTime(); 73 | } 74 | 75 | 76 | public long getCurrentTime() { 77 | return getCurrentTime(true); 78 | } 79 | 80 | public long getCurrentTime(boolean waitOnInitialization) { 81 | if (currentTime == 0 && waitOnInitialization) { 82 | // if tick has never been called, wait for it to be called once 83 | this.waitForTime(1); 84 | } 85 | return currentTime; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/clock/EventClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.util.clock; 18 | 19 | import io.s4.collector.EventWrapper; 20 | import io.s4.schema.Schema; 21 | import io.s4.schema.Schema.Property; 22 | import io.s4.schema.SchemaContainer; 23 | 24 | import java.util.HashMap; 25 | import java.util.Map; 26 | 27 | import org.apache.log4j.Logger; 28 | 29 | public class EventClock extends DrivenClock { 30 | 31 | private static Logger logger = Logger.getLogger(EventClock.class); 32 | 33 | Map eventClockStreamsMap = new HashMap(); 34 | SchemaContainer schemaContainer = new SchemaContainer(); 35 | 36 | public void update(EventWrapper eventWrapper) { 37 | long eventTime = -1; 38 | String streamName = eventWrapper.getStreamName(); 39 | String fieldName = eventClockStreamsMap.get(streamName); 40 | if (fieldName != null) { 41 | Object event = eventWrapper.getEvent(); 42 | Schema schema = schemaContainer.getSchema(event.getClass()); 43 | Property property = schema.getProperties().get(fieldName); 44 | if (property != null 45 | && (property.getType().equals(Long.TYPE) || property 46 | .getType().equals(Long.class))) { 47 | try { 48 | eventTime = (Long) property.getGetterMethod().invoke(event); 49 | updateTime(eventTime); 50 | } catch (Exception e) { 51 | throw new RuntimeException(e); 52 | } 53 | } 54 | } 55 | } 56 | 57 | public void addEventClockStream(String streamName, String fieldName) { 58 | String fieldNameInStream = eventClockStreamsMap.get(streamName); 59 | if (fieldNameInStream != null) { 60 | if (!fieldNameInStream.equals(fieldName)) { 61 | // we can add an runtime exception over error messages for 62 | // making debugging easy 63 | logger.error("Stream " + streamName 64 | + " already has a timestamp field defined " 65 | + eventClockStreamsMap.get(streamName)); 66 | logger.error("Stream " + streamName 67 | + " is updating the timestamp field to " + fieldName); 68 | eventClockStreamsMap.put(streamName, fieldName); 69 | } 70 | } else { 71 | eventClockStreamsMap.put(streamName, fieldName); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/clock/TimerRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.util.clock; 18 | 19 | import java.util.concurrent.BlockingQueue; 20 | import java.util.concurrent.LinkedBlockingQueue; 21 | 22 | public class TimerRequest { 23 | private long targetTime; 24 | private BlockingQueue blockingQueue = new LinkedBlockingQueue(); 25 | 26 | public TimerRequest(long targetTime) { 27 | this.targetTime = targetTime; 28 | } 29 | 30 | public long getTargetTime() { 31 | return targetTime; 32 | } 33 | 34 | public void wakeUp(long currentTime) { 35 | blockingQueue.add(currentTime); 36 | } 37 | 38 | public long waitForTargetTime() { 39 | try { 40 | return blockingQueue.take(); 41 | } catch (InterruptedException ie) { 42 | return -1; 43 | } 44 | } 45 | } 46 | 47 | -------------------------------------------------------------------------------- /src/main/java/io/s4/util/clock/WallClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.util.clock; 18 | 19 | 20 | public class WallClock implements Clock { 21 | 22 | @Override 23 | public long waitForTime(long targetTime) { 24 | long interval = (targetTime - getCurrentTime()); 25 | if (interval > 0) { 26 | try { 27 | Thread.sleep(interval); 28 | } catch (InterruptedException ie) { 29 | Thread.currentThread().interrupt(); 30 | } 31 | } 32 | return getCurrentTime(); 33 | } 34 | 35 | @Override 36 | public long getCurrentTime() { 37 | // TODO Auto-generated method stub 38 | return System.currentTimeMillis(); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/resources/adapter_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | classpath:adapter.properties 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /src/main/resources/client_adapter_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | classpath:adapter.properties 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /src/main/resources/client_stub_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | classpath:adapter.properties 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/main/resources/clusters_example.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 0 5 | localhost 6 | 5077 7 | s4node-0 8 | 9 | 10 | 11 | 12 | localhost 13 | adapter-0 14 | 15 | 16 | 17 | 18 | 0 19 | localhost 20 | client-adapter-0 21 | 6077 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/resources/event_clock.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /src/main/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/main/resources/s4_core.properties_header_redbutton: -------------------------------------------------------------------------------- 1 | gc_opts=-server -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit 2 | mem_opts=-Xms800m -Xmx2000m 3 | remote_debug_enabled=no 4 | pe_container_max_queue_size=8000 5 | listener_max_queue_size=8000 6 | commlayer_mode=static 7 | zk_session_timeout=5000 8 | deamon_mode_enabled=false 9 | -------------------------------------------------------------------------------- /src/main/resources/s4_core.properties_header_typical: -------------------------------------------------------------------------------- 1 | gc_opts=-server -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit 2 | mem_opts=-Xms800m -Xmx2000m 3 | remote_debug_enabled=no 4 | pe_container_max_queue_size=8000 5 | listener_max_queue_size=8000 6 | commlayer_mode=dynamic 7 | zk_session_timeout=5000 8 | deamon_mode_enabled=false 9 | zk_address=localhost:2181 10 | -------------------------------------------------------------------------------- /src/main/resources/wall_clock.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /src/test/java/io/s4/processor/MockPE.java: -------------------------------------------------------------------------------- 1 | package io.s4.processor; 2 | 3 | /** 4 | * Mock PE for asserting correct behavior of AbstractPE 5 | */ 6 | public class MockPE extends AbstractPE { 7 | 8 | private int initializeCount = 0; 9 | 10 | public void testInitialize() { 11 | initializeCount++; 12 | } 13 | 14 | public void processEvent(Object obj) { 15 | } 16 | 17 | @Override 18 | public String getId() { 19 | return null; 20 | } 21 | 22 | @Override 23 | public void output() { 24 | 25 | } 26 | 27 | /** 28 | * @return the initializeCount 29 | */ 30 | public int getInitializeCount() { 31 | return initializeCount; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/io/s4/processor/TestPrototypeWrapper.java: -------------------------------------------------------------------------------- 1 | package io.s4.processor; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.assertNotNull; 5 | import io.s4.util.clock.WallClock; 6 | 7 | import org.junit.Test; 8 | 9 | public class TestPrototypeWrapper 10 | { 11 | 12 | /** 13 | * Verifies ability to set an initialize method that will be called when 14 | * a new PE is instantiated 15 | */ 16 | @Test 17 | public void testCloneAndInitialize() { 18 | MockPE prototype = new MockPE(); 19 | prototype.setInitMethod("testInitialize"); 20 | 21 | PrototypeWrapper prototypeWrapper = new PrototypeWrapper(prototype, new WallClock()); 22 | 23 | assertEquals(0, prototype.getInitializeCount()); 24 | MockPE instance = (MockPE)prototypeWrapper.getPE("asd"); 25 | assertNotNull(instance); 26 | 27 | assertEquals(0, prototype.getInitializeCount()); 28 | assertEquals(1, instance.getInitializeCount()); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | --------------------------------------------------------------------------------