├── .gitignore ├── LICENSE ├── README.md ├── bin ├── bagheera ├── consumer ├── create_tables.hbql ├── init.d │ └── bagheera ├── start_consumers └── stop_consumers ├── conf ├── bagheera.metrics.properties ├── bagheera.properties ├── kafka.consumer.properties ├── kafka.producer.properties └── log4j.properties ├── pom.xml └── src ├── assembly └── dist.xml ├── main ├── java │ └── com │ │ └── mozilla │ │ └── bagheera │ │ ├── cli │ │ ├── App.java │ │ ├── Option.java │ │ └── OptionFactory.java │ │ ├── consumer │ │ ├── Consumer.java │ │ ├── KafkaConsumer.java │ │ ├── KafkaHBaseConsumer.java │ │ ├── KafkaLoggerConsumer.java │ │ ├── KafkaReplayConsumer.java │ │ ├── KafkaSequenceFileConsumer.java │ │ └── validation │ │ │ ├── JsonValidator.java │ │ │ ├── ValidationPipeline.java │ │ │ └── Validator.java │ │ ├── http │ │ ├── AccessFilter.java │ │ ├── Bagheera.java │ │ ├── BagheeraHttpRequest.java │ │ ├── BagheeraHttpRequestDecoder.java │ │ ├── ContentEncodingCorrector.java │ │ ├── ContentLengthFilter.java │ │ ├── HttpSecurityException.java │ │ ├── HttpServerPipelineFactory.java │ │ ├── InvalidPathException.java │ │ ├── PathDecoder.java │ │ ├── RootResponse.java │ │ ├── SubmissionHandler.java │ │ └── json │ │ │ ├── InvalidJsonException.java │ │ │ └── JsonFilter.java │ │ ├── metrics │ │ ├── HttpMetric.java │ │ └── MetricsManager.java │ │ ├── producer │ │ ├── KafkaProducer.java │ │ └── Producer.java │ │ ├── serializer │ │ ├── BagheeraDecoder.java │ │ └── BagheeraEncoder.java │ │ ├── sink │ │ ├── HBaseSink.java │ │ ├── KeyValueSink.java │ │ ├── KeyValueSinkFactory.java │ │ ├── LoggerSink.java │ │ ├── ReplaySink.java │ │ ├── SequenceFileSink.java │ │ ├── Sink.java │ │ └── SinkConfiguration.java │ │ ├── util │ │ ├── HttpUtil.java │ │ ├── IdUtil.java │ │ ├── ShutdownHook.java │ │ └── WildcardProperties.java │ │ └── validation │ │ └── Validator.java └── proto │ └── bagheera_message.proto └── test ├── java ├── com │ └── mozilla │ │ └── bagheera │ │ ├── cli │ │ ├── OptionFactoryTest.java │ │ └── OptionTest.java │ │ ├── http │ │ ├── AccessFilterTest.java │ │ ├── BagheeraHttpRequestTest.java │ │ ├── BagheeraTest.java │ │ ├── ContentLengthFilterTest.java │ │ ├── PathDecoderTest.java │ │ └── SubmissionHandlerTest.java │ │ ├── producer │ │ └── ProducerTest.java │ │ ├── sink │ │ ├── HBaseSinkTest.java │ │ └── ReplaySinkTest.java │ │ ├── util │ │ ├── HttpUtilTest.java │ │ ├── IdUtilTest.java │ │ └── WildcardPropertiesTest.java │ │ └── validation │ │ └── ValidatorTest.java └── org │ └── jboss │ └── netty │ └── channel │ └── FakeChannelHandlerContext.java └── python ├── http_test.py ├── metrics_ping_mechanizer.py ├── telemetry_mechanizer.py └── testpilot_mechanizer.py /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | target 3 | .project 4 | .classpath 5 | .settings 6 | 7 | # Compiled source # 8 | ################### 9 | *.com 10 | *.class 11 | *.dll 12 | *.exe 13 | *.o 14 | *.so 15 | 16 | # Packages # 17 | ############ 18 | # it's better to unpack these files and commit the raw source 19 | # git has its own built in compression methods 20 | *.7z 21 | *.dmg 22 | *.gz 23 | *.iso 24 | *.rar 25 | *.tar 26 | *.zip 27 | 28 | # Logs and databases # 29 | ###################### 30 | logs 31 | *.log 32 | *.sql 33 | *.sqlite 34 | 35 | # OS generated files # 36 | ###################### 37 | .DS_Store? 38 | ehthumbs.db 39 | Icon? 40 | Thumbs.db 41 | 42 | # IntelliJ Idea project files 43 | .idea 44 | *.iml 45 | 46 | # Clover 47 | .clover 48 | report 49 | 50 | # Editor files 51 | .*.sw* 52 | *~ 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bagheera # 2 | 3 | Version: 0.15-SNAPSHOT 4 | 5 | #### REST service for Mozilla Metrics. This service currently uses Apache Kafka as its backing data store, then provides a few implementations of Kafka consumers to pull and persist to various data sinks. #### 6 | 7 | 8 | ### Version Compatability ### 9 | This code is built with the following assumptions. You may get mixed results if you deviate from these versions. 10 | 11 | * [Kafka](http://incubator.apache.org/kafka) 0.7.1+ 12 | * [Protocol Buffers](https://developers.google.com/protocol-buffers) 2.4.1+ 13 | * [Hadoop](http://hadoop.apache.org) 0.20.2+ 14 | * [HBase](http://hbase.apache.org) 0.90+ 15 | 16 | ### Prerequisites ### 17 | * Protocol Buffers 18 | * Zookeeper (for Kafka) 19 | * Kafka 20 | * Hadoop (if using HDFS based consumer) 21 | * HBase (if using HBase based consumer) 22 | 23 | ### Building ### 24 | To make a jar you can do: 25 | 26 | `mvn package` 27 | 28 | The jar file is then located under `target`. 29 | 30 | ### Running an instance ### 31 | **Make sure your Kafka and Zookeeper servers are running first (see Kafka documentation)** 32 | 33 | In order to run bagheera on another machine you will probably want to use the _dist_ assembly like so: 34 | 35 | `mvn assembly:assembly` 36 | 37 | The zip file now under the `target` directory should be deployed to `BAGHEERA_HOME` on the remote server. 38 | 39 | To run Bagheera you can use `bin/bagheera` or copy the init.d script by the same name from `bin/init.d` to `/etc/init.d`. The init script assumes an installation of bagheera at `/usr/lib/bagheera`, but this can be modified by changing the `BAGHEERA_HOME` variable near the top of that script. Here is an example of using the regular bagheera script: 40 | 41 | `bin/bagheera 8080` 42 | 43 | ### REST Request Format ### 44 | #####URI _/submit/namespace/id_ | _/1.0/submit/namespace/id_##### 45 | POST/PUT 46 | 47 | * The _namespace_ is required and is only accepted if it is in the configured white-list. 48 | * The _id_ is optional although if you provide it currently it needs to be a valid UUID unless id validation is disabled on the _namespace_. 49 | * The payload content length must be less than the configured maximum. 50 | 51 | DELETE 52 | 53 | * The _namespace_ is required and is only accepted if it is in the configured white-list. 54 | * The _id_ is required although if you provide it currently it needs to be a valid UUID unless id validation is disabled on the _namespace_. 55 | 56 | Here's the list of HTTP response codes that Bagheera could send back: 57 | 58 | * 201 Created - Returns the id submitted/generated. (default) 59 | * 403 Forbidden - Violated access restrictions. Most likely because of the method used. 60 | * 413 Request Too Large - Request payload was larger than the configured maximum. 61 | * 400 Bad Request - Returned if the POST/PUT failed validation in some manner. 62 | * 404 Not Found - Returned if the URI path doesn't exist or if the URI was not in the proper format. 63 | * 500 Server Error - General server error. Someone with access should look at the logs for more details. 64 | 65 | ### Example Bagheera Configuration (conf/bagheera.properties) ### 66 | # valid namespaces (whitelist only, comma separated) 67 | valid.namespaces=mynamespace,othernamespace 68 | max.content.length=1048576 69 | 70 | ### Example Kafka Producer Configuration (conf/kafka.producer.properties) ### 71 | # comma delimited list of ZK servers 72 | zk.connect=127.0.0.1:2181 73 | # use bagheera message encoder 74 | serializer.class=com.mozilla.bagheera.serializer.BagheeraEncoder 75 | # asynchronous producer 76 | producer.type=async 77 | # compression.code (0=uncompressed,1=gzip,2=snappy) 78 | compression.codec=2 79 | # batch size (one of many knobs to turn in kafka depending on expected data size and request rate) 80 | batch.size=100 81 | 82 | ### Example Kafka Consumer Configuration (conf/kafka.consumer.properties) ### 83 | # kafka consumer properties 84 | zk.connect=127.0.0.1:2181 85 | fetch.size=1048576 86 | #serializer.class=com.mozilla.bagheera.serializer.BagheeraDecoder 87 | # bagheera specific kafka consumer properties 88 | consumer.threads=2 89 | 90 | ### Notes on consumers ### 91 | We currently use the consumers implemented here, but it may also be of interest to look at systems such as [Storm](https://github.com/nathanmarz/storm) to process the messages. Storm contains a Kafka spout (consumer) and there are at least a couple of HBase bolts (processing/sink) already out there. 92 | 93 | ### License ### 94 | All aspects of this software are distributed under Apache Software License 2.0. See LICENSE file for full license text. 95 | 96 | ### Contributors ### 97 | 98 | * Xavier Stevens ([@xstevens](http://twitter.com/xstevens)) 99 | * Daniel Einspanjer ([@deinspanjer](http://twitter.com/deinspanjer)) 100 | * Anurag Phadke ([@anuragphadke](http://twitter.com/anuragphadke)) 101 | * Mark Reid ([@reid_write](http://twitter.com/reid_write)) 102 | * Harsha Chintalapani 103 | -------------------------------------------------------------------------------- /bin/bagheera: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function usage() { 4 | echo "Usage: $0 " 5 | exit 1 6 | } 7 | 8 | # Print usage if incorrect number of args 9 | [[ $# -ne 1 ]] && usage 10 | 11 | bin=`dirname "$0"` 12 | bin=`cd "$bin"; pwd` 13 | 14 | SERVER_PORT=$1 15 | SERVER_CLASS_NAME="com.mozilla.bagheera.http.Bagheera" 16 | NOW=`date "+%Y%m%d%H%M%S"` 17 | BAGHEERA_OPTS="-Dserver.port=$SERVER_PORT -Dbagheera.log.dir=$bin/../logs" 18 | GC_OPTS="-XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$bin/../logs/gc.log.$NOW" 19 | JAVA_OPTS="$BAGHEERA_OPTS -Xss128k -Xms1024m -Xmx1024m -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:NewRatio=3 -XX:+UseCompressedOops $GC_OPTS" 20 | 21 | export MALLOC_ARENA_MAX=2 22 | 23 | if [ "$BAGHEERA_USER" = "" ]; then 24 | BAGHEERA_USER="$USER" 25 | fi 26 | 27 | if [ -d "/var/run/bagheera" ]; then 28 | PIDFILE="/var/run/bagheera/bagheera.pid" 29 | else 30 | PIDFILE="$bin/../bagheera.pid" 31 | fi 32 | 33 | # if this is a developer then use the main jar in the build directory 34 | if [ -d $bin/../target ]; then 35 | MAIN_JAR_PATH="$bin/../target/bagheera-*.jar" 36 | if [ "$DAEMON_DETACHED" = "" ]; then 37 | DAEMON_DETACHED=false 38 | fi 39 | else 40 | MAIN_JAR_PATH="$bin/../bagheera-*.jar" 41 | if [ "$DAEMON_DETACHED" = "" ]; then 42 | DAEMON_DETACHED=true 43 | fi 44 | fi 45 | 46 | CLASSPATH="$bin/../conf":"$HADOOP_CONF":"$HBASE_CONF" 47 | # add main jar 48 | for lib in `ls $MAIN_JAR_PATH`; do 49 | CLASSPATH=${CLASSPATH}:$lib 50 | done 51 | 52 | # add dependency libs 53 | for lib in `ls $bin/../lib/*.jar`; do 54 | CLASSPATH=${CLASSPATH}:$lib 55 | done 56 | 57 | # create logs dir if it doesn't exist 58 | if [ ! -d $bin/../logs ]; then 59 | mkdir -p $bin/../logs 60 | fi 61 | 62 | if [ "$DAEMON_DETACHED" = false ]; then 63 | java $JAVA_OPTS -cp $CLASSPATH $SERVER_CLASS_NAME 64 | RETVAL=$? 65 | else 66 | nohup java $JAVA_OPTS -cp $CLASSPATH $SERVER_CLASS_NAME > $bin/../logs/bagheera.out 2>&1 < /dev/null & 67 | PID=$! 68 | RETVAL=$? 69 | 70 | echo $PID > $PIDFILE 71 | fi 72 | 73 | exit $RETVAL -------------------------------------------------------------------------------- /bin/consumer: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bin=`dirname "$0"` 4 | bin=`cd "$bin"; pwd` 5 | 6 | 7 | # To use a different distro, set this outside. 8 | if [ -z "$HADOOP_CLUSTER" ]; then 9 | HADOOP_CLUSTER="PEACH" 10 | fi 11 | 12 | NOW=`date "+%Y%m%d%H%M%S"` 13 | CONSUMER_HASH=`echo -n \"$@\" | md5sum | cut -f1 -d' '` 14 | CONSUMER_OPTS="-Dbagheera.log.dir=$bin/../logs -Dconsumer.hash=$CONSUMER_HASH" 15 | GC_OPTS="-XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$bin/../logs/consumer-$CONSUMER_HASH-gc.log.$NOW" 16 | 17 | if [ "$1" == "com.mozilla.fhr.consumer.FHRConsumer" ]; then 18 | JAVA_OPTS="$CONSUMER_OPTS -Xmx1024m -XX:+UseParallelGC" 19 | else 20 | JAVA_OPTS="$CONSUMER_OPTS -Xmx1024m -XX:+UseParallelGC" 21 | fi 22 | 23 | HADOOP_CONF="$bin/../conf/$HADOOP_CLUSTER" 24 | if [ "$ENABLE_YJP" = true ]; then 25 | JAVA_OPTS="${JAVA_OPTS} -agentpath:/usr/lib/yjp-11.0.10/bin/linux-x86-64/libyjpagent.so" 26 | fi 27 | 28 | export MALLOC_ARENA_MAX=4 29 | 30 | if [ "$BAGHEERA_USER" = "" ]; then 31 | BAGHEERA_USER="$USER" 32 | fi 33 | 34 | if [ "$HADOOP_CONF" = "" ]; then 35 | HADOOP_CONF="/etc/hadoop/conf" 36 | fi 37 | 38 | if [ -d "/var/run/bagheera" ]; then 39 | PIDFILE="/var/run/bagheera/consumer-$CONSUMER_HASH.pid" 40 | else 41 | PIDFILE="$bin/../consumer-$CONSUMER_HASH.pid" 42 | fi 43 | 44 | # if this is a developer then use the classes directory in the build directory 45 | if [ -d $bin/../target/classes ]; then 46 | MAIN_JAR_PATH="$bin/../target/classes" 47 | if [ "$DAEMON_DETACHED" = "" ]; then 48 | DAEMON_DETACHED=false 49 | fi 50 | else 51 | MAIN_JAR_PATH="$bin/../bagheera-*.jar" 52 | if [ "$DAEMON_DETACHED" = "" ]; then 53 | DAEMON_DETACHED=true 54 | fi 55 | fi 56 | 57 | CLASSPATH="$bin/../conf":"$HADOOP_CONF" 58 | # add main jar 59 | for lib in `ls $MAIN_JAR_PATH`; do 60 | CLASSPATH=${CLASSPATH}:$lib 61 | done 62 | 63 | 64 | # add dependency libs 65 | CLASSPATH=${CLASSPATH}:"$bin/../lib/*":"$bin/../lib/hadoop/*" 66 | # create logs dir if it doesn't exist 67 | if [ ! -d $bin/../logs ]; then 68 | mkdir -p $bin/../logs 69 | fi 70 | 71 | if [ "$DAEMON_DETACHED" = false ]; then 72 | java $JAVA_OPTS $YJP_OPTS -cp $CLASSPATH "$@" 73 | RETVAL=$? 74 | else 75 | nohup java $JAVA_OPTS $YJP_OPTS -cp $CLASSPATH "$@" > $bin/../logs/consumer-$CONSUMER_HASH.out 2>&1 < /dev/null & 76 | PID=$! 77 | RETVAL=$? 78 | 79 | echo $PID > $PIDFILE 80 | fi 81 | 82 | exit $RETVAL 83 | -------------------------------------------------------------------------------- /bin/create_tables.hbql: -------------------------------------------------------------------------------- 1 | create 'telemetry', {NAME => 'data', COMPRESSION => 'SNAPPY', VERSIONS => '1', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'} 2 | create 'metrics', {NAME => 'data', COMPRESSION => 'SNAPPY', VERSIONS => '1', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'} 3 | create 'android_release_review', {NAME => 'data', COMPRESSION => 'SNAPPY', VERSIONS => '1', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'} 4 | create 'android_beta_review', {NAME => 'data', COMPRESSION => 'SNAPPY', VERSIONS => '1', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'} -------------------------------------------------------------------------------- /bin/init.d/bagheera: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # (c) Copyright 2011 Mozilla Foundation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Starts a bagheera server 18 | 19 | # 20 | # bagheera: Starts the bagheera daemon 21 | # 22 | # chkconfig: 2345 95 20 23 | # description: A REST server for Mozilla Metrics 24 | # processname: java 25 | # pidfile: /var/run/bagheera/bagheera.pid 26 | 27 | # Source function library 28 | . /etc/rc.d/init.d/functions 29 | 30 | RETVAL_SUCCESS=0 31 | RETVAL=0 32 | BAGHEERA_HOME="/usr/lib/bagheera" 33 | PIDFILE="/var/run/bagheera/bagheera.pid" 34 | LOCKFILE="/var/lock/subsys/bagheera" 35 | NAME="bagheera" 36 | PORT=8080 37 | if [ "$BAGHEERA_USER" = "" ]; then 38 | BAGHEERA_USER="bagheera" 39 | fi 40 | 41 | # create PID dir if it doesn't exist already 42 | PIDDIR=`dirname "$PIDFILE"` 43 | if [ ! -d "$PIDDIR" ]; then 44 | mkdir -p "$PIDDIR" 45 | chown -R "$BAGHEERA_USER"."$BAGHEERA_USER" "$PIDDIR" 46 | fi 47 | 48 | start() { 49 | echo -n $"Starting $NAME: " 50 | daemon --user="$BAGHEERA_USER" $BAGHEERA_HOME/bin/bagheera $PORT 51 | RETVAL=$? 52 | echo 53 | [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE 54 | } 55 | 56 | stop() { 57 | echo -n $"Stopping $NAME: " 58 | killproc -p "$PIDFILE" 59 | RETVAL=$? 60 | echo 61 | [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE 62 | } 63 | 64 | restart() { 65 | stop 66 | start 67 | } 68 | 69 | checkstatus() { 70 | status -p $PIDFILE 71 | RETVAL=$? 72 | } 73 | 74 | condrestart() { 75 | [ -e $LOCKFILE ] && restart || : 76 | } 77 | 78 | case "$1" in 79 | start) 80 | start 81 | ;; 82 | stop) 83 | stop 84 | ;; 85 | status) 86 | checkstatus 87 | ;; 88 | restart) 89 | restart 90 | ;; 91 | condrestart) 92 | condrestart 93 | ;; 94 | *) 95 | echo $"Usage: $0 {start|stop|status|condrestart|restart}" 96 | RETVAL=3 97 | ;; 98 | esac 99 | 100 | exit $RETVAL 101 | -------------------------------------------------------------------------------- /bin/start_consumers: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bin=`dirname "$0"` 4 | bin=`cd "$bin"; pwd` 5 | 6 | CONSUMER_PROPS=$bin/../conf/kafka.consumer.properties 7 | HBASE_CONSUMER_CLASS=com.mozilla.bagheera.consumer.KafkaHBaseConsumer 8 | HDFS_CONSUMER_CLASS=com.mozilla.bagheera.consumer.KafkaSequenceFileConsumer 9 | HDFS_DELETE_CONSUMER_CLASS=com.mozilla.bagheera.consumer.KafkaDeleteSequenceFileConsumer 10 | # FHR consumer comes from fhr-toolbox which needs its jar deployed to bagheera/lib in order to use it 11 | FHR_CONSUMER_CLASS=com.mozilla.fhr.consumer.FHRConsumer 12 | 13 | # UserProfile 14 | HADOOP_CLUSTER=PEACH $bin/consumer $HBASE_CONSUMER_CLASS -t userprofile -gid userprofile-hbase-prod -p $CONSUMER_PROPS --validatejson --table user_profile --family data --qualifier json --prefixdate --batchsize 100 15 | # FHR (does its own json validation so option isn't needed) 16 | HADOOP_CLUSTER=PEACH $bin/consumer $FHR_CONSUMER_CLASS -t metrics -gid fhr-hbase-prod-cdh4 -p $CONSUMER_PROPS --table metrics --family data --qualifier json --numthreads 1 --batchsize 2000 -t 1 17 | HADOOP_CLUSTER=PEACH $bin/consumer $FHR_CONSUMER_CLASS -t metrics -gid fhr-hbase-prod-cdh4 -p $CONSUMER_PROPS --table metrics --family data --qualifier json --numthreads 1 --batchsize 2000 -t 2 18 | HADOOP_CLUSTER=PEACH $bin/consumer $FHR_CONSUMER_CLASS -t metrics -gid fhr-hbase-prod-cdh4 -p $CONSUMER_PROPS --table metrics --family data --qualifier json --numthreads 1 --batchsize 2000 -t 3 19 | HADOOP_CLUSTER=PEACH $bin/consumer $FHR_CONSUMER_CLASS -t metrics -gid fhr-hbase-prod-cdh4 -p $CONSUMER_PROPS --table metrics --family data --qualifier json --numthreads 1 --batchsize 2000 -t 4 20 | 21 | # FHR delete 22 | #HADOOP_CLUSTER=PEACH $bin/consumer $HDFS_DELETE_CONSUMER_CLASS -t metrics -gid fhr-delete-hdfs-prod -p $CONSUMER_PROPS --validatejson 23 | # Testpilot 24 | HADOOP_CLUSTER=PEACH $bin/consumer $HDFS_CONSUMER_CLASS -t testpilot_.+ -gid testpilot-hdfs-prod -p $CONSUMER_PROPS --validatejson 25 | # Marketplace 26 | HADOOP_CLUSTER=PEACH $bin/consumer $HDFS_CONSUMER_CLASS -t marketplace_.+ -gid marketplace-hdfs-prod -p $CONSUMER_PROPS --validatejson 27 | # Android Reviews 28 | HADOOP_CLUSTER=PEACH $bin/consumer $HBASE_CONSUMER_CLASS -t android_release_review -gid android_release_review-hbase-prod -p $CONSUMER_PROPS --table android_release_review --family data --qualifier json 29 | HADOOP_CLUSTER=PEACH $bin/consumer $HBASE_CONSUMER_CLASS -t android_beta_review -gid android_beta_review-hbase-prod -p $CONSUMER_PROPS --table android_beta_review --family data --qualifier json 30 | # Lightbeam 31 | HADOOP_CLUSTER=PEACH $bin/consumer $HDFS_CONSUMER_CLASS -t lightbeam -gid lightbeam-hdfs-prod -p $CONSUMER_PROPS --validatejson --addtimestamp 32 | -------------------------------------------------------------------------------- /bin/stop_consumers: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ps ax | grep -i 'com.mozilla.bagheera.consumer.' | grep -v grep | awk '{print $1}' | xargs kill 4 | ps ax | grep -i 'com.mozilla.fhr.consumer.' | grep -v grep | awk '{print $1}' | xargs kill -------------------------------------------------------------------------------- /conf/bagheera.metrics.properties: -------------------------------------------------------------------------------- 1 | # Ganglia 2 | bagheera.metrics.ganglia.enable=false 3 | bagheera.metrics.ganglia.host=239.2.12.79 4 | bagheera.metrics.ganglia.port=8649 5 | bagheera.metrics.ganglia.update.secs=15 6 | # Graphite 7 | bagheera.metrics.graphite.enable=false 8 | bagheera.metrics.graphite.host=localhost 9 | bagheera.metrics.graphite.port=2003 10 | bagheera.metrics.graphite.update.secs=60 -------------------------------------------------------------------------------- /conf/bagheera.properties: -------------------------------------------------------------------------------- 1 | # General 2 | valid.namespaces=telemetry,testpilot_*,marketplace_*,metrics 3 | max.content.length=1048576 4 | 5 | # Metrics 6 | metrics.allow.delete.access=true -------------------------------------------------------------------------------- /conf/kafka.consumer.properties: -------------------------------------------------------------------------------- 1 | # kafka consumer properties 2 | zk.connect=127.0.0.1:2181 3 | fetch.size=1048576 4 | #serializer.class=com.mozilla.bagheera.serializer.BagheeraDecoder 5 | # bagheera specific kafka consumer properties 6 | consumer.threads=1 7 | 8 | # Uncomment the following to enable MaxMind GeoIP Lookups 9 | #maxmind.db.path=/usr/local/share/GeoIP/GeoIP.dat -------------------------------------------------------------------------------- /conf/kafka.producer.properties: -------------------------------------------------------------------------------- 1 | #zk.connect=localhost:2181 2 | broker.list=0:localhost:9092 3 | serializer.class=com.mozilla.bagheera.serializer.BagheeraEncoder 4 | producer.type=async 5 | compression.codec=2 6 | batch.size=100 -------------------------------------------------------------------------------- /conf/log4j.properties: -------------------------------------------------------------------------------- 1 | #log4j.debug=true 2 | 3 | log4j.rootCategory=INFO, stdout, DRFA 4 | 5 | # Stdout 6 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 9 | # Debugging pattern 10 | #log4j.appender.stdout.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n 11 | 12 | # File 13 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 14 | log4j.appender.DRFA.File=${bagheera.log.dir}/bagheera.log 15 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 16 | log4j.appender.DRFA.Append=true 17 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 18 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 19 | # Debugging Pattern format 20 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 21 | 22 | # File for Consumers 23 | log4j.appender.CONSUMER=org.apache.log4j.DailyRollingFileAppender 24 | log4j.appender.CONSUMER.File=${bagheera.log.dir}/consumer-${consumer.hash}.log 25 | log4j.appender.CONSUMER.DatePattern=.yyyy-MM-dd 26 | log4j.appender.CONSUMER.Append=true 27 | log4j.appender.CONSUMER.layout=org.apache.log4j.PatternLayout 28 | log4j.appender.CONSUMER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 29 | 30 | log4j.logger.com.mozilla.bagheera.consumer=INFO, CONSUMER 31 | log4j.logger.com.mozilla.fhr.consumer=INFO, CONSUMER 32 | -------------------------------------------------------------------------------- /src/assembly/dist.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | dist 9 | 10 | zip 11 | 12 | 13 | 14 | 15 | ${basedir}/*.txt 16 | 17 | 18 | 19 | conf 20 | 21 | 22 | bin 23 | 755 24 | 25 | 26 | target 27 | / 28 | 29 | ${project.name}-${project.version}.jar 30 | 31 | 32 | 33 | 34 | 35 | /lib 36 | false 37 | runtime 38 | 39 | ${project.groupId}:${project.artifactId} 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/cli/App.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | package com.mozilla.bagheera.cli; 22 | 23 | import com.yammer.metrics.HealthChecks; 24 | import com.yammer.metrics.util.DeadlockHealthCheck; 25 | 26 | /** 27 | * Base class for apps with a main method. 28 | */ 29 | public abstract class App { 30 | private static boolean healthChecksInitialized = false; 31 | 32 | public static synchronized void prepareHealthChecks() { 33 | if (healthChecksInitialized) { 34 | return; 35 | } 36 | HealthChecks.register(new DeadlockHealthCheck()); 37 | healthChecksInitialized = true; 38 | } 39 | } 40 | 41 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/cli/Option.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.cli; 21 | 22 | /** 23 | * This Option class extends Apache Commons every so slightly to make required options 24 | * more convienent to setup. 25 | */ 26 | public class Option extends org.apache.commons.cli.Option { 27 | 28 | private static final long serialVersionUID = 4764162254116593978L; 29 | 30 | public Option(String opt, String longOpt, boolean hasArg, String description) { 31 | this(opt, longOpt, hasArg, description, false); 32 | } 33 | 34 | public Option(String opt, String longOpt, boolean hasArg, String description, boolean required) { 35 | super(opt, longOpt, hasArg, description); 36 | setRequired(required); 37 | } 38 | 39 | public Option required() { 40 | setRequired(true); 41 | return this; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/cli/OptionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.cli; 21 | 22 | /** 23 | * A simple factory for Option creation. 24 | */ 25 | public class OptionFactory { 26 | 27 | private static OptionFactory INSTANCE; 28 | 29 | private OptionFactory() { 30 | } 31 | 32 | public static OptionFactory getInstance() { 33 | if (INSTANCE == null) { 34 | INSTANCE = new OptionFactory(); 35 | } 36 | 37 | return INSTANCE; 38 | } 39 | 40 | public Option create(String opt, String longOpt, boolean hasArg, String description) { 41 | return new Option(opt, longOpt, hasArg, description); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/consumer/Consumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.consumer; 21 | 22 | import java.io.Closeable; 23 | 24 | public interface Consumer extends Closeable { 25 | 26 | public void poll(); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/consumer/KafkaHBaseConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.consumer; 21 | 22 | import org.apache.commons.cli.CommandLine; 23 | import org.apache.commons.cli.CommandLineParser; 24 | import org.apache.commons.cli.GnuParser; 25 | import org.apache.commons.cli.HelpFormatter; 26 | import org.apache.commons.cli.Options; 27 | import org.apache.commons.cli.ParseException; 28 | import org.apache.log4j.Logger; 29 | 30 | import com.mozilla.bagheera.cli.App; 31 | import com.mozilla.bagheera.cli.OptionFactory; 32 | import com.mozilla.bagheera.metrics.MetricsManager; 33 | import com.mozilla.bagheera.sink.HBaseSink; 34 | import com.mozilla.bagheera.sink.KeyValueSinkFactory; 35 | import com.mozilla.bagheera.sink.SinkConfiguration; 36 | import com.mozilla.bagheera.util.ShutdownHook; 37 | 38 | /** 39 | * Basic HBase Kafka consumer. This class can be utilized as is but if you want more 40 | * sophisticated logic consider creating your own consumer. 41 | */ 42 | public final class KafkaHBaseConsumer extends App { 43 | 44 | private static final Logger LOG = Logger.getLogger(KafkaHBaseConsumer.class); 45 | 46 | public static void main(String[] args) { 47 | OptionFactory optFactory = OptionFactory.getInstance(); 48 | Options options = KafkaConsumer.getOptions(); 49 | options.addOption(optFactory.create("tbl", "table", true, "HBase table name.").required()); 50 | options.addOption(optFactory.create("f", "family", true, "Column family.")); 51 | options.addOption(optFactory.create("q", "qualifier", true, "Column qualifier.")); 52 | options.addOption(optFactory.create("b", "batchsize", true, "Batch size (number of messages per HBase flush).")); 53 | options.addOption(optFactory.create("pd", "prefixdate", false, "Prefix key with salted date.")); 54 | 55 | CommandLineParser parser = new GnuParser(); 56 | ShutdownHook sh = ShutdownHook.getInstance(); 57 | try { 58 | // Parse command line options 59 | CommandLine cmd = parser.parse(options, args); 60 | 61 | final KafkaConsumer consumer = KafkaConsumer.fromOptions(cmd); 62 | sh.addFirst(consumer); 63 | 64 | // Create a sink for storing data 65 | SinkConfiguration sinkConfig = new SinkConfiguration(); 66 | if (cmd.hasOption("numthreads")) { 67 | sinkConfig.setInt("hbasesink.hbase.numthreads", Integer.parseInt(cmd.getOptionValue("numthreads"))); 68 | } 69 | if (cmd.hasOption("batchsize")) { 70 | sinkConfig.setInt("hbasesink.hbase.batchsize", Integer.parseInt(cmd.getOptionValue("batchsize"))); 71 | } 72 | sinkConfig.setString("hbasesink.hbase.tablename", cmd.getOptionValue("table")); 73 | sinkConfig.setString("hbasesink.hbase.column.family", cmd.getOptionValue("family", "data")); 74 | sinkConfig.setString("hbasesink.hbase.column.qualifier", cmd.getOptionValue("qualifier", "json")); 75 | sinkConfig.setBoolean("hbasesink.hbase.rowkey.prefixdate", cmd.hasOption("prefixdate")); 76 | KeyValueSinkFactory sinkFactory = KeyValueSinkFactory.getInstance(HBaseSink.class, sinkConfig); 77 | sh.addLast(sinkFactory); 78 | 79 | // Set the sink factory for consumer storage 80 | consumer.setSinkFactory(sinkFactory); 81 | 82 | prepareHealthChecks(); 83 | 84 | // Initialize metrics collection, reporting, etc. 85 | final MetricsManager manager = MetricsManager.getDefaultMetricsManager(); 86 | 87 | // Begin polling 88 | consumer.poll(); 89 | } catch (ParseException e) { 90 | LOG.error("Error parsing command line options", e); 91 | HelpFormatter formatter = new HelpFormatter(); 92 | formatter.printHelp(KafkaHBaseConsumer.class.getName(), options); 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/consumer/KafkaLoggerConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.consumer; 21 | 22 | import org.apache.commons.cli.CommandLine; 23 | import org.apache.commons.cli.CommandLineParser; 24 | import org.apache.commons.cli.GnuParser; 25 | import org.apache.commons.cli.HelpFormatter; 26 | import org.apache.commons.cli.Options; 27 | import org.apache.commons.cli.ParseException; 28 | import org.apache.log4j.Logger; 29 | 30 | import com.mozilla.bagheera.cli.App; 31 | import com.mozilla.bagheera.cli.OptionFactory; 32 | import com.mozilla.bagheera.sink.LoggerSink; 33 | import com.mozilla.bagheera.sink.SinkConfiguration; 34 | import com.mozilla.bagheera.sink.KeyValueSinkFactory; 35 | import com.mozilla.bagheera.util.ShutdownHook; 36 | import com.mozilla.bagheera.metrics.MetricsManager; 37 | 38 | public class KafkaLoggerConsumer extends App { 39 | 40 | private static final Logger LOG = Logger.getLogger(KafkaLoggerConsumer.class); 41 | 42 | public static void main(String[] args) { 43 | OptionFactory optFactory = OptionFactory.getInstance(); 44 | Options options = KafkaConsumer.getOptions(); 45 | options.addOption(optFactory.create("lv", "logvalues", false, "Log values.")); 46 | 47 | CommandLineParser parser = new GnuParser(); 48 | ShutdownHook sh = ShutdownHook.getInstance(); 49 | try { 50 | // Parse command line options 51 | CommandLine cmd = parser.parse(options, args); 52 | 53 | final KafkaConsumer consumer = KafkaConsumer.fromOptions(cmd); 54 | sh.addFirst(consumer); 55 | 56 | // Create a sink for storing data 57 | SinkConfiguration sinkConfig = new SinkConfiguration(); 58 | sinkConfig.setBoolean("loggersink.logvalues", cmd.hasOption("logvalues")); 59 | KeyValueSinkFactory sinkFactory = KeyValueSinkFactory.getInstance(LoggerSink.class, sinkConfig); 60 | sh.addLast(sinkFactory); 61 | 62 | // Set the sink for consumer storage 63 | consumer.setSinkFactory(sinkFactory); 64 | 65 | prepareHealthChecks(); 66 | 67 | // Initialize metrics collection, reporting, etc. 68 | final MetricsManager manager = MetricsManager.getDefaultMetricsManager(); 69 | 70 | // Begin polling 71 | consumer.poll(); 72 | } catch (ParseException e) { 73 | LOG.error("Error parsing command line options", e); 74 | HelpFormatter formatter = new HelpFormatter(); 75 | formatter.printHelp(KafkaHBaseConsumer.class.getName(), options); 76 | } 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/consumer/KafkaReplayConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.consumer; 21 | 22 | import org.apache.commons.cli.CommandLine; 23 | import org.apache.commons.cli.CommandLineParser; 24 | import org.apache.commons.cli.GnuParser; 25 | import org.apache.commons.cli.HelpFormatter; 26 | import org.apache.commons.cli.Options; 27 | import org.apache.commons.cli.ParseException; 28 | import org.apache.log4j.Logger; 29 | 30 | import com.mozilla.bagheera.cli.App; 31 | import com.mozilla.bagheera.cli.OptionFactory; 32 | import com.mozilla.bagheera.sink.KeyValueSinkFactory; 33 | import com.mozilla.bagheera.sink.ReplaySink; 34 | import com.mozilla.bagheera.sink.SinkConfiguration; 35 | import com.mozilla.bagheera.util.ShutdownHook; 36 | import com.mozilla.bagheera.metrics.MetricsManager; 37 | 38 | /** 39 | * Kafka consumer which reads from one kafka queue and re-creates requests to send elsewhere. 40 | */ 41 | public final class KafkaReplayConsumer extends App { 42 | 43 | private static final Logger LOG = Logger.getLogger(KafkaReplayConsumer.class); 44 | 45 | public static void main(String[] args) { 46 | OptionFactory optFactory = OptionFactory.getInstance(); 47 | Options options = KafkaConsumer.getOptions(); 48 | options.addOption(optFactory.create("k", "copy-keys", true, "Whether or not to copy keys from the source data")); 49 | options.addOption(optFactory.create("d", "dest", true, "Destination host / url pattern (include '" + ReplaySink.KEY_PLACEHOLDER + "' for key placeholder)").required()); 50 | options.addOption(optFactory.create("s", "sample", true, "Rate at which to sample the source data (defaults to using all data)")); 51 | options.addOption(optFactory.create("D", "delete", true, "Also replay deletes (using the source keys by necessity)")); 52 | 53 | 54 | CommandLineParser parser = new GnuParser(); 55 | ShutdownHook sh = ShutdownHook.getInstance(); 56 | try { 57 | // Parse command line options 58 | CommandLine cmd = parser.parse(options, args); 59 | 60 | final KafkaConsumer consumer = KafkaConsumer.fromOptions(cmd); 61 | sh.addFirst(consumer); 62 | 63 | // Create a sink for storing data 64 | SinkConfiguration sinkConfig = new SinkConfiguration(); 65 | if (cmd.hasOption("numthreads")) { 66 | sinkConfig.setInt("hbasesink.hbase.numthreads", Integer.parseInt(cmd.getOptionValue("numthreads"))); 67 | } 68 | sinkConfig.setString("replaysink.keys", cmd.getOptionValue("copy-keys", "true")); 69 | sinkConfig.setString("replaysink.dest", cmd.getOptionValue("dest", "http://bogus:8080/submit/endpoint/" + ReplaySink.KEY_PLACEHOLDER)); 70 | sinkConfig.setString("replaysink.sample", cmd.getOptionValue("sample", "1")); 71 | sinkConfig.setString("replaysink.delete", cmd.getOptionValue("delete", "true")); 72 | KeyValueSinkFactory sinkFactory = KeyValueSinkFactory.getInstance(ReplaySink.class, sinkConfig); 73 | sh.addLast(sinkFactory); 74 | 75 | // Set the sink factory for consumer storage 76 | consumer.setSinkFactory(sinkFactory); 77 | 78 | prepareHealthChecks(); 79 | 80 | // Initialize metrics collection, reporting, etc. 81 | final MetricsManager manager = MetricsManager.getDefaultMetricsManager(); 82 | 83 | // Begin polling 84 | consumer.poll(); 85 | } catch (ParseException e) { 86 | LOG.error("Error parsing command line options", e); 87 | HelpFormatter formatter = new HelpFormatter(); 88 | formatter.printHelp(KafkaReplayConsumer.class.getName(), options); 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/consumer/KafkaSequenceFileConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.consumer; 21 | 22 | import org.apache.commons.cli.CommandLine; 23 | import org.apache.commons.cli.CommandLineParser; 24 | import org.apache.commons.cli.GnuParser; 25 | import org.apache.commons.cli.HelpFormatter; 26 | import org.apache.commons.cli.Options; 27 | import org.apache.commons.cli.ParseException; 28 | import org.apache.log4j.Logger; 29 | 30 | import com.mozilla.bagheera.cli.App; 31 | import com.mozilla.bagheera.cli.OptionFactory; 32 | import com.mozilla.bagheera.sink.SequenceFileSink; 33 | import com.mozilla.bagheera.sink.SinkConfiguration; 34 | import com.mozilla.bagheera.sink.KeyValueSinkFactory; 35 | import com.mozilla.bagheera.util.ShutdownHook; 36 | import com.mozilla.bagheera.metrics.MetricsManager; 37 | 38 | /** 39 | * Basic SequenceFile (HDFS) Kafka consumer. This class can be utilized as is but if you want more 40 | * sophisticated logic consider creating your own consumer. 41 | */ 42 | public final class KafkaSequenceFileConsumer extends App { 43 | 44 | private static final Logger LOG = Logger.getLogger(KafkaSequenceFileConsumer.class); 45 | 46 | public static void main(String[] args) { 47 | OptionFactory optFactory = OptionFactory.getInstance(); 48 | Options options = KafkaConsumer.getOptions(); 49 | options.addOption(optFactory.create("o", "output", true, "HDFS base path for output.")); 50 | options.addOption(optFactory.create("df", "dateformat", true, "Date format for the date subdirectories.")); 51 | options.addOption(optFactory.create("fs", "filesize", true, "Max file size for output files.")); 52 | options.addOption(optFactory.create("b", "usebytes", false, "Use BytesWritable for value rather than Text.")); 53 | options.addOption(optFactory.create("ts", "addtimestamp", false, "Adds bagheera timestamp to the json")); 54 | 55 | CommandLineParser parser = new GnuParser(); 56 | ShutdownHook sh = ShutdownHook.getInstance(); 57 | try { 58 | // Parse command line options 59 | CommandLine cmd = parser.parse(options, args); 60 | 61 | final KafkaConsumer consumer = KafkaConsumer.fromOptions(cmd); 62 | sh.addFirst(consumer); 63 | 64 | // Create a sink for storing data 65 | SinkConfiguration sinkConfig = new SinkConfiguration(); 66 | sinkConfig.setString("hdfssink.hdfs.basedir.path", cmd.getOptionValue("output", "/bagheera")); 67 | sinkConfig.setString("hdfssink.hdfs.date.format", cmd.getOptionValue("dateformat", "yyyy-MM-dd")); 68 | sinkConfig.setLong("hdfssink.hdfs.max.filesize", Long.parseLong(cmd.getOptionValue("filesize", "536870912"))); 69 | sinkConfig.setBoolean("hdfssink.hdfs.usebytes", cmd.hasOption("usebytes")); 70 | if(cmd.hasOption("addtimestamp")) { 71 | sinkConfig.setBoolean("hdfssink.hdfs.addtimestamp", true); 72 | } 73 | KeyValueSinkFactory sinkFactory = KeyValueSinkFactory.getInstance(SequenceFileSink.class, sinkConfig); 74 | sh.addLast(sinkFactory); 75 | 76 | // Set the sink for consumer storage 77 | consumer.setSinkFactory(sinkFactory); 78 | 79 | // Initialize metrics collection, reporting, etc. 80 | final MetricsManager manager = MetricsManager.getDefaultMetricsManager(); 81 | 82 | prepareHealthChecks(); 83 | 84 | // Begin polling 85 | consumer.poll(); 86 | } catch (ParseException e) { 87 | LOG.error("Error parsing command line options", e); 88 | HelpFormatter formatter = new HelpFormatter(); 89 | formatter.printHelp(KafkaSequenceFileConsumer.class.getName(), options); 90 | } catch (NumberFormatException e) { 91 | LOG.error("Failed to parse filesize option", e); 92 | } 93 | } 94 | } 95 | 96 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/consumer/validation/JsonValidator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.consumer.validation; 21 | 22 | import java.io.IOException; 23 | 24 | import org.apache.log4j.Logger; 25 | 26 | import com.fasterxml.jackson.core.JsonFactory; 27 | import com.fasterxml.jackson.core.JsonParseException; 28 | import com.fasterxml.jackson.core.JsonParser; 29 | 30 | public class JsonValidator implements Validator { 31 | 32 | private static final Logger LOG = Logger.getLogger(JsonValidator.class); 33 | 34 | private final JsonFactory jsonFactory = new JsonFactory(); 35 | 36 | @Override 37 | public boolean isValid(byte[] data) { 38 | boolean isValid = false; 39 | JsonParser parser = null; 40 | try { 41 | parser = jsonFactory.createJsonParser(data); 42 | while (parser.nextToken() != null) { 43 | // noop 44 | } 45 | isValid = true; 46 | } catch (JsonParseException ex) { 47 | LOG.error("JSON parse error"); 48 | } catch (IOException e) { 49 | LOG.error("JSON IO error"); 50 | } catch (Exception e) { 51 | LOG.error("Generic error during validation: ", e); 52 | } finally { 53 | if (parser != null) { 54 | try { 55 | parser.close(); 56 | } catch (IOException e) { 57 | LOG.error("Error closing JSON parser", e); 58 | } 59 | } 60 | } 61 | 62 | return isValid; 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/consumer/validation/ValidationPipeline.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.consumer.validation; 21 | 22 | import java.util.LinkedList; 23 | import java.util.List; 24 | 25 | public class ValidationPipeline { 26 | 27 | private List validators = new LinkedList(); 28 | 29 | public void addFirst(Validator validator) { 30 | validators.add(0, validator); 31 | } 32 | 33 | public void addLast(Validator validator) { 34 | validators.add(validator); 35 | } 36 | 37 | public boolean isValid(byte[] data) { 38 | boolean valid = false; 39 | for (Validator validator : validators) { 40 | valid = validator.isValid(data); 41 | if (!valid) { 42 | break; 43 | } 44 | } 45 | 46 | return valid; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/consumer/validation/Validator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.consumer.validation; 21 | 22 | public interface Validator { 23 | 24 | public boolean isValid(byte[] data); 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/AccessFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import java.net.InetSocketAddress; 23 | 24 | import org.jboss.netty.channel.ChannelHandlerContext; 25 | import org.jboss.netty.channel.Channels; 26 | import org.jboss.netty.channel.MessageEvent; 27 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 28 | import org.jboss.netty.handler.codec.http.HttpMethod; 29 | import org.jboss.netty.handler.codec.http.HttpRequest; 30 | 31 | import com.mozilla.bagheera.util.HttpUtil; 32 | import com.mozilla.bagheera.util.WildcardProperties; 33 | import com.mozilla.bagheera.validation.Validator; 34 | 35 | public class AccessFilter extends SimpleChannelUpstreamHandler { 36 | 37 | private static final String ALLOW_DELETE_ACCESS = ".allow.delete.access"; 38 | private static final String ID_VALIDATION = ".id.validation"; 39 | 40 | private final Validator validator; 41 | private final WildcardProperties props; 42 | 43 | public AccessFilter(Validator validator, WildcardProperties props) { 44 | this.validator = validator; 45 | this.props = props; 46 | } 47 | 48 | private String buildErrorMessage(String msg, HttpRequest request, MessageEvent e) { 49 | return String.format("%s: %s %s - \"%s\" \"%s\"", msg, request.getMethod().getName(), request.getUri(), 50 | HttpUtil.getRemoteAddr(request, ((InetSocketAddress)e.getChannel().getRemoteAddress()).toString()), 51 | request.getHeader(HttpUtil.USER_AGENT)); 52 | } 53 | 54 | @Override 55 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 56 | Object msg = e.getMessage(); 57 | if (msg instanceof BagheeraHttpRequest) { 58 | BagheeraHttpRequest request = (BagheeraHttpRequest)msg; 59 | // Check Namespace 60 | if (request.getNamespace() == null || !validator.isValidNamespace(request.getNamespace())) { 61 | throw new InvalidPathException(buildErrorMessage("Tried to access invalid resource", request, e)); 62 | } 63 | // Check Id 64 | boolean validateId = Boolean.parseBoolean(props.getWildcardProperty(request.getNamespace() + ID_VALIDATION, "true")); 65 | if (request.getId() != null && validateId && !validator.isValidId(request.getId())) { 66 | throw new InvalidPathException(buildErrorMessage("Submitted an invalid ID", request, e)); 67 | } 68 | // Check POST/GET/DELETE Access 69 | if (request.getMethod() == HttpMethod.POST || request.getMethod() == HttpMethod.PUT) { 70 | // noop 71 | } else if (request.getMethod() == HttpMethod.GET) { 72 | throw new HttpSecurityException(buildErrorMessage("Tried to access GET method for resource", request, e)); 73 | } else if (request.getMethod() == HttpMethod.DELETE) { 74 | boolean allowDelAccess = Boolean.parseBoolean(props.getWildcardProperty(request.getNamespace() + ALLOW_DELETE_ACCESS, "false")); 75 | if (!allowDelAccess) { 76 | throw new HttpSecurityException(buildErrorMessage("Tried to access DELETE method for resource", request, e)); 77 | } 78 | } else { 79 | throw new HttpSecurityException(buildErrorMessage("Tried to access invalid method for resource", request, e)); 80 | } 81 | Channels.fireMessageReceived(ctx, request, e.getRemoteAddress()); 82 | } else { 83 | ctx.sendUpstream(e); 84 | } 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/Bagheera.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | package com.mozilla.bagheera.http; 22 | 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.net.InetSocketAddress; 26 | import java.net.URL; 27 | import java.util.Properties; 28 | import java.util.concurrent.Executors; 29 | 30 | import org.apache.log4j.Logger; 31 | import org.jboss.netty.bootstrap.ServerBootstrap; 32 | import org.jboss.netty.channel.Channel; 33 | import org.jboss.netty.channel.group.ChannelGroup; 34 | import org.jboss.netty.channel.group.DefaultChannelGroup; 35 | import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; 36 | 37 | import com.mozilla.bagheera.cli.App; 38 | import com.mozilla.bagheera.metrics.MetricsManager; 39 | import com.mozilla.bagheera.producer.KafkaProducer; 40 | import com.mozilla.bagheera.producer.Producer; 41 | import com.mozilla.bagheera.util.WildcardProperties; 42 | 43 | /** 44 | * Front-end class to a Bagheera server instance. 45 | * 46 | * Either create a server using `startServer`, or allow the main method to do so. 47 | */ 48 | public class Bagheera extends App { 49 | 50 | private static final Logger LOG = Logger.getLogger(Bagheera.class); 51 | 52 | public static final String PROPERTIES_RESOURCE_NAME = "/bagheera.properties"; 53 | public static final String KAFKA_PROPERTIES_RESOURCE_NAME = "/kafka.producer.properties"; 54 | 55 | private static final int DEFAULT_IO_THREADS = Runtime.getRuntime().availableProcessors() * 2; 56 | 57 | public static NioServerSocketChannelFactory getChannelFactory() { 58 | return new NioServerSocketChannelFactory(Executors.newCachedThreadPool(), 59 | Executors.newFixedThreadPool(DEFAULT_IO_THREADS)); 60 | } 61 | 62 | /** 63 | * Immutable record of the persistent state around a Bagheera server. 64 | */ 65 | public static class BagheeraServerState { 66 | public final int port; 67 | public final Producer producer; 68 | public final NioServerSocketChannelFactory channelFactory; 69 | public final Channel channel; 70 | public final ChannelGroup channelGroup; 71 | 72 | public BagheeraServerState(final int port, 73 | final Producer producer, 74 | final NioServerSocketChannelFactory channelFactory, 75 | final Channel channel, 76 | final ChannelGroup channelGroup) { 77 | this.port = port; 78 | this.producer = producer; 79 | this.channelFactory = channelFactory; 80 | this.channel = channel; 81 | this.channelGroup = channelGroup; 82 | } 83 | 84 | public void close() { 85 | // Close our channels. 86 | this.channelGroup.close().awaitUninterruptibly(); 87 | this.channel.close().awaitUninterruptibly(); 88 | 89 | // The caller is responsible for releasing resources from the channel factory. 90 | 91 | // Shut down producer. 92 | if (this.producer != null) { 93 | LOG.info("Closing producer resource..."); 94 | try { 95 | this.producer.close(); 96 | } catch (IOException e) { 97 | LOG.error("Error closing producer.", e); 98 | } 99 | } 100 | } 101 | } 102 | 103 | /** 104 | * Start a Bagheera server with the provided settings. 105 | * Throws if the server could not be started. 106 | * The caller is responsible for closing the returned instance, and the 107 | * channel factory if desired. 108 | */ 109 | public static BagheeraServerState startServer(final int port, 110 | final boolean tcpNoDelay, 111 | final WildcardProperties props, 112 | final Producer producer, 113 | final NioServerSocketChannelFactory channelFactory, 114 | final String channelGroupName, 115 | final MetricsManager manager) 116 | throws Exception { 117 | 118 | prepareHealthChecks(); 119 | 120 | // HTTP server setup. 121 | final ChannelGroup channelGroup = new DefaultChannelGroup(channelGroupName); 122 | final ServerBootstrap server = new ServerBootstrap(channelFactory); 123 | final HttpServerPipelineFactory pipeFactory = new HttpServerPipelineFactory(props, producer, channelGroup, manager); 124 | server.setPipelineFactory(pipeFactory); 125 | server.setOption("tcpNoDelay", tcpNoDelay); 126 | 127 | // Disable keep-alive so client connections don't hang around. 128 | server.setOption("keepAlive", false); 129 | 130 | final Channel channel = server.bind(new InetSocketAddress(port)); 131 | return new BagheeraServerState(port, producer, channelFactory, channel, channelGroup); 132 | } 133 | 134 | /** 135 | * A simple front-end that configures a new server from properties files, 136 | * waiting until runtime shutdown to clean up. 137 | */ 138 | public static void main(String[] args) throws Exception { 139 | final int port = Integer.parseInt(System.getProperty("server.port", "8080")); 140 | final boolean tcpNoDelay = Boolean.parseBoolean(System.getProperty("server.tcpnodelay", "false")); 141 | 142 | // Initalize properties and producer. 143 | final WildcardProperties props = getDefaultProperties(); 144 | final Properties kafkaProps = getDefaultKafkaProperties(); 145 | final Producer producer = new KafkaProducer(kafkaProps); 146 | final MetricsManager manager = MetricsManager.getDefaultMetricsManager(); 147 | 148 | final BagheeraServerState server = startServer(port, 149 | tcpNoDelay, 150 | props, 151 | producer, 152 | getChannelFactory(), 153 | Bagheera.class.getName(), 154 | manager); 155 | 156 | Runtime.getRuntime().addShutdownHook(new Thread() { 157 | @Override 158 | public void run() { 159 | server.close(); 160 | server.channelFactory.releaseExternalResources(); 161 | } 162 | }); 163 | } 164 | 165 | protected static Properties getDefaultKafkaProperties() throws Exception { 166 | final Properties props = new Properties(); 167 | final URL propUrl = Bagheera.class.getResource(KAFKA_PROPERTIES_RESOURCE_NAME); 168 | if (propUrl == null) { 169 | throw new IllegalArgumentException("Could not find the properties file: " + KAFKA_PROPERTIES_RESOURCE_NAME); 170 | } 171 | 172 | final InputStream in = propUrl.openStream(); 173 | try { 174 | props.load(in); 175 | } finally { 176 | in.close(); 177 | } 178 | 179 | return props; 180 | } 181 | 182 | protected static WildcardProperties getDefaultProperties() throws Exception { 183 | final WildcardProperties props = new WildcardProperties(); 184 | final URL propUrl = Bagheera.class.getResource(PROPERTIES_RESOURCE_NAME); 185 | if (propUrl == null) { 186 | throw new IllegalArgumentException("Could not find the properties file: " + PROPERTIES_RESOURCE_NAME); 187 | } 188 | 189 | final InputStream in = propUrl.openStream(); 190 | try { 191 | props.load(in); 192 | } finally { 193 | in.close(); 194 | } 195 | 196 | return props; 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/BagheeraHttpRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.UUID; 25 | import java.util.regex.Pattern; 26 | 27 | import org.jboss.netty.handler.codec.http.DefaultHttpRequest; 28 | import org.jboss.netty.handler.codec.http.HttpMethod; 29 | import org.jboss.netty.handler.codec.http.HttpRequest; 30 | import org.jboss.netty.handler.codec.http.HttpVersion; 31 | 32 | public class BagheeraHttpRequest extends DefaultHttpRequest { 33 | 34 | // API Version 35 | private static final Pattern VERSION_PATTERN = Pattern.compile("^[0-9]+([.][0-9]+)?$"); 36 | 37 | // REST path indices 38 | public static final int ENDPOINT_PATH_IDX = 0; 39 | public static final int NAMESPACE_PATH_IDX = 1; 40 | public static final int ID_PATH_IDX = 2; 41 | 42 | private final String apiVersion; 43 | private final String endpoint; 44 | private final String namespace; 45 | private final String id; 46 | private final List partitions; 47 | 48 | public BagheeraHttpRequest(HttpVersion httpVersion, HttpMethod method, String uri) { 49 | this(httpVersion, method, uri, new PathDecoder(uri)); 50 | } 51 | 52 | public BagheeraHttpRequest(HttpRequest request) { 53 | this(request.getProtocolVersion(), request.getMethod(), request.getUri()); 54 | } 55 | 56 | public BagheeraHttpRequest(HttpVersion httpVersion, HttpMethod method, String uri, 57 | PathDecoder pathDecoder) { 58 | super(httpVersion, method, uri); 59 | int idxOffset = 0; 60 | 61 | String apiVersionIn = pathDecoder.getPathElement(0); 62 | // If API version is in the path then offset the path indices 63 | if (isApiVersion(apiVersionIn)) { 64 | idxOffset = 1; 65 | apiVersion = apiVersionIn; 66 | } else { 67 | apiVersion = null; 68 | } 69 | endpoint = pathDecoder.getPathElement(ENDPOINT_PATH_IDX + idxOffset); 70 | namespace = pathDecoder.getPathElement(NAMESPACE_PATH_IDX + idxOffset); 71 | String incomingId = pathDecoder.getPathElement(ID_PATH_IDX + idxOffset); 72 | if (incomingId == null) { 73 | incomingId = UUID.randomUUID().toString(); 74 | } 75 | id = incomingId; 76 | 77 | int partitionOffset = idxOffset + 3; 78 | int numPartitions = pathDecoder.size() - partitionOffset; 79 | if (numPartitions < 0) { 80 | numPartitions = 0; 81 | } 82 | partitions = new ArrayList(numPartitions); 83 | for (int i = partitionOffset; i < pathDecoder.size(); i++) { 84 | partitions.add(pathDecoder.getPathElement(i)); 85 | } 86 | } 87 | 88 | public boolean isApiVersion(String possibleApiVersion) { 89 | return VERSION_PATTERN.matcher(possibleApiVersion).matches(); 90 | } 91 | 92 | public String getEndpoint() { 93 | return endpoint; 94 | } 95 | 96 | public String getNamespace() { 97 | return namespace; 98 | } 99 | 100 | public String getId() { 101 | return id; 102 | } 103 | 104 | public String getApiVersion() { 105 | return apiVersion; 106 | } 107 | 108 | public List getPartitions() { 109 | return partitions; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/BagheeraHttpRequestDecoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import org.jboss.netty.handler.codec.http.HttpMessage; 23 | import org.jboss.netty.handler.codec.http.HttpMethod; 24 | import org.jboss.netty.handler.codec.http.HttpRequestDecoder; 25 | import org.jboss.netty.handler.codec.http.HttpVersion; 26 | 27 | public class BagheeraHttpRequestDecoder extends HttpRequestDecoder { 28 | 29 | @Override 30 | protected HttpMessage createMessage(String[] initialLine) throws Exception { 31 | return new BagheeraHttpRequest(HttpVersion.valueOf(initialLine[2]), HttpMethod.valueOf(initialLine[0]), initialLine[1]); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/ContentEncodingCorrector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import org.jboss.netty.channel.ChannelHandlerContext; 23 | import org.jboss.netty.channel.Channels; 24 | import org.jboss.netty.channel.MessageEvent; 25 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 26 | import org.jboss.netty.handler.codec.http.HttpHeaders; 27 | import org.jboss.netty.handler.codec.http.HttpMessage; 28 | 29 | /** 30 | * If Content-Encoding isn't specified this checks the Content-Type to see 31 | * if there is any indication that content is compressed even though encoding 32 | * isn't set. 33 | */ 34 | public class ContentEncodingCorrector extends SimpleChannelUpstreamHandler { 35 | 36 | public static final String MIME_TYPE_JSON_ZLIB = "application/json+zlib"; 37 | 38 | @Override 39 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 40 | Object msg = e.getMessage(); 41 | if (msg instanceof HttpMessage) { 42 | HttpMessage m = (HttpMessage) msg; 43 | String contentEncoding = m.getHeader(HttpHeaders.Names.CONTENT_ENCODING); 44 | if (contentEncoding == null) { 45 | String contentType = m.getHeader(HttpHeaders.Names.CONTENT_TYPE); 46 | if (contentType != null && contentType.startsWith(MIME_TYPE_JSON_ZLIB)) { 47 | m.setHeader(HttpHeaders.Names.CONTENT_ENCODING,"deflate"); 48 | } 49 | } 50 | Channels.fireMessageReceived(ctx, m, e.getRemoteAddress()); 51 | } else { 52 | ctx.sendUpstream(e); 53 | } 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/ContentLengthFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import org.jboss.netty.channel.ChannelHandlerContext; 23 | import org.jboss.netty.channel.Channels; 24 | import org.jboss.netty.channel.MessageEvent; 25 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 26 | import org.jboss.netty.handler.codec.frame.TooLongFrameException; 27 | import org.jboss.netty.handler.codec.http.HttpMessage; 28 | 29 | public class ContentLengthFilter extends SimpleChannelUpstreamHandler { 30 | 31 | private final int maxContentLength; 32 | 33 | public ContentLengthFilter(int maxContentLength) { 34 | if (maxContentLength <= 0) { 35 | throw new IllegalArgumentException("maxContentLength must be a positive integer: " + maxContentLength); 36 | } 37 | this.maxContentLength = maxContentLength; 38 | } 39 | 40 | @Override 41 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 42 | Object msg = e.getMessage(); 43 | if (msg instanceof HttpMessage) { 44 | HttpMessage m = (HttpMessage) msg; 45 | if (m.getContent().readableBytes() > maxContentLength) { 46 | throw new TooLongFrameException("HTTP content length exceeded: " + maxContentLength + " bytes."); 47 | } 48 | Channels.fireMessageReceived(ctx, m, e.getRemoteAddress()); 49 | } else { 50 | ctx.sendUpstream(e); 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/HttpSecurityException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | public class HttpSecurityException extends Exception { 23 | 24 | private static final long serialVersionUID = 480106127340879943L; 25 | 26 | public HttpSecurityException(String message) { 27 | super(message); 28 | } 29 | 30 | public HttpSecurityException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/HttpServerPipelineFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import java.io.IOException; 23 | 24 | import org.jboss.netty.channel.ChannelPipeline; 25 | import org.jboss.netty.channel.ChannelPipelineFactory; 26 | import org.jboss.netty.channel.Channels; 27 | import org.jboss.netty.channel.group.ChannelGroup; 28 | import org.jboss.netty.handler.codec.http.HttpChunkAggregator; 29 | import org.jboss.netty.handler.codec.http.HttpContentDecompressor; 30 | import org.jboss.netty.handler.codec.http.HttpResponseEncoder; 31 | 32 | import com.mozilla.bagheera.metrics.MetricsManager; 33 | import com.mozilla.bagheera.producer.Producer; 34 | import com.mozilla.bagheera.util.WildcardProperties; 35 | import com.mozilla.bagheera.validation.Validator; 36 | 37 | public class HttpServerPipelineFactory implements ChannelPipelineFactory { 38 | 39 | private final WildcardProperties props; 40 | private final int maxContentLength; 41 | private final Validator validator; 42 | private final Producer producer; 43 | private final ChannelGroup channelGroup; 44 | private final MetricsManager metricsManager; 45 | 46 | public HttpServerPipelineFactory(WildcardProperties props, 47 | Producer producer, 48 | ChannelGroup channelGroup, 49 | MetricsManager metricsManager) 50 | throws IOException { 51 | this.props = props; 52 | String validNsStr = props.getProperty("valid.namespaces"); 53 | if (validNsStr == null || validNsStr.length() == 0) { 54 | throw new IllegalArgumentException("No valid.namespaces in properties"); 55 | } 56 | this.validator = new Validator(validNsStr.split(",")); 57 | this.maxContentLength = Integer.parseInt(props.getProperty("max.content.length","1048576")); 58 | this.producer = producer; 59 | this.channelGroup = channelGroup; 60 | this.metricsManager = metricsManager; 61 | } 62 | 63 | /* (non-Javadoc) 64 | * @see org.jboss.netty.channel.ChannelPipelineFactory#getPipeline() 65 | */ 66 | public ChannelPipeline getPipeline() throws Exception { 67 | ChannelPipeline pipeline = Channels.pipeline(); 68 | 69 | pipeline.addLast("decoder", new BagheeraHttpRequestDecoder()); 70 | pipeline.addLast("aggregator", new HttpChunkAggregator(maxContentLength)); 71 | pipeline.addLast("contentLengthFilter", new ContentLengthFilter(maxContentLength)); 72 | pipeline.addLast("rootResponse", new RootResponse()); 73 | pipeline.addLast("accessFilter", new AccessFilter(validator, props)); 74 | pipeline.addLast("encodingCorrector", new ContentEncodingCorrector()); 75 | pipeline.addLast("inflater", new HttpContentDecompressor()); 76 | pipeline.addLast("encoder", new HttpResponseEncoder()); 77 | pipeline.addLast("handler", new SubmissionHandler(validator, producer, this.channelGroup, this.metricsManager)); 78 | 79 | return pipeline; 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/InvalidPathException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | public class InvalidPathException extends Exception { 23 | 24 | private static final long serialVersionUID = -2753373949363309083L; 25 | 26 | public InvalidPathException(String message) { 27 | super(message); 28 | } 29 | 30 | public InvalidPathException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/PathDecoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.regex.Matcher; 25 | import java.util.regex.Pattern; 26 | 27 | public class PathDecoder { 28 | 29 | private static final Pattern PATH_PATTERN = Pattern.compile("/([^/]+)"); 30 | 31 | private List pathElements = new ArrayList(); 32 | 33 | public PathDecoder(String uri) { 34 | Matcher m = PATH_PATTERN.matcher(uri); 35 | while (m.find()) { 36 | if (m.groupCount() > 0) { 37 | pathElements.add(m.group(1)); 38 | } 39 | } 40 | } 41 | 42 | public String getPathElement(int idx) { 43 | return idx < pathElements.size() ? pathElements.get(idx) : null; 44 | } 45 | 46 | public int size() { 47 | return pathElements.size(); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/RootResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK; 23 | import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; 24 | 25 | import org.jboss.netty.channel.ChannelFuture; 26 | import org.jboss.netty.channel.ChannelFutureListener; 27 | import org.jboss.netty.channel.ChannelHandlerContext; 28 | import org.jboss.netty.channel.Channels; 29 | import org.jboss.netty.channel.MessageEvent; 30 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 31 | import org.jboss.netty.handler.codec.http.DefaultHttpResponse; 32 | import org.jboss.netty.handler.codec.http.HttpMessage; 33 | import org.jboss.netty.handler.codec.http.HttpRequest; 34 | import org.jboss.netty.handler.codec.http.HttpResponse; 35 | 36 | public class RootResponse extends SimpleChannelUpstreamHandler { 37 | 38 | private static final String ROOT_PATH = "/"; 39 | 40 | @Override 41 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 42 | Object msg = e.getMessage(); 43 | if (msg instanceof HttpMessage) { 44 | HttpRequest request = (HttpRequest) msg; 45 | if (ROOT_PATH.equals(request.getUri()) || request.getUri().isEmpty()) { 46 | HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); 47 | ChannelFuture future = e.getChannel().write(response); 48 | future.addListener(ChannelFutureListener.CLOSE); 49 | } else { 50 | Channels.fireMessageReceived(ctx, request, e.getRemoteAddress()); 51 | } 52 | } else { 53 | ctx.sendUpstream(e); 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/json/InvalidJsonException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http.json; 21 | 22 | import java.io.IOException; 23 | 24 | public class InvalidJsonException extends IOException { 25 | 26 | private static final long serialVersionUID = -3575427769547846171L; 27 | 28 | public InvalidJsonException(String message) { 29 | super(message); 30 | } 31 | 32 | public InvalidJsonException(String message, Throwable cause) { 33 | super(message, cause); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/http/json/JsonFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http.json; 21 | 22 | import org.jboss.netty.buffer.ChannelBuffer; 23 | import org.jboss.netty.channel.ChannelHandlerContext; 24 | import org.jboss.netty.channel.Channels; 25 | import org.jboss.netty.channel.MessageEvent; 26 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 27 | import org.jboss.netty.handler.codec.http.HttpRequest; 28 | import org.jboss.netty.util.CharsetUtil; 29 | 30 | import com.mozilla.bagheera.validation.Validator; 31 | 32 | public class JsonFilter extends SimpleChannelUpstreamHandler { 33 | 34 | private final Validator validator; 35 | 36 | public JsonFilter(Validator validator) { 37 | this.validator = validator; 38 | } 39 | 40 | @Override 41 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 42 | Object msg = e.getMessage(); 43 | if (msg instanceof HttpRequest) { 44 | HttpRequest request = (HttpRequest)e.getMessage(); 45 | ChannelBuffer content = request.getContent(); 46 | if (content.readable()) { 47 | if (!validator.isValidJson(content.toString(CharsetUtil.UTF_8))) { 48 | throw new InvalidJsonException("Invalid JSON"); 49 | } 50 | } 51 | Channels.fireMessageReceived(ctx, request, e.getRemoteAddress()); 52 | } else { 53 | ctx.sendUpstream(e); 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/metrics/HttpMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.metrics; 21 | 22 | import java.util.concurrent.ConcurrentHashMap; 23 | import java.util.concurrent.ConcurrentMap; 24 | import java.util.concurrent.TimeUnit; 25 | 26 | import com.yammer.metrics.Metrics; 27 | import com.yammer.metrics.core.Counter; 28 | import com.yammer.metrics.core.Meter; 29 | import com.yammer.metrics.core.MetricName; 30 | 31 | public class HttpMetric { 32 | 33 | private static final String DEFAULT_GROUP = "bagheera"; 34 | private static final String DEFAULT_TYPE = "http"; 35 | private final String id; 36 | 37 | private Meter requests, throughput; 38 | private ConcurrentMap methods; 39 | private ConcurrentMap responseCodeCounts; 40 | 41 | HttpMetric(String id) { 42 | this.id = id; 43 | configureMetrics(); 44 | } 45 | 46 | private void configureMetrics() { 47 | requests = Metrics.newMeter(new MetricName(DEFAULT_GROUP, DEFAULT_TYPE, this.id + ".requests"), "requests", TimeUnit.SECONDS); 48 | throughput = Metrics.newMeter(new MetricName(DEFAULT_GROUP, DEFAULT_TYPE, this.id + ".throughput"), "bytes", TimeUnit.SECONDS); 49 | methods = new ConcurrentHashMap(); 50 | responseCodeCounts = new ConcurrentHashMap(); 51 | } 52 | 53 | public void updateRequestMetrics(String method, int contentSize) { 54 | requests.mark(); 55 | throughput.mark(contentSize); 56 | if (methods.containsKey(method)) { 57 | methods.get(method).mark(); 58 | } else { 59 | Meter methodMeter = Metrics.newMeter(new MetricName(DEFAULT_GROUP, DEFAULT_TYPE, this.id + ".method." + method), "requests", TimeUnit.SECONDS); 60 | methodMeter.mark(); 61 | methods.put(method, methodMeter); 62 | } 63 | } 64 | 65 | public void updateResponseMetrics(int status) { 66 | if (responseCodeCounts.containsKey(status)) { 67 | responseCodeCounts.get(status).inc(); 68 | } else { 69 | Counter statusCounter = Metrics.newCounter(new MetricName(DEFAULT_GROUP, DEFAULT_TYPE, this.id + ".response." + status)); 70 | statusCounter.inc(); 71 | responseCodeCounts.put(status, statusCounter); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/metrics/MetricsManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.metrics; 21 | 22 | import java.io.IOException; 23 | import java.io.InputStream; 24 | import java.util.Properties; 25 | import java.util.concurrent.ConcurrentHashMap; 26 | import java.util.concurrent.ConcurrentMap; 27 | import java.util.concurrent.TimeUnit; 28 | 29 | import com.yammer.metrics.reporting.ConsoleReporter; 30 | import com.yammer.metrics.reporting.GangliaReporter; 31 | import com.yammer.metrics.reporting.GraphiteReporter; 32 | 33 | public class MetricsManager { 34 | private static final String DEFAULT_METRICS_PROPERTIES_RESOURCE_NAME = "/bagheera.metrics.properties"; 35 | private static final String DEFAULT_METRICS_PROPERTIES_PREFIX = "bagheera.metrics."; 36 | private static final String GLOBAL_HTTP_METRIC_ID = "global"; 37 | 38 | private ConcurrentMap httpMetrics = new ConcurrentHashMap(); 39 | 40 | public static MetricsManager getDefaultMetricsManager() { 41 | final Properties properties = readProperties(DEFAULT_METRICS_PROPERTIES_RESOURCE_NAME); 42 | return new MetricsManager(properties, DEFAULT_METRICS_PROPERTIES_PREFIX); 43 | } 44 | 45 | /** 46 | * Each MetricsManager instance currently shares state (Ganglia, Graphite). 47 | * Alas. 48 | */ 49 | public MetricsManager(final Properties properties, final String propertiesPrefix) { 50 | configureReporters(properties, propertiesPrefix); 51 | HttpMetric h = new HttpMetric(GLOBAL_HTTP_METRIC_ID); 52 | httpMetrics.put(GLOBAL_HTTP_METRIC_ID, h); 53 | } 54 | 55 | private static String getProp(final Properties props, final String prefix, final String key) { 56 | return props.getProperty(prefix + key, null); 57 | } 58 | 59 | private static boolean getBoolean(final Properties props, final String prefix, final String key) { 60 | String value = props.getProperty(prefix + key, null); 61 | if (value == null) { 62 | return false; 63 | } 64 | return Boolean.parseBoolean(value); 65 | } 66 | 67 | private static long getLong(final Properties props, final String prefix, final String key) { 68 | String value = props.getProperty(prefix + key, null); 69 | if (value == null) { 70 | return 0; 71 | } 72 | return Long.parseLong(value); 73 | } 74 | 75 | private static int getInt(final Properties props, final String prefix, final String key) { 76 | String value = props.getProperty(prefix + key, null); 77 | if (value == null) { 78 | return 0; 79 | } 80 | return Integer.parseInt(value); 81 | } 82 | 83 | private static void configureReporters(final Properties props, final String prefix) { 84 | if (getBoolean(props, prefix, "ganglia.enable")) { 85 | GangliaReporter.enable(getLong(props, prefix, "ganglia.update.secs"), 86 | TimeUnit.SECONDS, 87 | getProp(props, prefix, "ganglia.host"), 88 | getInt(props, prefix, "ganglia.port")); 89 | } 90 | if (getBoolean(props, prefix, "graphite.enable")) { 91 | GraphiteReporter.enable(getLong(props, prefix, "graphite.update.secs"), 92 | TimeUnit.SECONDS, 93 | getProp(props, prefix, "graphite.host"), 94 | getInt(props, prefix, "graphite.port")); 95 | } 96 | if (getBoolean(props, prefix, "consolereporter.enable")) { 97 | ConsoleReporter.enable(getLong(props, prefix, "consolereporter.update.secs"), 98 | TimeUnit.SECONDS); 99 | } 100 | } 101 | 102 | public HttpMetric getGlobalHttpMetric() { 103 | return getHttpMetricForNamespace(GLOBAL_HTTP_METRIC_ID); 104 | } 105 | 106 | public HttpMetric getHttpMetricForNamespace(final String ns) { 107 | final HttpMetric metric = httpMetrics.get(ns); 108 | if (metric != null) { 109 | return metric; 110 | } 111 | 112 | final HttpMetric newMetric = httpMetrics.putIfAbsent(ns, new HttpMetric(ns)); 113 | if (newMetric == null) { 114 | return httpMetrics.get(ns); 115 | } 116 | return newMetric; 117 | } 118 | 119 | /** 120 | * Utility to read a properties file from a path. 121 | */ 122 | public static Properties readProperties(final String path) { 123 | final Properties properties = new Properties(); 124 | final InputStream in = MetricsManager.class.getResourceAsStream(path); 125 | try { 126 | try { 127 | properties.load(in); 128 | } finally { 129 | in.close(); 130 | } 131 | } catch (IOException e) { 132 | throw new IllegalArgumentException("Could not find the properties file: " + path); 133 | } catch (Exception e) { 134 | throw new IllegalArgumentException("Exception reading " + path + ": " + e); 135 | } 136 | return properties; 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/producer/KafkaProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.producer; 21 | 22 | import java.util.Properties; 23 | import java.util.List; 24 | import java.util.ArrayList; 25 | import kafka.javaapi.producer.Producer; 26 | import kafka.javaapi.producer.ProducerData; 27 | import kafka.producer.ProducerConfig; 28 | 29 | import com.mozilla.bagheera.BagheeraProto.BagheeraMessage; 30 | 31 | public class KafkaProducer implements com.mozilla.bagheera.producer.Producer { 32 | 33 | private final Producer producer; 34 | 35 | public KafkaProducer(Properties props) { 36 | ProducerConfig config = new ProducerConfig(props); 37 | producer = new Producer(config); 38 | } 39 | 40 | /* (non-Javadoc) 41 | * @see com.mozilla.bagheera.producer.Producer#close() 42 | */ 43 | public void close() { 44 | if (producer != null) { 45 | producer.close(); 46 | } 47 | } 48 | 49 | /* (non-Javadoc) 50 | * @see com.mozilla.bagheera.producer.Producer#send(com.mozilla.bagheera.BagheeraProto.BagheeraMessage) 51 | */ 52 | @Override 53 | public void send(BagheeraMessage msg) { 54 | List list = new ArrayList(); 55 | list.add(msg); 56 | producer.send(new ProducerData(msg.getNamespace(), msg.getId(),list)); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/producer/Producer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.producer; 21 | 22 | import java.io.Closeable; 23 | 24 | import com.mozilla.bagheera.BagheeraProto.BagheeraMessage; 25 | 26 | public interface Producer extends Closeable { 27 | 28 | public void send(BagheeraMessage msg); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/serializer/BagheeraDecoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.serializer; 21 | 22 | import kafka.message.Message; 23 | import kafka.serializer.Decoder; 24 | 25 | import org.apache.log4j.Logger; 26 | 27 | import com.google.protobuf.ByteString; 28 | import com.google.protobuf.InvalidProtocolBufferException; 29 | import com.mozilla.bagheera.BagheeraProto; 30 | import com.mozilla.bagheera.BagheeraProto.BagheeraMessage; 31 | 32 | public class BagheeraDecoder implements Decoder { 33 | 34 | private static final Logger LOG = Logger.getLogger(BagheeraDecoder.class); 35 | 36 | @Override 37 | public BagheeraMessage toEvent(Message msg) { 38 | BagheeraMessage bmsg = null; 39 | try { 40 | bmsg = BagheeraProto.BagheeraMessage.parseFrom(ByteString.copyFrom(msg.payload())); 41 | } catch (InvalidProtocolBufferException e) { 42 | LOG.error("Received unparseable message", e); 43 | } 44 | 45 | return bmsg; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/serializer/BagheeraEncoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.serializer; 21 | 22 | import com.mozilla.bagheera.BagheeraProto.BagheeraMessage; 23 | 24 | import kafka.message.Message; 25 | import kafka.serializer.Encoder; 26 | 27 | public class BagheeraEncoder implements Encoder { 28 | 29 | /* (non-Javadoc) 30 | * @see kafka.serializer.Encoder#toMessage(java.lang.Object) 31 | */ 32 | @Override 33 | public Message toMessage(BagheeraMessage bmsg) { 34 | return new Message(bmsg.toByteArray()); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/sink/KeyValueSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import java.io.Closeable; 23 | import java.io.IOException; 24 | 25 | public interface KeyValueSink extends Closeable { 26 | 27 | public void store(String key, byte[] data) throws IOException; 28 | public void store(String key, byte[] data, long timestamp) throws IOException; 29 | public void delete(String key) throws IOException; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/sink/KeyValueSinkFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import java.io.Closeable; 23 | import java.io.IOException; 24 | import java.lang.reflect.Constructor; 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | 28 | import org.apache.log4j.Logger; 29 | 30 | public class KeyValueSinkFactory implements Closeable { 31 | 32 | private static final Logger LOG = Logger.getLogger(KeyValueSinkFactory.class); 33 | 34 | private static KeyValueSinkFactory INSTANCE; 35 | private Map sinkMap; 36 | private SinkConfiguration sinkConfiguration; 37 | private Class sinkClazz; 38 | 39 | /** 40 | * @param sinkConfiguration 41 | */ 42 | private KeyValueSinkFactory(Class sinkClazz, SinkConfiguration sinkConfiguration) { 43 | this.sinkMap = new HashMap(); 44 | this.sinkClazz = sinkClazz; 45 | this.sinkConfiguration = sinkConfiguration; 46 | } 47 | 48 | /** 49 | * @param sinkConfiguration 50 | * @return 51 | */ 52 | public static KeyValueSinkFactory getInstance(Class sinkClazz, SinkConfiguration sinkConfiguration) { 53 | if (INSTANCE == null) { 54 | INSTANCE = new KeyValueSinkFactory(sinkClazz, sinkConfiguration); 55 | } 56 | 57 | return INSTANCE; 58 | } 59 | 60 | /** 61 | * @param namespace 62 | * @return 63 | */ 64 | public KeyValueSink getSink(String namespace) { 65 | if (namespace == null) { 66 | throw new IllegalArgumentException("Namespace cannot be null"); 67 | } 68 | 69 | if (!sinkMap.containsKey(namespace)) { 70 | sinkConfiguration.setString("namespace", namespace); 71 | KeyValueSink sink; 72 | for (Constructor constructor : sinkClazz.getConstructors()) { 73 | Class[] paramTypes = constructor.getParameterTypes(); 74 | if (KeyValueSink.class.isAssignableFrom(sinkClazz) && 75 | paramTypes.length == 1 && paramTypes[0] == SinkConfiguration.class) { 76 | try { 77 | sink = (KeyValueSink)constructor.newInstance(sinkConfiguration); 78 | sinkMap.put(namespace, sink); 79 | } catch (Exception e) { 80 | LOG.error("Error constructing new instance of sink class for namespace: " + namespace, e); 81 | } 82 | } 83 | } 84 | } 85 | return sinkMap.get(namespace); 86 | } 87 | 88 | /* (non-Javadoc) 89 | * @see java.io.Closeable#close() 90 | */ 91 | public void close() throws IOException { 92 | for (Map.Entry entry : sinkMap.entrySet()) { 93 | try { 94 | LOG.info("Closing sink for namespace: " + entry.getKey()); 95 | entry.getValue().close(); 96 | } catch (IOException e) { 97 | LOG.info("Error closing sink for namespace: " + entry.getKey()); 98 | } 99 | } 100 | } 101 | 102 | } -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/sink/LoggerSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import java.io.IOException; 23 | 24 | import org.apache.log4j.Logger; 25 | 26 | public class LoggerSink implements Sink, KeyValueSink { 27 | 28 | private static final Logger LOG = Logger.getLogger(LoggerSink.class); 29 | 30 | private boolean logValues; 31 | 32 | public LoggerSink(SinkConfiguration sinkConfiguration) { 33 | this(sinkConfiguration.getBoolean("loggersink.logvalues", false)); 34 | } 35 | 36 | public LoggerSink(boolean logValues) { 37 | this.logValues = logValues; 38 | } 39 | 40 | @Override 41 | public void close() { 42 | LOG.info("Called close()"); 43 | } 44 | 45 | @Override 46 | public void store(byte[] data) throws IOException { 47 | LOG.info("Called store(data)"); 48 | LOG.info("store(d) data length:" + data.length); 49 | if (logValues) { 50 | LOG.info("store(d) data: " + new String(data, "UTF-8")); 51 | } 52 | } 53 | 54 | @Override 55 | public void store(String key, byte[] data) throws IOException { 56 | LOG.info("Called store(key,data)"); 57 | LOG.info("store(k,d) key length: " + key.length()); 58 | LOG.info("store(k,d) data length:" + data.length); 59 | if (logValues) { 60 | LOG.info("store(k,d) key: " + key); 61 | LOG.info("store(k,d) data: " + new String(data, "UTF-8")); 62 | } 63 | } 64 | 65 | @Override 66 | public void store(String key, byte[] data, long timestamp) throws IOException { 67 | LOG.info("Called store(key,data,timestamp)"); 68 | LOG.info("store(k,d,t) key length: " + key.length()); 69 | LOG.info("store(k,d,t) data length:" + data.length); 70 | LOG.info("store(k,d,t) timestamp: " + timestamp); 71 | if (logValues) { 72 | LOG.info("store(k,d,t) key: " + key); 73 | LOG.info("store(k,d,t) data: " + new String(data, "UTF-8")); 74 | } 75 | } 76 | 77 | @Override 78 | public void delete(String key) { 79 | LOG.info("Called delete(key)"); 80 | LOG.info("delete(k) key length: " + key.length()); 81 | if (logValues) { 82 | LOG.info("delete(k) key: " + key); 83 | } 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/sink/ReplaySink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import java.io.BufferedReader; 23 | import java.io.DataOutputStream; 24 | import java.io.IOException; 25 | import java.io.InputStream; 26 | import java.io.InputStreamReader; 27 | import java.net.HttpURLConnection; 28 | import java.net.URL; 29 | import java.util.UUID; 30 | 31 | import org.apache.log4j.Logger; 32 | 33 | // TODO: should we queue up deletes and combine them with "store" requests using the "X-Obsolete-Doc" header? 34 | // TODO: create placeholders for API version, partitions, etc. 35 | 36 | public class ReplaySink implements Sink, KeyValueSink { 37 | 38 | public static final String KEY_PLACEHOLDER = "%k"; 39 | 40 | private static final Logger LOG = Logger.getLogger(ReplaySink.class); 41 | 42 | protected final String destinationPattern; 43 | protected final String destPrefix; 44 | protected final String destSuffix; 45 | protected final boolean useSuffix; 46 | protected final boolean useKey; 47 | protected final boolean sample; 48 | protected final double sampleRate; 49 | protected final boolean copyKeys; 50 | protected final boolean replayDeletes; 51 | 52 | public ReplaySink(SinkConfiguration sinkConfiguration) { 53 | this(sinkConfiguration.getString("replaysink.dest"), 54 | sinkConfiguration.getString("replaysink.sample"), 55 | sinkConfiguration.getString("replaysink.keys"), 56 | sinkConfiguration.getString("replaysink.delete")); 57 | } 58 | 59 | public ReplaySink(String destinationPattern, String sampleRate, String copyKeys, String replayDeletes) { 60 | this.destinationPattern = destinationPattern; 61 | int keyPlaceholderLocation = destinationPattern.indexOf(KEY_PLACEHOLDER); 62 | boolean tmpUseKey = true; 63 | if (keyPlaceholderLocation == -1) { 64 | this.destPrefix = destinationPattern; 65 | this.destSuffix = ""; 66 | this.useSuffix = false; 67 | tmpUseKey = false; 68 | } else if (keyPlaceholderLocation == (destinationPattern.length() - 3)) { 69 | this.destPrefix = destinationPattern.substring(0, keyPlaceholderLocation); 70 | this.destSuffix = ""; 71 | this.useSuffix = false; 72 | } else { 73 | this.destPrefix = destinationPattern.substring(0, keyPlaceholderLocation); 74 | this.destSuffix = destinationPattern.substring(keyPlaceholderLocation + 2); 75 | this.useSuffix = true; 76 | } 77 | this.sampleRate = Double.parseDouble(sampleRate); 78 | this.copyKeys = Boolean.parseBoolean(copyKeys); 79 | this.replayDeletes = Boolean.parseBoolean(replayDeletes); 80 | if ("1".equals(sampleRate)) { 81 | sample = false; 82 | } else { 83 | sample = true; 84 | } 85 | 86 | this.useKey = tmpUseKey; 87 | } 88 | 89 | public String getDest(String key) { 90 | if (useKey && useSuffix) { 91 | return destPrefix + key + destSuffix; 92 | } else if (useKey) { 93 | return destPrefix + key; 94 | } else { 95 | return destinationPattern; 96 | } 97 | } 98 | 99 | @Override 100 | public void close() { 101 | // nothing to do. 102 | } 103 | 104 | @Override 105 | public void store(byte[] data) throws IOException { 106 | this.store("", data); 107 | } 108 | 109 | @Override 110 | public void store(String key, byte[] data) throws IOException { 111 | boolean go = true; 112 | if (this.sample) { 113 | go = (Math.random() < sampleRate); 114 | } 115 | 116 | if (go) { 117 | String newKey = key; 118 | if (!copyKeys) { 119 | // Assign new UUIDs 120 | newKey = UUID.randomUUID().toString(); 121 | } 122 | 123 | if (LOG.isDebugEnabled()) { 124 | LOG.debug(String.format("Attempting to replay key '%s' (using '%s' on dest)", key, newKey)); 125 | } 126 | 127 | // TODO: replay it. 128 | replay("POST", newKey, data); 129 | } else { 130 | LOG.debug("Record skipped due to sampling."); 131 | } 132 | } 133 | 134 | // Connect to the specified server and replay the given request 135 | private void replay(String method, String key, byte[] data) { 136 | URL url; 137 | HttpURLConnection connection = null; 138 | try { 139 | // Create connection 140 | url = new URL(getDest(key)); 141 | connection = (HttpURLConnection)url.openConnection(); 142 | connection.setRequestMethod(method); 143 | 144 | connection.setUseCaches (false); 145 | connection.setDoInput(true); 146 | connection.setDoOutput(true); 147 | 148 | // Send request (if need be) 149 | if (data != null && data.length > 0) { 150 | connection.setRequestProperty("Content-Length", String.valueOf(data.length)); 151 | DataOutputStream wr = new DataOutputStream(connection.getOutputStream()); 152 | wr.write(data); 153 | wr.flush(); 154 | wr.close(); 155 | } 156 | 157 | // Get Response 158 | // TODO: do we care about the response? 159 | InputStream is = connection.getInputStream(); 160 | BufferedReader rd = new BufferedReader(new InputStreamReader(is)); 161 | String line; 162 | StringBuffer response = new StringBuffer(); 163 | while((line = rd.readLine()) != null) { 164 | response.append(line); 165 | response.append('\n'); 166 | } 167 | rd.close(); 168 | LOG.debug(response.toString()); 169 | } catch (IOException e) { 170 | LOG.error("Error replaying request", e); 171 | } finally { 172 | if(connection != null) { 173 | connection.disconnect(); 174 | } 175 | } 176 | } 177 | 178 | @Override 179 | public void store(String key, byte[] data, long timestamp) throws IOException { 180 | store(key, data); 181 | } 182 | 183 | @Override 184 | public void delete(String key) { 185 | // Note that this breaks the "copyKeys" contract - it would be quite 186 | // useless to generate random delete keys. 187 | 188 | // Whether or not we process deletes is controlled by a config setting. 189 | if (replayDeletes) { 190 | replay("DELETE", key, null); 191 | } 192 | } 193 | 194 | } 195 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/sink/SequenceFileSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import java.io.IOException; 23 | import java.text.SimpleDateFormat; 24 | import java.util.Calendar; 25 | import java.util.Date; 26 | import java.util.UUID; 27 | import java.util.concurrent.Semaphore; 28 | import java.util.concurrent.TimeUnit; 29 | import java.util.concurrent.atomic.AtomicLong; 30 | 31 | import org.apache.hadoop.conf.Configuration; 32 | import org.apache.hadoop.fs.FileSystem; 33 | import org.apache.hadoop.fs.Path; 34 | import org.apache.hadoop.io.BytesWritable; 35 | import org.apache.hadoop.io.SequenceFile; 36 | import org.apache.hadoop.io.SequenceFile.CompressionType; 37 | import org.apache.hadoop.io.Text; 38 | import org.apache.log4j.Logger; 39 | 40 | import com.yammer.metrics.Metrics; 41 | import com.yammer.metrics.core.Meter; 42 | import com.yammer.metrics.core.MetricName; 43 | 44 | import com.fasterxml.jackson.core.JsonParseException; 45 | import com.fasterxml.jackson.databind.JsonMappingException; 46 | import com.fasterxml.jackson.databind.ObjectMapper; 47 | import com.fasterxml.jackson.databind.node.ObjectNode; 48 | 49 | 50 | public class SequenceFileSink implements KeyValueSink { 51 | private static final Logger LOG = Logger.getLogger(SequenceFileSink.class); 52 | private ObjectMapper jsonMapper = new ObjectMapper(); 53 | 54 | protected static final long DAY_IN_MILLIS = 86400000L; 55 | 56 | // HDFS related member vars 57 | protected final Semaphore lock = new Semaphore(1, true); 58 | protected final Configuration conf; 59 | protected final FileSystem hdfs; 60 | protected SequenceFile.Writer writer; 61 | protected Path baseDir; 62 | protected Path outputPath; 63 | protected boolean useBytesValue; 64 | protected boolean addTimestamp; 65 | protected long nextRolloverMillis = 0L; 66 | protected AtomicLong bytesWritten = new AtomicLong(); 67 | protected long maxFileSize = 0L; 68 | protected final SimpleDateFormat sdf; 69 | 70 | protected Meter stored; 71 | 72 | public static final String SINK_TIMESTAMP_FIELD = "BAGHEERA_TS"; 73 | 74 | public SequenceFileSink(SinkConfiguration config) throws IOException { 75 | this(config.getString("namespace"), 76 | config.getString("hdfssink.hdfs.basedir.path", "/bagheera"), 77 | config.getString("hdfssink.hdfs.date.format", "yyyy-MM-dd"), 78 | config.getLong("hdfssink.hdfs.max.filesize", 536870912), 79 | config.getBoolean("hdfssink.hdfs.usebytes", false), 80 | config.getBoolean("hdfssink.hdfs.addtimestamp",false)); 81 | } 82 | 83 | public SequenceFileSink(String namespace, String baseDirPath, String dateFormat, long maxFileSize, 84 | boolean useBytesValue,boolean addTimestamp) throws IOException { 85 | LOG.info("Initializing writer for namespace: " + namespace); 86 | conf = new Configuration(); 87 | conf.setBoolean("fs.automatic.close", false); 88 | hdfs = FileSystem.newInstance(conf); 89 | this.useBytesValue = useBytesValue; 90 | this.maxFileSize = maxFileSize; 91 | this.addTimestamp = addTimestamp; 92 | sdf = new SimpleDateFormat(dateFormat); 93 | if (!baseDirPath.endsWith(Path.SEPARATOR)) { 94 | baseDir = new Path(baseDirPath + Path.SEPARATOR + namespace + Path.SEPARATOR + 95 | sdf.format(new Date(System.currentTimeMillis()))); 96 | } else { 97 | baseDir = new Path(baseDirPath + namespace + Path.SEPARATOR + 98 | sdf.format(new Date(System.currentTimeMillis()))); 99 | } 100 | initWriter(); 101 | stored = Metrics.newMeter(new MetricName("bagheera", "sink.hdfs.", namespace + ".stored"), "messages", 102 | TimeUnit.SECONDS); 103 | } 104 | 105 | private void initWriter() throws IOException { 106 | if (LOG.isDebugEnabled()) { 107 | LOG.debug("Thread " + Thread.currentThread().getId() + " - initWriter() called"); 108 | } 109 | 110 | if (!hdfs.exists(baseDir)) { 111 | hdfs.mkdirs(baseDir); 112 | } 113 | 114 | outputPath = new Path(baseDir, new Path(UUID.randomUUID().toString())); 115 | LOG.info("Opening file handle to: " + outputPath.toString()); 116 | 117 | if (useBytesValue) { 118 | writer = SequenceFile.createWriter(hdfs, conf, outputPath, Text.class, BytesWritable.class, 119 | CompressionType.BLOCK); 120 | } else { 121 | writer = SequenceFile.createWriter(hdfs, conf, outputPath, Text.class, Text.class, CompressionType.BLOCK); 122 | } 123 | 124 | // Get time in millis at a day resolution 125 | Calendar prev = Calendar.getInstance(); 126 | prev.set(Calendar.HOUR_OF_DAY, 0); 127 | prev.set(Calendar.MINUTE, 0); 128 | prev.set(Calendar.SECOND, 0); 129 | prev.set(Calendar.MILLISECOND, 0); 130 | nextRolloverMillis = prev.getTimeInMillis() + DAY_IN_MILLIS; 131 | } 132 | 133 | private void checkRollover() throws IOException { 134 | boolean getNewFile = false; 135 | long now = System.currentTimeMillis(); 136 | if (maxFileSize != 0 && bytesWritten.get() >= maxFileSize) { 137 | getNewFile = true; 138 | } else if (now > nextRolloverMillis) { 139 | getNewFile = true; 140 | baseDir = new Path(baseDir.getParent(), new Path(sdf.format(new Date(now)))); 141 | } 142 | 143 | if (writer == null || getNewFile) { 144 | closeWriter(); 145 | initWriter(); 146 | } 147 | } 148 | 149 | private void closeWriter() throws IOException { 150 | if (writer != null) { 151 | writer.close(); 152 | writer = null; 153 | } 154 | bytesWritten.set(0); 155 | } 156 | 157 | @Override 158 | public void close() { 159 | try { 160 | lock.acquire(); 161 | LOG.info("Closing file handle to: " + outputPath.toString()); 162 | try { 163 | closeWriter(); 164 | } catch (IOException e) { 165 | LOG.error("Error closing writer", e); 166 | } 167 | 168 | if (hdfs != null) { 169 | try { 170 | LOG.info("fs.automatic.close = " + hdfs.getConf().get("fs.automatic.close")); 171 | hdfs.close(); 172 | } catch (IOException e) { 173 | LOG.error("Error closing HDFS handle", e); 174 | } 175 | } 176 | } catch (InterruptedException ex) { 177 | LOG.error("Interrupted while closing HDFS handle", ex); 178 | } finally { 179 | lock.release(); 180 | } 181 | } 182 | 183 | @Override 184 | public void store(String key, byte[] data) { 185 | try { 186 | lock.acquire(); 187 | checkRollover(); 188 | if (useBytesValue) { 189 | writer.append(new Text(key), new BytesWritable(data)); 190 | } else { 191 | writer.append(new Text(key), new Text(data)); 192 | } 193 | stored.mark(); 194 | bytesWritten.getAndAdd(key.length() + data.length); 195 | } catch (IOException e) { 196 | LOG.error("IOException while writing key/value pair", e); 197 | throw new RuntimeException(e); 198 | } catch (InterruptedException e) { 199 | LOG.error("Interrupted while writing key/value pair", e); 200 | } finally { 201 | lock.release(); 202 | } 203 | } 204 | 205 | @Override 206 | public void store(String key, byte[] data, long timestamp) throws IOException { 207 | try { 208 | lock.acquire(); 209 | checkRollover(); 210 | if(addTimestamp) { 211 | data = addTimestampToJson(data,timestamp); 212 | } 213 | if (useBytesValue) { 214 | writer.append(new Text(key), new BytesWritable(data)); 215 | } else { 216 | writer.append(new Text(key), new Text(data)); 217 | } 218 | stored.mark(); 219 | bytesWritten.getAndAdd(key.length() + data.length); 220 | } catch (IOException e) { 221 | LOG.error("IOException while writing key/value pair", e); 222 | throw new RuntimeException(e); 223 | } catch (InterruptedException e) { 224 | LOG.error("Interrupted while writing key/value pair", e); 225 | } finally { 226 | lock.release(); 227 | } 228 | } 229 | 230 | public byte[] addTimestampToJson(byte[] data, long timestamp) throws IOException { 231 | // TODO: add metrics/counters for failures 232 | try { 233 | ObjectNode document = jsonMapper.readValue(data, ObjectNode.class); 234 | document.put(SINK_TIMESTAMP_FIELD, timestamp); 235 | return(jsonMapper.writeValueAsBytes(document)); 236 | } catch (JsonParseException e) { 237 | LOG.error("Invalid JSON", e); 238 | LOG.debug(data); 239 | } catch (JsonMappingException e) { 240 | LOG.error("Invalid JSON", e); 241 | LOG.debug(data); 242 | } 243 | 244 | throw new IOException("Invalid JSON"); 245 | } 246 | 247 | @Override 248 | public void delete(String key) { 249 | // TODO: Throw error or just ignore? 250 | // NOOP 251 | } 252 | 253 | } 254 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/sink/Sink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import java.io.IOException; 23 | 24 | public interface Sink { 25 | 26 | public void store(byte[] data) throws IOException; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/sink/SinkConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import java.util.Properties; 23 | 24 | public class SinkConfiguration { 25 | 26 | private Properties props; 27 | 28 | public SinkConfiguration() { 29 | this(new Properties()); 30 | } 31 | 32 | public SinkConfiguration(Properties props) { 33 | this.props = props; 34 | } 35 | 36 | public String getString(String key) { 37 | return props.getProperty(key); 38 | } 39 | 40 | public String getString(String key, String def) { 41 | return props.containsKey(key) ? props.getProperty(key) : def; 42 | } 43 | 44 | public void setString(String key, String value) { 45 | if (key == null || value == null) { 46 | return; 47 | } 48 | props.setProperty(key, value); 49 | } 50 | 51 | public int getInt(String key) { 52 | return Integer.parseInt(props.getProperty(key)); 53 | } 54 | 55 | public int getInt(String key, int def) { 56 | return props.containsKey(key) ? Integer.parseInt(props.getProperty(key)) : def; 57 | } 58 | 59 | public void setInt(String key, int value) { 60 | props.setProperty(key, String.valueOf(value)); 61 | } 62 | 63 | public long getLong(String key) { 64 | return Long.parseLong(props.getProperty(key)); 65 | } 66 | 67 | public long getLong(String key, long def) { 68 | return props.containsKey(key) ? Long.parseLong(props.getProperty(key)) : def; 69 | } 70 | 71 | public void setLong(String key, long value) { 72 | props.setProperty(key, String.valueOf(value)); 73 | } 74 | 75 | public boolean getBoolean(String key) { 76 | return Boolean.parseBoolean(props.getProperty(key)); 77 | } 78 | 79 | public boolean getBoolean(String key, boolean def) { 80 | return props.containsKey(key) ? Boolean.parseBoolean(props.getProperty(key)) : def; 81 | } 82 | 83 | public void setBoolean(String key, boolean value) { 84 | props.setProperty(key, String.valueOf(value)); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/util/HttpUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.util; 21 | 22 | import java.net.InetAddress; 23 | import java.net.UnknownHostException; 24 | 25 | import org.jboss.netty.handler.codec.http.HttpRequest; 26 | 27 | public class HttpUtil { 28 | 29 | // header fields 30 | public static final String USER_AGENT = "User-Agent"; 31 | public static final String X_FORWARDED_FOR = "X-Forwarded-For"; 32 | 33 | public static String getUserAgent(HttpRequest request) { 34 | return request.getHeader(USER_AGENT); 35 | } 36 | 37 | private static String getForwardedAddr(String forwardedAddr) { 38 | if (forwardedAddr == null) { 39 | return null; 40 | } 41 | 42 | String clientAddr = null; 43 | int idx = forwardedAddr.indexOf(","); 44 | if (idx > 0) { 45 | clientAddr = forwardedAddr.substring(0, idx); 46 | } else { 47 | clientAddr = forwardedAddr; 48 | } 49 | 50 | return clientAddr; 51 | } 52 | 53 | public static String getRemoteAddr(HttpRequest request, String channelRemoteAddr) { 54 | String forwardedAddr = getForwardedAddr(request.getHeader(X_FORWARDED_FOR)); 55 | return forwardedAddr == null ? channelRemoteAddr : forwardedAddr; 56 | } 57 | 58 | public static byte[] getRemoteAddr(HttpRequest request, InetAddress channelRemoteAddr) { 59 | String forwardedAddr = getForwardedAddr(request.getHeader(X_FORWARDED_FOR)); 60 | byte[] addrBytes = null; 61 | if (forwardedAddr != null) { 62 | InetAddress addr; 63 | try { 64 | addr = InetAddress.getByName(forwardedAddr); 65 | addrBytes = addr.getAddress(); 66 | } catch (UnknownHostException e) { 67 | // going to swallow this for now 68 | } 69 | } 70 | // if we're still null here then use the remote addr bytes 71 | if (addrBytes == null) { 72 | addrBytes = channelRemoteAddr.getAddress(); 73 | } 74 | 75 | return addrBytes; 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/util/IdUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.util; 21 | 22 | import java.io.IOException; 23 | import java.nio.ByteBuffer; 24 | import java.text.SimpleDateFormat; 25 | import java.util.Calendar; 26 | import java.util.Date; 27 | 28 | import org.apache.commons.lang.StringUtils; 29 | 30 | /** 31 | * Utility class for id generation and bucketing 32 | */ 33 | public class IdUtil { 34 | 35 | public static final SimpleDateFormat SDF = new SimpleDateFormat("yyyyMMdd"); 36 | 37 | /** 38 | * @param id 39 | * @param timestamp 40 | * @return 41 | * @throws IOException 42 | */ 43 | public static byte[] bucketizeId(String id, long timestamp) throws IOException { 44 | if (id == null) { 45 | throw new IllegalArgumentException("id cannot be null"); 46 | } 47 | 48 | return nonRandByteBucketizeId(id, new Date(timestamp)); 49 | } 50 | 51 | /** 52 | * Takes a given id and prefixes it with a byte character and the date in a non-random fashion 53 | * This method expects id to be something like a UUID consisting only of hex characters. If id 54 | * is something else this will produce unpredictable results. 55 | * @param id 56 | * @param d 57 | * @return 58 | * @throws IOException 59 | */ 60 | public static byte[] nonRandByteBucketizeId(String id, Date d) throws IOException { 61 | if (StringUtils.isBlank(id)) { 62 | throw new IllegalArgumentException("id cannot be null or empty"); 63 | } 64 | if (d == null) { 65 | throw new IllegalArgumentException("date cannot be null"); 66 | } 67 | 68 | // bucket byte + SDF bytes + id bytes 69 | ByteBuffer buf = ByteBuffer.allocate(9 + id.length()); 70 | int bucket = 0; 71 | if (id.length() >= 2) { 72 | // Munge two hex characters into the range of a single byte 73 | bucket = Integer.parseInt(id.substring(0, 2), 16) - 128; 74 | } else { 75 | bucket = Integer.parseInt(id, 16) - 128; 76 | } 77 | buf.put((byte)bucket); 78 | buf.put(SDF.format(d).getBytes()); 79 | buf.put(id.getBytes()); 80 | 81 | return buf.array(); 82 | } 83 | 84 | /** 85 | * Takes a given id and gives you an hbase shell compatible string that you can 86 | * use for get command. 87 | * @param id 88 | * @param d 89 | * @return 90 | * @throws IOException 91 | */ 92 | public static String hbaseShellId(String id, Date d) throws IOException { 93 | byte[] idBytes = IdUtil.nonRandByteBucketizeId(id, d); 94 | StringBuilder sb = new StringBuilder("\"\\x"); 95 | sb.append(String.format("%02x", idBytes[0])); 96 | sb.append((new String(idBytes)).substring(1)); 97 | sb.append("\""); 98 | return sb.toString(); 99 | } 100 | 101 | public static String hbaseShellId(byte[] idBytes) throws IOException { 102 | StringBuilder sb = new StringBuilder("\"\\x"); 103 | sb.append(String.format("%02x", idBytes[0])); 104 | sb.append((new String(idBytes)).substring(1)); 105 | sb.append("\""); 106 | return sb.toString(); 107 | } 108 | 109 | public static void main(String[] args) throws IOException { 110 | String id = "06b87727-2946-4b1e-b654-8bafb60bb995"; 111 | long timestamp = 1360493115891L; 112 | Calendar cal = Calendar.getInstance(); 113 | //cal.set(2013, Calendar.FEBRUARY, 10); 114 | cal.setTimeInMillis(timestamp); 115 | System.out.println(IdUtil.hbaseShellId(id, cal.getTime())); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/util/ShutdownHook.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.util; 21 | 22 | import java.io.Closeable; 23 | import java.io.IOException; 24 | import java.util.LinkedList; 25 | import java.util.List; 26 | 27 | import org.apache.log4j.Logger; 28 | 29 | public class ShutdownHook { 30 | 31 | private static final Logger LOG = Logger.getLogger(ShutdownHook.class); 32 | 33 | private static ShutdownHook INSTANCE; 34 | private List closeables = new LinkedList(); 35 | 36 | private ShutdownHook() { 37 | Runtime.getRuntime().addShutdownHook(new Thread() { 38 | public void run() { 39 | for (Closeable c : closeables) { 40 | try { 41 | c.close(); 42 | } catch (IOException e) { 43 | LOG.error("Error while closing", e); 44 | } 45 | } 46 | } 47 | }); 48 | } 49 | 50 | public static ShutdownHook getInstance() { 51 | if (INSTANCE == null) { 52 | INSTANCE = new ShutdownHook(); 53 | } 54 | 55 | return INSTANCE; 56 | } 57 | 58 | public void addFirst(Closeable c) { 59 | if (c != null) { 60 | closeables.add(0, c); 61 | } 62 | } 63 | 64 | public void addLast(Closeable c) { 65 | if (c != null) { 66 | closeables.add(c); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/util/WildcardProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.util; 21 | 22 | import java.io.BufferedReader; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.io.InputStreamReader; 26 | import java.util.HashSet; 27 | import java.util.Properties; 28 | import java.util.Set; 29 | 30 | public class WildcardProperties extends Properties { 31 | 32 | private static final long serialVersionUID = 8726833938438520686L; 33 | 34 | private static final String WILDCARD = "*"; 35 | private static final char WILDCARD_CHAR = '*'; 36 | private Set wildcardKeys = new HashSet(); 37 | 38 | public void load(InputStream is) throws IOException { 39 | BufferedReader reader = new BufferedReader(new InputStreamReader(is)); 40 | String line = null; 41 | while ((line = reader.readLine()) != null) { 42 | if (!line.startsWith("#") && !line.trim().isEmpty()) { 43 | String[] splits = line.split("="); 44 | String k = splits[0]; 45 | if (k.contains(WILDCARD)) { 46 | int starIdx = k.indexOf(WILDCARD_CHAR); 47 | if (starIdx >= -1) { 48 | wildcardKeys.add(k); 49 | } 50 | } 51 | put(k, splits[1]); 52 | } 53 | } 54 | } 55 | 56 | /** 57 | * Get a property if it exists. If not check for wildcard property matches. 58 | * 59 | * @param name 60 | * @return null if property does not exist 61 | * @return String if property exists 62 | */ 63 | public String getWildcardProperty(String name) { 64 | String v = null; 65 | if (containsKey(name)) { 66 | v = getProperty(name); 67 | } else { 68 | for (String pk : wildcardKeys) { 69 | int starIdx = pk.indexOf(WILDCARD_CHAR); 70 | if (name.startsWith(pk.substring(0, starIdx)) && 71 | name.endsWith(pk.substring(starIdx+1))) { 72 | v = getProperty(pk); 73 | setProperty(name, v); 74 | break; 75 | } 76 | } 77 | } 78 | 79 | return v; 80 | } 81 | 82 | /** 83 | * @param name 84 | * @param defaultValue 85 | * @return 86 | */ 87 | public String getWildcardProperty(String name, String defaultValue) { 88 | String v = getWildcardProperty(name); 89 | return v == null ? defaultValue : v; 90 | } 91 | 92 | /** 93 | * @param name 94 | * @param defaultValue 95 | * @return 96 | */ 97 | public boolean getWildcardProperty(String name, boolean defaultValue) { 98 | String v = getWildcardProperty(name); 99 | return v == null ? defaultValue : Boolean.parseBoolean(v); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/com/mozilla/bagheera/validation/Validator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.validation; 21 | 22 | import java.io.IOException; 23 | import java.util.UUID; 24 | import java.util.regex.Pattern; 25 | 26 | import org.apache.log4j.Logger; 27 | 28 | import com.fasterxml.jackson.core.JsonFactory; 29 | import com.fasterxml.jackson.core.JsonParseException; 30 | import com.fasterxml.jackson.core.JsonParser; 31 | 32 | public class Validator { 33 | 34 | private static final Logger LOG = Logger.getLogger(Validator.class); 35 | 36 | private final Pattern validNamespacePattern; 37 | private final JsonFactory jsonFactory; 38 | 39 | public Validator(final String[] validNamespaces) { 40 | if (validNamespaces == null || validNamespaces.length == 0) { 41 | throw new IllegalArgumentException("No valid namespace was specified"); 42 | } 43 | StringBuilder nsPatternBuilder = new StringBuilder("("); 44 | int i=0, size=validNamespaces.length; 45 | for (String name : validNamespaces) { 46 | nsPatternBuilder.append(name.replaceAll("\\*", ".+")); 47 | if ((i+1) < size) { 48 | nsPatternBuilder.append("|"); 49 | } 50 | i++; 51 | } 52 | nsPatternBuilder.append(")"); 53 | LOG.info("Namespace pattern: " + nsPatternBuilder.toString()); 54 | validNamespacePattern = Pattern.compile(nsPatternBuilder.toString()); 55 | 56 | jsonFactory = new JsonFactory(); 57 | } 58 | 59 | public boolean isValidNamespace(String ns) { 60 | return validNamespacePattern.matcher(ns).find(); 61 | } 62 | 63 | public boolean isValidJson(String json) { 64 | boolean isValid = false; 65 | JsonParser parser = null; 66 | try { 67 | parser = jsonFactory.createJsonParser(json); 68 | while (parser.nextToken() != null) { 69 | // noop 70 | } 71 | isValid = true; 72 | } catch (JsonParseException ex) { 73 | LOG.error("JSON parse error"); 74 | } catch (IOException e) { 75 | LOG.error("JSON IO error"); 76 | } finally { 77 | if (parser != null) { 78 | try { 79 | parser.close(); 80 | } catch (IOException e) { 81 | LOG.error("Error closing JSON parser", e); 82 | } 83 | } 84 | } 85 | 86 | return isValid; 87 | } 88 | 89 | public boolean isValidId(String id) { 90 | boolean isValid = false; 91 | try { 92 | UUID.fromString(id); 93 | isValid = true; 94 | } catch (IllegalArgumentException e) { 95 | LOG.error("Invalid ID: " + id); 96 | } 97 | 98 | return isValid; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/proto/bagheera_message.proto: -------------------------------------------------------------------------------- 1 | package bagheera_data_definition; 2 | 3 | option java_package = "com.mozilla.bagheera"; 4 | option java_outer_classname = "BagheeraProto"; 5 | option optimize_for = SPEED; 6 | 7 | message BagheeraMessage { 8 | 9 | // 1 is reserved in case we ever want a different id scheme later on 10 | optional string namespace = 2; 11 | optional string id = 3; 12 | optional bytes ip_addr = 4; 13 | optional bytes payload = 5; 14 | optional sint64 timestamp = 6; 15 | enum Operation { 16 | CREATE_UPDATE = 0; 17 | DELETE = 1; 18 | } 19 | optional Operation operation = 7 [default = CREATE_UPDATE]; 20 | optional string api_version = 8; 21 | repeated string partition = 9; 22 | } -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/cli/OptionFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.cli; 21 | 22 | import static org.junit.Assert.*; 23 | 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | 27 | public class OptionFactoryTest { 28 | 29 | private OptionFactory optionFactory; 30 | 31 | @Before 32 | public void setup() { 33 | optionFactory = OptionFactory.getInstance(); 34 | } 35 | 36 | @Test 37 | public void testCreate() { 38 | Option opt = optionFactory.create("i", "input", true, "Input description"); 39 | assertEquals("i", opt.getOpt()); 40 | assertEquals("input", opt.getLongOpt()); 41 | assertTrue(opt.hasArg()); 42 | assertEquals("Input description", opt.getDescription()); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/cli/OptionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.cli; 21 | 22 | import static org.junit.Assert.*; 23 | 24 | import org.junit.Test; 25 | 26 | public class OptionTest { 27 | 28 | @Test 29 | public void testRequired() { 30 | Option opt = new Option("i", "input", true, "Input description").required(); 31 | assertTrue(opt.isRequired()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/http/AccessFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import static org.easymock.EasyMock.createMock; 23 | import static org.easymock.EasyMock.expect; 24 | import static org.easymock.EasyMock.replay; 25 | import static org.jboss.netty.handler.codec.http.HttpMethod.DELETE; 26 | import static org.jboss.netty.handler.codec.http.HttpMethod.GET; 27 | import static org.jboss.netty.handler.codec.http.HttpMethod.POST; 28 | import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; 29 | import static org.junit.Assert.assertTrue; 30 | 31 | import java.io.ByteArrayInputStream; 32 | import java.io.IOException; 33 | import java.io.InputStream; 34 | import java.net.InetSocketAddress; 35 | import java.util.UUID; 36 | 37 | import org.jboss.netty.channel.Channel; 38 | import org.jboss.netty.channel.ChannelHandlerContext; 39 | import org.jboss.netty.channel.DefaultChannelFuture; 40 | import org.jboss.netty.channel.FakeChannelHandlerContext; 41 | import org.jboss.netty.channel.MessageEvent; 42 | import org.jboss.netty.channel.UpstreamMessageEvent; 43 | import org.jboss.netty.handler.codec.http.HttpMethod; 44 | import org.jboss.netty.handler.codec.http.HttpVersion; 45 | import org.jboss.netty.handler.execution.ExecutionHandler; 46 | import org.jboss.netty.handler.execution.OrderedMemoryAwareThreadPoolExecutor; 47 | import org.junit.Before; 48 | import org.junit.Test; 49 | 50 | import com.mozilla.bagheera.util.WildcardProperties; 51 | import com.mozilla.bagheera.validation.Validator; 52 | 53 | public class AccessFilterTest { 54 | 55 | private ChannelHandlerContext ctx; 56 | private InetSocketAddress remoteAddr; 57 | private AccessFilter filter; 58 | 59 | @Before 60 | public void setup() throws IOException { 61 | String[] namespaces = new String[] { "foo_*", "bar" }; 62 | WildcardProperties props = new WildcardProperties(); 63 | String propsFileStr = "foo_*.allow.delete.access=false\n" + 64 | "foo_*.id.validation=true\n" + 65 | "bar.allow.delete.access=true\n" + 66 | "bar.id.validation=false"; 67 | InputStream is = new ByteArrayInputStream(propsFileStr.getBytes("UTF-8")); 68 | props.load(is); 69 | filter = new AccessFilter(new Validator(namespaces), props); 70 | 71 | remoteAddr = InetSocketAddress.createUnresolved("192.168.1.1", 51723); 72 | 73 | Channel channel = createMock(Channel.class); 74 | expect(channel.getCloseFuture()).andReturn(new DefaultChannelFuture(channel, false)); 75 | expect(channel.getRemoteAddress()).andReturn(remoteAddr); 76 | 77 | OrderedMemoryAwareThreadPoolExecutor executor = new OrderedMemoryAwareThreadPoolExecutor(10, 0L, 0L); 78 | final ExecutionHandler handler = new ExecutionHandler(executor, true, true); 79 | 80 | ctx = new FakeChannelHandlerContext(channel, handler); 81 | } 82 | 83 | private MessageEvent createMockEvent(Channel channel, HttpVersion protocolVersion, HttpMethod method, String uri) { 84 | MessageEvent event = createMock(UpstreamMessageEvent.class); 85 | expect(event.getChannel()).andReturn(channel).anyTimes(); 86 | expect(event.getFuture()).andReturn(new DefaultChannelFuture(channel,false)).anyTimes(); 87 | expect(event.getRemoteAddress()).andReturn(remoteAddr); 88 | expect(event.getMessage()).andReturn(new BagheeraHttpRequest(protocolVersion, method, uri)); 89 | replay(channel, event); 90 | 91 | return event; 92 | } 93 | 94 | @Test 95 | public void testInvalidNamespace() throws Exception { 96 | boolean success = false; 97 | try { 98 | filter.messageReceived(ctx, createMockEvent(ctx.getChannel(), HTTP_1_1, POST, "/bad")); 99 | } catch (InvalidPathException e) { 100 | success = true; 101 | } 102 | assertTrue(success); 103 | } 104 | 105 | @Test 106 | public void testInvalidId() throws Exception { 107 | boolean success = false; 108 | try { 109 | filter.messageReceived(ctx, createMockEvent(ctx.getChannel(), HTTP_1_1, POST, "/submit/foo_blah/fakeid")); 110 | } catch (InvalidPathException e) { 111 | success = true; 112 | } 113 | assertTrue(success); 114 | } 115 | 116 | @Test 117 | public void testGetAccessDenied() throws Exception { 118 | boolean success = false; 119 | try { 120 | filter.messageReceived(ctx, createMockEvent(ctx.getChannel(), HTTP_1_1, GET, "/submit/foo_blah/" + UUID.randomUUID().toString())); 121 | } catch (HttpSecurityException e) { 122 | success = true; 123 | } 124 | assertTrue(success); 125 | } 126 | 127 | @Test 128 | public void testDeleteAccessGranted() throws Exception { 129 | boolean success = false; 130 | try { 131 | filter.messageReceived(ctx, createMockEvent(ctx.getChannel(), HTTP_1_1, DELETE, "/submit/bar/" + UUID.randomUUID().toString())); 132 | success = true; 133 | } catch (Exception e) { 134 | } 135 | assertTrue(success); 136 | } 137 | 138 | @Test 139 | public void testDeleteAccessDenied() throws Exception { 140 | boolean success = false; 141 | try { 142 | filter.messageReceived(ctx, createMockEvent(ctx.getChannel(), HTTP_1_1, DELETE, "/submit/foo_blah/" + UUID.randomUUID().toString())); 143 | } catch (HttpSecurityException e) { 144 | success = true; 145 | } 146 | assertTrue(success); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/http/BagheeraTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import static org.junit.Assert.assertEquals; 23 | import static org.junit.Assert.assertNotSame; 24 | 25 | import java.io.IOException; 26 | import java.util.Date; 27 | import java.util.LinkedList; 28 | import java.util.Properties; 29 | import java.util.UUID; 30 | 31 | import org.junit.After; 32 | import org.junit.Before; 33 | import org.junit.Test; 34 | 35 | import com.mozilla.bagheera.BagheeraProto.BagheeraMessage; 36 | import com.mozilla.bagheera.http.Bagheera.BagheeraServerState; 37 | import com.mozilla.bagheera.metrics.MetricsManager; 38 | import com.mozilla.bagheera.producer.Producer; 39 | import com.mozilla.bagheera.sink.ReplaySink; 40 | import com.mozilla.bagheera.util.WildcardProperties; 41 | 42 | 43 | public class BagheeraTest { 44 | private static final int BAGHEERA_PORT = 8999; 45 | private static final String TEST_NAMESPACE = "test"; 46 | BagheeraServerState state; 47 | SimpleProducer producer = new SimpleProducer(); 48 | WildcardProperties props = new WildcardProperties(); 49 | Properties metricsProps = new Properties(); 50 | MetricsManager manager = new MetricsManager(metricsProps, "foo."); 51 | 52 | String json = "{\"test\":\"bagheera\"}"; 53 | String key = UUID.randomUUID().toString(); 54 | long timestamp = new Date().getTime(); 55 | 56 | @Before 57 | public void setup() throws Exception { 58 | props.put("valid.namespaces", TEST_NAMESPACE); 59 | props.put(TEST_NAMESPACE + ".allow.delete.access", "true"); 60 | 61 | state = Bagheera.startServer(BAGHEERA_PORT, false, props, producer, 62 | Bagheera.getChannelFactory(), Bagheera.class.getName(), manager); 63 | } 64 | 65 | @Test 66 | public void testBasicMessage() throws IOException, InterruptedException { 67 | // Use a ReplaySink to send messages 68 | String destPattern = String.format("http://localhost:%d/%s/%s/%s", BAGHEERA_PORT, SubmissionHandler.ENDPOINT_SUBMIT, TEST_NAMESPACE, ReplaySink.KEY_PLACEHOLDER); 69 | ReplaySink sink = new ReplaySink(destPattern, "1", "true", "true"); 70 | sink.store(key, json.getBytes(), timestamp); 71 | 72 | assertEquals(1, producer.queueSize()); 73 | BagheeraMessage message = producer.getQueue().poll(); 74 | String payload = message.getPayload().toStringUtf8(); 75 | assertEquals(json, payload); 76 | 77 | // ReplaySink doesn't preserve timestamps. 78 | assertNotSame(timestamp, message.getTimestamp()); 79 | 80 | assertEquals(key, message.getId()); 81 | assertEquals(BagheeraMessage.Operation.CREATE_UPDATE, message.getOperation()); 82 | assertEquals("", message.getApiVersion()); 83 | assertEquals(0, message.getPartitionCount()); 84 | } 85 | 86 | @Test 87 | public void testMessageWithPartitions() throws IOException, InterruptedException { 88 | // Use a ReplaySink to send messages 89 | String destPattern = String.format("http://localhost:%d/%s/%s/%s/partition1/partition2", BAGHEERA_PORT, SubmissionHandler.ENDPOINT_SUBMIT, TEST_NAMESPACE, ReplaySink.KEY_PLACEHOLDER); 90 | ReplaySink sink = new ReplaySink(destPattern, "1", "true", "true"); 91 | sink.store(key, json.getBytes(), timestamp); 92 | 93 | assertEquals(1, producer.queueSize()); 94 | BagheeraMessage message = producer.getQueue().poll(); 95 | String payload = message.getPayload().toStringUtf8(); 96 | assertEquals(json, payload); 97 | 98 | // ReplaySink doesn't preserve timestamps. 99 | assertNotSame(timestamp, message.getTimestamp()); 100 | 101 | assertEquals(key, message.getId()); 102 | assertEquals(BagheeraMessage.Operation.CREATE_UPDATE, message.getOperation()); 103 | 104 | // Ensure that partition information comes through. 105 | assertEquals(2, message.getPartitionCount()); 106 | assertEquals("partition1", message.getPartition(0)); 107 | assertEquals("partition2", message.getPartition(1)); 108 | 109 | } 110 | 111 | @Test 112 | public void testDeleteWithPartitions() throws IOException, InterruptedException { 113 | // Use a ReplaySink to send messages 114 | String destPattern = String.format("http://localhost:%d/%s/%s/%s/partition1/partition2", BAGHEERA_PORT, SubmissionHandler.ENDPOINT_SUBMIT, TEST_NAMESPACE, ReplaySink.KEY_PLACEHOLDER); 115 | ReplaySink sink = new ReplaySink(destPattern, "1", "true", "true"); 116 | sink.delete(key); 117 | 118 | assertEquals(1, producer.queueSize()); 119 | BagheeraMessage message = producer.getQueue().poll(); 120 | String payload = message.getPayload().toStringUtf8(); 121 | assertEquals("", payload); 122 | 123 | // ReplaySink doesn't preserve timestamps. 124 | assertNotSame(timestamp, message.getTimestamp()); 125 | 126 | assertEquals(key, message.getId()); 127 | assertEquals(BagheeraMessage.Operation.DELETE, message.getOperation()); 128 | 129 | // Ensure that partition information comes through. 130 | assertEquals(2, message.getPartitionCount()); 131 | assertEquals("partition1", message.getPartition(0)); 132 | assertEquals("partition2", message.getPartition(1)); 133 | 134 | } 135 | 136 | @Test 137 | public void testMessageWithApiVersion() throws IOException, InterruptedException { 138 | // Use a ReplaySink to send messages 139 | String destPattern = String.format("http://localhost:%d/5.5/%s/%s/%s/partition1/partition2", BAGHEERA_PORT, SubmissionHandler.ENDPOINT_SUBMIT, TEST_NAMESPACE, ReplaySink.KEY_PLACEHOLDER); 140 | ReplaySink sink = new ReplaySink(destPattern, "1", "true", "true"); 141 | sink.store(key, json.getBytes(), timestamp); 142 | 143 | assertEquals(1, producer.queueSize()); 144 | BagheeraMessage message = producer.getQueue().poll(); 145 | String payload = message.getPayload().toStringUtf8(); 146 | assertEquals(json, payload); 147 | 148 | // ReplaySink doesn't preserve timestamps. 149 | assertNotSame(timestamp, message.getTimestamp()); 150 | 151 | assertEquals(key, message.getId()); 152 | 153 | // Ensure that partition information comes through. 154 | assertEquals(2, message.getPartitionCount()); 155 | assertEquals("partition1", message.getPartition(0)); 156 | assertEquals("partition2", message.getPartition(1)); 157 | 158 | assertEquals("5.5", message.getApiVersion()); 159 | } 160 | 161 | @After 162 | public void tearDown() { 163 | state.close(); 164 | } 165 | } 166 | 167 | class SimpleProducer implements Producer { 168 | private final int maxQueueSize; 169 | private final LinkedList queue = new LinkedList(); 170 | 171 | public SimpleProducer(int queueSize) { 172 | maxQueueSize = queueSize; 173 | } 174 | 175 | public SimpleProducer() { 176 | this(1000); 177 | } 178 | 179 | public LinkedList getQueue() { 180 | return queue; 181 | } 182 | 183 | public int queueSize() { 184 | return queue.size(); 185 | } 186 | 187 | @Override 188 | public void close() throws IOException { } 189 | 190 | @Override 191 | public void send(BagheeraMessage msg) { 192 | // Remove & discard some to make room. 193 | while (queue.size() >= maxQueueSize) { 194 | queue.poll(); 195 | } 196 | queue.offer(msg); 197 | } 198 | } -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/http/ContentLengthFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import static org.easymock.EasyMock.createMock; 23 | import static org.easymock.EasyMock.expect; 24 | import static org.easymock.EasyMock.replay; 25 | import static org.jboss.netty.handler.codec.http.HttpMethod.POST; 26 | import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1; 27 | import static org.junit.Assert.assertTrue; 28 | 29 | import java.net.InetSocketAddress; 30 | 31 | import org.jboss.netty.buffer.ChannelBuffers; 32 | import org.jboss.netty.channel.Channel; 33 | import org.jboss.netty.channel.ChannelHandlerContext; 34 | import org.jboss.netty.channel.DefaultChannelFuture; 35 | import org.jboss.netty.channel.FakeChannelHandlerContext; 36 | import org.jboss.netty.channel.MessageEvent; 37 | import org.jboss.netty.channel.UpstreamMessageEvent; 38 | import org.jboss.netty.handler.codec.frame.TooLongFrameException; 39 | import org.jboss.netty.handler.codec.http.DefaultHttpRequest; 40 | import org.jboss.netty.handler.codec.http.HttpMethod; 41 | import org.jboss.netty.handler.codec.http.HttpRequest; 42 | import org.jboss.netty.handler.codec.http.HttpVersion; 43 | import org.jboss.netty.handler.execution.ExecutionHandler; 44 | import org.jboss.netty.handler.execution.OrderedMemoryAwareThreadPoolExecutor; 45 | import org.junit.Before; 46 | import org.junit.Test; 47 | 48 | public class ContentLengthFilterTest { 49 | 50 | private ChannelHandlerContext ctx; 51 | private InetSocketAddress remoteAddr; 52 | 53 | @Before 54 | public void setup() { 55 | Channel channel = createMock(Channel.class); 56 | expect(channel.getCloseFuture()).andReturn(new DefaultChannelFuture(channel, false)); 57 | expect(channel.getRemoteAddress()).andReturn(InetSocketAddress.createUnresolved("192.168.1.1", 51723)); 58 | 59 | OrderedMemoryAwareThreadPoolExecutor executor = new OrderedMemoryAwareThreadPoolExecutor(10, 0L, 0L); 60 | final ExecutionHandler handler = new ExecutionHandler(executor, true, true); 61 | 62 | ctx = new FakeChannelHandlerContext(channel, handler); 63 | } 64 | 65 | @Test 66 | public void testNegativeLengthFilter() { 67 | boolean success = false; 68 | try { 69 | @SuppressWarnings("unused") 70 | ContentLengthFilter filter = new ContentLengthFilter(-1); 71 | } catch (IllegalArgumentException e) { 72 | success = true; 73 | } 74 | assertTrue(success); 75 | } 76 | 77 | private MessageEvent createMockEvent(Channel channel, HttpVersion protocolVersion, HttpMethod method, String uri, byte[] contentBytes) { 78 | MessageEvent event = createMock(UpstreamMessageEvent.class); 79 | expect(event.getChannel()).andReturn(channel).anyTimes(); 80 | expect(event.getFuture()).andReturn(new DefaultChannelFuture(channel,false)).anyTimes(); 81 | expect(event.getRemoteAddress()).andReturn(remoteAddr); 82 | HttpRequest req = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/"); 83 | req.setChunked(false); 84 | req.setContent(ChannelBuffers.copiedBuffer(contentBytes)); 85 | expect(event.getMessage()).andReturn(req); 86 | replay(channel, event); 87 | 88 | return event; 89 | } 90 | 91 | @Test 92 | public void testRequestTooLong() { 93 | ContentLengthFilter filter = new ContentLengthFilter(1); 94 | byte[] contentBytes = new String("foo").getBytes(); 95 | boolean success = false; 96 | try { 97 | filter.messageReceived(ctx, createMockEvent(ctx.getChannel(), HTTP_1_1, POST, "/", contentBytes)); 98 | } catch (TooLongFrameException e) { 99 | success = true; 100 | } catch (Exception e) { 101 | // TODO Auto-generated catch block 102 | e.printStackTrace(); 103 | } 104 | assertTrue(success); 105 | } 106 | 107 | @Test 108 | public void testFilterSuccess() { 109 | ContentLengthFilter filter = new ContentLengthFilter(4); 110 | byte[] contentBytes = new String("foo").getBytes(); 111 | boolean success = false; 112 | try { 113 | filter.messageReceived(ctx, createMockEvent(ctx.getChannel(), HTTP_1_1, POST, "/", contentBytes)); 114 | success = true; 115 | } catch (Exception e) { 116 | // TODO Auto-generated catch block 117 | e.printStackTrace(); 118 | } 119 | assertTrue(success); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/http/PathDecoderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import static org.junit.Assert.assertEquals; 23 | import static org.junit.Assert.assertNull; 24 | 25 | import org.junit.Test; 26 | 27 | public class PathDecoderTest { 28 | 29 | @Test 30 | public void testPathDecoder() { 31 | PathDecoder pd = new PathDecoder("/submit/foo/fakeid"); 32 | assertEquals("submit", pd.getPathElement(0)); 33 | assertEquals("foo", pd.getPathElement(1)); 34 | assertEquals("fakeid", pd.getPathElement(2)); 35 | assertNull(pd.getPathElement(3)); 36 | 37 | assertEquals(3, pd.size()); 38 | 39 | 40 | pd = new PathDecoder("/1/2/3/4/5/6/7/8/9/10"); 41 | for (int i = 1; i <= 10; i++) { 42 | assertEquals(String.valueOf(i), pd.getPathElement(i-1)); 43 | } 44 | assertEquals(10, pd.size()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/http/SubmissionHandlerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.http; 21 | 22 | import static org.junit.Assert.assertEquals; 23 | 24 | import java.net.InetSocketAddress; 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | 28 | import org.jboss.netty.channel.Channel; 29 | import org.jboss.netty.channel.MessageEvent; 30 | import org.junit.Test; 31 | import org.mockito.Mockito; 32 | 33 | import com.mozilla.bagheera.BagheeraProto.BagheeraMessage; 34 | import com.mozilla.bagheera.util.HttpUtil; 35 | 36 | 37 | public class SubmissionHandlerTest { 38 | 39 | @Test 40 | public void testSetFields() throws Exception { 41 | SubmissionHandler handler = new SubmissionHandler(null, null, null, null); 42 | BagheeraMessage.Builder builder = BagheeraMessage.newBuilder(); 43 | BagheeraMessage before = builder.buildPartial(); 44 | 45 | assertEquals("", before.getId()); 46 | assertEquals("", before.getNamespace()); 47 | assertEquals("", before.getApiVersion()); 48 | assertEquals(0, before.getPartitionCount()); 49 | assertEquals(0l, before.getTimestamp()); 50 | 51 | String expectedNamespace = "test"; 52 | String expectedApiVersion = "2.5"; 53 | String expectedId = "hello there"; 54 | long expectedTimestamp = System.currentTimeMillis(); 55 | List expectedPartitions = new ArrayList(); 56 | 57 | BagheeraHttpRequest request = Mockito.mock(BagheeraHttpRequest.class); 58 | Mockito.when(request.getNamespace()).thenReturn(expectedNamespace); 59 | Mockito.when(request.getApiVersion()).thenReturn(expectedApiVersion); 60 | Mockito.when(request.getId()).thenReturn(expectedId); 61 | Mockito.when(request.getPartitions()).thenReturn(expectedPartitions); 62 | 63 | // Make sure we don't interrogate the mocked InetSocketAddress below. 64 | Mockito.when(request.getHeader(HttpUtil.X_FORWARDED_FOR)).thenReturn("123.123.123.123"); 65 | 66 | MessageEvent event = Mockito.mock(MessageEvent.class); 67 | Channel channel = Mockito.mock(Channel.class); 68 | Mockito.when(event.getChannel()).thenReturn(channel); 69 | 70 | InetSocketAddress address = Mockito.mock(InetSocketAddress.class); 71 | Mockito.when(channel.getRemoteAddress()).thenReturn(address); 72 | 73 | // Do not set the ID 74 | //handler.setMessageFields(request, event, builder, expectedTimestamp, false); 75 | 76 | BagheeraMessage after = builder.build(); 77 | assertEquals("", after.getId()); // <-- missing ID 78 | //assertEquals(expectedNamespace, after.getNamespace()); 79 | // assertEquals(expectedApiVersion, after.getApiVersion()); 80 | // assertEquals(0, after.getPartitionCount()); 81 | // assertEquals(expectedTimestamp, after.getTimestamp()); 82 | 83 | builder = BagheeraMessage.newBuilder(); 84 | // This time, *do* set the ID 85 | // handler.setMessageFields(request, event, builder, expectedTimestamp, true); 86 | 87 | // after = builder.build(); 88 | // assertEquals(expectedId, after.getId()); // <-- ID has been set. 89 | // assertEquals(expectedNamespace, after.getNamespace()); 90 | // assertEquals(expectedApiVersion, after.getApiVersion()); 91 | // assertEquals(0, after.getPartitionCount()); 92 | // assertEquals(expectedTimestamp, after.getTimestamp()); 93 | 94 | // // Test without specifying an apiVersion 95 | // Mockito.when(request.getApiVersion()).thenReturn(null); 96 | // builder = BagheeraMessage.newBuilder(); 97 | 98 | // handler.setMessageFields(request, event, builder, expectedTimestamp, true); 99 | 100 | // after = builder.build(); 101 | // assertEquals(expectedId, after.getId()); // <-- ID has been set. 102 | // assertEquals(expectedNamespace, after.getNamespace()); 103 | // assertEquals("", after.getApiVersion()); 104 | // assertEquals(0, after.getPartitionCount()); 105 | // assertEquals(expectedTimestamp, after.getTimestamp()); 106 | 107 | // Test with some partitions 108 | // Expectedpartitions.add("hello"); 109 | // expectedPartitions.add("goodbye"); 110 | 111 | // Mockito.when(request.getPartitions()).thenReturn(expectedPartitions); 112 | 113 | // builder = BagheeraMessage.newBuilder(); 114 | 115 | // assertEquals(0, builder.getPartitionCount()); 116 | // handler.setMessageFields(request, event, builder, expectedTimestamp, true); 117 | // after = builder.build(); 118 | // assertEquals(expectedPartitions.size(), after.getPartitionCount()); 119 | // for (int i = 0; i < expectedPartitions.size(); i++) { 120 | // assertEquals(expectedPartitions.get(i), after.getPartition(i)); 121 | // } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/sink/HBaseSinkTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import java.io.IOException; 23 | import java.util.ArrayList; 24 | import java.util.Date; 25 | import java.util.concurrent.ConcurrentLinkedQueue; 26 | 27 | import org.apache.commons.cli.ParseException; 28 | import org.apache.hadoop.hbase.HServerAddress; 29 | import org.apache.hadoop.hbase.client.HTable; 30 | import org.apache.hadoop.hbase.client.HTablePool; 31 | import org.apache.hadoop.hbase.client.Put; 32 | import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; 33 | import org.apache.hadoop.hbase.client.Row; 34 | import org.junit.Before; 35 | import org.junit.Test; 36 | import org.mockito.Mockito; 37 | import org.mockito.invocation.InvocationOnMock; 38 | import org.mockito.stubbing.Answer; 39 | 40 | public class HBaseSinkTest { 41 | SinkConfiguration sinkConfig; 42 | KeyValueSinkFactory sinkFactory; 43 | HTablePool hbasePool; 44 | HTable htable; 45 | 46 | @Before 47 | public void setup() throws IOException { 48 | sinkConfig = new SinkConfiguration(); 49 | sinkConfig.setString("hbasesink.hbase.tablename", "test"); 50 | sinkConfig.setString("hbasesink.hbase.column.family", "data"); 51 | sinkConfig.setString("hbasesink.hbase.column.qualifier", "json"); 52 | sinkConfig.setBoolean("hbasesink.hbase.rowkey.prefixdate", false); 53 | sinkFactory = KeyValueSinkFactory.getInstance(HBaseSink.class, sinkConfig); 54 | 55 | hbasePool = Mockito.mock(HTablePool.class); 56 | htable = Mockito.mock(HTable.class); 57 | 58 | Mockito.when(hbasePool.getTable("test".getBytes())).thenReturn(htable); 59 | 60 | Mockito.doAnswer(new Answer() { 61 | int count = 0; 62 | @Override 63 | public Object answer(InvocationOnMock invocation) throws Throwable { 64 | count++; 65 | // Force code to retry once. 66 | if (count <= 1) { 67 | throw new RetriesExhaustedWithDetailsException(new ArrayList(), new ArrayList(), new ArrayList()); 68 | } 69 | return null; 70 | } 71 | }).when(htable).put(Mockito.anyListOf(Put.class)); 72 | } 73 | 74 | /* removing from test since we don't have clearRegionCache in 75 | Hbase 0.94 on HTableInterface */ 76 | public void testRetry() throws ParseException, IOException { 77 | HBaseSink sink = (HBaseSink) sinkFactory.getSink("test"); 78 | sink.setRetrySleepSeconds(1); 79 | sink.hbasePool = hbasePool; 80 | sink.flush(); 81 | Mockito.verify(htable, Mockito.times(1)).clearRegionCache(); 82 | } 83 | 84 | @Test 85 | public void testLargePut() throws IOException { 86 | HBaseSink sink = (HBaseSink) sinkFactory.getSink("test"); 87 | 88 | @SuppressWarnings("unchecked") 89 | ConcurrentLinkedQueue rowQueue = Mockito.mock(ConcurrentLinkedQueue.class); 90 | 91 | sink.rowQueue = rowQueue; 92 | byte[] theArray = new byte[50*1000*1000]; // 50MB 93 | for (int i = 0; i < theArray.length; i++) { 94 | theArray[i] = (byte)(i % 256 - 128); 95 | } 96 | 97 | sink.store("test1", theArray); 98 | Mockito.verify(rowQueue, Mockito.times(0)).add((Put)Mockito.any()); 99 | sink.store("test2", "acceptable".getBytes()); 100 | Mockito.verify(rowQueue, Mockito.times(1)).add((Put)Mockito.any()); 101 | sink.store("test3", theArray); 102 | Mockito.verify(rowQueue, Mockito.times(1)).add((Put)Mockito.any()); 103 | 104 | sink.store("test4", theArray, new Date().getTime()); 105 | Mockito.verify(rowQueue, Mockito.times(1)).add((Put)Mockito.any()); 106 | sink.store("test5", "acceptable".getBytes(), new Date().getTime()); 107 | Mockito.verify(rowQueue, Mockito.times(2)).add((Put)Mockito.any()); 108 | sink.store("test6", theArray, new Date().getTime()); 109 | Mockito.verify(rowQueue, Mockito.times(2)).add((Put)Mockito.any()); 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/sink/ReplaySinkTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.sink; 21 | 22 | import static org.junit.Assert.assertEquals; 23 | import static org.junit.Assert.assertTrue; 24 | 25 | import java.io.IOException; 26 | import java.io.OutputStream; 27 | import java.net.InetSocketAddress; 28 | import java.net.URI; 29 | 30 | import org.junit.After; 31 | import org.junit.Before; 32 | import org.junit.Test; 33 | 34 | import com.sun.net.httpserver.HttpExchange; 35 | import com.sun.net.httpserver.HttpHandler; 36 | import com.sun.net.httpserver.HttpServer; 37 | 38 | // Don't care that this httpserver stuff is restricted. 39 | @SuppressWarnings("restriction") 40 | public class ReplaySinkTest { 41 | int fakePort = 9888; 42 | String fakePath = "/submit/test"; 43 | MyHandler requestHandler = new MyHandler(); 44 | HttpServer server; 45 | 46 | @Before 47 | public void setup() throws IOException { 48 | // Set up a basic server: 49 | // See docs here: http://docs.oracle.com/javase/6/docs/jre/api/net/httpserver/spec/com/sun/net/httpserver/package-summary.html 50 | server = HttpServer.create(new InetSocketAddress(fakePort), 0); 51 | server.createContext(fakePath, requestHandler); 52 | server.setExecutor(null); // default executor 53 | server.start(); 54 | } 55 | 56 | @After 57 | public void tearDown() { 58 | if (server != null) { 59 | server.stop(0); 60 | } 61 | } 62 | 63 | @Test 64 | public void testReplayWithoutSampling() throws IOException { 65 | ReplaySink sink = new ReplaySink(getDestConfig("http://localhost:" + fakePort + fakePath + "/" + ReplaySink.KEY_PLACEHOLDER)); 66 | 67 | sink.store("foo", "bar".getBytes()); 68 | assertEquals(fakePath + "/foo", requestHandler.lastRequestURI.toString()); 69 | 70 | // Make sure we see each request. 71 | int counter = 0; 72 | int max = 50; 73 | for (int i = 0; i < max; i++) { 74 | String key = "test" + i; 75 | byte[] payload = ("bar" + i).getBytes(); 76 | sink.store(key, payload); 77 | String expectedURI = fakePath + "/" + key; 78 | if (expectedURI.equals(requestHandler.lastRequestURI.toString())) { 79 | counter++; 80 | } 81 | } 82 | 83 | // Without sampling, we should see all requests. 84 | assertEquals(max, counter); 85 | } 86 | 87 | @Test 88 | public void testReplayWithSampling() throws IOException { 89 | SinkConfiguration config = getDestConfig("http://localhost:" + fakePort + fakePath + "/" + ReplaySink.KEY_PLACEHOLDER); 90 | // Override sample rate: 91 | config.setString("replaysink.sample", "0.1"); 92 | ReplaySink sink = new ReplaySink(config); 93 | 94 | // Make sure we see some, but not all, requests 95 | int counter = 0; 96 | int max = 500; 97 | for (int i = 0; i < max; i++) { 98 | String key = "test" + i; 99 | byte[] payload = ("bar" + i).getBytes(); 100 | sink.store(key, payload); 101 | String expectedURI = fakePath + "/" + key; 102 | String actualURI = ""; 103 | if (requestHandler.lastRequestURI != null) { 104 | actualURI = requestHandler.lastRequestURI.toString(); 105 | } 106 | if (expectedURI.equals(actualURI)) { 107 | counter++; 108 | } 109 | } 110 | 111 | // With sampling, we should see some requests, but not all of them. 112 | assertTrue(counter > 0); 113 | assertTrue(counter < max); 114 | } 115 | 116 | @Test 117 | public void testDestNoSuffix() throws IOException { 118 | SinkConfiguration config = getDestConfig("http://localhost:8080/submit/foof/%k"); 119 | ReplaySink sink = new ReplaySink(config); 120 | 121 | assertEquals("http://localhost:8080/submit/foof/test1", sink.getDest("test1")); 122 | assertEquals("http://localhost:8080/submit/foof/test2", sink.getDest("test2")); 123 | assertEquals("http://localhost:8080/submit/foof/a/b/c", sink.getDest("a/b/c")); 124 | } 125 | 126 | @Test 127 | public void testDestNoKey() throws IOException { 128 | SinkConfiguration config = getDestConfig("I am a test"); 129 | ReplaySink sink = new ReplaySink(config); 130 | 131 | assertEquals("I am a test", sink.getDest("test1")); 132 | assertEquals("I am a test", sink.getDest("test2")); 133 | assertEquals("I am a test", sink.getDest("a/b/c")); 134 | } 135 | 136 | // FIXME We can't use the KeyValueSinkFactory because it gets stuck with the original config :( 137 | public SinkConfiguration getDestConfig(String destPattern) { 138 | SinkConfiguration config = new SinkConfiguration(); 139 | config.setString("replaysink.keys", "true"); 140 | config.setString("replaysink.delete", "true"); 141 | config.setString("replaysink.sample", "1"); 142 | config.setString("replaysink.dest", destPattern); 143 | 144 | return config; 145 | } 146 | 147 | @Test 148 | public void testDestSuffix() throws IOException { 149 | SinkConfiguration config = getDestConfig("foo %k bar"); 150 | ReplaySink sink = new ReplaySink(config); 151 | 152 | assertEquals("foo test1 bar", sink.getDest("test1")); 153 | assertEquals("foo test2 bar", sink.getDest("test2")); 154 | assertEquals("foo a/b/c bar", sink.getDest("a/b/c")); 155 | } 156 | 157 | @Test 158 | public void testDeleteWithoutSampling() throws IOException { 159 | ReplaySink sink = new ReplaySink(getDestConfig("http://localhost:" + fakePort + fakePath + "/" + ReplaySink.KEY_PLACEHOLDER)); 160 | 161 | // Make sure we see each request. 162 | int counter = 0; 163 | int max = 50; 164 | for (int i = 0; i < max; i++) { 165 | String key = "delete" + i; 166 | sink.delete(key); 167 | String expectedURI = fakePath + "/" + key; 168 | if (expectedURI.equals(requestHandler.lastRequestURI.toString())) { 169 | counter++; 170 | } 171 | } 172 | 173 | // Without sampling, we should see all delete requests. 174 | assertEquals(max, counter); 175 | } 176 | 177 | @Test 178 | public void testDisabledDeletes() throws IOException { 179 | SinkConfiguration config = getDestConfig("http://localhost:" + fakePort + fakePath + "/" + ReplaySink.KEY_PLACEHOLDER); 180 | config.setString("replaysink.delete", "false"); 181 | ReplaySink sink = new ReplaySink(config); 182 | 183 | // Make sure we don't process any delete requests. 184 | int counter = 0; 185 | int max = 10; 186 | for (int i = 0; i < max; i++) { 187 | String key = "delete" + i; 188 | sink.delete(key); 189 | String expectedURI = fakePath + "/" + key; 190 | String actualURI = ""; 191 | if (requestHandler.lastRequestURI != null) { 192 | actualURI = requestHandler.lastRequestURI.toString(); 193 | } 194 | if (expectedURI.equals(actualURI)) { 195 | counter++; 196 | } 197 | } 198 | 199 | // Without sampling, we should see all delete requests. 200 | assertEquals(0, counter); 201 | } 202 | 203 | class MyHandler implements HttpHandler { 204 | public URI lastRequestURI; 205 | @Override 206 | public void handle(HttpExchange t) throws IOException { 207 | // TODO: this doesn't include the full URL, can we do better? 208 | lastRequestURI = t.getRequestURI(); 209 | 210 | String response = "This is the response"; 211 | t.sendResponseHeaders(200, response.length()); 212 | OutputStream os = t.getResponseBody(); 213 | os.write(response.getBytes()); 214 | os.close(); 215 | } 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/util/HttpUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.util; 21 | 22 | import static org.junit.Assert.assertEquals; 23 | import static org.junit.Assert.assertNotNull; 24 | 25 | import java.net.InetAddress; 26 | import java.net.UnknownHostException; 27 | 28 | import org.jboss.netty.handler.codec.http.DefaultHttpRequest; 29 | import org.jboss.netty.handler.codec.http.HttpMethod; 30 | import org.jboss.netty.handler.codec.http.HttpVersion; 31 | import org.junit.Before; 32 | import org.junit.Test; 33 | 34 | public class HttpUtilTest { 35 | 36 | private DefaultHttpRequest request; 37 | private InetAddress addr; 38 | 39 | @Before 40 | public void setup() { 41 | request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/foo"); 42 | request.setHeader(HttpUtil.USER_AGENT, "Dummy Agent"); 43 | request.setHeader(HttpUtil.X_FORWARDED_FOR, "127.0.0.2, 127.0.0.3, 127.0.0.4"); 44 | try { 45 | addr = InetAddress.getLocalHost(); 46 | } catch (UnknownHostException e) { 47 | // TODO Auto-generated catch block 48 | e.printStackTrace(); 49 | } 50 | assertNotNull(addr); 51 | } 52 | 53 | @Test 54 | public void testGetUserAgent() { 55 | assertEquals("Dummy Agent", HttpUtil.getUserAgent(request)); 56 | } 57 | 58 | @Test 59 | public void testGetRemoteAddrBytes() throws UnknownHostException { 60 | byte[] addrBytes = HttpUtil.getRemoteAddr(request, addr); 61 | assertEquals(4, addrBytes.length); 62 | assertEquals("127.0.0.2", InetAddress.getByAddress(addrBytes).getHostAddress()); 63 | } 64 | 65 | @Test 66 | public void testGetRemoteAddrString() throws UnknownHostException { 67 | String addrStr = HttpUtil.getRemoteAddr(request, addr.getHostAddress()); 68 | assertEquals("127.0.0.2", addrStr); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/util/IdUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.util; 21 | 22 | import static org.junit.Assert.assertNotNull; 23 | import static org.junit.Assert.assertTrue; 24 | 25 | import java.io.IOException; 26 | import java.util.Calendar; 27 | import java.util.Date; 28 | import java.util.UUID; 29 | 30 | import org.junit.Test; 31 | 32 | public class IdUtilTest { 33 | 34 | @Test 35 | public void testBucketizeIdWithTimestamp2() throws IOException { 36 | UUID uuid = UUID.randomUUID(); 37 | long ts = System.currentTimeMillis(); 38 | byte[] idBytes = IdUtil.bucketizeId(uuid.toString(), ts); 39 | assertNotNull(idBytes); 40 | String bucketIdStr = new String(idBytes); 41 | assertTrue(bucketIdStr.endsWith(IdUtil.SDF.format(new Date(ts)) + uuid.toString())); 42 | } 43 | 44 | @Test 45 | public void testNonRandByteBucketizeId1() throws IOException { 46 | boolean caughtException = false; 47 | try { 48 | IdUtil.nonRandByteBucketizeId(null, Calendar.getInstance().getTime()); 49 | } catch (IllegalArgumentException e) { 50 | caughtException = true; 51 | } 52 | assertTrue(caughtException); 53 | } 54 | 55 | @Test 56 | public void testNonRandByteBucketizeId2() throws IOException { 57 | boolean caughtException = false; 58 | try { 59 | UUID uuid = UUID.randomUUID(); 60 | IdUtil.nonRandByteBucketizeId(uuid.toString(), null); 61 | } catch (IllegalArgumentException e) { 62 | caughtException = true; 63 | } 64 | assertTrue(caughtException); 65 | } 66 | 67 | @Test 68 | public void testNonRandByteBucketizeId3() throws IOException { 69 | UUID uuid = UUID.randomUUID(); 70 | Date d = Calendar.getInstance().getTime(); 71 | byte[] idBytes = IdUtil.nonRandByteBucketizeId(uuid.toString(), d); 72 | assertNotNull(idBytes); 73 | String bucketIdStr = new String(idBytes); 74 | assert(bucketIdStr.endsWith(IdUtil.SDF.format(d) + uuid.toString())); 75 | } 76 | 77 | @Test 78 | public void testNonRandByteBucketizeId4() throws IOException { 79 | UUID uuid = UUID.randomUUID(); 80 | Date d = Calendar.getInstance().getTime(); 81 | byte[] idBytes1 = IdUtil.nonRandByteBucketizeId(uuid.toString(), d); 82 | assertNotNull(idBytes1); 83 | String bucketIdStr1 = new String(idBytes1); 84 | 85 | byte[] idBytes2 = IdUtil.nonRandByteBucketizeId(uuid.toString(), d); 86 | assertNotNull(idBytes2); 87 | String bucketIdStr2 = new String(idBytes2); 88 | 89 | assertTrue(bucketIdStr1.equals(bucketIdStr2)); 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/util/WildcardPropertiesTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.util; 21 | 22 | import static org.junit.Assert.assertEquals; 23 | 24 | import java.io.ByteArrayInputStream; 25 | import java.io.IOException; 26 | import java.io.InputStream; 27 | 28 | import org.junit.Test; 29 | 30 | public class WildcardPropertiesTest { 31 | 32 | @Test 33 | public void testWildcardProperties() throws IOException { 34 | String propsFileStr = "foo_*.baz=7"; 35 | InputStream is = new ByteArrayInputStream(propsFileStr.getBytes("UTF-8")); 36 | WildcardProperties wcProps = new WildcardProperties(); 37 | wcProps.load(is); 38 | assertEquals("7", wcProps.getWildcardProperty("foo_bar.baz")); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/com/mozilla/bagheera/validation/ValidatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package com.mozilla.bagheera.validation; 21 | 22 | import static org.junit.Assert.assertFalse; 23 | import static org.junit.Assert.assertTrue; 24 | 25 | import java.util.UUID; 26 | 27 | import org.junit.Before; 28 | import org.junit.Test; 29 | 30 | public class ValidatorTest { 31 | 32 | private Validator validator; 33 | 34 | @Before 35 | public void setup() { 36 | validator = new Validator(new String[] { "foo", "bar" }); 37 | } 38 | 39 | @Test 40 | public void testConstructorNullArg() { 41 | boolean success = false; 42 | try { 43 | @SuppressWarnings("unused") 44 | Validator v = new Validator(null); 45 | } catch (IllegalArgumentException e) { 46 | success = true; 47 | } 48 | assertTrue(success); 49 | } 50 | 51 | @Test 52 | public void testConstructorZeroLength() { 53 | boolean success = false; 54 | try { 55 | @SuppressWarnings("unused") 56 | Validator v = new Validator(new String[0]); 57 | } catch (IllegalArgumentException e) { 58 | success = true; 59 | } 60 | assertTrue(success); 61 | } 62 | 63 | @Test 64 | public void testIsValidJson() { 65 | assertTrue(validator.isValidJson("{ \"baz\" : \"blah\" }")); 66 | assertFalse(validator.isValidJson("{ \"baz : 7 }")); 67 | } 68 | 69 | @Test 70 | public void testIsValidNamespace() { 71 | assertTrue(validator.isValidNamespace("foo")); 72 | assertFalse(validator.isValidNamespace("baz")); 73 | } 74 | 75 | @Test 76 | public void testIsValidId() { 77 | String id = UUID.randomUUID().toString(); 78 | assertTrue(validator.isValidId(id)); 79 | assertFalse(validator.isValidId("fakeid")); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/org/jboss/netty/channel/FakeChannelHandlerContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Mozilla Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package org.jboss.netty.channel; 21 | 22 | import org.jboss.netty.handler.execution.ExecutionHandler; 23 | 24 | public class FakeChannelHandlerContext implements ChannelHandlerContext { 25 | 26 | private Channel channel; 27 | private final ExecutionHandler handler; 28 | 29 | public FakeChannelHandlerContext(Channel channel, ExecutionHandler handler) { 30 | this.channel = channel; 31 | this.handler = handler; 32 | } 33 | 34 | @Override 35 | public boolean canHandleDownstream() { 36 | return true; 37 | } 38 | 39 | @Override 40 | public boolean canHandleUpstream() { 41 | return true; 42 | } 43 | 44 | @Override 45 | public Object getAttachment() { 46 | // TODO Auto-generated method stub 47 | return null; 48 | } 49 | 50 | @Override 51 | public Channel getChannel() { 52 | return channel; 53 | } 54 | 55 | @Override 56 | public ChannelHandler getHandler() { 57 | return handler; 58 | } 59 | 60 | @Override 61 | public String getName() { 62 | return handler.getClass().getName(); 63 | } 64 | 65 | @Override 66 | public ChannelPipeline getPipeline() { 67 | return null; 68 | } 69 | 70 | @Override 71 | public void sendDownstream(ChannelEvent e) { 72 | // NOOP 73 | } 74 | 75 | @Override 76 | public void sendUpstream(ChannelEvent e) { 77 | // NOOP 78 | } 79 | 80 | @Override 81 | public void setAttachment(Object o) { 82 | // NOOP 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/test/python/http_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2011 Xavier Stevens 3 | 4 | This file is provided to you under the Apache License, 5 | Version 2.0 (the "License"); you may not use this file 6 | except in compliance with the License. You may obtain 7 | a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | """ 18 | import sys 19 | import getopt 20 | import urllib2 21 | import time 22 | import uuid 23 | import numpy 24 | from scipy import stats 25 | 26 | help_message = ''' 27 | Flags: 28 | -p 29 | -n (default: 1) 30 | -T (default: text/plain) 31 | -h|--help prints this message 32 | 33 | Example Usage: 34 | python http-test.py -n 100 -T application/json -p data/small.json http://localhost:5701/hazelcast/rest/maps/metrics_ping 35 | ''' 36 | 37 | times = [] 38 | 39 | def timeit(method): 40 | 41 | def timed(*args, **kw): 42 | ts = time.time() 43 | result = method(*args, **kw) 44 | te = time.time() 45 | 46 | times.append((te-ts)*1000.0) 47 | 48 | #print '%r (%r, %r) %2.4f sec' % (method.__name__, args, kw, te-ts) 49 | 50 | return result 51 | 52 | return timed 53 | 54 | class Usage(Exception): 55 | def __init__(self, msg): 56 | self.msg = msg 57 | 58 | 59 | @timeit 60 | def post(url, post_headers, post_data): 61 | request = urllib2.Request(url, post_data, post_headers) 62 | code = 200 63 | try: 64 | response = urllib2.urlopen(request) 65 | except URLError, e: 66 | code = e.code 67 | return code 68 | 69 | def post_requests(url, n, content_type, post_data): 70 | post_headers = { "Content-Type": content_type } 71 | ret_codes = [] 72 | for i in range(0,n): 73 | post_url = "%s/%s" % (url, str(uuid.uuid4())) 74 | ret_codes.append(post(post_url, post_headers, post_data)) 75 | return ret_codes 76 | 77 | def read_file(post_file): 78 | f = open(post_file, 'r') 79 | data = f.read() 80 | f.close() 81 | return data 82 | 83 | def print_time_stats(num_requests, ret_codes): 84 | print "%d requests" % (num_requests) 85 | print "%d responses" % (len(ret_codes)) 86 | print "Response code distribution:" 87 | for item in stats.itemfreq(ret_codes): 88 | print "\t%d => %d" % (item[0], item[1]) 89 | print "Request times (ms):" 90 | print "\tmin: %.5f" % (min(times)) 91 | print "\tmean: %.5f" % (numpy.mean(times)) 92 | print "\tmedian: %.5f" % (numpy.median(times)) 93 | print "\tmax: %.5f" % (max(times)) 94 | print "\tstddev: %.5f" % (numpy.std(times)) 95 | print "Quantiles of request times (ms):" 96 | print "\t25%%: %.5f" % (stats.scoreatpercentile(times, 25)) 97 | print "\t50%%: %.5f" % (stats.scoreatpercentile(times, 50)) 98 | print "\t75%%: %.5f" % (stats.scoreatpercentile(times, 75)) 99 | print "\t90%%: %.5f" % (stats.scoreatpercentile(times, 90)) 100 | print "\t95%%: %.5f" % (stats.scoreatpercentile(times, 95)) 101 | print "\t99%%: %.5f" % (stats.scoreatpercentile(times, 99)) 102 | 103 | def main(argv=None): 104 | if argv is None: 105 | argv = sys.argv 106 | try: 107 | try: 108 | opts, args = getopt.getopt(argv[1:], "hn:p:T:", ["help"]) 109 | except getopt.error, msg: 110 | raise Usage(msg) 111 | 112 | # option processing 113 | post_file = None 114 | num_requests = 1 115 | content_type = "text/plain" 116 | for option, value in opts: 117 | if option in ("-h", "--help"): 118 | raise Usage(help_message) 119 | elif option == "-p": 120 | post_file = value 121 | elif option == "-n": 122 | num_requests = int(value) 123 | elif option == "-T": 124 | content_type = value 125 | 126 | url = None 127 | if len(args) > 0: 128 | url = args[0] 129 | 130 | if url == None or post_file == None: 131 | raise Usage(help_message) 132 | 133 | data = read_file(post_file) 134 | ret_codes = post_requests(url, num_requests, content_type, data) 135 | print_time_stats(num_requests, ret_codes) 136 | except Usage, err: 137 | print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg) 138 | print >> sys.stderr, "\t for help use --help" 139 | return 2 140 | 141 | 142 | if __name__ == "__main__": 143 | sys.exit(main()) 144 | -------------------------------------------------------------------------------- /src/test/python/metrics_ping_mechanizer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2011 Xavier Stevens 3 | 4 | This file is provided to you under the Apache License, 5 | Version 2.0 (the "License"); you may not use this file 6 | except in compliance with the License. You may obtain 7 | a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | """ 18 | import mechanize 19 | import urllib2 20 | import uuid 21 | import random 22 | import time 23 | 24 | class Transaction: 25 | 26 | def __init__(self): 27 | self.custom_timers = {} 28 | self.servers = ["localhost"] 29 | self.url = "http://%s:8080/submit/metrics" 30 | self.content_type = "application/json" 31 | self.post_headers = { "Content-Type": self.content_type } 32 | self.post_data = read_data("metrics_ping.js") 33 | 34 | def read_data(self, filename): 35 | fin = open(filename, "r") 36 | data = fin.read() 37 | fin.close() 38 | return data 39 | 40 | def run(self): 41 | base_url = self.url % random.choice(self.servers) 42 | post_url = "%s/%s" % (base_url, str(uuid.uuid4())) 43 | request = urllib2.Request(post_url, self.post_data, self.post_headers) 44 | start_timer = time.time() 45 | response = urllib2.urlopen(request) 46 | latency = time.time() - start_timer 47 | #self.custom_timers['response time'] = latency 48 | timer_key = 'response time %d' % (response.code) 49 | self.custom_timers[timer_key] = latency 50 | assert(response.code == 200 or response.code == 201 or response.code == 204), 'Bad HTTP Response: %d' % (response.code) 51 | 52 | if __name__ == '__main__': 53 | trans = Transaction() 54 | trans.run() 55 | 56 | for t in trans.custom_timers.iterkeys(): 57 | print '%s: %.5f secs' % (t, trans.custom_timers[t]) 58 | -------------------------------------------------------------------------------- /src/test/python/telemetry_mechanizer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2011 Xavier Stevens 3 | 4 | This file is provided to you under the Apache License, 5 | Version 2.0 (the "License"); you may not use this file 6 | except in compliance with the License. You may obtain 7 | a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | """ 18 | import mechanize 19 | import urllib2 20 | import random 21 | import time 22 | 23 | class Transaction(object): 24 | 25 | def __init__(self): 26 | self.custom_timers = {} 27 | self.servers = ["localhost"] 28 | self.url = "http://%s:8080/submit/telemetry" 29 | self.post_headers = { "Content-Type": "application/json", "Content-Encoding": "gzip" } 30 | self.post_data = self.read_data("telemetry.js.gz") 31 | 32 | def read_data(self, filename): 33 | fin = open(filename, "rb") 34 | data = fin.read() 35 | fin.close() 36 | return data 37 | 38 | def run(self): 39 | post_url = self.url % random.choice(self.servers) 40 | request = urllib2.Request(post_url, self.post_data, self.post_headers) 41 | start_timer = time.time() 42 | response = urllib2.urlopen(request) 43 | latency = time.time() - start_timer 44 | #self.custom_timers['response time'] = latency 45 | timer_key = 'response time %d' % (response.code) 46 | self.custom_timers[timer_key] = latency 47 | assert(response.code == 201), 'Bad HTTP Response: %d' % (response.code) 48 | 49 | if __name__ == '__main__': 50 | trans = Transaction() 51 | trans.run() 52 | 53 | for t in trans.custom_timers.iterkeys(): 54 | print '%s: %.5f secs' % (t, trans.custom_timers[t]) 55 | -------------------------------------------------------------------------------- /src/test/python/testpilot_mechanizer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2011 Xavier Stevens 3 | 4 | This file is provided to you under the Apache License, 5 | Version 2.0 (the "License"); you may not use this file 6 | except in compliance with the License. You may obtain 7 | a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | """ 18 | import mechanize 19 | import urllib2 20 | import uuid 21 | import random 22 | import time 23 | 24 | class Transaction: 25 | 26 | def __init__(self): 27 | self.custom_timers = {} 28 | self.servers = ["localhost"] 29 | self.url = "http://%s:8080/submit/testpilot_test" 30 | self.content_type = "application/json" 31 | self.post_headers = { "Content-Type": self.content_type } 32 | self.post_data = read_data("testpilot.js") 33 | 34 | def read_data(self, filename): 35 | fin = open(filename, "r") 36 | data = fin.read() 37 | fin.close() 38 | return data 39 | 40 | def run(self): 41 | base_url = self.url % random.choice(self.servers) 42 | post_url = "%s/%s" % (base_url, str(uuid.uuid4())) 43 | request = urllib2.Request(post_url, self.post_data, self.post_headers) 44 | start_timer = time.time() 45 | response = urllib2.urlopen(request) 46 | latency = time.time() - start_timer 47 | #self.custom_timers['response time'] = latency 48 | timer_key = 'response time %d' % (response.code) 49 | self.custom_timers[timer_key] = latency 50 | assert(response.code == 200 or response.code == 201 or response.code == 204), 'Bad HTTP Response: %d' % (response.code) 51 | 52 | if __name__ == '__main__': 53 | trans = Transaction() 54 | trans.run() 55 | 56 | for t in trans.custom_timers.iterkeys(): 57 | print '%s: %.5f secs' % (t, trans.custom_timers[t]) 58 | --------------------------------------------------------------------------------