├── .travis.yml ├── VERSION ├── bin ├── jafka-linux-x86-32 ├── jafka-linux-x86-64 ├── optional │ ├── wrapper.jar │ ├── libwrapper-linux-x86-32.so │ ├── libwrapper-linux-x86-64.so │ ├── wrapper-windows-x86-32.dll │ ├── libwrapper-macosx-universal-32.jnilib │ └── libwrapper-macosx-universal-64.jnilib ├── jafka-macosx-universal-32 ├── jafka-macosx-universal-64 ├── jafka-windows-x86-32.exe ├── env.sh ├── server.sh ├── dumper.sh ├── admin-console.sh ├── consumer-console.sh ├── getoffset-console.sh ├── producer-console.sh ├── simple-consumer-console.sh ├── zookeeper-server.sh └── jafka.conf ├── .gitignore ├── clients └── python │ └── consume.py ├── src ├── main │ ├── java │ │ └── io │ │ │ └── jafka │ │ │ ├── consumer │ │ │ ├── IMessageListener.java │ │ │ ├── TopicEventHandler.java │ │ │ ├── FetchedDataChunk.java │ │ │ ├── Consumer.java │ │ │ ├── ConsumerConnector.java │ │ │ └── MessageStream.java │ │ │ ├── log │ │ │ ├── RollingStrategy.java │ │ │ ├── FixedSizeRollingStrategy.java │ │ │ ├── LogSegmentFilter.java │ │ │ └── ILog.java │ │ │ ├── utils │ │ │ ├── TopicNameValidator.java │ │ │ ├── Range.java │ │ │ └── zookeeper │ │ │ │ ├── ZkGroupTopicDirs.java │ │ │ │ └── ZkGroupDirs.java │ │ │ ├── mx │ │ │ ├── IMBeanName.java │ │ │ ├── ConsumerTopicStatMBean.java │ │ │ ├── AsyncProducerQueueSizeStatsMBean.java │ │ │ ├── AsyncProducerStatsMBean.java │ │ │ ├── ServerInfoMBean.java │ │ │ ├── BrokerTopicStatMBean.java │ │ │ ├── LogFlushStatsMBean.java │ │ │ ├── SyncProducerStatsMBean.java │ │ │ ├── LogStatsMBean.java │ │ │ ├── SocketServerStatsMBean.java │ │ │ ├── AsyncProducerQueueSizeStats.java │ │ │ ├── AsyncProducerStats.java │ │ │ ├── SyncProducerStats.java │ │ │ ├── ConsumerTopicStat.java │ │ │ └── LogFlushStats.java │ │ │ ├── common │ │ │ ├── annotations │ │ │ │ ├── ServerSide.java │ │ │ │ ├── ClientSide.java │ │ │ │ ├── ThreadSafe.java │ │ │ │ └── NotThreadSafe.java │ │ │ ├── MessageSizeTooLargeException.java │ │ │ ├── InvalidConfigException.java │ │ │ ├── UnKnownCodecException.java │ │ │ ├── UnknownMagicByteException.java │ │ │ ├── InvalidMessageSizeException.java │ │ │ ├── OffsetOutOfRangeException.java │ │ │ ├── NoBrokersForPartitionException.java │ │ │ ├── ConsumerRebalanceFailedException.java │ │ │ ├── QueueFullException.java │ │ │ ├── InvalidPartitionException.java │ │ │ ├── ConsumerTimeoutException.java │ │ │ ├── UnavailableProducerException.java │ │ │ ├── UnkonwnException.java │ │ │ ├── IllegalQueueStateException.java │ │ │ ├── QueueClosedException.java │ │ │ ├── AsyncProducerInterruptedException.java │ │ │ └── ErrorMappingException.java │ │ │ ├── producer │ │ │ ├── serializer │ │ │ │ ├── DefaultEncoders.java │ │ │ │ ├── Decoder.java │ │ │ │ ├── Encoder.java │ │ │ │ ├── StringEncoder.java │ │ │ │ ├── MessageEncoders.java │ │ │ │ ├── StringDecoder.java │ │ │ │ └── ByteArrayEncoders.java │ │ │ ├── async │ │ │ │ ├── AbstractCallbackHandler.java │ │ │ │ ├── AsyncProducerConfigShared.java │ │ │ │ ├── QueueItem.java │ │ │ │ └── EventHandler.java │ │ │ ├── IStringProducer.java │ │ │ ├── Partitioner.java │ │ │ ├── DefaultPartitioner.java │ │ │ ├── ProducerPoolData.java │ │ │ ├── StringProducerData.java │ │ │ ├── SyncProducerConfigShared.java │ │ │ ├── StringProducer.java │ │ │ └── IProducer.java │ │ │ ├── api │ │ │ ├── PartitionChooser.java │ │ │ ├── ICalculable.java │ │ │ ├── RequestKeys.java │ │ │ └── DeleterRequest.java │ │ │ ├── console │ │ │ ├── MessageReader.java │ │ │ ├── MessageFormatter.java │ │ │ ├── NewlineMessageFormatter.java │ │ │ ├── ChecksumMessageFormatter.java │ │ │ ├── LineMessageReader.java │ │ │ └── MyFormatter.java │ │ │ ├── network │ │ │ ├── Send.java │ │ │ ├── Receive.java │ │ │ ├── RequestHandlerFactory.java │ │ │ ├── AbstractSend.java │ │ │ ├── RequestHandler.java │ │ │ ├── Transmission.java │ │ │ ├── handlers │ │ │ │ ├── AbstractHandler.java │ │ │ │ ├── OffsetsHandler.java │ │ │ │ ├── DeleterHandler.java │ │ │ │ ├── CreaterHandler.java │ │ │ │ ├── MultiProduceHandler.java │ │ │ │ └── MultiFetchHandler.java │ │ │ ├── InvalidRequestException.java │ │ │ ├── Request.java │ │ │ ├── AbstractTransmission.java │ │ │ ├── ByteBufferSend.java │ │ │ ├── MultiMessageSetSend.java │ │ │ └── OffsetArraySend.java │ │ │ ├── message │ │ │ ├── MessageLengthException.java │ │ │ ├── InvalidMessageException.java │ │ │ ├── MessageAndOffset.java │ │ │ ├── compress │ │ │ │ ├── GZIPCompression.java │ │ │ │ ├── CompressionFacade.java │ │ │ │ └── CompressionFactory.java │ │ │ ├── ByteBufferBackedInputStream.java │ │ │ └── CompressionCodec.java │ │ │ ├── server │ │ │ └── TopicTask.java │ │ │ ├── http │ │ │ └── HttpServerInitializer.java │ │ │ └── cluster │ │ │ └── Cluster.java │ └── assembly │ │ └── assembly.xml └── test │ ├── java │ └── io │ │ └── jafka │ │ ├── utils │ │ ├── TopicNameValidatorTest.java │ │ ├── zookeeper │ │ │ ├── ZkGroupDirsTest.java │ │ │ └── ZkGroupTopicDirsTest.java │ │ ├── KVTest.java │ │ └── Gateway.java │ │ ├── PortUtils.java │ │ ├── TestUtil.java │ │ ├── common │ │ └── ErrorMappingTest.java │ │ ├── JafkaTest.java │ │ ├── cluster │ │ └── PartitionTest.java │ │ ├── ZkServerTestUtil.java │ │ └── DataLogCleaner.java │ └── resources │ └── log4j.properties ├── INSTALL └── conf ├── zookeeper.properties └── log4j.properties.sample /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | ${project.version} build ${timestamp} -------------------------------------------------------------------------------- /bin/jafka-linux-x86-32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/jafka-linux-x86-32 -------------------------------------------------------------------------------- /bin/jafka-linux-x86-64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/jafka-linux-x86-64 -------------------------------------------------------------------------------- /bin/optional/wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/optional/wrapper.jar -------------------------------------------------------------------------------- /bin/jafka-macosx-universal-32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/jafka-macosx-universal-32 -------------------------------------------------------------------------------- /bin/jafka-macosx-universal-64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/jafka-macosx-universal-64 -------------------------------------------------------------------------------- /bin/jafka-windows-x86-32.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/jafka-windows-x86-32.exe -------------------------------------------------------------------------------- /bin/optional/libwrapper-linux-x86-32.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/optional/libwrapper-linux-x86-32.so -------------------------------------------------------------------------------- /bin/optional/libwrapper-linux-x86-64.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/optional/libwrapper-linux-x86-64.so -------------------------------------------------------------------------------- /bin/optional/wrapper-windows-x86-32.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/optional/wrapper-windows-x86-32.dll -------------------------------------------------------------------------------- /bin/optional/libwrapper-macosx-universal-32.jnilib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/optional/libwrapper-macosx-universal-32.jnilib -------------------------------------------------------------------------------- /bin/optional/libwrapper-macosx-universal-64.jnilib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adyliu/jafka/HEAD/bin/optional/libwrapper-macosx-universal-64.jnilib -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .project 3 | .settings 4 | *.tmp 5 | target 6 | *.log 7 | build-dist.sh 8 | *.tgz 9 | /jafka-1.0* 10 | /data 11 | /build 12 | /docs 13 | /doc 14 | __pycache__ 15 | *.pyc 16 | *.swp 17 | /.idea 18 | *.iml 19 | pom-*.xml 20 | dist 21 | release 22 | -------------------------------------------------------------------------------- /clients/python/consume.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import jafka 4 | 5 | consumer = jafka.Consumer('localhost',9092) 6 | offsets = consumer.getoffsetsbefore('demo',0,-2,100) 7 | print('offsets',offsets) 8 | 9 | ms = consumer.fetch('demo',0,offsets[0],1024) 10 | for offset,message in ms: 11 | print(offset,message.decode('utf-8')) 12 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/consumer/IMessageListener.java: -------------------------------------------------------------------------------- 1 | package io.jafka.consumer; 2 | 3 | /** 4 | * Simple message callback 5 | * 6 | * @author adyliu (imxylz@gmail.com) 7 | * @since 1.8.0 8 | */ 9 | public interface IMessageListener { 10 | 11 | /** 12 | * consume the message 13 | *

14 | * This method should never throw any exception; otherwise this will interupt the stream 15 | *

16 | * 17 | * @param message message body 18 | */ 19 | void onMessage(T message); 20 | } -------------------------------------------------------------------------------- /src/main/java/io/jafka/log/RollingStrategy.java: -------------------------------------------------------------------------------- 1 | package io.jafka.log; 2 | 3 | import java.io.Closeable; 4 | 5 | 6 | /** 7 | * log rolling strategy 8 | * 9 | * @author adyliu (imxylz@gmail.com) 10 | * @since 1.0 11 | */ 12 | public interface RollingStrategy extends Closeable{ 13 | 14 | /** 15 | * the the last LogSegment whether needing rolling over 16 | * 17 | * @param lastSegment the last segment 18 | * @return true meaning rolling over and false doing nothing 19 | */ 20 | boolean check(LogSegment lastSegment); 21 | } 22 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/utils/TopicNameValidatorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package io.jafka.utils; 5 | 6 | import org.junit.Test; 7 | 8 | import static org.junit.Assert.*; 9 | 10 | /** 11 | * 12 | * @author adyliu(imxylz@gmail.com) 13 | * @since 2012-11-20 14 | */ 15 | public class TopicNameValidatorTest { 16 | 17 | /** 18 | * Test method for {@link TopicNameValidator#validate(java.lang.String)}. 19 | */ 20 | @Test 21 | public void testValidate() { 22 | final char[] badchars = {'/', '\u0000', '\u0001', '\u0018', '\u001F', '\u008F', '\uD805', '\uFFFA'}; 23 | for(char c:badchars) { 24 | final String badTopicName = "bad"+c+"topicname"; 25 | try { 26 | TopicNameValidator.validate(badTopicName); 27 | fail("topic name is illegal"); 28 | } catch (IllegalArgumentException e) { 29 | //ignore 30 | } 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/log/FixedSizeRollingStrategy.java: -------------------------------------------------------------------------------- 1 | package io.jafka.log; 2 | 3 | import static java.lang.String.format; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * This strategy will be rolling file while it reaches the max file size. 9 | * 10 | * @author adyliu (imxylz@gmail.com) 11 | * @since 1.0 12 | */ 13 | public class FixedSizeRollingStrategy implements RollingStrategy { 14 | 15 | private final int maxFileSize; 16 | 17 | public FixedSizeRollingStrategy(int maxFileSize) { 18 | this.maxFileSize = maxFileSize; 19 | } 20 | 21 | @Override 22 | public boolean check(LogSegment lastSegment) { 23 | return lastSegment.getMessageSet().getSizeInBytes() > maxFileSize; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return format("FixedSizeRollingStrategy [maxFileSize=%d bytes(%dMB)", maxFileSize, maxFileSize / (1024 * 1024)); 29 | } 30 | 31 | @Override 32 | public void close() throws IOException { 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/utils/TopicNameValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package io.jafka.utils; 5 | 6 | import java.util.regex.Pattern; 7 | 8 | /** 9 | * Topic name validator 10 | * 11 | * @author adyliu (imxylz@gmail.com) 12 | * @since 1.3 13 | */ 14 | public class TopicNameValidator { 15 | 16 | private static final String illegalChars = "/" + '\u0000' + '\u0001' + "-" + '\u001F' + '\u007F' + "-" + '\u009F' + '\uD800' + "-" + '\uF8FF' + '\uFFF0' 17 | + "-" + '\uFFFF'; 18 | private static final Pattern p = Pattern.compile("(^\\.{1,2}$)|[" + illegalChars + "]"); 19 | 20 | public static void validate(String topic) { 21 | if (topic.length() == 0) { 22 | throw new IllegalArgumentException("topic name is emtpy"); 23 | } 24 | if (topic.length() > 255) { 25 | throw new IllegalArgumentException("topic name is too long"); 26 | } 27 | if (p.matcher(topic).find()) { 28 | throw new IllegalArgumentException("topic name [" + topic + "] is illegal"); 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/IMBeanName.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public interface IMBeanName { 25 | 26 | String getMbeanName(); 27 | } 28 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Simple Install Guide 2 | ====== 3 | 4 | The package can be built on any Java enviroment. 5 | 6 | Require 7 | ====== 8 | 9 | * JDK 1.6+ (maybe works on JDK 1.5+ after changing the pom) 10 | * Maven 2.2+ (3.0+ better) 11 | 12 | Build 13 | ====== 14 | 15 | Build the source with maven: 16 | 17 | mvn clean package assembly:single -Dmaven.test.skip=true 18 | 19 | We recommend you to run the unit test. 20 | 21 | mvn clean package assembly:single 22 | 23 | Copy and unpack the package 'target/jafka-1.x.x-all.tar.gz' while 1.x.x is the version. 24 | 25 | Install & Upgrade 26 | ====== 27 | 28 | copy 'server.properties' and 'log4j.properties' from sample or previous installation: 29 | 30 | cp conf/server.properties.sample conf/server.properties 31 | cp conf/log4j.properties.sample conf/log4j.properties 32 | 33 | 34 | Run 35 | ====== 36 | 37 | Start the service with wrapper script: 38 | 39 | chmod +x bin/run.sh 40 | bin/run.sh console 41 | 42 | ---- 43 | 44 | Full install guide: https://github.com/adyliu/jafka/wiki/install 45 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/annotations/ServerSide.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common.annotations; 19 | 20 | 21 | /** 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public @interface ServerSide { 26 | 27 | } 28 | -------------------------------------------------------------------------------- /bin/env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | JVM_ARGS="" 18 | CLASSPATH="$CLASSPATH:${BIN_HOME}" 19 | for jar in `ls ${JAFKA_HOME}/lib/*.jar`;do 20 | CLASSPATH=$CLASSPATH:$jar 21 | done 22 | 23 | #echo "APP_HOME: $APP_HOME" 24 | #echo "CLASSPATH: $CLASSPATH" 25 | 26 | export CLASSPATH JVM_ARGS 27 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/annotations/ClientSide.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common.annotations; 19 | 20 | 21 | /** Client side code mark 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public @interface ClientSide { 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/log/LogSegmentFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.log; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public interface LogSegmentFilter { 25 | 26 | boolean filter(LogSegment segment); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/ConsumerTopicStatMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public interface ConsumerTopicStatMBean { 25 | 26 | long getMessagesPerTopic(); 27 | } 28 | -------------------------------------------------------------------------------- /conf/zookeeper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # the directory where the snapshot is stored. 18 | dataDir=/tmp/zookeeper 19 | # the port at which the clients will connect 20 | clientPort=2181 21 | # disable the per-ip limit on the number of connections since this is a non-production config 22 | maxClientCnxns=0 23 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/annotations/ThreadSafe.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common.annotations; 19 | 20 | /** 21 | * make an object thread-safe. 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public @interface ThreadSafe { 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/AsyncProducerQueueSizeStatsMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public interface AsyncProducerQueueSizeStatsMBean { 25 | 26 | int getAsyncProducerQueueSize(); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/annotations/NotThreadSafe.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common.annotations; 19 | 20 | 21 | /** 22 | * an annotation of thread-safe markup 23 | * 24 | * @author adyliu (imxylz@gmail.com) 25 | * @since 1.0 26 | */ 27 | public @interface NotThreadSafe { 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/serializer/DefaultEncoders.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.serializer; 19 | 20 | /** 21 | * byte array en/decoder 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.1 25 | */ 26 | public class DefaultEncoders extends ByteArrayEncoders { 27 | } 28 | -------------------------------------------------------------------------------- /bin/server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if readlink -f "$0" > /dev/null 2>&1 18 | then 19 | SERVER_BIN=`readlink -f "$0"` 20 | else 21 | SERVER_BIN="$0" 22 | fi 23 | BIN_HOME=`dirname ${SERVER_BIN}` 24 | JAFKA_HOME=`dirname ${BIN_HOME}` 25 | export JAFKA_HOME 26 | 27 | 28 | . ${BIN_HOME}/env.sh 29 | java $JVM_ARGS io.jafka.Jafka $* 30 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/api/PartitionChooser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.api; 19 | 20 | /** 21 | * Choose the partition index for the topic 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public interface PartitionChooser { 26 | 27 | int choosePartition(String topic); 28 | } 29 | -------------------------------------------------------------------------------- /bin/dumper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if readlink -f "$0" > /dev/null 2>&1 18 | then 19 | SERVER_BIN=`readlink -f "$0"` 20 | else 21 | SERVER_BIN="$0" 22 | fi 23 | BIN_HOME=`dirname ${SERVER_BIN}` 24 | JAFKA_HOME=`dirname ${BIN_HOME}` 25 | export JAFKA_HOME 26 | 27 | 28 | . ${BIN_HOME}/env.sh 29 | 30 | java io.jafka.console.Dumper $* 31 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/AsyncProducerStatsMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public interface AsyncProducerStatsMBean { 25 | 26 | int getAsyncProducerEvents(); 27 | 28 | int getAsyncProducerDroppedEvents(); 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/consumer/TopicEventHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.consumer; 19 | 20 | import java.util.List; 21 | 22 | 23 | /** 24 | * @author adyliu (imxylz@gmail.com) 25 | * @since 1.0 26 | */ 27 | public interface TopicEventHandler { 28 | 29 | void handleTopicEvent(List allTopics); 30 | } 31 | -------------------------------------------------------------------------------- /bin/admin-console.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if readlink -f "$0" > /dev/null 2>&1 18 | then 19 | SERVER_BIN=`readlink -f "$0"` 20 | else 21 | SERVER_BIN="$0" 22 | fi 23 | BIN_HOME=`dirname ${SERVER_BIN}` 24 | JAFKA_HOME=`dirname ${BIN_HOME}` 25 | export JAFKA_HOME 26 | 27 | 28 | . ${BIN_HOME}/env.sh 29 | 30 | java io.jafka.console.AdminConsole $* 31 | -------------------------------------------------------------------------------- /bin/consumer-console.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if readlink -f "$0" > /dev/null 2>&1 18 | then 19 | SERVER_BIN=`readlink -f "$0"` 20 | else 21 | SERVER_BIN="$0" 22 | fi 23 | BIN_HOME=`dirname ${SERVER_BIN}` 24 | JAFKA_HOME=`dirname ${BIN_HOME}` 25 | export JAFKA_HOME 26 | 27 | 28 | . ${BIN_HOME}/env.sh 29 | 30 | java io.jafka.console.ConsoleConsumer $* 31 | -------------------------------------------------------------------------------- /bin/getoffset-console.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if readlink -f "$0" > /dev/null 2>&1 18 | then 19 | SERVER_BIN=`readlink -f "$0"` 20 | else 21 | SERVER_BIN="$0" 22 | fi 23 | BIN_HOME=`dirname ${SERVER_BIN}` 24 | JAFKA_HOME=`dirname ${BIN_HOME}` 25 | export JAFKA_HOME 26 | 27 | 28 | . ${BIN_HOME}/env.sh 29 | 30 | java io.jafka.console.GetOffsetShell $* 31 | -------------------------------------------------------------------------------- /bin/producer-console.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if readlink -f "$0" > /dev/null 2>&1 18 | then 19 | SERVER_BIN=`readlink -f "$0"` 20 | else 21 | SERVER_BIN="$0" 22 | fi 23 | BIN_HOME=`dirname ${SERVER_BIN}` 24 | JAFKA_HOME=`dirname ${BIN_HOME}` 25 | export JAFKA_HOME 26 | 27 | 28 | . ${BIN_HOME}/env.sh 29 | 30 | java io.jafka.console.ConsoleProducer $* 31 | -------------------------------------------------------------------------------- /bin/simple-consumer-console.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if readlink -f "$0" > /dev/null 2>&1 18 | then 19 | SERVER_BIN=`readlink -f "$0"` 20 | else 21 | SERVER_BIN="$0" 22 | fi 23 | BIN_HOME=`dirname ${SERVER_BIN}` 24 | JAFKA_HOME=`dirname ${BIN_HOME}` 25 | export JAFKA_HOME 26 | 27 | 28 | . ${BIN_HOME}/env.sh 29 | 30 | java io.jafka.console.SimipleConsoleConsumer $* 31 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/async/AbstractCallbackHandler.java: -------------------------------------------------------------------------------- 1 | package io.jafka.producer.async; 2 | 3 | import java.util.Collections; 4 | import java.util.List; 5 | import java.util.Properties; 6 | 7 | /** 8 | * Empty handler 9 | * 10 | * @author adyliu (imxylz@gmail.com) 11 | * @since 1.4.0 12 | */ 13 | public abstract class AbstractCallbackHandler implements CallbackHandler { 14 | @Override 15 | public void init(Properties properties) { 16 | } 17 | 18 | @Override 19 | public QueueItem beforeEnqueue(QueueItem data) { 20 | return data; 21 | } 22 | 23 | @Override 24 | public QueueItem afterEnqueue(QueueItem data, boolean added) { 25 | return data; 26 | } 27 | 28 | @Override 29 | public List> afterDequeuingExistingData(QueueItem data) { 30 | return Collections.emptyList(); 31 | } 32 | 33 | @Override 34 | public List> beforeSendingData(List> data) { 35 | return data; 36 | } 37 | 38 | @Override 39 | public List> lastBatchBeforeClose() { 40 | return Collections.emptyList(); 41 | } 42 | 43 | @Override 44 | public void close() { 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/PortUtils.java: -------------------------------------------------------------------------------- 1 | package io.jafka; 2 | 3 | import io.jafka.utils.Closer; 4 | 5 | import java.io.IOException; 6 | import java.net.ServerSocket; 7 | 8 | /** 9 | * @author adyliu (imxylz@gmail.com) 10 | * @since 2013-04-25 11 | */ 12 | public class PortUtils { 13 | 14 | /** 15 | * check available port on the machine
16 | * This will check the next port if the port is already used. 17 | * @param port the checking port 18 | * @return a available port 19 | */ 20 | public static int checkAvailablePort(int port) { 21 | while (port < 65500) { 22 | ServerSocket serverSocket = null; 23 | try { 24 | serverSocket = new ServerSocket(port); 25 | return port; 26 | } catch (IOException e) { 27 | //ignore error 28 | } finally { 29 | Closer.closeQuietly(serverSocket); 30 | } 31 | port++; 32 | } 33 | throw new RuntimeException("no available port"); 34 | } 35 | 36 | public static void main(String[] args) { 37 | int port = checkAvailablePort(80); 38 | System.out.println("The available port is " + port); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/ServerInfoMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * Server information 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.1 24 | */ 25 | public interface ServerInfoMBean { 26 | 27 | String getVersion(); 28 | 29 | String getStartupTime(); 30 | 31 | String getStartedTime(); 32 | 33 | String getRunningTime(); 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/serializer/Decoder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.serializer; 19 | 20 | import io.jafka.message.Message; 21 | 22 | /** 23 | * convert Message to <Object> 24 | * 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | * @see Message 28 | * @see Encoder 29 | */ 30 | public interface Decoder { 31 | 32 | T toEvent(Message message); 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/serializer/Encoder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.serializer; 19 | 20 | import io.jafka.message.Message; 21 | 22 | /** 23 | * convert Object<T> to message 24 | * 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | * @see Message 28 | * @see Decoder 29 | */ 30 | public interface Encoder { 31 | 32 | Message toMessage(T event); 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/BrokerTopicStatMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public interface BrokerTopicStatMBean { 25 | 26 | long getMessagesIn(); 27 | 28 | long getBytesIn(); 29 | 30 | long getBytesOut(); 31 | 32 | long getFailedProduceRequest(); 33 | 34 | long getFailedFetchRequest(); 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/LogFlushStatsMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | 21 | /** 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public interface LogFlushStatsMBean { 26 | 27 | double getFlushesPerSecond(); 28 | 29 | double getAvgFlushMs(); 30 | 31 | long getTotalFlushMs(); 32 | 33 | double getMaxFlushMs(); 34 | 35 | long getNumFlushes(); 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/api/ICalculable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.api; 19 | 20 | /** 21 | * Mark a calculable object(request/message/data...) 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public interface ICalculable { 27 | 28 | /** 29 | * get the size of current object(in bytes) 30 | * 31 | * @return the object size 32 | */ 33 | int getSizeInBytes(); 34 | } 35 | -------------------------------------------------------------------------------- /bin/zookeeper-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [ $# -ne 1 ]; 18 | then 19 | echo "USAGE: $0 config/zookeeper.properties" 20 | exit 1 21 | fi 22 | 23 | if readlink -f "$0" > /dev/null 2>&1 24 | then 25 | SERVER_BIN=`readlink -f "$0"` 26 | else 27 | SERVER_BIN="$0" 28 | fi 29 | BIN_HOME=`dirname ${SERVER_BIN}` 30 | JAFKA_HOME=`dirname ${BIN_HOME}` 31 | export JAFKA_HOME 32 | 33 | 34 | . ${BIN_HOME}/env.sh 35 | 36 | java org.apache.zookeeper.server.quorum.QuorumPeerMain $@ 37 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/SyncProducerStatsMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * file{producer/SyncProducer.scala#SyncProducerStatsMBean} 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public interface SyncProducerStatsMBean { 27 | 28 | double getProduceRequestsPerSecond(); 29 | 30 | double getAvgProduceRequestMs(); 31 | 32 | double getMaxProduceRequestMs(); 33 | 34 | long getNumProduceRequests(); 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/console/MessageReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.console; 19 | 20 | import java.io.Closeable; 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | import java.util.Properties; 24 | 25 | /** 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | public interface MessageReader extends Closeable { 30 | 31 | void init(InputStream inputStream, Properties props); 32 | 33 | String readMessage() throws IOException; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/Send.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | import java.io.IOException; 21 | 22 | /** 23 | * A transmission that is being sent out to the channel 24 | * @author adyliu (imxylz@gmail.com) 25 | * @since 1.0 26 | */ 27 | public interface Send extends Transmission { 28 | 29 | int writeTo(java.nio.channels.GatheringByteChannel channel) throws IOException; 30 | 31 | int writeCompletely(java.nio.channels.GatheringByteChannel channel) throws IOException; 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/console/MessageFormatter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.console; 19 | 20 | import java.io.Closeable; 21 | import java.io.PrintStream; 22 | import java.util.Properties; 23 | 24 | import io.jafka.message.Message; 25 | 26 | /** 27 | * @author adyliu (imxylz@gmail.com) 28 | * @since 1.0 29 | */ 30 | public interface MessageFormatter extends Closeable { 31 | 32 | void writeTo(Message message, PrintStream output); 33 | 34 | void init(Properties props); 35 | 36 | void close(); 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/MessageSizeTooLargeException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public class MessageSizeTooLargeException extends RuntimeException { 26 | 27 | private static final long serialVersionUID = 1L; 28 | 29 | public MessageSizeTooLargeException() { 30 | super(); 31 | } 32 | 33 | public MessageSizeTooLargeException(String message) { 34 | super(message); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/Receive.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | import java.io.IOException; 21 | import java.nio.ByteBuffer; 22 | import java.nio.channels.ReadableByteChannel; 23 | 24 | /** 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public interface Receive extends Transmission { 29 | 30 | ByteBuffer buffer(); 31 | 32 | int readFrom(ReadableByteChannel channel) throws IOException; 33 | 34 | int readCompletely(ReadableByteChannel channel) throws IOException; 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/serializer/StringEncoder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.serializer; 19 | 20 | import io.jafka.message.Message; 21 | import io.jafka.utils.Utils; 22 | 23 | /** 24 | * UTF-8 bytes encoder 25 | * 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | * @see StringDecoder 29 | */ 30 | public class StringEncoder implements Encoder { 31 | 32 | public Message toMessage(String event) { 33 | return new Message(Utils.getBytes(event, "UTF-8")); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/message/MessageLengthException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.message; 19 | 20 | /** 21 | * Indicates the presense of a message that exceeds the maximum acceptable 22 | * length (whatever that happens to be) 23 | * 24 | * @author adyliu (imxylz@gmail.com) 25 | * @since 1.0 26 | */ 27 | public class MessageLengthException extends RuntimeException { 28 | 29 | private static final long serialVersionUID = 1L; 30 | 31 | public MessageLengthException(String message) { 32 | super(message); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/InvalidConfigException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | 21 | /** 22 | * Indicates that the given config parameter has invalid value 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class InvalidConfigException extends RuntimeException { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | public InvalidConfigException() { 31 | super(); 32 | } 33 | 34 | public InvalidConfigException(String message) { 35 | super(message); 36 | } 37 | 38 | } -------------------------------------------------------------------------------- /src/main/java/io/jafka/message/InvalidMessageException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.message; 19 | 20 | 21 | /** 22 | * Indicates that a message failed its checksum and is corrupt 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class InvalidMessageException extends RuntimeException { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | public InvalidMessageException() { 31 | super(); 32 | } 33 | 34 | public InvalidMessageException(String message) { 35 | super(message); 36 | } 37 | } -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/IStringProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer; 19 | 20 | import io.jafka.common.InvalidPartitionException; 21 | import io.jafka.common.NoBrokersForPartitionException; 22 | 23 | /** 24 | * utf-8 string producer 25 | * 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.2 28 | */ 29 | public interface IStringProducer extends IProducer { 30 | 31 | @Override 32 | public void send(ProducerData data) throws NoBrokersForPartitionException, 33 | InvalidPartitionException; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/UnKnownCodecException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * Indicates the client has requested a range no longer available on the server 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class UnKnownCodecException extends RuntimeException { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | public UnKnownCodecException() { 31 | super(); 32 | } 33 | 34 | public UnKnownCodecException(String message) { 35 | super(message); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/RequestHandlerFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | import io.jafka.api.RequestKeys; 21 | 22 | /** 23 | * factory for creating request handler 24 | * 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public interface RequestHandlerFactory { 29 | 30 | /** 31 | * mapping RequestHandler for request 32 | * 33 | * @param id request type 34 | * @param request request body 35 | * @return handler for the request 36 | */ 37 | RequestHandler mapping(RequestKeys id, Receive request); 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/UnknownMagicByteException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | 21 | /** 22 | * Indicates the client has requested a range no longer available on the server 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class UnknownMagicByteException extends RuntimeException { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | public UnknownMagicByteException() { 31 | super(); 32 | } 33 | 34 | public UnknownMagicByteException(String message) { 35 | super(message); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | log4j.rootLogger=INFO,stdout 16 | 17 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 18 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 19 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%C.java:%L)%n 20 | 21 | #log4j.logger.jafka.request.logger=DEBUG 22 | log4j.logger.org.apache.zookeeper=WARN 23 | log4j.logger.com.github.zkclient=WARN 24 | #log4j.logger.io.jafka.producer=DEBUG 25 | #log4j.logger.Log=DEBUG 26 | #log4j.logger.io.jafka=DEBUG 27 | #log4j.logger.io.jafka.log=DEBUG 28 | #log4j.logger.ZookeeperConsumerConnector=DEBUG 29 | #log4j.logger.ProducerPool=DEBUG -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/InvalidMessageSizeException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | 21 | /** 22 | * Indicates the client has requested a range no longer available on the server 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class InvalidMessageSizeException extends RuntimeException { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | public InvalidMessageSizeException() { 31 | super(); 32 | } 33 | 34 | public InvalidMessageSizeException(String message) { 35 | super(message); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/OffsetOutOfRangeException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * Indicates the client has requested a range no longer available on the 22 | * server 23 | * 24 | * @author adyliu (imxylz@gmail.com) 25 | * @since 1.0 26 | */ 27 | public class OffsetOutOfRangeException extends RuntimeException { 28 | 29 | private static final long serialVersionUID = 1L; 30 | 31 | public OffsetOutOfRangeException() { 32 | super(); 33 | } 34 | 35 | public OffsetOutOfRangeException(String message) { 36 | super(message); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/NoBrokersForPartitionException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | 21 | /**Thrown when a request is made for broker but no brokers with that topic 22 | * exist. 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class NoBrokersForPartitionException extends RuntimeException { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | public NoBrokersForPartitionException() { 31 | super(); 32 | } 33 | 34 | public NoBrokersForPartitionException(String message) { 35 | super(message); 36 | } 37 | 38 | } -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/AbstractSend.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | import java.io.IOException; 21 | import java.nio.channels.GatheringByteChannel; 22 | 23 | 24 | /** 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public abstract class AbstractSend extends AbstractTransmission implements Send { 29 | 30 | public int writeCompletely(GatheringByteChannel channel) throws IOException { 31 | int written = 0; 32 | while(!complete()) { 33 | written += writeTo(channel); 34 | } 35 | return written; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/ConsumerRebalanceFailedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * Thrown when a request is made for broker but no brokers with that topic 22 | * exist. 23 | * 24 | * @author adyliu (imxylz@gmail.com) 25 | * @since 1.0 26 | */ 27 | public class ConsumerRebalanceFailedException extends RuntimeException { 28 | 29 | private static final long serialVersionUID = 1L; 30 | 31 | public ConsumerRebalanceFailedException() { 32 | } 33 | 34 | public ConsumerRebalanceFailedException(String message) { 35 | super(message); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/RequestHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | import io.jafka.api.RequestKeys; 21 | import io.jafka.common.annotations.ServerSide; 22 | 23 | /** 24 | * handle request from client 25 | * 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | @ServerSide 30 | public interface RequestHandler { 31 | 32 | /** 33 | * handling the request 34 | * 35 | * @param requestType request type 36 | * @param request request body 37 | * @return handling response 38 | */ 39 | Send handler(RequestKeys requestType, Receive request); 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/Partitioner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer; 19 | 20 | /** 21 | * some messages to special broker(server) 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public interface Partitioner { 27 | 28 | /** 29 | * Uses the key to calculate a partition bucket id for routing the data to the appropriate 30 | * broker partition 31 | * @param key key of partition name 32 | * @param numPartitions number of paritition 33 | * @return an integer between 0 and numPartitions-1 34 | */ 35 | int partition(T key, int numPartitions); 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/serializer/MessageEncoders.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.serializer; 19 | 20 | import io.jafka.message.Message; 21 | 22 | /** 23 | * {@link Message} en/decoder(nothing to do) 24 | * 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.1 27 | * @see Message 28 | */ 29 | public class MessageEncoders implements Encoder, Decoder { 30 | 31 | @Override 32 | public Message toMessage(Message event) { 33 | return event; 34 | } 35 | 36 | @Override 37 | public Message toEvent(Message message) { 38 | return message; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/Transmission.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | /** 21 | * transmission interface 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public interface Transmission { 26 | /** 27 | * expect task has not been finished othewise throwing an exception 28 | */ 29 | void expectIncomplete(); 30 | /** 31 | * expect task has been finished othewise throwing an exception 32 | */ 33 | void expectComplete(); 34 | /** 35 | * check task whether it has been finished 36 | * @return true while finished or false 37 | */ 38 | boolean complete(); 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/DefaultPartitioner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer; 19 | 20 | 21 | import java.util.Random; 22 | 23 | import io.jafka.common.annotations.ClientSide; 24 | 25 | /** 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | @ClientSide 30 | public class DefaultPartitioner implements Partitioner { 31 | 32 | private final Random random = new Random(); 33 | 34 | public int partition(T key, int numPartitions) { 35 | if (key == null) { 36 | return random.nextInt(numPartitions); 37 | } 38 | return Math.abs(key.hashCode()) % numPartitions; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/async/AsyncProducerConfigShared.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.async; 19 | 20 | import java.util.Properties; 21 | 22 | /** 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public interface AsyncProducerConfigShared { 27 | 28 | Properties getProperties(); 29 | 30 | int getQueueTime(); 31 | 32 | int getQueueSize(); 33 | 34 | int getEnqueueTimeoutMs(); 35 | 36 | int getBatchSize(); 37 | 38 | String getCbkHandler(); 39 | 40 | Properties getCbkHandlerProperties(); 41 | 42 | String getEventHandler(); 43 | 44 | Properties getEventHandlerProperties(); 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/serializer/StringDecoder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.serializer; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import io.jafka.message.Message; 23 | import io.jafka.utils.Utils; 24 | 25 | /** 26 | * UTF-8 bytes decoder 27 | * 28 | * @author adyliu (imxylz@gmail.com) 29 | * @since 1.0 30 | * @see StringEncoder 31 | */ 32 | public class StringDecoder implements Decoder { 33 | 34 | public String toEvent(Message message) { 35 | ByteBuffer buf = message.payload(); 36 | byte[] b = new byte[buf.remaining()]; 37 | buf.get(b); 38 | return Utils.fromBytes(b); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/QueueFullException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public class QueueFullException extends RuntimeException { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | public QueueFullException() { 29 | } 30 | 31 | public QueueFullException(String message) { 32 | super(message); 33 | } 34 | 35 | public QueueFullException(Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | public QueueFullException(String message, Throwable cause) { 40 | super(message, cause); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/LogStatsMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * a partition topic status 22 | *

23 | * For the last writeable file: 24 | *

    25 | *
  • size
    • 26 | *
27 | * 28 | * @author adyliu (imxylz@gmail.com) 29 | * @since 1.0 30 | */ 31 | public interface LogStatsMBean { 32 | 33 | String getName(); 34 | 35 | String getLastFlushedTime(); 36 | 37 | int getSegmentNum(); 38 | 39 | long getStartingAppendedMessagesNum(); 40 | 41 | long getLastSegmentSize(); 42 | 43 | long getLastSegmentAddressingSize(); 44 | 45 | long getTotalSegmentSize(); 46 | 47 | long getTotalOffset(); 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/consumer/FetchedDataChunk.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.consumer; 19 | 20 | import io.jafka.message.ByteBufferMessageSet; 21 | 22 | /** 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class FetchedDataChunk { 27 | 28 | public final ByteBufferMessageSet messages; 29 | 30 | public final PartitionTopicInfo topicInfo; 31 | 32 | public final long fetchOffset; 33 | 34 | public FetchedDataChunk(ByteBufferMessageSet messages, PartitionTopicInfo topicInfo, long fetchOffset) { 35 | super(); 36 | this.messages = messages; 37 | this.topicInfo = topicInfo; 38 | this.fetchOffset = fetchOffset; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/handlers/AbstractHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network.handlers; 19 | 20 | 21 | import io.jafka.log.LogManager; 22 | import io.jafka.network.RequestHandler; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * the base request handler 28 | * 29 | * @author adyliu (imxylz@gmail.com) 30 | * @since 1.0 31 | */ 32 | public abstract class AbstractHandler implements RequestHandler { 33 | 34 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 35 | 36 | protected final LogManager logManager; 37 | 38 | public AbstractHandler(LogManager logManager) { 39 | this.logManager = logManager; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/async/QueueItem.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.async; 19 | 20 | /** 21 | * 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public class QueueItem { 26 | 27 | public final T data; 28 | 29 | public final int partition; 30 | 31 | public final String topic; 32 | 33 | public QueueItem(T data, int partition, String topic) { 34 | super(); 35 | this.data = data; 36 | this.partition = partition; 37 | this.topic = topic; 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return String.format("QueueItem [data=%s, partition=%s, topic=%s]", data, partition, topic); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/InvalidPartitionException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * Indicates that the partition id is not between 0 and numPartitions-1 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class InvalidPartitionException extends ErrorMappingException { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | public InvalidPartitionException() { 31 | super(); 32 | } 33 | 34 | public InvalidPartitionException(String message) { 35 | super(message); 36 | } 37 | 38 | @Override 39 | public ErrorMapping getErrorMapping() { 40 | return ErrorMapping.WrongPartitionCode; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/InvalidRequestException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public class InvalidRequestException extends RuntimeException { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | public InvalidRequestException() { 29 | super(); 30 | } 31 | 32 | public InvalidRequestException(String message, Throwable cause) { 33 | super(message, cause); 34 | } 35 | 36 | public InvalidRequestException(String message) { 37 | super(message); 38 | } 39 | 40 | public InvalidRequestException(Throwable cause) { 41 | super(cause); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/message/MessageAndOffset.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.message; 19 | 20 | /** 21 | * Represents message and offset of the next message. This is used in the 22 | * MessageSet to iterate over it 23 | * 24 | * @author adyliu (imxylz@gmail.com) 25 | * @since 1.0 26 | */ 27 | public class MessageAndOffset { 28 | 29 | public final Message message; 30 | 31 | public final long offset; 32 | 33 | public MessageAndOffset(Message message, long offset) { 34 | this.message = message; 35 | this.offset = offset; 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return String.format("MessageAndOffset [offset=%s, message=%s]", offset, message); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/ConsumerTimeoutException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | 21 | /** 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public class ConsumerTimeoutException extends RuntimeException { 26 | 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | 31 | public ConsumerTimeoutException() { 32 | } 33 | 34 | 35 | public ConsumerTimeoutException(String message) { 36 | super(message); 37 | } 38 | 39 | 40 | public ConsumerTimeoutException(Throwable cause) { 41 | super(cause); 42 | } 43 | 44 | public ConsumerTimeoutException(String message, Throwable cause) { 45 | super(message, cause); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/UnavailableProducerException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public class UnavailableProducerException extends RuntimeException { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | public UnavailableProducerException() { 29 | } 30 | 31 | 32 | public UnavailableProducerException(String message) { 33 | super(message); 34 | } 35 | 36 | public UnavailableProducerException(Throwable cause) { 37 | super(cause); 38 | } 39 | 40 | public UnavailableProducerException(String message, Throwable cause) { 41 | super(message, cause); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/UnkonwnException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | 21 | /** 22 | * If we don't know what else it is, call it this 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class UnkonwnException extends RuntimeException{ 27 | private static final long serialVersionUID = 1L; 28 | 29 | public UnkonwnException() { 30 | super(); 31 | } 32 | 33 | public UnkonwnException(String message, Throwable cause) { 34 | super(message, cause); 35 | } 36 | 37 | public UnkonwnException(String message) { 38 | super(message); 39 | } 40 | 41 | public UnkonwnException(Throwable cause) { 42 | super(cause); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/IllegalQueueStateException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | 21 | /** 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.0 24 | */ 25 | public class IllegalQueueStateException extends RuntimeException{ 26 | 27 | private static final long serialVersionUID = 1L; 28 | 29 | public IllegalQueueStateException() { 30 | super(); 31 | } 32 | 33 | public IllegalQueueStateException(String message, Throwable cause) { 34 | super(message, cause); 35 | } 36 | 37 | public IllegalQueueStateException(String message) { 38 | super(message); 39 | } 40 | 41 | public IllegalQueueStateException(Throwable cause) { 42 | super(cause); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/serializer/ByteArrayEncoders.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.serializer; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import io.jafka.message.Message; 23 | 24 | /** 25 | * a bytes en/decoder 26 | * 27 | * @author adyliu (imxylz@gmail.com) 28 | * @since 1.1 29 | */ 30 | public class ByteArrayEncoders implements Encoder, Decoder { 31 | 32 | @Override 33 | public Message toMessage(byte[] event) { 34 | return new Message(event); 35 | } 36 | 37 | @Override 38 | public byte[] toEvent(Message message) { 39 | ByteBuffer buf = message.payload(); 40 | byte[] b = new byte[buf.remaining()]; 41 | buf.get(b); 42 | return b; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/message/compress/GZIPCompression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.message.compress; 19 | 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | import java.util.zip.GZIPInputStream; 24 | import java.util.zip.GZIPOutputStream; 25 | 26 | /** 27 | * @author adyliu (imxylz@gmail.com) 28 | * @since 1.0 29 | */ 30 | public class GZIPCompression extends CompressionFacade { 31 | 32 | public GZIPCompression(InputStream inputStream, ByteArrayOutputStream outputStream) throws IOException { 33 | super(inputStream != null ? new GZIPInputStream(inputStream) : null, // 34 | outputStream != null ? new GZIPOutputStream(outputStream) : null); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/QueueClosedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * Indicates that client is sending event to a closed queue 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class QueueClosedException extends RuntimeException { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | public QueueClosedException() { 31 | } 32 | 33 | public QueueClosedException(String message) { 34 | super(message); 35 | } 36 | 37 | public QueueClosedException(Throwable cause) { 38 | super(cause); 39 | } 40 | 41 | public QueueClosedException(String message, Throwable cause) { 42 | super(message, cause); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/console/NewlineMessageFormatter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.console; 19 | 20 | import java.io.PrintStream; 21 | import java.nio.ByteBuffer; 22 | import java.util.Properties; 23 | 24 | import io.jafka.message.Message; 25 | 26 | /** 27 | * @author adyliu (imxylz@gmail.com) 28 | * @since 1.0 29 | */ 30 | public class NewlineMessageFormatter implements MessageFormatter { 31 | 32 | public void writeTo(Message message, PrintStream output) { 33 | ByteBuffer buffer = message.payload(); 34 | output.write(buffer.array(), buffer.arrayOffset(), buffer.limit()); 35 | output.write('\n'); 36 | } 37 | 38 | public void init(Properties props) { 39 | } 40 | 41 | public void close() { 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/server/TopicTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.server; 19 | 20 | /** 21 | * Task for topic processor(create/delete/enlarge/shutdown/...) 22 | * @author adyliu (imxylz@gmail.com) 23 | * @since 1.2 24 | */ 25 | public class TopicTask { 26 | 27 | public static enum TaskType { 28 | CREATE, // 29 | DELETE, // 30 | ENLARGE, // 31 | SHUTDOWN; 32 | } 33 | 34 | public final String topic; 35 | 36 | public final TaskType type; 37 | 38 | public TopicTask(TaskType type, String topic) { 39 | this.type = type; 40 | this.topic = topic; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return "TopicTask [type=" + type + ", topic=" + topic + "]"; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/utils/zookeeper/ZkGroupDirsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.utils.zookeeper; 19 | 20 | import static org.junit.Assert.*; 21 | 22 | import org.junit.Test; 23 | 24 | /** 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public class ZkGroupDirsTest { 29 | 30 | /** 31 | * Test method for 32 | * {@link ZkGroupDirs#ZkGroupDirs(java.lang.String)} 33 | * . 34 | */ 35 | @Test 36 | public void testZkGroupDirs() { 37 | ZkGroupDirs dirs = new ZkGroupDirs("demo"); 38 | assertEquals("demo", dirs.group); 39 | assertEquals(ZkUtils.ConsumersPath + "/demo", dirs.consumerGroupDir); 40 | assertEquals(ZkUtils.ConsumersPath + "/demo/ids", dirs.consumerRegistryDir); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/AsyncProducerInterruptedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public class AsyncProducerInterruptedException extends RuntimeException { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | public AsyncProducerInterruptedException() { 29 | super(); 30 | } 31 | 32 | public AsyncProducerInterruptedException(String message, Throwable cause) { 33 | super(message, cause); 34 | } 35 | 36 | public AsyncProducerInterruptedException(String message) { 37 | super(message); 38 | } 39 | 40 | public AsyncProducerInterruptedException(Throwable cause) { 41 | super(cause); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/common/ErrorMappingException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.1 23 | */ 24 | public class ErrorMappingException extends RuntimeException { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | public ErrorMappingException() { 29 | } 30 | 31 | public ErrorMappingException(String message) { 32 | super(message); 33 | } 34 | 35 | public ErrorMappingException(Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | public ErrorMappingException(String message, Throwable cause) { 40 | super(message, cause); 41 | } 42 | 43 | public ErrorMapping getErrorMapping() { 44 | return ErrorMapping.UnkonwCode; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/console/ChecksumMessageFormatter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.console; 19 | 20 | import java.io.PrintStream; 21 | import java.util.Properties; 22 | 23 | import io.jafka.message.Message; 24 | 25 | /** 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | public class ChecksumMessageFormatter implements MessageFormatter { 30 | 31 | private String topicStr = null; 32 | 33 | public void writeTo(Message message, PrintStream output) { 34 | output.println(topicStr + "checksum: " + message.checksum()); 35 | } 36 | 37 | public void init(Properties props) { 38 | topicStr = props.getProperty("topic"); 39 | topicStr = topicStr == null ? "" : topicStr + "-"; 40 | } 41 | 42 | public void close() { 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 4 | all 5 | 6 | tar.gz 7 | 8 | /${artifactId}-${version} 9 | 10 | 11 | LICENSE 12 | / 13 | 14 | 15 | VERSION 16 | / 17 | true 18 | 19 | 20 | 21 | 22 | bin 23 | /bin 24 | 25 | 26 | conf 27 | /conf 28 | 29 | 30 | src/main/assembly 31 | /logs 32 | 33 | * 34 | 35 | 36 | 37 | 38 | 39 | /lib 40 | 41 | mx4j:mx4j-tools 42 | 43 | 44 | 45 | /lib/optional 46 | 47 | mx4j:mx4j-tools 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /conf/log4j.properties.sample: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | log4j.rootLogger=INFO,stdout 17 | 18 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 19 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 20 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p %C{1} - %m%n 21 | 22 | log4j.appender.file=org.apache.log4j.FileAppender 23 | log4j.appender.file.File=logs/jafka.log 24 | log4j.appender.file.Encoding=UTF-8 25 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 26 | log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p %C{1} - %m%n 27 | 28 | 29 | 30 | #log4j.logger.jafka.request.logger=DEBUG 31 | log4j.logger.org.apache.zookeeper.ZooKeeper=WARN 32 | #log4j.logger.io.jafka.producer=DEBUG 33 | #log4j.logger.Log=DEBUG 34 | #log4j.logger.io.jafka=DEBUG 35 | #log4j.logger.io.jafka.log=DEBUG 36 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/utils/zookeeper/ZkGroupTopicDirsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.utils.zookeeper; 19 | 20 | import static org.junit.Assert.*; 21 | 22 | import org.junit.Test; 23 | 24 | 25 | /** 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | public class ZkGroupTopicDirsTest { 30 | 31 | /** 32 | * Test method for {@link ZkGroupTopicDirs#ZkGroupTopicDirs(java.lang.String, java.lang.String)}. 33 | */ 34 | @Test 35 | public void testZkGroupTopicDirs() { 36 | ZkGroupTopicDirs dirs = new ZkGroupTopicDirs("demo", "topic1"); 37 | assertEquals("topic1", dirs.topic); 38 | assertEquals(ZkUtils.ConsumersPath+"/demo/offsets/topic1", dirs.consumerOffsetDir); 39 | assertEquals(ZkUtils.ConsumersPath+"/demo/owners/topic1", dirs.consumerOwnerDir); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/utils/Range.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.utils; 19 | 20 | /** 21 | * A generic range value with a start and end 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public interface Range { 27 | 28 | /** The first index in the range 29 | * @return range of start 30 | */ 31 | long start(); 32 | 33 | /** The total number of indexes in the range 34 | * @return size of range 35 | */ 36 | long size(); 37 | 38 | /** Return true iff the range is empty 39 | * @return check the range is emtpy 40 | */ 41 | boolean isEmpty(); 42 | 43 | /** if value is in range 44 | * @param value value for check 45 | * @return check the value in range 46 | */ 47 | boolean contains(long value); 48 | 49 | String toString(); 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/Request.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import io.jafka.api.ICalculable; 23 | import io.jafka.api.RequestKeys; 24 | import io.jafka.common.annotations.ClientSide; 25 | import io.jafka.common.annotations.ServerSide; 26 | 27 | /** 28 | * Client-Server communication 29 | * 30 | * @author adyliu (imxylz@gmail.com) 31 | * @since 1.0 32 | */ 33 | @ClientSide 34 | @ServerSide 35 | public interface Request extends ICalculable { 36 | 37 | /** 38 | * request type 39 | * 40 | * @return request type 41 | * @see RequestKeys 42 | */ 43 | RequestKeys getRequestKey(); 44 | 45 | /** 46 | * write the request data to buffer 47 | * 48 | * @param buffer data buffer 49 | */ 50 | void writeTo(ByteBuffer buffer); 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/utils/KVTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.utils; 19 | 20 | import static org.junit.Assert.*; 21 | 22 | import org.junit.Test; 23 | 24 | 25 | /** 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 28 | */ 29 | public class KVTest { 30 | 31 | /** 32 | * Test method for {@link KV#KV(java.lang.Object, java.lang.Object)}. 33 | */ 34 | @Test 35 | public void testKV() { 36 | KV kv = new KV("demo",100L); 37 | assertEquals("demo", kv.k); 38 | assertEquals(100L, kv.v.longValue()); 39 | } 40 | 41 | /** 42 | * Test method for {@link KV#equals(java.lang.Object)}. 43 | */ 44 | @Test 45 | public void testEqualsObject() { 46 | assertEquals(new KV("demo",100L),new KV("demo",100L)); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/SocketServerStatsMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | /** 21 | * @author adyliu (imxylz@gmail.com) 22 | * @since 1.0 23 | */ 24 | public interface SocketServerStatsMBean { 25 | 26 | double getProduceRequestsPerSecond(); 27 | 28 | double getFetchRequestsPerSecond(); 29 | 30 | double getAvgProduceRequestMs(); 31 | 32 | double getMaxProduceRequestMs(); 33 | 34 | double getAvgFetchRequestMs(); 35 | 36 | double getMaxFetchRequestMs(); 37 | 38 | double getBytesReadPerSecond(); 39 | 40 | double getBytesWrittenPerSecond(); 41 | 42 | long getNumFetchRequests(); 43 | 44 | long getNumProduceRequests(); 45 | 46 | long getTotalBytesRead(); 47 | 48 | long getTotalBytesWritten(); 49 | 50 | long getTotalFetchRequestMs(); 51 | 52 | long getTotalProduceRequestMs(); 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/TestUtil.java: -------------------------------------------------------------------------------- 1 | package io.jafka; 2 | 3 | import org.junit.Ignore; 4 | 5 | import java.util.concurrent.Callable; 6 | import java.util.concurrent.TimeUnit; 7 | 8 | @Ignore 9 | public class TestUtil { 10 | 11 | static { 12 | System.setProperty("zookeeper.preAllocSize", "1024");//1M data log 13 | } 14 | /** 15 | * This waits until the provided {@link Callable} returns an object that is equals to the given expected value or 16 | * the timeout has been reached. In both cases this method will return the return value of the latest 17 | * {@link Callable} execution. 18 | * 19 | * @param expectedValue 20 | * The expected value of the callable. 21 | * @param callable 22 | * The callable. 23 | * @param 24 | * The return type of the callable. 25 | * @param timeUnit 26 | * The timeout timeunit. 27 | * @param timeout 28 | * The timeout. 29 | * @return the return value of the latest {@link Callable} execution. 30 | * @throws Exception 31 | * @throws InterruptedException 32 | */ 33 | public static T waitUntil(T expectedValue, Callable callable, TimeUnit timeUnit, long timeout) throws Exception { 34 | long startTime = System.currentTimeMillis(); 35 | do { 36 | T actual = callable.call(); 37 | if (expectedValue.equals(actual)) { 38 | return actual; 39 | } 40 | if (System.currentTimeMillis() > startTime + timeUnit.toMillis(timeout)) { 41 | return actual; 42 | } 43 | Thread.sleep(50); 44 | } while (true); 45 | } 46 | 47 | } -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/handlers/OffsetsHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network.handlers; 19 | 20 | import io.jafka.api.OffsetRequest; 21 | import io.jafka.api.RequestKeys; 22 | import io.jafka.log.LogManager; 23 | import io.jafka.network.OffsetArraySend; 24 | import io.jafka.network.Receive; 25 | import io.jafka.network.Send; 26 | 27 | /** 28 | * handler for offsets request 29 | * 30 | * @author adyliu (imxylz@gmail.com) 31 | * @since 1.0 32 | */ 33 | public class OffsetsHandler extends AbstractHandler { 34 | 35 | public OffsetsHandler(LogManager logManager) { 36 | super(logManager); 37 | } 38 | 39 | public Send handler(RequestKeys requestType, Receive request) { 40 | OffsetRequest offsetRequest = OffsetRequest.readFrom(request.buffer()); 41 | return new OffsetArraySend(logManager.getOffsets(offsetRequest)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/consumer/Consumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.consumer; 19 | 20 | /** 21 | * Consumer factory 22 | * 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class Consumer { 27 | 28 | /** 29 | * create a ConsumerConnector 30 | * 31 | * @param config at the minium, need to specify these properties: 32 | * 33 | * 
34 |      *  groupid: the consumer group name
35 |      *  zk.connect: the zookeeper connection string
36 |      *
37 | * @return a zookeeper consumer connector 38 | */ 39 | public static ConsumerConnector create(ConsumerConfig config) { 40 | ConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config); 41 | //register mbean 42 | //Utils.registerMBean(consumerConnector, "jafka:type=jafka.ConsumerStats"); 43 | return consumerConnector; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/ProducerPoolData.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer; 19 | 20 | import java.util.List; 21 | 22 | import io.jafka.cluster.Partition; 23 | 24 | /** 25 | * message with topic and partition 26 | * 27 | * @author adyliu (imxylz@gmail.com) 28 | * @since 1.0 29 | * @see Partition 30 | */ 31 | public class ProducerPoolData { 32 | 33 | public final String topic; 34 | 35 | public final Partition partition; 36 | 37 | public final List data; 38 | 39 | public ProducerPoolData(String topic, Partition partition, List data) { 40 | super(); 41 | this.topic = topic; 42 | this.partition = partition; 43 | this.data = data; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return "ProducerPoolData [topic=" + topic + ", partition=" + partition + ", data size=" + (data != null ? data.size() : -1) + "]"; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/utils/zookeeper/ZkGroupTopicDirs.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.utils.zookeeper; 19 | 20 | import io.jafka.consumer.ZookeeperConsumerConnector; 21 | 22 | /** 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | * @see ZookeeperConsumerConnector 26 | */ 27 | public class ZkGroupTopicDirs extends ZkGroupDirs { 28 | 29 | /** topic name */ 30 | public final String topic; 31 | 32 | /** '/consumers/<group>/offsets/<topic>' */ 33 | public final String consumerOffsetDir; 34 | 35 | /** '/consumers/<group>/owners/<topic>' */ 36 | public final String consumerOwnerDir; 37 | 38 | public ZkGroupTopicDirs(String group, String topic) { 39 | super(group); 40 | this.topic = topic; 41 | this.consumerOffsetDir = consumerGroupDir + "/offsets/" + topic; 42 | this.consumerOwnerDir = consumerGroupDir + "/owners/" + topic; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/api/RequestKeys.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.api; 19 | 20 | import io.jafka.common.annotations.ClientSide; 21 | import io.jafka.common.annotations.ServerSide; 22 | 23 | /** 24 | * Request Type 25 | * 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | @ClientSide 30 | @ServerSide 31 | public enum RequestKeys { 32 | PRODUCE, //0 33 | FETCH, //1 34 | MULTIFETCH, //2 35 | MULTIPRODUCE, //3 36 | OFFSETS,//4 37 | /** create more partitions 38 | * @since 1.2 39 | */ 40 | CREATE,//5 41 | /** 42 | * delete unused topic 43 | * @since 1.2 44 | */ 45 | DELETE;//6 46 | 47 | // 48 | public int value = ordinal(); 49 | 50 | // 51 | final static int size = values().length; 52 | 53 | public static RequestKeys valueOf(int ordinal) { 54 | if (ordinal < 0 || ordinal >= size) return null; 55 | return values()[ordinal]; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/StringProducerData.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | /** 24 | * string data 25 | * 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | public class StringProducerData extends ProducerData { 30 | 31 | 32 | public StringProducerData(String topic, String key, List data) { 33 | super(topic, key, data); 34 | } 35 | 36 | public StringProducerData(String topic, List data) { 37 | super(topic, data); 38 | } 39 | 40 | public StringProducerData(String topic, String data) { 41 | super(topic, data); 42 | } 43 | 44 | public StringProducerData(String topic) { 45 | this(topic, new ArrayList()); 46 | } 47 | 48 | public StringProducerData add(String message) { 49 | getData().add(message); 50 | return this; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/utils/zookeeper/ZkGroupDirs.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.utils.zookeeper; 19 | 20 | import io.jafka.consumer.ZookeeperConsumerConnector; 21 | 22 | /** 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | * @see ZookeeperConsumerConnector 26 | */ 27 | public class ZkGroupDirs { 28 | 29 | /** group name */ 30 | public final String group; 31 | 32 | /** '/consumers' */ 33 | public final String consumerDir; 34 | 35 | /** '/consumers/<group>' */ 36 | public final String consumerGroupDir; 37 | 38 | /** '/consumers/<group>/ids' */ 39 | public final String consumerRegistryDir; 40 | 41 | public ZkGroupDirs(String group) { 42 | super(); 43 | this.group = group; 44 | this.consumerDir = ZkUtils.ConsumersPath; 45 | this.consumerGroupDir = this.consumerDir + "/" + group; 46 | this.consumerRegistryDir = this.consumerGroupDir + "/ids"; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/SyncProducerConfigShared.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer; 19 | 20 | import java.util.Properties; 21 | 22 | /** 23 | * Config for sync producer 24 | * 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public interface SyncProducerConfigShared { 29 | 30 | Properties getProperties(); 31 | 32 | /** 33 | * buffer size of producer 34 | * @return buffer size 35 | */ 36 | int getBufferSize(); 37 | 38 | /** 39 | * timeout of socket connection 40 | * @return connection timeout 41 | */ 42 | int getConnectTimeoutMs(); 43 | 44 | /** 45 | * timeout of socket data 46 | * @return timeout of socket 47 | */ 48 | int getSocketTimeoutMs(); 49 | 50 | 51 | //int getMaxMessageSize(); 52 | 53 | /** 54 | * default serializer 55 | * 56 | * @since 1.1 57 | * @return serializer class 58 | */ 59 | String getSerializerClass(); 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/http/HttpServerInitializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 The Netty Project 3 | * 4 | * The Netty Project licenses this file to you under the Apache License, 5 | * version 2.0 (the "License"); you may not use this file except in compliance 6 | * with the License. You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations 14 | * under the License. 15 | */ 16 | package io.jafka.http; 17 | 18 | import io.netty.channel.ChannelInitializer; 19 | import io.netty.channel.ChannelPipeline; 20 | import io.netty.channel.socket.SocketChannel; 21 | import io.netty.handler.codec.http.HttpRequestDecoder; 22 | import io.netty.handler.codec.http.HttpResponseEncoder; 23 | 24 | public class HttpServerInitializer extends ChannelInitializer { 25 | final HttpServer server; 26 | public HttpServerInitializer(HttpServer server){ 27 | this.server = server; 28 | } 29 | @Override 30 | public void initChannel(SocketChannel ch) throws Exception { 31 | ChannelPipeline p = ch.pipeline(); 32 | p.addLast(new HttpRequestDecoder()); 33 | // Uncomment the following line if you don't want to handle HttpChunks. 34 | //p.addLast(new HttpObjectAggregator(1048576)); 35 | p.addLast(new HttpResponseEncoder()); 36 | // Remove the following line if you don't want automatic content compression. 37 | //p.addLast(new HttpContentCompressor()); 38 | p.addLast(new HttpServerHandler(server)); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/utils/Gateway.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2010 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.jafka.utils; 17 | 18 | public class Gateway { 19 | 20 | private GatewayThread _thread; 21 | private final int _port; 22 | private final int _destinationPort; 23 | 24 | public Gateway(int port, int destinationPort) { 25 | _port = port; 26 | _destinationPort = destinationPort; 27 | } 28 | 29 | public synchronized void start() { 30 | if (_thread != null) { 31 | throw new IllegalStateException("Gateway already running"); 32 | } 33 | _thread = new GatewayThread(_port, _destinationPort); 34 | _thread.start(); 35 | _thread.awaitUp(); 36 | } 37 | 38 | public synchronized void stop() { 39 | if (_thread != null) { 40 | try { 41 | _thread.interruptAndJoin(); 42 | } catch (InterruptedException e) { 43 | Thread.currentThread().interrupt(); 44 | } 45 | _thread = null; 46 | } 47 | } 48 | public synchronized void supsend(long timeMs){ 49 | if(_thread != null){ 50 | _thread.suspendTimeoutMs = timeMs; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/AbstractTransmission.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | /** 24 | * @author adyliu (imxylz@gmail.com) 25 | * @since 1.0 26 | */ 27 | public class AbstractTransmission implements Transmission { 28 | 29 | private boolean done = false; 30 | 31 | final protected Logger logger = LoggerFactory.getLogger(getClass()); 32 | 33 | public void expectIncomplete() { 34 | if (complete()) { 35 | throw new IllegalStateException("This operation cannot be completed on a complete request."); 36 | } 37 | } 38 | 39 | public void expectComplete() { 40 | if (!complete()) { 41 | throw new IllegalStateException("This operation cannot be completed on an incomplete request."); 42 | } 43 | } 44 | 45 | public boolean complete() { 46 | return done; 47 | } 48 | 49 | public void setCompleted() { 50 | this.done = true; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/message/ByteBufferBackedInputStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.message; 19 | 20 | import java.io.IOException; 21 | import java.io.InputStream; 22 | import java.nio.ByteBuffer; 23 | 24 | /** 25 | * Convert ByteBuffer to InputStream 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | class ByteBufferBackedInputStream extends InputStream { 30 | 31 | private final ByteBuffer buffer; 32 | 33 | public ByteBufferBackedInputStream(ByteBuffer buffer) { 34 | this.buffer = buffer; 35 | } 36 | 37 | @Override 38 | public int read() throws IOException { 39 | return buffer.hasRemaining() ? (buffer.get() & 0xFF) : -1; 40 | } 41 | 42 | @Override 43 | public int read(byte[] b, int off, int len) throws IOException { 44 | if (buffer.hasRemaining()) { 45 | int realLen = Math.min(len, buffer.remaining()); 46 | buffer.get(b, off, realLen); 47 | return realLen; 48 | } 49 | return -1; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/handlers/DeleterHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network.handlers; 19 | 20 | import io.jafka.api.DeleterRequest; 21 | import io.jafka.api.RequestKeys; 22 | import io.jafka.log.LogManager; 23 | import io.jafka.network.Receive; 24 | import io.jafka.network.Send; 25 | import io.jafka.network.NumbersSend; 26 | 27 | /** 28 | * Delete Operation 29 | * @author adyliu (imxylz@gmail.com) 30 | * @since 1.2 31 | */ 32 | public class DeleterHandler extends AbstractHandler { 33 | 34 | public DeleterHandler(LogManager logManager) { 35 | super(logManager); 36 | } 37 | 38 | @Override 39 | public Send handler(RequestKeys requestType, Receive request) { 40 | DeleterRequest deleterRequest = DeleterRequest.readFrom(request.buffer()); 41 | if (logger.isDebugEnabled()) { 42 | logger.debug("delete request " + deleterRequest); 43 | } 44 | int value = logManager.deleteLogs(deleterRequest.topic,deleterRequest.password); 45 | return new NumbersSend.IntegersSend(value); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/handlers/CreaterHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network.handlers; 19 | 20 | import io.jafka.api.CreaterRequest; 21 | import io.jafka.api.RequestKeys; 22 | import io.jafka.log.LogManager; 23 | import io.jafka.network.Receive; 24 | import io.jafka.network.Send; 25 | import io.jafka.network.NumbersSend; 26 | 27 | /** 28 | * @author adyliu (imxylz@gmail.com) 29 | * @since 1.2 30 | */ 31 | public class CreaterHandler extends AbstractHandler { 32 | 33 | public CreaterHandler(LogManager logManager) { 34 | super(logManager); 35 | } 36 | 37 | @Override 38 | public Send handler(RequestKeys requestType, Receive request) { 39 | CreaterRequest createrRequest = CreaterRequest.readFrom(request.buffer()); 40 | if (logger.isDebugEnabled()) { 41 | logger.debug("Create request " + createrRequest); 42 | } 43 | int value = logManager.createLogs(createrRequest.topic, createrRequest.partitions, createrRequest.enlarge); 44 | return new NumbersSend.IntegersSend(value); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/AsyncProducerQueueSizeStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | import io.jafka.producer.async.QueueItem; 21 | 22 | import java.util.concurrent.BlockingQueue; 23 | 24 | /** 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public class AsyncProducerQueueSizeStats implements AsyncProducerQueueSizeStatsMBean, IMBeanName { 29 | 30 | private final BlockingQueue> queue; 31 | 32 | private String mbeanName; 33 | 34 | public AsyncProducerQueueSizeStats(BlockingQueue> queue) { 35 | super(); 36 | this.queue = queue; 37 | } 38 | 39 | public int getAsyncProducerQueueSize() { 40 | return queue.size(); 41 | } 42 | 43 | 44 | /** 45 | * @return the mbeanName 46 | */ 47 | public String getMbeanName() { 48 | return mbeanName; 49 | } 50 | 51 | 52 | /** 53 | * @param mbeanName the mbeanName to set 54 | */ 55 | public void setMbeanName(String mbeanName) { 56 | this.mbeanName = mbeanName; 57 | } 58 | 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/ByteBufferSend.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | import java.io.IOException; 21 | import java.nio.ByteBuffer; 22 | import java.nio.channels.GatheringByteChannel; 23 | 24 | /** 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public class ByteBufferSend extends AbstractSend { 29 | 30 | final ByteBuffer buffer; 31 | 32 | public ByteBufferSend(ByteBuffer buffer) { 33 | super(); 34 | this.buffer = buffer; 35 | } 36 | 37 | 38 | /** 39 | * @return the buffer 40 | */ 41 | public ByteBuffer getBuffer() { 42 | return buffer; 43 | } 44 | public ByteBufferSend(int size) { 45 | this(ByteBuffer.allocate(size)); 46 | } 47 | 48 | public int writeTo(GatheringByteChannel channel) throws IOException { 49 | expectIncomplete(); 50 | int written = 0; 51 | written += channel.write(buffer); 52 | if (!buffer.hasRemaining()) { 53 | setCompleted(); 54 | } 55 | return written; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/StringProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer; 19 | 20 | import io.jafka.producer.serializer.Encoder; 21 | import io.jafka.producer.serializer.StringEncoder; 22 | 23 | /** 24 | * UTF-8 String Producer 25 | * 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.2 28 | */ 29 | public class StringProducer extends Producer implements IStringProducer { 30 | 31 | /** 32 | * create a string producer with given partitioner rules 33 | * 34 | * @param config producer config 35 | * @param partitioner partitioner rules 36 | */ 37 | public StringProducer(ProducerConfig config, Partitioner partitioner) { 38 | super(config, partitioner, null, true, null); 39 | } 40 | 41 | /** 42 | * create a string producer 43 | * 44 | * @param config producer config 45 | */ 46 | public StringProducer(ProducerConfig config) { 47 | super(config); 48 | } 49 | 50 | @Override 51 | public Encoder getEncoder() { 52 | return new StringEncoder(); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/message/compress/CompressionFacade.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.message.compress; 19 | 20 | 21 | import java.io.Closeable; 22 | import java.io.IOException; 23 | import java.io.InputStream; 24 | import java.io.OutputStream; 25 | 26 | import io.jafka.utils.Closer; 27 | 28 | /** 29 | * message compression 30 | * 31 | * @author adyliu (imxylz@gmail.com) 32 | * @since 1.0 33 | */ 34 | public abstract class CompressionFacade implements Closeable { 35 | 36 | protected final InputStream inputStream; 37 | 38 | protected final OutputStream outputStream; 39 | 40 | public CompressionFacade(InputStream inputStream, OutputStream outputStream) { 41 | this.inputStream = inputStream; 42 | this.outputStream = outputStream; 43 | } 44 | 45 | public void close() { 46 | Closer.closeQuietly(inputStream); 47 | Closer.closeQuietly(outputStream); 48 | } 49 | 50 | public int read(byte[] b) throws IOException { 51 | return inputStream.read(b); 52 | } 53 | 54 | public void write(byte[] b) throws IOException { 55 | outputStream.write(b); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/cluster/Cluster.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.cluster; 19 | 20 | import java.util.LinkedHashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | /** 25 | * The set of active brokers in the cluster 26 | * 27 | * @author adyliu (imxylz@gmail.com) 28 | * @since 1.0 29 | */ 30 | public class Cluster { 31 | 32 | private final Map brokers = new LinkedHashMap(); 33 | 34 | public Cluster() {} 35 | public Cluster(List brokerList) { 36 | for (Broker broker : brokerList) { 37 | brokers.put(broker.id, broker); 38 | } 39 | } 40 | 41 | public Broker getBroker(Integer id) { 42 | return brokers.get(id); 43 | } 44 | 45 | public void add(Broker broker) { 46 | brokers.put(broker.id, broker); 47 | } 48 | 49 | public void remove(Integer id) { 50 | brokers.remove(id); 51 | } 52 | 53 | public int size() { 54 | return brokers.size(); 55 | } 56 | 57 | @Override 58 | public String toString() { 59 | return "Cluster(" + brokers.values() + ")"; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/handlers/MultiProduceHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network.handlers; 19 | 20 | import io.jafka.api.MultiProducerRequest; 21 | import io.jafka.api.ProducerRequest; 22 | import io.jafka.api.RequestKeys; 23 | import io.jafka.log.LogManager; 24 | import io.jafka.network.Receive; 25 | import io.jafka.network.Send; 26 | 27 | /** 28 | * handler for multi produce request 29 | * 30 | * @author adyliu (imxylz@gmail.com) 31 | * @since 1.0 32 | */ 33 | public class MultiProduceHandler extends ProducerHandler { 34 | 35 | public MultiProduceHandler(LogManager logManager) { 36 | super(logManager); 37 | } 38 | 39 | public Send handler(RequestKeys requestType, Receive receive) { 40 | MultiProducerRequest request = MultiProducerRequest.readFrom(receive.buffer()); 41 | if (logger.isDebugEnabled()) { 42 | logger.debug("Multiproducer request " + request); 43 | } 44 | for (ProducerRequest produce : request.produces) { 45 | handleProducerRequest(produce); 46 | } 47 | return null; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/IProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer; 19 | 20 | import java.io.Closeable; 21 | 22 | import io.jafka.common.InvalidPartitionException; 23 | import io.jafka.common.NoBrokersForPartitionException; 24 | import io.jafka.producer.serializer.Encoder; 25 | 26 | /** 27 | * Producer interface 28 | * 29 | * @author adyliu (imxylz@gmail.com) 30 | * @since 1.2 31 | */ 32 | public interface IProducer extends Closeable { 33 | 34 | /** 35 | * Send messages 36 | * 37 | * @param data message data 38 | * @throws NoBrokersForPartitionException no broker has this topic 39 | * @throws InvalidPartitionException partition is out of range 40 | */ 41 | void send(ProducerData data) throws NoBrokersForPartitionException, InvalidPartitionException; 42 | 43 | /** 44 | * get message encoder 45 | * 46 | * @return message encoder 47 | * @see Encoder 48 | */ 49 | Encoder getEncoder(); 50 | 51 | /** 52 | * get partition chooser 53 | * 54 | * @return partition chooser 55 | * @see Partitioner 56 | */ 57 | Partitioner getPartitioner(); 58 | } 59 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/common/ErrorMappingTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.common; 19 | 20 | import static org.junit.Assert.*; 21 | 22 | import org.junit.Test; 23 | 24 | /** 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public class ErrorMappingTest { 29 | 30 | /** 31 | * Test method for {@link ErrorMapping#valueOf(java.lang.Exception)}. 32 | */ 33 | @Test 34 | public void testValueOfException() { 35 | assertEquals(ErrorMapping.OffsetOutOfRangeCode, ErrorMapping.valueOf(new OffsetOutOfRangeException())); 36 | assertEquals(ErrorMapping.InvalidFetchSizeCode, ErrorMapping.valueOf(new InvalidMessageSizeException())); 37 | assertEquals(ErrorMapping.UnkonwCode, ErrorMapping.valueOf(new NullPointerException())); 38 | } 39 | 40 | /** 41 | * Test method for {@link ErrorMapping#valueOf(short)}. 42 | */ 43 | @Test 44 | public void testValueOfShort() { 45 | assertEquals(ErrorMapping.UnkonwCode, ErrorMapping.valueOf((short) 100)); 46 | for (ErrorMapping em : ErrorMapping.values()) { 47 | assertEquals(em, ErrorMapping.valueOf(em.code)); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/console/LineMessageReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.console; 19 | 20 | import java.io.BufferedReader; 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | import java.io.InputStreamReader; 24 | import java.io.UnsupportedEncodingException; 25 | import java.util.Properties; 26 | 27 | 28 | /** 29 | * @author adyliu (imxylz@gmail.com) 30 | * @since 1.0 31 | */ 32 | public class LineMessageReader implements MessageReader { 33 | 34 | private BufferedReader reader; 35 | boolean first = true; 36 | public void init(InputStream inputStream, Properties props) { 37 | try { 38 | reader = new BufferedReader(new InputStreamReader(inputStream,"UTF-8")); 39 | } catch (UnsupportedEncodingException e) { 40 | throw new RuntimeException(e); 41 | } 42 | } 43 | 44 | public String readMessage() throws IOException { 45 | if(first) { 46 | first = false; 47 | System.out.println("Enter you message and exit with empty string."); 48 | } 49 | System.out.print("> "); 50 | return reader.readLine(); 51 | } 52 | 53 | public void close() { 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/console/MyFormatter.java: -------------------------------------------------------------------------------- 1 | package io.jafka.console; 2 | 3 | import java.util.HashSet; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | import joptsimple.HelpFormatter; 8 | import joptsimple.OptionDescriptor; 9 | 10 | public class MyFormatter implements HelpFormatter { 11 | public String format( Map options ) { 12 | StringBuilder buffer = new StringBuilder(); 13 | for ( OptionDescriptor each : new HashSet( options.values() ) ) { 14 | buffer.append( lineFor( each ) ); 15 | } 16 | return buffer.toString(); 17 | } 18 | 19 | private String lineFor( OptionDescriptor descriptor ) { 20 | StringBuilder line = new StringBuilder(); 21 | for(String option:descriptor.options()) { 22 | line.append(option.length()>1?"--":"-"); 23 | line.append(option); 24 | line.append(','); 25 | } 26 | line.setCharAt(line.length()-1, ' '); 27 | line.append(" "); 28 | final int blankSize = line.length() + 2; 29 | if(descriptor.isRequired()) { 30 | line.append("REQUIRED: "); 31 | } 32 | line.append(descriptor.description()); 33 | List list = descriptor.defaultValues(); 34 | if(list!=null&&list.size()>0) { 35 | line.append("\n"); 36 | for(int i=0;i0)line.append(','); 42 | line.append(list.get(i).toString()); 43 | } 44 | line.append(')'); 45 | } 46 | line.append( System.getProperty( "line.separator" ) ); 47 | return line.toString(); 48 | } 49 | } -------------------------------------------------------------------------------- /src/test/java/io/jafka/JafkaTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka; 19 | 20 | import java.util.Properties; 21 | 22 | import org.junit.Test; 23 | 24 | import io.jafka.utils.Closer; 25 | 26 | /** 27 | * @author adyliu (imxylz@gmail.com) 28 | * @since 1.0 29 | */ 30 | public class JafkaTest { 31 | 32 | /** 33 | * Test method for 34 | * {@link Jafka#start(java.util.Properties, java.util.Properties, java.util.Properties)} 35 | * . 36 | * 37 | * @throws InterruptedException 38 | */ 39 | @Test 40 | public void testStartPropertiesPropertiesProperties() { 41 | DataLogCleaner.cleanDataLogDir(); 42 | Jafka jafka = new Jafka(); 43 | int port = PortUtils.checkAvailablePort(9092); 44 | try { 45 | Properties mainProperties = new Properties(); 46 | mainProperties.setProperty("port", "" + port); 47 | mainProperties.setProperty("brokerid", "0"); 48 | mainProperties.setProperty("log.dir", DataLogCleaner.defaultDataLogPath); 49 | jafka.start(mainProperties, null, null); 50 | } finally { 51 | Closer.closeQuietly(jafka); 52 | jafka.awaitShutdown(); 53 | } 54 | 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/MultiMessageSetSend.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import io.jafka.common.annotations.NotThreadSafe; 25 | 26 | /** 27 | * A set of message sets prefixed by size 28 | * 29 | * @author adyliu (imxylz@gmail.com) 30 | * @since 1.0 31 | */ 32 | @NotThreadSafe 33 | public class MultiMessageSetSend extends MultiSend { 34 | 35 | public MultiMessageSetSend(List sets) { 36 | super(); 37 | final ByteBufferSend sizeBuffer = new ByteBufferSend(6); 38 | List sends = new ArrayList(sets.size() + 1); 39 | sends.add(sizeBuffer); 40 | int allMessageSetSize = 0; 41 | for (MessageSetSend send : sets) { 42 | sends.add(send); 43 | allMessageSetSize += send.getSendSize(); 44 | } 45 | //write head size 46 | sizeBuffer.getBuffer().putInt(2 + allMessageSetSize);//4 47 | sizeBuffer.getBuffer().putShort((short) 0);//2 48 | sizeBuffer.getBuffer().rewind(); 49 | super.expectedBytesToWrite = 4 + 2 + allMessageSetSize; 50 | // 51 | super.setSends(sends); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/cluster/PartitionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.cluster; 19 | 20 | import java.util.Arrays; 21 | import java.util.List; 22 | import java.util.TreeSet; 23 | 24 | import org.junit.Assert; 25 | import org.junit.Test; 26 | 27 | /** 28 | * @author adyliu (imxylz@gmail.com) 29 | */ 30 | public class PartitionTest { 31 | 32 | /** 33 | * Test method for 34 | * {@link Partition#compareTo(Partition)} 35 | * . 36 | */ 37 | @Test 38 | public void testCompareTo() { 39 | Partition p1 = new Partition(1, 0); 40 | Partition p2 = new Partition(2, 2); 41 | Partition p3 = new Partition(1, 1); 42 | Partition p4 = new Partition(1, 2); 43 | Partition p5 = new Partition(2, 0); 44 | Partition p6 = new Partition(2, 1); 45 | Partition p7 = new Partition(1, 3); 46 | List partitions = Arrays.asList(p1, p2, p3, p4, p5, p6, p7); 47 | TreeSet set = new TreeSet(partitions); 48 | System.out.println(set); 49 | Partition[] expectPartitions = {p1,p3,p4,p7,p5,p6,p2}; 50 | Assert.assertArrayEquals(expectPartitions, set.toArray()); 51 | Assert.assertFalse(set.add(new Partition(2,1))); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/ZkServerTestUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka; 19 | 20 | import java.io.File; 21 | import java.io.IOException; 22 | 23 | import com.github.zkclient.ZkServer; 24 | 25 | /** 26 | * @author adyliu (imxylz@gmail.com) 27 | */ 28 | public class ZkServerTestUtil { 29 | 30 | static { 31 | System.setProperty("zookeeper.preAllocSize", "1024");//1M data log 32 | } 33 | 34 | 35 | public static ZkServer startZkServer(int port) throws IOException { 36 | final String dataPath=DataLogCleaner.defaultBuildPath+"/zk/default/data"; 37 | final String logPath=DataLogCleaner.defaultBuildPath+"/zk/default/log"; 38 | File dataDir = new File(dataPath); 39 | File logDir = new File(logPath); 40 | DataLogCleaner.cleanDataLogDir(dataDir); 41 | DataLogCleaner.cleanDataLogDir(logDir); 42 | dataDir.mkdirs(); 43 | logDir.mkdirs(); 44 | ZkServer zkServer = new ZkServer(dataPath, logPath, port, 45 | ZkServer.DEFAULT_TICK_TIME, 100); 46 | zkServer.start(); 47 | return zkServer; 48 | } 49 | 50 | public static void closeZkServer(ZkServer server) { 51 | if (server != null) { 52 | server.shutdown(); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/producer/async/EventHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.producer.async; 19 | 20 | 21 | import java.io.Closeable; 22 | import java.util.List; 23 | import java.util.Properties; 24 | 25 | import io.jafka.producer.SyncProducer; 26 | import io.jafka.producer.serializer.Encoder; 27 | 28 | /** 29 | * Handler that dispatches the batched data from the queue of the 30 | * asynchronous producer. 31 | * 32 | * @author adyliu (imxylz@gmail.com) 33 | * @since 1.0 34 | */ 35 | public interface EventHandler extends Closeable{ 36 | 37 | /** 38 | * Initializes the event handler using a Properties object 39 | * 40 | * @param properties the properties used to initialize the event 41 | * handler 42 | */ 43 | void init(Properties properties); 44 | 45 | /** 46 | * Callback to dispatch the batched data and send it to a Jafka server 47 | * 48 | * @param events the data sent to the producer 49 | * @param producer the low-level producer used to send the data 50 | * @param encoder data encoder 51 | */ 52 | void handle(List> events, SyncProducer producer, Encoder encoder); 53 | 54 | /** 55 | * Cleans up and shuts down the event handler 56 | */ 57 | void close(); 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/consumer/ConsumerConnector.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.consumer; 19 | 20 | 21 | import java.io.Closeable; 22 | import java.io.IOException; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | import io.jafka.producer.serializer.Decoder; 27 | 28 | /** 29 | * Main interface for consumer 30 | * 31 | * @author adyliu (imxylz@gmail.com) 32 | * @since 1.0 33 | */ 34 | public interface ConsumerConnector extends Closeable{ 35 | 36 | /** 37 | * Create a list of {@link MessageStream} for each topic 38 | * @param message type 39 | * @param topicCountMap a map of (topic,#streams) pair 40 | * @param decoder message decoder 41 | * @return a map of (topic,list of MessageStream) pair. The number of 42 | * items in the list is #streams. Each MessageStream supports 43 | * an iterator of messages. 44 | */ 45 | Map>> createMessageStreams(// 46 | Map topicCountMap, Decoder decoder); 47 | 48 | /** 49 | * Commit the offsets of all broker partitions connected by this 50 | * connector 51 | */ 52 | void commitOffsets(); 53 | 54 | /** 55 | * Shut down the connector 56 | */ 57 | public void close() throws IOException; 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/message/CompressionCodec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.message; 19 | 20 | import io.jafka.common.UnKnownCodecException; 21 | 22 | /** 23 | * message compression method 24 | * 25 | * @since 1.0 26 | * @author adyliu (imxylz@gmail.com) 27 | */ 28 | public enum CompressionCodec { 29 | /** 30 | * Not compression 31 | */ 32 | NoCompressionCodec(0), // 33 | /** 34 | * GZIP compression 35 | */ 36 | GZIPCompressionCodec(1), // 37 | /** 38 | * Snappy compression (NOT USED) 39 | */ 40 | SnappyCompressionCodec(2), // 41 | /** 42 | * DEFAULT compression(equals GZIPCompressionCode) 43 | */ 44 | DefaultCompressionCodec(1); 45 | 46 | /** 47 | * the codec value 48 | */ 49 | public final int codec; 50 | 51 | CompressionCodec(int codec) { 52 | this.codec = codec; 53 | } 54 | 55 | // 56 | public static CompressionCodec valueOf(int codec) { 57 | switch (codec) { 58 | case 0: 59 | return NoCompressionCodec; 60 | case 1: 61 | return GZIPCompressionCodec; 62 | case 2: 63 | return SnappyCompressionCodec; 64 | } 65 | throw new UnKnownCodecException("unkonw codec: " + codec); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /bin/jafka.conf: -------------------------------------------------------------------------------- 1 | #encoding=UTF-8 2 | #full configuration: http://wrapper.tanukisoftware.com/doc/english/properties.html 3 | 4 | set.JAFKA_HOME=%WRAPPER_BIN_DIR%%WRAPPER_FILE_SEPARATOR%.. 5 | wrapper.working.dir=%JAFKA_HOME% 6 | 7 | wrapper.java.command=java 8 | #set.JAVA_HOME=/java/path 9 | #wrapper.java.command=%JAVA_HOME%/bin/java 10 | 11 | wrapper.java.mainclass=org.tanukisoftware.wrapper.WrapperSimpleApp 12 | wrapper.java.classpath.1=%JAFKA_HOME%/lib/*.jar 13 | wrapper.java.classpath.2=%JAFKA_HOME%/bin/optional/*.jar 14 | wrapper.java.classpath.3=%JAFKA_HOME%/conf 15 | wrapper.java.classpath.4=%JAFKA_HOME%/lib/optional/*.jar 16 | 17 | wrapper.java.library.path.1=%JAFKA_HOME%/bin/optional 18 | 19 | wrapper.java.additional.auto_bits=TRUE 20 | 21 | wrapper.java.additional.1=-Xloggc:%JAFKA_HOME%/logs/gc.log 22 | wrapper.java.additional.2=-server 23 | wrapper.java.additional.3=-Djafka_mx4jenable=true 24 | wrapper.java.additional.4=-Djafka_mx4jport=9094 25 | wrapper.java.additional.5=-Djafka_mx4jaddress=0.0.0.0 26 | 27 | wrapper.java.initmemory=128 28 | wrapper.java.maxmemory=1024 29 | 30 | wrapper.app.parameter.1=io.jafka.Jafka 31 | wrapper.app.parameter.2=%JAFKA_HOME%/conf/server.properties 32 | 33 | #******************************************************************** 34 | # Wrapper Logging Properties 35 | #******************************************************************** 36 | wrapper.console.format=M 37 | wrapper.console.loglevel=INFO 38 | wrapper.logfile=%JAFKA_HOME%/logs/wrapper.log 39 | wrapper.logfile.format=M 40 | wrapper.logfile.loglevel=INFO 41 | wrapper.logfile.maxsize=1024m 42 | wrapper.logfile.maxfiles=7 43 | wrapper.syslog.loglevel=NONE 44 | wrapper.ignore_sequence_gaps=TRUE 45 | wrapper.pidfile.strict=TRUE 46 | wrapper.console.title=Jafka 47 | 48 | 49 | #******************************************************************** 50 | # Wrapper Windows NT/2000/XP Service Properties 51 | #******************************************************************** 52 | wrapper.name=jafka 53 | wrapper.displayname=jafka 54 | wrapper.description=a distributed messaging system 55 | wrapper.ntservice.dependency.1= 56 | wrapper.ntservice.starttype=AUTO_START 57 | wrapper.ntservice.interactive=false 58 | 59 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/AsyncProducerStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | import io.jafka.utils.Utils; 21 | 22 | import java.util.concurrent.atomic.AtomicInteger; 23 | 24 | /** 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public class AsyncProducerStats implements AsyncProducerStatsMBean, IMBeanName { 29 | 30 | private final AtomicInteger droppedEvents = new AtomicInteger(); 31 | 32 | private final AtomicInteger numEvents = new AtomicInteger(); 33 | 34 | public int getAsyncProducerEvents() { 35 | return numEvents.get(); 36 | } 37 | 38 | public int getAsyncProducerDroppedEvents() { 39 | return droppedEvents.get(); 40 | } 41 | 42 | public String getMbeanName() { 43 | return "jafka.producer.Producer:type=AsyncProducerStats"; 44 | } 45 | 46 | private static class AsyncProducerStatsHolder { 47 | 48 | static AsyncProducerStats instance = new AsyncProducerStats(); 49 | 50 | static { 51 | Utils.registerMBean(instance); 52 | } 53 | } 54 | 55 | public static void recordDroppedEvents() { 56 | AsyncProducerStatsHolder.instance.droppedEvents.incrementAndGet(); 57 | } 58 | 59 | public static void recordEvent() { 60 | AsyncProducerStatsHolder.instance.numEvents.incrementAndGet(); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/OffsetArraySend.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network; 19 | 20 | 21 | import java.io.IOException; 22 | import java.nio.ByteBuffer; 23 | import java.nio.channels.GatheringByteChannel; 24 | import java.util.List; 25 | 26 | import io.jafka.api.OffsetRequest; 27 | import io.jafka.common.ErrorMapping; 28 | 29 | 30 | /** 31 | * @author adyliu (imxylz@gmail.com) 32 | * @since 1.0 33 | */ 34 | public class OffsetArraySend extends AbstractSend { 35 | 36 | final ByteBuffer header = ByteBuffer.allocate(6); 37 | final ByteBuffer contentBuffer; 38 | public OffsetArraySend(List offsets) { 39 | header.putInt(4 + offsets.size()*8 +2); 40 | header.putShort(ErrorMapping.NoError.code); 41 | header.rewind(); 42 | contentBuffer = OffsetRequest.serializeOffsetArray(offsets); 43 | } 44 | public int writeTo(GatheringByteChannel channel) throws IOException { 45 | expectIncomplete(); 46 | int written = 0; 47 | if(header.hasRemaining()) { 48 | written += channel.write(header); 49 | } 50 | if(!header.hasRemaining() && contentBuffer.hasRemaining()) { 51 | written += channel.write(contentBuffer); 52 | } 53 | if(!contentBuffer.hasRemaining()) { 54 | setCompleted(); 55 | } 56 | return written; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/message/compress/CompressionFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.message.compress; 19 | 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.InputStream; 22 | 23 | import io.jafka.common.UnKnownCodecException; 24 | import io.jafka.message.CompressionCodec; 25 | 26 | 27 | /** 28 | * @author adyliu (imxylz@gmail.com) 29 | * @since 1.0 30 | */ 31 | public final class CompressionFactory { 32 | 33 | public static CompressionFacade create(CompressionCodec compressionCodec, ByteArrayOutputStream out) { 34 | return create(compressionCodec, null, out); 35 | } 36 | 37 | public static CompressionFacade create(CompressionCodec compressionCodec, InputStream in) { 38 | return create(compressionCodec, in, null); 39 | } 40 | 41 | private static CompressionFacade create(CompressionCodec compressionCodec,// 42 | InputStream in,// 43 | ByteArrayOutputStream out) { 44 | try { 45 | switch (compressionCodec) { 46 | case GZIPCompressionCodec: 47 | return new GZIPCompression(in, out); 48 | default: 49 | break; 50 | } 51 | } catch (Exception e) { 52 | throw new RuntimeException(e.getMessage(), e); 53 | } 54 | 55 | throw new UnKnownCodecException("Unknown Codec: " + compressionCodec); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/io/jafka/DataLogCleaner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka; 19 | 20 | import java.io.File; 21 | 22 | import io.jafka.utils.Utils; 23 | 24 | /** 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | public class DataLogCleaner { 29 | 30 | public static final String defaultBuildPath = "./build"; 31 | 32 | public static final String defaultDataLogPath = defaultBuildPath + "/data"; 33 | 34 | public static final File defaultBuildDir = Utils.getCanonicalFile(new File(defaultBuildPath)); 35 | 36 | public static final File defaultDataLogDir = Utils.getCanonicalFile(new File(defaultDataLogPath)); 37 | 38 | public static void cleanDataLogDir() { 39 | cleanDataLogDir(defaultBuildDir); 40 | } 41 | 42 | public static void cleanDataLogDir(File dir) { 43 | if (!dir.exists()) return; 44 | File[] subs = dir.listFiles(); 45 | if (subs != null) { 46 | for (File f : dir.listFiles()) { 47 | if (f.isFile()) { 48 | if(!f.delete()) { 49 | throw new IllegalStateException("delete file failed: "+f); 50 | } 51 | } else { 52 | cleanDataLogDir(f); 53 | } 54 | } 55 | } 56 | if(!dir.delete()) { 57 | throw new IllegalStateException("delete directory failed: "+dir); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/log/ILog.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.log; 19 | 20 | import java.io.Closeable; 21 | import java.io.IOException; 22 | import java.util.Collections; 23 | import java.util.List; 24 | 25 | import io.jafka.api.OffsetRequest; 26 | import io.jafka.message.ByteBufferMessageSet; 27 | import io.jafka.message.MessageSet; 28 | 29 | /** 30 | * log interface 31 | *

32 | * A log describes a topic with partition(default value is 0). 33 | *

34 | * 35 | * @author adyliu (imxylz@gmail.com) 36 | * @since 1.0 37 | */ 38 | public interface ILog extends Closeable { 39 | 40 | List EMPTY_OFFSETS = Collections.emptyList(); 41 | 42 | /** 43 | * read messages from log 44 | * 45 | * @param offset offset of messages 46 | * @param length the max messages size 47 | * @return message objects 48 | * @throws IOException any Exception 49 | */ 50 | MessageSet read(long offset, int length) throws IOException; 51 | 52 | /** 53 | * append messages to log 54 | * 55 | * @param messages message set 56 | * @return all message offsets or null if not supported 57 | */ 58 | List append(ByteBufferMessageSet messages); 59 | 60 | /** 61 | * get offsets of before the OffsetRequest's time 62 | * 63 | * @param offsetRequest offset request 64 | * @return the offsets earlier than the offset request 65 | */ 66 | List getOffsetsBefore(OffsetRequest offsetRequest); 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/consumer/MessageStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.consumer; 19 | 20 | 21 | import java.util.Iterator; 22 | import java.util.concurrent.BlockingQueue; 23 | 24 | import io.jafka.common.annotations.ClientSide; 25 | import io.jafka.producer.serializer.Decoder; 26 | 27 | /** 28 | * @author adyliu (imxylz@gmail.com) 29 | * @since 1.0 30 | */ 31 | @ClientSide 32 | public class MessageStream implements Iterable { 33 | 34 | final String topic; 35 | 36 | final BlockingQueue queue; 37 | 38 | final int consumerTimeoutMs; 39 | 40 | final Decoder decoder; 41 | 42 | private final ConsumerIterator consumerIterator; 43 | 44 | public MessageStream(String topic, BlockingQueue queue, int consumerTimeoutMs, Decoder decoder) { 45 | super(); 46 | this.topic = topic; 47 | this.queue = queue; 48 | this.consumerTimeoutMs = consumerTimeoutMs; 49 | this.decoder = decoder; 50 | this.consumerIterator = new ConsumerIterator(topic, queue, consumerTimeoutMs, decoder); 51 | } 52 | 53 | public Iterator iterator() { 54 | return consumerIterator; 55 | } 56 | 57 | /** 58 | * This method clears the queue being iterated during the consumer 59 | * rebalancing. This is mainly to reduce the number of duplicates 60 | * received by the consumer 61 | */ 62 | public void clear() { 63 | consumerIterator.clearCurrentChunk(); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/SyncProducerStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | import io.jafka.common.annotations.ThreadSafe; 21 | import io.jafka.utils.Utils; 22 | 23 | /** 24 | * 25 | * @author adyliu (imxylz@gmail.com) 26 | * @since 1.0 27 | */ 28 | @ThreadSafe 29 | public class SyncProducerStats implements SyncProducerStatsMBean, IMBeanName { 30 | 31 | private final SnapshotStats produceRequestStats = new SnapshotStats(); 32 | 33 | public double getProduceRequestsPerSecond() { 34 | return produceRequestStats.getRequestsPerSecond(); 35 | } 36 | 37 | public double getAvgProduceRequestMs() { 38 | return produceRequestStats.getAvgMetric() / (1000.0 * 1000.0); 39 | } 40 | 41 | public double getMaxProduceRequestMs() { 42 | return produceRequestStats.getMaxMetric() / (1000.0 * 1000.0); 43 | } 44 | 45 | public long getNumProduceRequests() { 46 | return produceRequestStats.getNumRequests(); 47 | } 48 | 49 | public String getMbeanName() { 50 | return "jafka:type=jafka.jafkaProducerStats"; 51 | } 52 | 53 | private static class SyncProducerStatsHolder { 54 | 55 | static SyncProducerStats instance = new SyncProducerStats(); 56 | static { 57 | Utils.registerMBean(instance); 58 | } 59 | } 60 | 61 | public static void recordProduceRequest(long requestMs) { 62 | SyncProducerStatsHolder.instance.produceRequestStats.recordRequestMetric(requestMs); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/ConsumerTopicStat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | import java.util.concurrent.atomic.AtomicLong; 21 | 22 | import io.jafka.utils.Pool; 23 | import io.jafka.utils.Utils; 24 | 25 | /** 26 | * @author adyliu (imxylz@gmail.com) 27 | * @since 1.0 28 | */ 29 | public class ConsumerTopicStat implements ConsumerTopicStatMBean, IMBeanName { 30 | 31 | private static final Pool instances = new Pool(); 32 | 33 | private final AtomicLong numCumulatedMessagesPerTopic = new AtomicLong(0); 34 | 35 | public long getMessagesPerTopic() { 36 | return numCumulatedMessagesPerTopic.get(); 37 | } 38 | 39 | public void recordMessagesPerTopic(int nMessages) { 40 | numCumulatedMessagesPerTopic.addAndGet(nMessages); 41 | } 42 | 43 | public String getMbeanName() { 44 | return mBeanName; 45 | } 46 | 47 | private String mBeanName; 48 | 49 | public static ConsumerTopicStat getConsumerTopicStat(String topic) { 50 | ConsumerTopicStat stat = instances.get(topic); 51 | if (stat == null) { 52 | stat = new ConsumerTopicStat(); 53 | stat.mBeanName = "jafka:type=jafka.ConsumerTopicStat." + topic; 54 | if (instances.putIfNotExists(topic, stat) == null) { 55 | Utils.registerMBean(stat); 56 | } else { 57 | stat = instances.get(topic); 58 | } 59 | } 60 | return stat; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/mx/LogFlushStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.mx; 19 | 20 | import io.jafka.utils.Utils; 21 | 22 | /** 23 | * @author adyliu (imxylz@gmail.com) 24 | * @since 1.0 25 | */ 26 | public class LogFlushStats implements LogFlushStatsMBean, IMBeanName { 27 | 28 | private final SnapshotStats flushRequestStats = new SnapshotStats(); 29 | 30 | private LogFlushStats() { 31 | } 32 | 33 | // public void recordFlushRequest(long requestNs) { 34 | // flushRequestStats.recordRequestMetric(requestNs); 35 | //} 36 | 37 | public double getFlushesPerSecond() { 38 | return flushRequestStats.getRequestsPerSecond(); 39 | } 40 | 41 | public double getAvgFlushMs() { 42 | return flushRequestStats.getAvgMetric(); 43 | } 44 | 45 | public long getTotalFlushMs() { 46 | return flushRequestStats.getTotalMetric(); 47 | } 48 | 49 | public double getMaxFlushMs() { 50 | return flushRequestStats.getMaxMetric(); 51 | } 52 | 53 | public long getNumFlushes() { 54 | return flushRequestStats.getNumRequests(); 55 | } 56 | 57 | public String getMbeanName() { 58 | return "jafka:type=jafka.LogFlushStats"; 59 | } 60 | 61 | private static class LogFlushStatsHolder { 62 | 63 | static LogFlushStats stats = new LogFlushStats(); 64 | static { 65 | Utils.registerMBean(stats); 66 | } 67 | } 68 | 69 | public static void recordFlushRequest(long requestMs) { 70 | LogFlushStatsHolder.stats.flushRequestStats.recordRequestMetric(requestMs); 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/api/DeleterRequest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.api; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import io.jafka.common.annotations.ClientSide; 23 | import io.jafka.common.annotations.ServerSide; 24 | import io.jafka.network.Request; 25 | import io.jafka.utils.Utils; 26 | 27 | /** 28 | * create a delete request 29 | * 30 | * @author adyliu (imxylz@gmail.com) 31 | * @since 1.2 32 | */ 33 | @ClientSide 34 | @ServerSide 35 | public class DeleterRequest implements Request { 36 | 37 | public final String topic; 38 | 39 | public final String password; 40 | 41 | public DeleterRequest(String topic, String password) { 42 | this.topic = topic; 43 | this.password = password; 44 | } 45 | 46 | @Override 47 | public int getSizeInBytes() { 48 | return Utils.caculateShortString(topic) + Utils.caculateShortString(password); 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return "DeleterRequest [topic=" + topic + ", password=" + password + "]"; 54 | } 55 | 56 | @Override 57 | public RequestKeys getRequestKey() { 58 | return RequestKeys.DELETE; 59 | } 60 | 61 | @Override 62 | public void writeTo(ByteBuffer buffer) { 63 | Utils.writeShortString(buffer, topic); 64 | Utils.writeShortString(buffer, password); 65 | } 66 | 67 | public static DeleterRequest readFrom(ByteBuffer buffer) { 68 | String topic = Utils.readShortString(buffer); 69 | String password = Utils.readShortString(buffer); 70 | return new DeleterRequest(topic, password); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/io/jafka/network/handlers/MultiFetchHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.jafka.network.handlers; 19 | 20 | import io.jafka.api.FetchRequest; 21 | import io.jafka.api.MultiFetchRequest; 22 | import io.jafka.api.RequestKeys; 23 | import io.jafka.log.LogManager; 24 | import io.jafka.network.MessageSetSend; 25 | import io.jafka.network.MultiMessageSetSend; 26 | import io.jafka.network.Receive; 27 | import io.jafka.network.Send; 28 | 29 | import java.util.ArrayList; 30 | import java.util.List; 31 | 32 | /** 33 | * handler for multi fetch request 34 | * 35 | * @author adyliu (imxylz@gmail.com) 36 | * @since 1.0 37 | */ 38 | public class MultiFetchHandler extends FetchHandler { 39 | 40 | public MultiFetchHandler(LogManager logManager) { 41 | super(logManager); 42 | } 43 | 44 | public Send handler(RequestKeys requestType, Receive request) { 45 | MultiFetchRequest multiFetchRequest = MultiFetchRequest.readFrom(request.buffer()); 46 | List fetches = multiFetchRequest.getFetches(); 47 | if (logger.isDebugEnabled()) { 48 | logger.debug("Multifetch request objects size: " + fetches.size()); 49 | for (FetchRequest fetch : fetches) { 50 | logger.debug(fetch.toString()); 51 | } 52 | } 53 | List responses = new ArrayList(fetches.size()); 54 | for (FetchRequest fetch : fetches) { 55 | responses.add(readMessageSet(fetch)); 56 | } 57 | return new MultiMessageSetSend(responses); 58 | } 59 | } --------------------------------------------------------------------------------