├── .gitignore ├── LICENSE ├── README.md ├── data-analysis ├── data_analysis.py └── log.txt ├── log ├── 0000 │ └── 78921g1clv.logs.tgz ├── 0001 │ └── 78921g1clv.logs.tgz ├── 0002 │ └── 78921g1clv.logs.tgz ├── 0003 │ └── 78921g1clv.logs.tgz ├── 0004 │ ├── 78921g1clv.logs.tgz │ ├── assessment.100000.log │ ├── dstat.log │ └── iostat.log ├── 0005 │ ├── 78921g1clv.logs.tgz │ ├── assessment.100000.log │ ├── dstat.log │ └── iostat.log ├── 0007 │ ├── 78921g1clv.logs.tgz │ ├── assessment.100000.log │ ├── dstat.log │ └── iostat.log ├── 0008 │ ├── 78921g1clv.logs.tgz │ ├── assessment.100000.log │ ├── dstat.log │ └── iostat.log ├── 78921g1clv.logs.tgz └── fetch_log.sh ├── pom.xml └── src ├── main ├── java │ ├── .DS_Store │ └── io │ │ ├── .DS_Store │ │ └── openmessaging │ │ ├── BatchToPartition.java │ │ ├── BytesMessage.java │ │ ├── Filters.java │ │ ├── InvokeContext.java │ │ ├── KeyValue.java │ │ ├── Message.java │ │ ├── MessageFactory.java │ │ ├── MessageHeader.java │ │ ├── MessageListener.java │ │ ├── MessagingAccessPoint.java │ │ ├── MessagingAccessPointManager.java │ │ ├── OnMessageContext.java │ │ ├── PartitionConsumer.java │ │ ├── PartitionIterator.java │ │ ├── Producer.java │ │ ├── Promise.java │ │ ├── PromiseListener.java │ │ ├── PullConsumer.java │ │ ├── PushConsumer.java │ │ ├── ResourceManager.java │ │ ├── ServiceEndPoint.java │ │ ├── ServiceLifecycle.java │ │ ├── ServiceLoadBalance.java │ │ ├── ServiceProperties.java │ │ ├── demo │ │ ├── ClientOMSException.java │ │ ├── Constants.java │ │ ├── DataDump.java │ │ ├── DefaultBytesMessage.java │ │ ├── DefaultKeyValue.java │ │ ├── DefaultMessageFactory.java │ │ ├── DefaultProducer.java │ │ ├── DefaultPullConsumer.java │ │ ├── NaiveDataReader.java │ │ ├── net │ │ │ └── jpountz │ │ │ │ ├── lz4 │ │ │ │ ├── LZ4BlockInputStream.java │ │ │ │ ├── LZ4BlockOutputStream.java │ │ │ │ ├── LZ4ByteBufferUtils.java │ │ │ │ ├── LZ4Compressor.java │ │ │ │ ├── LZ4Constants.java │ │ │ │ ├── LZ4Decompressor.java │ │ │ │ ├── LZ4Exception.java │ │ │ │ ├── LZ4Factory.java │ │ │ │ ├── LZ4FastDecompressor.java │ │ │ │ ├── LZ4FrameInputStream.java │ │ │ │ ├── LZ4FrameOutputStream.java │ │ │ │ ├── LZ4HCJNICompressor.java │ │ │ │ ├── LZ4HCJavaSafeCompressor.java │ │ │ │ ├── LZ4HCJavaUnsafeCompressor.java │ │ │ │ ├── LZ4JNI.java │ │ │ │ ├── LZ4JNICompressor.java │ │ │ │ ├── LZ4JNIFastDecompressor.java │ │ │ │ ├── LZ4JNISafeDecompressor.java │ │ │ │ ├── LZ4JavaSafeCompressor.java │ │ │ │ ├── LZ4JavaSafeFastDecompressor.java │ │ │ │ ├── LZ4JavaSafeSafeDecompressor.java │ │ │ │ ├── LZ4JavaUnsafeCompressor.java │ │ │ │ ├── LZ4JavaUnsafeFastDecompressor.java │ │ │ │ ├── LZ4JavaUnsafeSafeDecompressor.java │ │ │ │ ├── LZ4SafeDecompressor.java │ │ │ │ ├── LZ4SafeUtils.java │ │ │ │ ├── LZ4UnknownSizeDecompressor.java │ │ │ │ ├── LZ4UnsafeUtils.java │ │ │ │ ├── LZ4Utils.java │ │ │ │ └── package.html │ │ │ │ ├── util │ │ │ │ ├── ByteBufferUtils.java │ │ │ │ ├── Native.java │ │ │ │ ├── SafeUtils.java │ │ │ │ ├── UnsafeUtils.java │ │ │ │ ├── Utils.java │ │ │ │ └── package.html │ │ │ │ └── xxhash │ │ │ │ ├── AbstractStreamingXXHash32Java.java │ │ │ │ ├── AbstractStreamingXXHash64Java.java │ │ │ │ ├── StreamingXXHash32.java │ │ │ │ ├── StreamingXXHash32JNI.java │ │ │ │ ├── StreamingXXHash32JavaSafe.java │ │ │ │ ├── StreamingXXHash32JavaUnsafe.java │ │ │ │ ├── StreamingXXHash64.java │ │ │ │ ├── StreamingXXHash64JNI.java │ │ │ │ ├── StreamingXXHash64JavaSafe.java │ │ │ │ ├── StreamingXXHash64JavaUnsafe.java │ │ │ │ ├── XXHash32.java │ │ │ │ ├── XXHash32JNI.java │ │ │ │ ├── XXHash32JavaSafe.java │ │ │ │ ├── XXHash32JavaUnsafe.java │ │ │ │ ├── XXHash64.java │ │ │ │ ├── XXHash64JNI.java │ │ │ │ ├── XXHash64JavaSafe.java │ │ │ │ ├── XXHash64JavaUnsafe.java │ │ │ │ ├── XXHashConstants.java │ │ │ │ ├── XXHashFactory.java │ │ │ │ ├── XXHashJNI.java │ │ │ │ └── package.html │ │ └── unitTest │ │ │ └── MessageStringTest.java │ │ ├── exception │ │ ├── OMSException.java │ │ ├── OMSNotSupportedException.java │ │ ├── OMSResourceNotExistException.java │ │ └── OMSRuntimeException.java │ │ ├── filter │ │ └── FilterChain.java │ │ ├── internal │ │ ├── DefaultKeyValue.java │ │ └── MessagingAccessPointAdapter.java │ │ ├── observer │ │ └── Observer.java │ │ ├── official │ │ └── tester │ │ │ └── ProducerTester.java │ │ └── tester │ │ ├── Constants.java │ │ ├── ConsumerTester.java │ │ ├── ProducerAndConsumerTest.java │ │ └── ProducerTester.java └── resources │ ├── log4j.properties │ └── package.xml └── test └── java └── ProducerAndConsumerTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | target/* 3 | .idea 4 | *.iml 5 | 6 | # Compiled class file 7 | *.class 8 | 9 | # Log file 10 | #*.log 11 | 12 | # BlueJ files 13 | *.ctxt 14 | 15 | # Mobile Tools for Java (J2ME) 16 | .mtj.tmp/ 17 | 18 | # Package Files # 19 | *.jar 20 | *.war 21 | *.ear 22 | *.zip 23 | *.tar.gz 24 | *.rar 25 | 26 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 27 | hs_err_pid* 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 CHE Yulin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | >写在前面: 2 | > 1. 在开始coding前请仔细阅读以下内容 3 | 4 | 5 | ## 1. 题目背景 6 | 支撑阿里双十一的万亿级消息中间件RocketMQ在去年10月进入Apache基金会进行孵化。异步解耦,削峰填谷,让消息中间件在现代软件架构中扮演者举足轻重的地位。天下大势,分久必合,合久必分,软件领域也是如此。市场上消息中间件种类繁多,阿里消息团队在此当口,推出厂商无关的[Open-Messaging](https://openmessaging.github.io)规范,促进消息领域的进行一步发展。 7 | 8 | ## 2. 题目描述 9 | 10 | ### 2.1 题目内容 11 | 阅读Open-Messaging的接口代码(本工程内除了demo目录外的其他代码),了解Topic,Queue的基本概念,并实现一个进程内消息引擎。 12 | 13 | 提示:Topic类似于水坝(蓄积功能,消峰填谷之利器),Queue类似于水渠;每当新建一个Queue的时候,可以选择绑定到几个Topic,类似于水渠从水坝引水; 14 | 每个Topic可以被任意多个Queue绑定,这点与现实生活不太一样,因为数据可以多次拷贝; 15 | 在发送的时候,可以选择发送到Topic,也可以选择直接发送到Queue;直接发送到Queue的数据只能被对应Queue消费,不能被其他Queue读取到; 16 | 在消费的时候,除了要读取绑定的Topic的数据,还要去取直接发送到该Queue的数据; 17 | 18 | 19 | ### 2.2 语言限定 20 | 限定使用JAVA语言 21 | 22 | ## 3. 程序目标 23 | 24 | 你的coding目标是实现以下接口: 25 | 26 | * Producer的createBytesMessageToTopic(topic, body) 27 | * Producer的createBytesMessageToQueue(queue, body) 28 | * Producer的send(message) 29 | * PullConsumer的attachQueue(queue, topics) 30 | * PullConsumer的poll() 31 | 32 | 实现类名字分别叫做:DefaultProducer,DefaultPullConsumer,包名均是:io.messaging.demo 33 | 请参考demo目录,建议是名字不要改,修改实现内容即可 34 | 注意,消息内容需要存入磁盘中,并能再次读取出来 35 | 36 | 37 | ## 4.参赛方法说明 38 | 1. 在阿里天池找到"中间件性能挑战赛",并报名参加 39 | 2. 在code.aliyun.com注册一个账号,并新建一个仓库名,并将大赛官方账号middlewarerace2017添加为项目成员,权限为reporter 40 | 3. fork或者拷贝本仓库的代码到自己的仓库,并实现自己的逻辑 41 | 4. 在天池提交成绩的入口,提交自己的仓库git地址,等待评测结果 42 | 5. 坐等每天10点排名更新 43 | 44 | 45 | ## 4. 测试环境描述 46 | 测试环境为相同的4核虚拟机(暂定)。限定使用的最大JVM大小为4GB(-Xmx4g),磁盘使用不做限制。 47 | 48 | ## 5. 程序校验逻辑 49 | 50 | 1. 10~20个线程(位于同一进程中)各自独立调用Producer发送消息(每个线程启动一个Producer,每条消息随机发送到某个Topic或者Queue),持续时间**T1**,请注意把消息数据写入磁盘中 51 | 2. 强行kill Producer进程,未写入磁盘的消息都会丢失 52 | 3. 10~20个线程(位于同一进程中)独立调用Consumer收取消息(每个线程启动一个Consumer,attach到指定的Queue,不同的Consumer不会attach同一个Queue),验证消息顺序准确性,可靠性,消费持续的时间为**T2**,消费到的总消息数目为**N** 53 | 4. 以**N/(t1+t2)**来衡量性能 54 | 55 | PS:请仔细阅读本仓库内Demo代码,尤其是DemoTester 56 | 57 | ### 5.1 补充说明 58 | 1. 测试时,topic和queue的数目大约是100个(其中queue的数目与消费者线程数相等); 59 | 2. 测试时,消息大小不会超过256K; 60 | 3. 可靠性是指,消息不能丢失,且消息的内容不能被篡改;在测试消费的时候,会对消息的body,headers,properties的内容进行校验; 61 | 4. header与properties中key和value都不会插入null或空值 62 | 5. 发送消息时,消息一定要写入磁盘,最简单的组织形式就是,一个topic或者queue一个文件,每条消息一行(这样做性能可能较低,请自行优化设计) 63 | 64 | ### 5.2 消息顺序的说明 65 | 顺序只针对单个topic或者queue,不同topic,不同queue,topic与queue之间都不用考虑顺序 66 | 67 | 消息产品的一个重要特性是顺序保证,也就是消息消费的顺序要与发送的时间顺序保持一致;在多发送端的情况下,保证全局顺序代价比较大,只要求各个发送端的顺序有保障即可; 68 | 举个例子P1发送M11,M12,M13,P2发送M21,M22,M23,在消费的时候,只要求保证 M11,M12,M13(M21,M22,M23)的顺序,也就是说,实际消费顺序为: 69 | M11,M21,M12,M13,M22,M23 正确; 70 | M11,M21,M22,M12,M13,M23 正确; 71 | M11,M13,M21,M22,M23,M12 错误,M12与M13的顺序颠倒了; 72 | 73 | ## 6. 排名规则 74 | 75 | 在结果校验100%正确的前提下,按照N/(t1 + t2)从高到低来排名 76 | 77 | 78 | ## 7. 第二/三方库规约 79 | 80 | * 仅允许依赖JavaSE 8 包含的lib 81 | * 可以参考别人的实现,拷贝少量的代码 82 | * 我们会对排名靠前的代码进行review,如果发现大量拷贝别人的代码,将扣分 83 | 84 | 85 | -------------------------------------------------------------------------------- /data-analysis/data_analysis.py: -------------------------------------------------------------------------------- 1 | with open('log.txt', 'r') as ifs: 2 | lines = filter(lambda line: 'main' in line, ifs.readlines()) 3 | record_list = set(map(lambda line: line.split('main:')[1], lines)) 4 | record_pair_list = map(lambda record: record.strip().split(','), record_list) 5 | 6 | queue_list = filter(lambda ele: 'QUEUE' in ele[0], record_pair_list) 7 | topic_list = filter(lambda ele: 'TOPIC' in ele[0], record_pair_list) 8 | for queue_pair in sorted(queue_list, key=lambda pair: int(pair[0].split('_')[1])): 9 | print queue_pair 10 | 11 | print 12 | 13 | for topic_pair in sorted(topic_list, key=lambda pair: int(pair[0].split('_')[1])): 14 | print topic_pair 15 | 16 | key_value_list = map(lambda lst: (lst[0], int(lst[1])), record_pair_list) 17 | file_size_list = map(lambda my_pair: my_pair[1], key_value_list) 18 | 19 | total_size = sum(file_size_list) 20 | print total_size 21 | print float(total_size) / 1024 / 1024 / 1024 22 | 23 | print 4.66584196314 24 | -------------------------------------------------------------------------------- /data-analysis/log.txt: -------------------------------------------------------------------------------- 1 | [2017-05-31 10:25:19,527] main:TOPIC_12,71660520 2 | [2017-05-31 10:25:19,528] main:TOPIC_20,71668440 3 | [2017-05-31 10:25:19,528] main:TOPIC_34,71686260 4 | [2017-05-31 10:25:19,528] main:TOPIC_42,71620920 5 | [2017-05-31 10:25:19,529] main:TOPIC_45,71644680 6 | [2017-05-31 10:25:19,529] main:TOPIC_47,71591220 7 | [2017-05-31 10:25:19,529] main:TOPIC_55,71658540 8 | [2017-05-31 10:25:19,530] main:TOPIC_67,71694180 9 | [2017-05-31 10:25:19,530] main:TOPIC_86,71642700 10 | [2017-05-31 10:25:19,531] main:QUEUE_0,71254580 11 | 12 | [2017-05-31 10:25:19,546] main:TOPIC_13,71700120 13 | [2017-05-31 10:25:19,546] main:TOPIC_33,71650620 14 | [2017-05-31 10:25:19,547] main:TOPIC_45,71644680 15 | [2017-05-31 10:25:19,547] main:TOPIC_5,71268440 16 | [2017-05-31 10:25:19,548] main:TOPIC_54,71644680 17 | [2017-05-31 10:25:19,548] main:TOPIC_56,71684280 18 | [2017-05-31 10:25:19,548] main:TOPIC_60,71646660 19 | [2017-05-31 10:25:19,549] main:TOPIC_75,71680320 20 | [2017-05-31 10:25:19,549] main:TOPIC_80,71607060 21 | [2017-05-31 10:25:19,550] main:QUEUE_1,71304080 22 | 23 | [2017-05-31 10:25:19,566] main:TOPIC_0,71288240 24 | [2017-05-31 10:25:19,566] main:TOPIC_19,71654580 25 | [2017-05-31 10:25:19,567] main:TOPIC_28,71666460 26 | [2017-05-31 10:25:19,567] main:TOPIC_29,71737740 27 | [2017-05-31 10:25:19,567] main:TOPIC_59,71672400 28 | [2017-05-31 10:25:19,568] main:TOPIC_66,71642700 29 | [2017-05-31 10:25:19,568] main:TOPIC_70,71690220 30 | [2017-05-31 10:25:19,569] main:TOPIC_85,71702100 31 | [2017-05-31 10:25:19,569] main:TOPIC_87,71670420 32 | [2017-05-31 10:25:19,570] main:QUEUE_2,71325860 33 | 34 | [2017-05-31 10:25:19,581] main:TOPIC_20,71668440 35 | [2017-05-31 10:25:19,581] main:TOPIC_53,71654580 36 | [2017-05-31 10:25:19,582] main:TOPIC_63,71664480 37 | [2017-05-31 10:25:19,582] main:TOPIC_69,71688240 38 | [2017-05-31 10:25:19,583] main:TOPIC_76,71656560 39 | [2017-05-31 10:25:19,583] main:TOPIC_77,71577360 40 | [2017-05-31 10:25:19,583] main:TOPIC_79,71666460 41 | [2017-05-31 10:25:19,584] main:TOPIC_83,71700120 42 | [2017-05-31 10:25:19,584] main:TOPIC_9,71224880 43 | [2017-05-31 10:25:19,585] main:QUEUE_3,71286260 44 | 45 | [2017-05-31 10:25:19,597] main:TOPIC_1,71260520 46 | [2017-05-31 10:25:19,597] main:TOPIC_2,71250620 47 | [2017-05-31 10:25:19,598] main:TOPIC_31,71682300 48 | [2017-05-31 10:25:19,598] main:TOPIC_45,71644680 49 | [2017-05-31 10:25:19,599] main:TOPIC_46,71660520 50 | [2017-05-31 10:25:19,599] main:TOPIC_53,71654580 51 | [2017-05-31 10:25:19,599] main:TOPIC_54,71644680 52 | [2017-05-31 10:25:19,600] main:TOPIC_62,71654580 53 | [2017-05-31 10:25:19,600] main:TOPIC_81,71666460 54 | [2017-05-31 10:25:19,601] main:QUEUE_4,71282300 55 | 56 | [2017-05-31 10:25:19,607] main:TOPIC_0,71288240 57 | [2017-05-31 10:25:19,607] main:TOPIC_18,71644680 58 | [2017-05-31 10:25:19,608] main:TOPIC_33,71650620 59 | [2017-05-31 10:25:19,608] main:TOPIC_37,71725860 60 | [2017-05-31 10:25:19,608] main:TOPIC_5,71268440 61 | [2017-05-31 10:25:19,609] main:TOPIC_76,71656560 62 | [2017-05-31 10:25:19,609] main:TOPIC_83,71700120 63 | [2017-05-31 10:25:19,610] main:TOPIC_86,71642700 64 | [2017-05-31 10:25:19,610] main:TOPIC_88,71678340 65 | [2017-05-31 10:25:19,611] main:QUEUE_5,71284280 66 | 67 | [2017-05-31 10:25:19,614] main:TOPIC_2,71250620 68 | [2017-05-31 10:25:19,614] main:TOPIC_22,71664480 69 | [2017-05-31 10:25:19,615] main:TOPIC_24,71682300 70 | [2017-05-31 10:25:19,615] main:TOPIC_34,71686260 71 | [2017-05-31 10:25:19,616] main:TOPIC_51,71638740 72 | [2017-05-31 10:25:19,616] main:TOPIC_59,71672400 73 | [2017-05-31 10:25:19,616] main:TOPIC_68,71611020 74 | [2017-05-31 10:25:19,617] main:TOPIC_71,71585280 75 | [2017-05-31 10:25:19,617] main:TOPIC_78,71609040 76 | [2017-05-31 10:25:19,617] main:QUEUE_6,71290220 77 | 78 | [2017-05-31 10:25:19,624] main:TOPIC_19,71654580 79 | [2017-05-31 10:25:19,624] main:TOPIC_30,71611020 80 | [2017-05-31 10:25:19,624] main:TOPIC_4,71294180 81 | [2017-05-31 10:25:19,625] main:TOPIC_40,71654580 82 | [2017-05-31 10:25:19,625] main:TOPIC_63,71664480 83 | [2017-05-31 10:25:19,625] main:TOPIC_65,71660520 84 | [2017-05-31 10:25:19,626] main:TOPIC_66,71642700 85 | [2017-05-31 10:25:19,626] main:TOPIC_75,71680320 86 | [2017-05-31 10:25:19,627] main:TOPIC_88,71678340 87 | [2017-05-31 10:25:19,627] main:QUEUE_7,71260520 88 | 89 | [2017-05-31 10:25:19,633] main:TOPIC_1,71260520 90 | [2017-05-31 10:25:19,633] main:TOPIC_12,71660520 91 | [2017-05-31 10:25:19,634] main:TOPIC_20,71668440 92 | [2017-05-31 10:25:19,634] main:TOPIC_23,71650620 93 | [2017-05-31 10:25:19,635] main:TOPIC_33,71650620 94 | [2017-05-31 10:25:19,636] main:TOPIC_34,71686260 95 | [2017-05-31 10:25:19,636] main:TOPIC_39,71640720 96 | [2017-05-31 10:25:19,637] main:TOPIC_62,71654580 97 | [2017-05-31 10:25:19,637] main:TOPIC_71,71585280 98 | [2017-05-31 10:25:19,637] main:QUEUE_8,71240720 99 | 100 | [2017-05-31 10:25:19,640] main:TOPIC_14,71696160 101 | [2017-05-31 10:25:19,640] main:TOPIC_15,71650620 102 | [2017-05-31 10:25:19,640] main:TOPIC_21,71652600 103 | [2017-05-31 10:25:19,640] main:TOPIC_37,71725860 104 | [2017-05-31 10:25:19,641] main:TOPIC_43,71611020 105 | [2017-05-31 10:25:19,641] main:TOPIC_49,71702100 106 | [2017-05-31 10:25:19,642] main:TOPIC_5,71268440 107 | [2017-05-31 10:25:19,642] main:TOPIC_70,71690220 108 | [2017-05-31 10:25:19,642] main:TOPIC_76,71656560 109 | [2017-05-31 10:25:19,643] main:QUEUE_9,71234780 -------------------------------------------------------------------------------- /log/0000/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/0000/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/0001/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/0001/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/0002/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/0002/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/0003/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/0003/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/0004/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/0004/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/0005/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/0005/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/0007/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/0007/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/0008/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/0008/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/78921g1clv.logs.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/log/78921g1clv.logs.tgz -------------------------------------------------------------------------------- /log/fetch_log.sh: -------------------------------------------------------------------------------- 1 | wget http://middle2017.oss-cn-shanghai.aliyuncs.com/78921g1clv.logs.tgz 2 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | io.openmessaging 8 | open-messaging-demo 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | org.slf4j 14 | slf4j-log4j12 15 | 1.7.21 16 | 17 | 18 | junit 19 | junit 20 | 4.12 21 | 22 | 23 | 24 | 25 | 26 | maven-compiler-plugin 27 | 2.3.2 28 | 29 | 1.8 30 | 1.8 31 | utf-8 32 | 33 | 34 | 35 | maven-assembly-plugin 36 | 2.2.1 37 | 38 | 39 | org.apache.maven 40 | maven-core 41 | 2.2.1 42 | 43 | 44 | 45 | OpenMessagingDemo 46 | false 47 | 48 | src/main/resources/package.xml 49 | 50 | 51 | 52 | 53 | make-assembly 54 | package 55 | 56 | single 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/main/java/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/src/main/java/.DS_Store -------------------------------------------------------------------------------- /src/main/java/io/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RapidsBlink/OpenMessageShaping/ac07bb40d96c2b4e333bbdedecafe7ebc68188fc/src/main/java/io/.DS_Store -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/BatchToPartition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | */ 23 | public interface BatchToPartition { 24 | void send(BytesMessage message); 25 | 26 | void send(BytesMessage message, KeyValue properties); 27 | 28 | void commit(); 29 | 30 | void rollback(); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/BytesMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * The {@code BytesMessage} contains a stream of uninterpreted bytes. It inherits from the {@code Message} interface and 22 | * adds a bytes message body. 23 | *

24 | * The {@code BytesMessage} doesn't know the format or encoding Rules of the body, the provider and consumer decide the 25 | * interpretation of the bytes body. 26 | * 27 | * @author vintagewang@apache.org 28 | * @author yukon@apache.org 29 | * @version OMS 1.0 30 | * @since OMS 1.0 31 | */ 32 | public interface BytesMessage extends Message { 33 | /** 34 | * Returns the bytes message body. 35 | * 36 | * @return the bytes message body 37 | */ 38 | byte[] getBody(); 39 | 40 | /** 41 | * Sets the bytes message body. 42 | * 43 | * @param body the message body to be set 44 | */ 45 | BytesMessage setBody(final byte[] body); 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/Filters.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | * @version OMS 1.0 23 | * @since OMS 1.0 24 | */ 25 | public interface Filters { 26 | Filters addFilter(String sql); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/InvokeContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | * @version OMS 1.0 23 | * @since OMS 1.0 24 | */ 25 | public interface InvokeContext { 26 | KeyValue properties(); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/Message.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * The {@code Message} interface is the root interface of all OMS messages, and the most commonly used OMS message is 22 | * {@link BytesMessage}. 23 | *

24 | * Most message-oriented middleware (MOM) products treat messages as lightweight entities that consist of a header and a 25 | * body, like Apache RocketMQ. The header contains fields used for message 26 | * routing and identification; the body contains the application data being sent. 27 | *

28 | * The {@code Message} is a lightweight entity that only contains the property related information of a specific message 29 | * object, and the {@code Message} is composed of the following parts: 30 | *

31 | *

37 | *

38 | * The body part is placed in the implementation classes of {@code Message}. 39 | * 40 | * @author vintagewang@apache.org 41 | * @author yukon@apache.org 42 | * @version OMS 1.0 43 | * @since OMS 1.0 44 | */ 45 | public interface Message { 46 | /** 47 | * Returns all the header fields of the {@code Message} object as a {@code KeyValue}. 48 | * 49 | * @return the headers of a {@code Message} 50 | * @see MessageHeader 51 | */ 52 | KeyValue headers(); 53 | 54 | /** 55 | * Returns all the built-in property fields of the {@code Message} object as a {@code KeyValue}. 56 | * 57 | * @return the properties of a {@code Message} 58 | */ 59 | KeyValue properties(); 60 | 61 | /** 62 | * Puts a {@code String}-{@code int} {@code KeyValue} entry to he headers of a {@code Message}. 63 | * 64 | * @param key the key to be placed into the headers 65 | * @param value the value corresponding to key 66 | */ 67 | Message putHeaders(String key, int value); 68 | 69 | /** 70 | * Puts a {@code String}-{@code long} {@code KeyValue} entry to he headers of a {@code Message}. 71 | * 72 | * @param key the key to be placed into the headers 73 | * @param value the value corresponding to key 74 | */ 75 | Message putHeaders(String key, long value); 76 | 77 | /** 78 | * Puts a {@code String}-{@code double} {@code KeyValue} entry to he headers of a {@code Message}. 79 | * 80 | * @param key the key to be placed into the headers 81 | * @param value the value corresponding to key 82 | */ 83 | Message putHeaders(String key, double value); 84 | 85 | /** 86 | * Puts a {@code String}-{@code String} {@code KeyValue} entry to he headers of a {@code Message}. 87 | * 88 | * @param key the key to be placed into the headers 89 | * @param value the value corresponding to key 90 | */ 91 | Message putHeaders(String key, String value); 92 | 93 | /** 94 | * Puts a {@code String}-{@code int} {@code KeyValue} entry to he headers of a {@code Message}. 95 | * 96 | * @param key the key to be placed into the headers 97 | * @param value the value corresponding to key 98 | */ 99 | Message putProperties(String key, int value); 100 | 101 | /** 102 | * Puts a {@code String}-{@code long} {@code KeyValue} entry to he headers of a {@code Message}. 103 | * 104 | * @param key the key to be placed into the headers 105 | * @param value the value corresponding to key 106 | */ 107 | Message putProperties(String key, long value); 108 | 109 | /** 110 | * Puts a {@code String}-{@code double} {@code KeyValue} entry to he headers of a {@code Message}. 111 | * 112 | * @param key the key to be placed into the headers 113 | * @param value the value corresponding to key 114 | */ 115 | Message putProperties(String key, double value); 116 | 117 | /** 118 | * Puts a {@code String}-{@code String} {@code KeyValue} entry to he headers of a {@code Message}. 119 | * 120 | * @param key the key to be placed into the headers 121 | * @param value the value corresponding to key 122 | */ 123 | Message putProperties(String key, String value); 124 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/MessageFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | import io.openmessaging.exception.OMSRuntimeException; 21 | 22 | /** 23 | * A factory interface for creating {@code Message} objects. 24 | * 25 | * @author vintagewang@apache.org 26 | * @author yukon@apache.org 27 | */ 28 | public interface MessageFactory { 29 | /** 30 | * Creates a {@code BytesMessage} object. A {@code BytesMessage} object is used to send a message containing a 31 | * stream of uninterpreted bytes. 32 | *

33 | * The returned {@code BytesMessage} object only can be sent to the specified topic. 34 | * 35 | * @param topic the target topic to send 36 | * @param body the body data for a message 37 | * @return the created {@code BytesMessage} object 38 | * @throws OMSRuntimeException if the OMS provider fails to create this message due to some internal error. 39 | */ 40 | BytesMessage createBytesMessageToTopic(String topic, byte[] body); 41 | 42 | /** 43 | * Creates a {@code BytesMessage} object. A {@code BytesMessage} object is used to send a message containing a 44 | * stream of uninterpreted bytes. 45 | *

46 | * The returned {@code BytesMessage} object only can be sent to the specified queue. 47 | * 48 | * @param queue the target queue to send 49 | * @param body the body data for a message 50 | * @return the created {@code BytesMessage} object 51 | * @throws OMSRuntimeException if the OMS provider fails to create this message due to some internal error. 52 | */ 53 | BytesMessage createBytesMessageToQueue(String queue, byte[] body); 54 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/MessageListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * The message listener interface. A message listener must implement this {@code MessageListener} interface and register 22 | * itself to a consumer instance to asynchronously receive messages. 23 | * 24 | * @author vintagewang@apache.org 25 | * @author yukon@apache.org 26 | * @version OMS 1.0 27 | * @since OMS 1.0 28 | */ 29 | public interface MessageListener { 30 | /** 31 | * Callback method to receive incoming messages. 32 | *

33 | * A message listener should handle different types of {@code Message}. 34 | * 35 | * @param message the received Message object 36 | * @param context the context delivered to the consume thread 37 | */ 38 | void onMessage(Message message, OnMessageContext context); 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/MessagingAccessPoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * The {@code MessagingAccessPoint} obtained from {@link MessagingAccessPointManager} is capable of creating {@code 22 | * Producer}, {@code Consumer}, {@code ServiceEndPoint}, and so on. 23 | *

For example: 24 | *

 25 |  * MessagingAccessPoint messagingAccessPoint = MessagingAccessPointManager.getMessagingAccessPoint("openmessaging:rocketmq://localhost:10911/namespace");
 26 |  * Producer producer = messagingAccessPoint.createProducer();
 27 |  * producer.send(producer.createBytesMessageToTopic("HELLO_TOPIC", "HELLO_BODY".getBytes(Charset.forName("UTF-8"))));
 28 |  * 
29 | * 30 | * @author vintagewang@apache.org 31 | * @author yukon@apache.org 32 | * @version OMS 1.0 33 | * @since OMS 1.0 34 | */ 35 | public interface MessagingAccessPoint extends ServiceLifecycle { 36 | /** 37 | * Creates a new {@code Producer} for the specified {@code MessagingAccessPoint}. 38 | * 39 | * @return the created producer 40 | */ 41 | Producer createProducer(); 42 | 43 | /** 44 | * Creates a new {@code Producer} for the specified {@code MessagingAccessPoint} with some preset properties. 45 | * 46 | * @param properties the preset properties 47 | * @return the created producer 48 | */ 49 | Producer createProducer(KeyValue properties); 50 | 51 | /** 52 | * Creates a new {@code PushConsumer} for the specified {@code MessagingAccessPoint}. 53 | * The returned {@code PushConsumer} isn't attached to any queue, 54 | * uses {@link PushConsumer#attachQueue(String, MessageListener)} to attach queues. 55 | * 56 | * @return the created {@code PushConsumer} 57 | */ 58 | PushConsumer createPushConsumer(); 59 | 60 | /** 61 | * Creates a new {@code PushConsumer} for the specified {@code MessagingAccessPoint} with some preset properties. 62 | * 63 | * @param properties the preset properties 64 | * @return the created {@code PushConsumer} 65 | */ 66 | PushConsumer createPushConsumer(KeyValue properties); 67 | 68 | /** 69 | * Creates a new {@code PullConsumer} for the specified {@code MessagingAccessPoint} with the specified queue. 70 | * 71 | * @param queueName the only attached queue for this {@code PullConsumer} 72 | * @return the created {@code PullConsumer} 73 | */ 74 | PullConsumer createPullConsumer(String queueName); 75 | 76 | /** 77 | * Creates a new {@code PullConsumer} for the specified {@code MessagingAccessPoint} with some preset properties. 78 | * 79 | * @param queueName the only attached queue for this {@code PullConsumer} 80 | * @param properties the preset properties 81 | * @return the created {@code PullConsumer} 82 | */ 83 | PullConsumer createPullConsumer(String queueName, KeyValue properties); 84 | 85 | /** 86 | * Creates a new {@code PartitionConsumer} for the specified {@code MessagingAccessPoint}. 87 | * 88 | * @param queueName the only attached queue for this {@code PartitionConsumer} 89 | * @return the created {@code PartitionConsumer} 90 | */ 91 | PartitionConsumer createPartitionConsumer(String queueName); 92 | 93 | /** 94 | * Creates a new {@code PartitionConsumer} for the specified {@code MessagingAccessPoint} with some preset properties. 95 | * 96 | * @param queueName the only attached queue for this {@code PartitionConsumer} 97 | * @param properties the preset properties 98 | * @return the created consumer 99 | */ 100 | PartitionConsumer createPartitionConsumer(String queueName, KeyValue properties); 101 | 102 | /** 103 | * Create a new {@code ResourceManager} for the specified {@code MessagingAccessPoint}. 104 | * 105 | * @return the created {@code ResourceManager} 106 | */ 107 | ResourceManager createResourceManager(); 108 | 109 | /** 110 | * Create a new {@code Filters} for the specified {@code MessagingAccessPoint}. 111 | * 112 | * @return the created {@code Filters} 113 | */ 114 | Filters createFilters(); 115 | 116 | /** 117 | * Create a new {@code ServiceEndPoint} for the specified {@code MessagingAccessPoint}. 118 | * 119 | * @return the created {@code ServiceEndPoint} 120 | */ 121 | ServiceEndPoint createServiceEndPoint(); 122 | 123 | /** 124 | * Create a new {@code ServiceEndPoint} for the specified {@code MessagingAccessPoint} with some preset properties. 125 | * 126 | * @param properties the preset properties 127 | * @return the created {@code ServiceEndPoint} 128 | */ 129 | ServiceEndPoint createServiceEndPoint(KeyValue properties); 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/MessagingAccessPointManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | * @version OMS 1.0 23 | * @since OMS 1.0 24 | */ 25 | public class MessagingAccessPointManager { 26 | public static MessagingAccessPoint getMessagingAccessPoint(String url) { 27 | return getMessagingAccessPoint(url, null); 28 | } 29 | 30 | public static MessagingAccessPoint getMessagingAccessPoint(String url, KeyValue properties) { 31 | return null; 32 | } 33 | 34 | public static KeyValue buildKeyValue() { 35 | return null; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/OnMessageContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | * @version OMS 1.0 23 | * @since OMS 1.0 24 | */ 25 | public interface OnMessageContext { 26 | KeyValue properties(); 27 | 28 | void ack(); 29 | 30 | void ack(final KeyValue properties); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/PartitionConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | import java.util.List; 21 | 22 | /** 23 | * A {@code Queue} may have multiple partitions, each partition supports streaming consumption. 24 | *

25 | * A {@code PartitionConsumer} object supports consume messages from all the partitions of a 26 | * specified queue by streaming way. 27 | * 28 | * @author vintagewang@apache.org 29 | * @author yukon@apache.org 30 | * @version OMS 1.0 31 | * @see MessagingAccessPoint#createPartitionConsumer(String) 32 | * @since OMS 1.0 33 | */ 34 | public interface PartitionConsumer { 35 | /** 36 | * Returns the properties of this {@code PartitionConsumer} instance. 37 | * Changes to the return {@code KeyValue} are not reflected in physical {@code PartitionConsumer}, 38 | * and use {@link ResourceManager#setConsumerProperties(String, KeyValue)} to modify. 39 | * 40 | * @return the properties 41 | */ 42 | KeyValue properties(); 43 | 44 | /** 45 | * Fetches the partition list of the specified queue, which related to this {@code PartitionConsumer} 46 | * 47 | * @return the partition list of queue 48 | */ 49 | List partitionList(); 50 | 51 | /** 52 | * Creates a {@code PartitionIterator} from the specified partition. 53 | *

54 | * If the specified partition doesn't exist, create it automatically. 55 | * 56 | * @param partitionName the specified partition name 57 | * @return the created {@code PartitionIterator} 58 | */ 59 | PartitionIterator partitionIterator(String partitionName); 60 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/PartitionIterator.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging; 2 | 3 | import io.openmessaging.exception.OMSRuntimeException; 4 | 5 | /** 6 | * A {@code PartitionIterator} over a partition of queue, created by 7 | * {@link PartitionConsumer#partitionIterator(String)}, supports consume 8 | * messages bilaterally. 9 | * 10 | * @author vintagewang@apache.org 11 | * @author yukon@apache.org 12 | * @version OMS 1.0 13 | * @since OMS 1.0 14 | */ 15 | public interface PartitionIterator { 16 | /** 17 | * Fetches the current offset of this partition iterator. 18 | * 19 | * @return the current offset, return -1 if the iterator is first created. 20 | */ 21 | long currentOffset(); 22 | 23 | /** 24 | * Fetches the first offset of this partition iterator. 25 | * 26 | * @return the first offset, return -1 if the partition has no message. 27 | */ 28 | long firstOffset(); 29 | 30 | /** 31 | * Fetches the last offset of this partition iterator. 32 | * 33 | * @return the last offset, return 0 if the iterator is first created. 34 | */ 35 | long lastOffset(); 36 | 37 | /** 38 | * Moves the current offset to the specified timestamp. 39 | *

40 | * Moves the current offset to the first offset, if the given timestamp 41 | * is earlier than the first message's store timestamp in this partition iterator. 42 | *

43 | * Moves the current offset to the last offset, if the given timestamp 44 | * is later than the last message's store timestamp in this partition iterator. 45 | * 46 | * @param timestamp the specified timestamp 47 | */ 48 | void seekByTime(long timestamp); 49 | 50 | /** 51 | * Moves the current offset to the specified offset. 52 | * 53 | * @param offset the specified offset 54 | */ 55 | void seekByOffset(long offset); 56 | 57 | /** 58 | * Persist this iterator to local or remote server, that depends on specified 59 | * implementation of {@link PartitionConsumer}. 60 | */ 61 | void persist(); 62 | 63 | /** 64 | * Returns {@code true} if this partition iterator has more messages when 65 | * traversing the iterator in the forward direction. 66 | * 67 | * @return {@code true} if the partition iterator has more messages when 68 | * traversing the iterator in the forward direction 69 | */ 70 | boolean hasNext(); 71 | 72 | /** 73 | * Returns the next message in the iteration and advances the offset position. 74 | *

75 | * This method may be called repeatedly to iterate through the iteration, 76 | * or intermixed with calls to {@link #previous} to go back and forth. 77 | * 78 | * @return the next message in the list 79 | * @throws OMSRuntimeException if the iteration has no more message 80 | */ 81 | Message next(); 82 | 83 | /** 84 | * Returns {@code true} if this partition iterator has more messages when 85 | * traversing the iterator in the reverse direction. 86 | * 87 | * @return {@code true} if the partition iterator has more messages when 88 | * traversing the iterator in the reverse direction 89 | */ 90 | boolean hasPrevious(); 91 | 92 | /** 93 | * Returns the previous message in the iteration and moves the offset 94 | * position backwards. 95 | *

96 | * This method may be called repeatedly to iterate through the iteration backwards, 97 | * or intermixed with calls to {@link #next} to go back and forth. 98 | * 99 | * @return the previous message in the list 100 | * @throws OMSRuntimeException if the iteration has no previous message 101 | */ 102 | Message previous(); 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/Promise.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * A {@code Promise} represents the result of an asynchronous computation. Methods are provided to check if the 22 | * computation is complete, to wait for its completion, and to retrieve the result of the computation. The result can 23 | * only be retrieved using method {@code get} when the computation has completed, blocking if necessary until it is 24 | * ready. Cancellation is performed by the {@code cancel} method. Additional methods are provided to determine if the 25 | * task completed normally or was cancelled. Once a computation has completed, the computation cannot be cancelled. If 26 | * you would like to use a {@code Promise} for the sake of cancellability but not provide a usable result, you can 27 | * declare types of the form {@code Promise} and return {@code null} as a result of the underlying task. 28 | * 29 | * @author vintagewang@apache.org 30 | * @author yukon@apache.org 31 | * @version OMS 1.0 32 | * @since OMS 1.0 33 | */ 34 | public interface Promise { 35 | 36 | /** 37 | * Attempts to cancel execution of this task. This attempt will fail if the task has already completed, has already 38 | * been cancelled, or could not be cancelled for some other reason. If successful, and this task has not started 39 | * when {@code cancel} is called, this task should never run. If the task has already started, then the {@code 40 | * mayInterruptIfRunning} parameter determines whether the thread executing this task should be interrupted in an 41 | * attempt to stop the task. 42 | *

43 | * After this method returns, subsequent calls to {@link #isDone} will always return {@code true}. Subsequent calls 44 | * to {@link #isCancelled} will always return {@code true} if this method returned {@code true}. 45 | * 46 | * @param mayInterruptIfRunning {@code true} if the thread executing this task should be interrupted; otherwise, 47 | * in-progress tasks are allowed to complete 48 | * @return {@code false} if the task could not be cancelled, typically because it has already completed normally; 49 | * {@code true} otherwise 50 | */ 51 | boolean cancel(boolean mayInterruptIfRunning); 52 | 53 | /** 54 | * Returns {@code true} if this task was cancelled before it completed 55 | * normally. 56 | * 57 | * @return {@code true} if this task was cancelled before it completed 58 | */ 59 | boolean isCancelled(); 60 | 61 | /** 62 | * Returns {@code true} if this task completed. 63 | *

64 | * Completion may be due to normal termination, an exception, or 65 | * cancellation -- in all of these cases, this method will return 66 | * {@code true}. 67 | * 68 | * @return {@code true} if this task completed 69 | */ 70 | boolean isDone(); 71 | 72 | /** 73 | * Waits if necessary for the computation to complete, and then 74 | * retrieves its result. 75 | * 76 | * @return the computed result 77 | */ 78 | V get(); 79 | 80 | /** 81 | * Waits if necessary for at most the given time for the computation 82 | * to complete, and then retrieves its result, if available. 83 | * 84 | * @param timeout the maximum time to wait 85 | * @return the computed result

if the computation was cancelled 86 | */ 87 | V get(long timeout); 88 | 89 | /** 90 | * Set the value to this promise and mark it completed if set successfully. 91 | * 92 | * @param value Value 93 | * @return Whether set is success 94 | */ 95 | boolean set(V value); 96 | 97 | /** 98 | * Adds the specified listener to this promise. The specified listener is notified when this promise is done. If this 99 | * promise is already completed, the specified listener is notified immediately. 100 | * 101 | * @param listener PromiseListener 102 | */ 103 | void addListener(PromiseListener listener); 104 | 105 | /** 106 | * @return a throwable caught by the promise 107 | */ 108 | Throwable getThrowable(); 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/PromiseListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | * @author yukon@apache.org 23 | * @version OMS 1.0 24 | * @since OMS 1.0 25 | */ 26 | public interface PromiseListener { 27 | /** 28 | * Invoked when the operation associated with the {@code Promise} has been completed successfully. 29 | * 30 | * @param promise the source {@code Promise} which called this callback 31 | */ 32 | void operationComplete(Promise promise); 33 | 34 | /** 35 | * Invoked when the operation associated with the {@code Promise} has been completed unsuccessfully. 36 | * 37 | * @param promise the source {@code Promise} which called this callback 38 | */ 39 | void operationFailed(Promise promise); 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/PullConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | import io.openmessaging.exception.OMSRuntimeException; 21 | 22 | import java.util.Collection; 23 | 24 | /** 25 | * A {@code PullConsumer} object can poll messages from the specified queue, 26 | * and supports submit the consume result by acknowledgement. 27 | * 28 | * @author vintagewang@apache.org 29 | * @author yukon@apache.org 30 | * @version OMS 1.0 31 | * @see MessagingAccessPoint#createPullConsumer(String) 32 | * @since OMS 1.0 33 | */ 34 | public interface PullConsumer { 35 | /** 36 | * Returns the properties of this {@code PullConsumer} instance. 37 | * Changes to the return {@code KeyValue} are not reflected in physical {@code PullConsumer}, 38 | * and use {@link ResourceManager#setConsumerProperties(String, KeyValue)} to modify. 39 | * 40 | * @return the properties 41 | */ 42 | KeyValue properties(); 43 | 44 | /** 45 | * Polls the next message produced for this {@code PullConsumer}. 46 | *

47 | * This call blocks indefinitely until a message is produced or until this {@code PullConsumer} is shut down. 48 | * 49 | * @return the next message produced for this {@code PullConsumer}, or null if this {@code PullConsumer} is 50 | * concurrently shut down 51 | * @throws OMSRuntimeException if this {@code PullConsumer} fails to pull the next message due to some internal 52 | * error. 53 | */ 54 | /** 55 | * 规范要求实现阻塞的接口,由properties来设置阻塞时间,但本赛题不需要用到该特性,请实现一个非阻塞(也即阻塞时间为0)调用, 也即没有消息则返回null 56 | * 57 | * @return 58 | */ 59 | Message poll(); 60 | 61 | /** 62 | * Polls the next message produced for this {@code PullConsumer}, using the specified properties. 63 | *

64 | * This call blocks indefinitely until a message is produced or until this {@code PullConsumer} is shut down. 65 | * 66 | * @param properties the specified properties 67 | * @return the next message produced for this {@code PullConsumer}, or null if this {@code PullConsumer} is 68 | * concurrently shut down 69 | * @throws OMSRuntimeException if this {@code PullConsumer} fails to pull the next message due to some internal 70 | * error. 71 | */ 72 | Message poll(final KeyValue properties); 73 | 74 | /** 75 | * Acknowledges the specified and consumed message, with unique message id. 76 | *

77 | * Messages that have been received but not acknowledged may be redelivered. 78 | * 79 | * @throws OMSRuntimeException if the consumer fails to acknowledge the messages due to some internal error. 80 | */ 81 | void ack(String messageId); 82 | 83 | /** 84 | * Acknowledges the specified and consumed message with the specified properties. 85 | *

86 | * Messages that have been received but not acknowledged may be redelivered. 87 | * 88 | * @throws OMSRuntimeException if the consumer fails to acknowledge the messages due to some internal error. 89 | */ 90 | void ack(String messageId, final KeyValue properties); 91 | 92 | /** 93 | * 绑定到一个Queue,并订阅topics,即从这些topic读取消息 94 | * 95 | * @param queueName 96 | * @param topics 97 | */ 98 | void attachQueue(String queueName, Collection topics); 99 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/PushConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | import io.openmessaging.exception.OMSResourceNotExistException; 21 | import io.openmessaging.exception.OMSRuntimeException; 22 | 23 | /** 24 | * A {@code PushConsumer} object to receive messages from multiple queue, these messages are pushed from 25 | * MOM server to {@code PushConsumer} client. 26 | * 27 | * @author vintagewang@apache.org 28 | * @author yukon@apache.org 29 | * @version OMS 1.0 30 | * @see MessagingAccessPoint#createPushConsumer() 31 | * @since OMS 1.0 32 | */ 33 | public interface PushConsumer extends ServiceLifecycle { 34 | /** 35 | * Resumes the {@code PushConsumer} after a suspend. 36 | *

37 | * This method resumes the {@code PushConsumer} instance after it was suspended. 38 | * The instance will not receive new messages between the suspend and resume calls. 39 | * 40 | * @throws OMSRuntimeException if the instance has not been suspended. 41 | * @see PushConsumer#suspend() 42 | */ 43 | void resume(); 44 | 45 | /** 46 | * Suspends the {@code PushConsumer} for later resumption. 47 | *

48 | * This method suspends the {@code PushConsumer} instance until it is resumed. 49 | * The instance will not receive new messages between the suspend and resume calls. 50 | * 51 | * @throws OMSRuntimeException if the instance is not currently running. 52 | * @see PushConsumer#resume() 53 | */ 54 | void suspend(); 55 | 56 | /** 57 | * This method is used to find out whether the {@code PushConsumer} is suspended. 58 | * 59 | * @return true if this {@code PushConsumer} is suspended, false otherwise 60 | */ 61 | boolean isSuspended(); 62 | 63 | /** 64 | * Returns the properties of this {@code PushConsumer} instance. 65 | * Changes to the return {@code KeyValue} are not reflected in physical {@code PushConsumer}, 66 | * and use {@link ResourceManager#setConsumerProperties(String, KeyValue)} to modify. 67 | * 68 | * @return the properties 69 | */ 70 | KeyValue properties(); 71 | 72 | /** 73 | * Attaches the {@code PushConsumer} to a specified queue, with a {@code MessageListener}. 74 | * {@link MessageListener#onMessage(Message, OnMessageContext)} will be called when new 75 | * delivered message is coming. 76 | * 77 | * @param queueName a specified queue 78 | * @param listener a specified listener to receive new message 79 | * @throws OMSResourceNotExistException if the specified queue is not exists 80 | */ 81 | PushConsumer attachQueue(String queueName, MessageListener listener) throws OMSResourceNotExistException; 82 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/ServiceEndPoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | import io.openmessaging.observer.Observer; 21 | 22 | /** 23 | * @author vintagewang@apache.org 24 | * @version OMS 1.0 25 | * @since OMS 1.0 26 | */ 27 | public interface ServiceEndPoint extends ServiceLifecycle { 28 | /** 29 | * Register/re-register a service in a serviceEndPoint object 30 | * if service has been registered in serviceEndPoint object, it will be failed when registering delicately 31 | * 32 | * @param service the service to publish in serviceEndPoint 33 | */ 34 | void publish(Object service); 35 | 36 | /** 37 | * Like {@link #publish(Object)} but specifying {@code properties} 38 | * that can be used to configure the service published 39 | * 40 | * @param service the service to publish in serviceEndPoint 41 | * @param properties the service published properties 42 | */ 43 | 44 | void publish(Object service, KeyValue properties); 45 | 46 | /** 47 | * Bind a service object to serviceEndPoint, which can directly call services provided by service object 48 | * 49 | * @param type service type to bind in serviceEndPoint 50 | * @return service proxy object to bind 51 | */ 52 | T bind(Class type); 53 | 54 | /** 55 | * Like {@link #bind(Class)} but specifying {@code properties} that can be used to configure the service band 56 | * 57 | * @param type service type to bind in serviceEndPoint 58 | * @param properties the service bind properties 59 | * @param service proxy object to bind 60 | * @return service proxy object to bind 61 | */ 62 | T bind(Class type, KeyValue properties); 63 | 64 | /** 65 | * Like {@link #bind(Class, KeyValue)} but specifying {@code serviceLoadBalance} that can be used to select 66 | * endPoint target 67 | * 68 | * @param type service type to bind in serviceConsumer 69 | * @param properties the service band properties 70 | * @param serviceLoadBalance select endPoint target algorithm 71 | * @param service proxy object to bind 72 | * @return service proxy object to bind 73 | */ 74 | T bind(Class type, KeyValue properties, ServiceLoadBalance serviceLoadBalance); 75 | 76 | /** 77 | * Register an observer in an serviceEndPoint object. Whenever serviceEndPoint object publish or bind an service 78 | * object, it will be notified to the list of observer object registered before 79 | * 80 | * @param observer observer event object to an serviceEndPoint object 81 | */ 82 | void addObserver(Observer observer); 83 | 84 | /** 85 | * Removes the given observer from the list of observer 86 | *

87 | * If the given observer has not been previously registered (i.e. it was 88 | * never added) then this method call is a no-op. If it had been previously 89 | * added then it will be removed. If it had been added more than once, then 90 | * only the first occurrence will be removed. 91 | * 92 | * @param observer The observer to remove 93 | */ 94 | void deleteObserver(Observer observer); 95 | 96 | /** 97 | * @return 98 | */ 99 | InvokeContext invokeContext(); 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/ServiceLifecycle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | import javax.annotation.PostConstruct; 21 | import javax.annotation.PreDestroy; 22 | 23 | /** 24 | * The {@code ServiceLifecycle} defines a lifecycle interface for a OMS related service endpoint, like {@link Producer}, 25 | * {@link PushConsumer}, and so on. 26 | *

27 | * If the service endpoint class implements the {@code ServiceLifecycle} interface, most of the containers can manage 28 | * the lifecycle of the corresponding service endpoint objects easily. 29 | *

30 | * Any service endpoint should support repeated restart if it implements the {@code ServiceLifecycle} interface. 31 | * 32 | * @author vintagewang@apache.org 33 | * @author yukon@apache.org 34 | * @version OMS 1.0 35 | * @since OMS 1.0 36 | */ 37 | public interface ServiceLifecycle { 38 | /** 39 | * Used for start or initialization of a service endpoint. A service endpoint instance will be in a ready state 40 | * after this method has been completed. 41 | */ 42 | @PostConstruct 43 | void start(); 44 | 45 | /** 46 | * Notify a service instance of the end of its life cycle. Once this method completes, the service endpoint could be 47 | * destroyed and eligible for garbage collection. 48 | */ 49 | @PreDestroy 50 | void shutdown(); 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/ServiceLoadBalance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | import java.util.Set; 21 | 22 | /** 23 | * @author vintagewang@apache.org 24 | * @version OMS 1.0 25 | * @since OMS 1.0 26 | */ 27 | public interface ServiceLoadBalance { 28 | /** 29 | * Select a collection of eligible providerServicePoint object from the the list of providerServicePoint provided 30 | * According to different selection strategies to select providerServicePoint that satisfied with application needs, 31 | * such as RoundRobin or Random etc. 32 | * 33 | * @param servicePropertiesList providerServicePoint to choose from. 34 | * @return a collection of eligible providerServicePoint object 35 | */ 36 | Set select(Set servicePropertiesList); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/ServiceProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging; 19 | 20 | /** 21 | * @author vintagewang@apache. 22 | * @version OMS 1.0 23 | * @since OMS 1.0 24 | */ 25 | public interface ServiceProperties { 26 | String id(); 27 | 28 | void id(String id); 29 | 30 | String relayAddress(); 31 | 32 | void relayAddress(String address); 33 | 34 | String providerId(); 35 | 36 | void providerId(String id); 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/ClientOMSException.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo; 2 | 3 | import io.openmessaging.exception.OMSRuntimeException; 4 | 5 | public class ClientOMSException extends OMSRuntimeException { 6 | 7 | public String message; 8 | 9 | public ClientOMSException(String message) { 10 | this.message = message; 11 | } 12 | 13 | public ClientOMSException(String message, Throwable throwable) { 14 | this.initCause(throwable); 15 | this.message = message; 16 | } 17 | 18 | @Override 19 | public String getMessage() { 20 | return message; 21 | } 22 | 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/Constants.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo; 2 | 3 | /** 4 | * Created by yche on 6/3/17. 5 | */ 6 | public class Constants { 7 | static final char OBJ_SPLITTER = '\t'; 8 | static final char ITEM_SPLITTER = '\f'; 9 | static final char PAIR_SPLITTER = ' ' ; 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/DataDump.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo; 2 | 3 | import io.openmessaging.demo.net.jpountz.lz4.LZ4BlockOutputStream; 4 | import io.openmessaging.demo.net.jpountz.lz4.LZ4JavaUnsafeCompressor; 5 | 6 | import java.io.*; 7 | import java.util.HashMap; 8 | import java.util.HashSet; 9 | import java.util.Set; 10 | import java.util.concurrent.locks.ReentrantReadWriteLock; 11 | 12 | /** 13 | * Created by will on 25/5/2017. 14 | */ 15 | 16 | 17 | public class DataDump { 18 | private static Set folderSet = new HashSet<>(); 19 | private static ReentrantReadWriteLock folderRWLock = new ReentrantReadWriteLock(); 20 | 21 | private final String rootPath; 22 | private HashMap myFileName; 23 | 24 | public DataDump(String folderRootPath) { 25 | rootPath = folderRootPath; 26 | myFileName = new HashMap<>(); 27 | } 28 | 29 | public void writeToFile(String topicName, DefaultBytesMessage message) { 30 | if (!myFileName.containsKey(topicName)) { 31 | createFile(topicName); 32 | } 33 | BufferedWriter bw = myFileName.get(topicName); 34 | try { 35 | bw.write(message.toString()); 36 | bw.newLine(); 37 | } catch (IOException e) { 38 | e.printStackTrace(); 39 | } 40 | } 41 | 42 | private void createFile(String folderName) { 43 | folderRWLock.readLock().lock(); 44 | if (!folderSet.contains(folderName)) { 45 | folderRWLock.readLock().unlock(); 46 | createFolder(folderName); 47 | } else { 48 | folderRWLock.readLock().unlock(); 49 | } 50 | 51 | String fileName = Thread.currentThread().getName(); 52 | 53 | try { 54 | LZ4BlockOutputStream zip = new LZ4BlockOutputStream(new FileOutputStream( 55 | new File(rootPath + File.separator + folderName + File.separator + fileName)), 16 * 1024, new LZ4JavaUnsafeCompressor()); 56 | BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zip)); 57 | myFileName.put(folderName, writer); 58 | } catch (IOException e) { 59 | e.printStackTrace(); 60 | } 61 | } 62 | 63 | public void close() { 64 | myFileName.forEach((topicName, bw) -> { 65 | try { 66 | bw.close(); 67 | } catch (IOException e) { 68 | e.printStackTrace(); 69 | } 70 | }); 71 | } 72 | 73 | private void createFolder(String folderName) { 74 | folderRWLock.writeLock().lock(); 75 | 76 | File dir = new File(rootPath + File.separator + folderName); 77 | if (!dir.exists()) { 78 | dir.mkdir(); 79 | } 80 | folderSet.add(folderName); 81 | 82 | folderRWLock.writeLock().unlock(); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/DefaultKeyValue.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo; 2 | 3 | import io.openmessaging.KeyValue; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.Set; 8 | 9 | public class DefaultKeyValue implements KeyValue { 10 | final Map kvs = new HashMap<>(); 11 | 12 | @Override 13 | public KeyValue put(String key, int value) { 14 | kvs.put(key, Integer.toString(value)); 15 | return this; 16 | } 17 | 18 | @Override 19 | public KeyValue put(String key, long value) { 20 | kvs.put(key, Long.toString(value)); 21 | return this; 22 | } 23 | 24 | @Override 25 | public KeyValue put(String key, double value) { 26 | kvs.put(key, Double.toString(value)); 27 | return this; 28 | } 29 | 30 | @Override 31 | public KeyValue put(String key, String value) { 32 | kvs.put(key, value); 33 | return this; 34 | } 35 | 36 | @Override 37 | public int getInt(String key) { 38 | return Integer.valueOf(kvs.getOrDefault(key, "0")); 39 | } 40 | 41 | @Override 42 | public long getLong(String key) { 43 | return Long.valueOf(kvs.getOrDefault(key, "0")); 44 | } 45 | 46 | @Override 47 | public double getDouble(String key) { 48 | return Double.valueOf(kvs.getOrDefault(key, "0.0")); 49 | } 50 | 51 | @Override 52 | public String getString(String key) { 53 | return kvs.getOrDefault(key, null); 54 | } 55 | 56 | @Override 57 | public Set keySet() { 58 | return kvs.keySet(); 59 | } 60 | 61 | @Override 62 | public boolean containsKey(String key) { 63 | return kvs.containsKey(key); 64 | } 65 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/DefaultMessageFactory.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo; 2 | 3 | import io.openmessaging.BytesMessage; 4 | import io.openmessaging.MessageFactory; 5 | import io.openmessaging.MessageHeader; 6 | 7 | public class DefaultMessageFactory implements MessageFactory { 8 | 9 | @Override 10 | public BytesMessage createBytesMessageToTopic(String topic, byte[] body) { 11 | DefaultBytesMessage defaultBytesMessage = new DefaultBytesMessage(body); 12 | defaultBytesMessage.putHeaders(MessageHeader.TOPIC, topic); 13 | return defaultBytesMessage; 14 | } 15 | 16 | @Override 17 | public BytesMessage createBytesMessageToQueue(String queue, byte[] body) { 18 | DefaultBytesMessage defaultBytesMessage = new DefaultBytesMessage(body); 19 | defaultBytesMessage.putHeaders(MessageHeader.QUEUE, queue); 20 | return defaultBytesMessage; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/DefaultProducer.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo; 2 | 3 | import io.openmessaging.*; 4 | 5 | public class DefaultProducer implements Producer { 6 | private MessageFactory messageFactory = new DefaultMessageFactory(); 7 | private KeyValue properties; 8 | private DataDump dataDump; 9 | 10 | public DefaultProducer(KeyValue properties) { 11 | this.properties = properties; 12 | this.dataDump = new DataDump(properties.getString("STORE_PATH")); 13 | } 14 | 15 | @Override 16 | public BytesMessage createBytesMessageToTopic(String topic, byte[] body) { 17 | return messageFactory.createBytesMessageToTopic(topic, body); 18 | } 19 | 20 | @Override 21 | public BytesMessage createBytesMessageToQueue(String queue, byte[] body) { 22 | return messageFactory.createBytesMessageToQueue(queue, body); 23 | } 24 | 25 | @Override 26 | public void start() { 27 | 28 | } 29 | 30 | @Override 31 | public void shutdown() { 32 | 33 | } 34 | 35 | @Override 36 | public KeyValue properties() { 37 | return properties; 38 | } 39 | 40 | @Override 41 | public void send(Message message) { 42 | String topic = message.headers().getString(MessageHeader.TOPIC); 43 | String queue = message.headers().getString(MessageHeader.QUEUE); 44 | dataDump.writeToFile(topic != null ? topic : queue, (DefaultBytesMessage) message); 45 | } 46 | 47 | @Override 48 | public void send(Message message, KeyValue properties) { 49 | throw new UnsupportedOperationException("Unsupported"); 50 | } 51 | 52 | @Override 53 | public Promise sendAsync(Message message) { 54 | throw new UnsupportedOperationException("Unsupported"); 55 | } 56 | 57 | @Override 58 | public Promise sendAsync(Message message, KeyValue properties) { 59 | throw new UnsupportedOperationException("Unsupported"); 60 | } 61 | 62 | @Override 63 | public void sendOneway(Message message) { 64 | throw new UnsupportedOperationException("Unsupported"); 65 | } 66 | 67 | @Override 68 | public void sendOneway(Message message, KeyValue properties) { 69 | throw new UnsupportedOperationException("Unsupported"); 70 | } 71 | 72 | @Override 73 | public BatchToPartition createBatchToPartition(String partitionName) { 74 | throw new UnsupportedOperationException("Unsupported"); 75 | } 76 | 77 | @Override 78 | public BatchToPartition createBatchToPartition(String partitionName, KeyValue properties) { 79 | throw new UnsupportedOperationException("Unsupported"); 80 | } 81 | 82 | @Override 83 | public void flush() { 84 | dataDump.close(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/DefaultPullConsumer.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo; 2 | 3 | import io.openmessaging.KeyValue; 4 | import io.openmessaging.Message; 5 | import io.openmessaging.PullConsumer; 6 | 7 | import java.util.Collection; 8 | 9 | public class DefaultPullConsumer implements PullConsumer { 10 | private KeyValue properties; 11 | private NaiveDataReader naiveDataReader; 12 | 13 | public DefaultPullConsumer(KeyValue properties) { 14 | this.properties = properties; 15 | this.naiveDataReader = new NaiveDataReader(properties.getString("STORE_PATH")); 16 | } 17 | 18 | @Override 19 | public KeyValue properties() { 20 | return properties; 21 | } 22 | 23 | @Override 24 | public Message poll() { 25 | if (naiveDataReader.hasNext()) { 26 | return naiveDataReader.next(); 27 | } else { 28 | return null; 29 | } 30 | } 31 | 32 | @Override 33 | public Message poll(KeyValue properties) { 34 | throw new UnsupportedOperationException("Unsupported"); 35 | } 36 | 37 | @Override 38 | public void ack(String messageId) { 39 | throw new UnsupportedOperationException("Unsupported"); 40 | } 41 | 42 | @Override 43 | public void ack(String messageId, KeyValue properties) { 44 | throw new UnsupportedOperationException("Unsupported"); 45 | } 46 | 47 | @Override 48 | public void attachQueue(String queueName, Collection topics) { 49 | naiveDataReader.attachNames(queueName, topics); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/NaiveDataReader.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo; 2 | 3 | import io.openmessaging.demo.net.jpountz.lz4.LZ4BlockInputStream; 4 | import io.openmessaging.demo.net.jpountz.lz4.LZ4Factory; 5 | import io.openmessaging.demo.net.jpountz.lz4.LZ4JavaUnsafeCompressor; 6 | import io.openmessaging.demo.net.jpountz.lz4.LZ4JavaUnsafeFastDecompressor; 7 | 8 | import java.io.*; 9 | import java.util.ArrayList; 10 | import java.util.Collection; 11 | import java.util.Collections; 12 | import java.util.Iterator; 13 | 14 | /** 15 | * Created by yche on 5/27/17. 16 | */ 17 | public class NaiveDataReader implements Iterator { 18 | private final String storePath; 19 | 20 | // 1st-level all folders 21 | private ArrayList folderFilesList; 22 | private int folderIndex; 23 | 24 | // 2nd-level one folder 25 | private File[] files; 26 | private int fileIndex; 27 | 28 | // 3rd-level one file 29 | private BufferedReader bufferedReader; 30 | private String tmpBinString; 31 | 32 | public NaiveDataReader(String storePath) { 33 | this.storePath = storePath; 34 | } 35 | 36 | private void fetchNextFolder() { 37 | files = folderFilesList.get(folderIndex); 38 | fileIndex = 0; 39 | } 40 | 41 | private void fetchNextFile() { 42 | try { 43 | // LZ4Factory factory = LZ4Factory.fastestJavaInstance(); 44 | LZ4BlockInputStream zip = new LZ4BlockInputStream(new FileInputStream(files[fileIndex]), new LZ4JavaUnsafeFastDecompressor()); 45 | bufferedReader = new BufferedReader(new InputStreamReader(zip)); 46 | } catch (IOException e) { 47 | e.printStackTrace(); 48 | } 49 | } 50 | 51 | public void attachNames(String queueName, Collection topicNameList) { 52 | // folder_list 53 | ArrayList folderNameList; 54 | folderNameList = new ArrayList<>(topicNameList.size() + 1); 55 | folderNameList.addAll(topicNameList); 56 | Collections.sort(folderNameList); 57 | 58 | folderNameList.add(queueName); 59 | 60 | // 1st-level: all folders 61 | folderFilesList = new ArrayList<>(folderNameList.size()); 62 | folderNameList.forEach((folderString) -> { 63 | File[] files = new File(storePath + File.separator + folderString).listFiles(File::isFile); 64 | folderFilesList.add(files); 65 | }); 66 | folderIndex = 0; 67 | 68 | // 2nd-level: one folder 69 | fetchNextFolder(); 70 | 71 | // 3rd-level: one file 72 | try { 73 | fetchNextFile(); 74 | tmpBinString = bufferedReader.readLine(); 75 | } catch (IOException e) { 76 | e.printStackTrace(); 77 | } 78 | } 79 | 80 | private void nextAndUpdateIteratorStates() throws IOException { 81 | tmpBinString = bufferedReader.readLine(); 82 | // 3rd-level: reach end of file 83 | if (tmpBinString == null) { 84 | 85 | // 2nd-level: reach end of files in this folder 86 | fileIndex++; 87 | if (fileIndex >= files.length) { 88 | folderIndex++; 89 | 90 | // 1st-level: go through all the files already 91 | if (folderIndex >= folderFilesList.size()) { 92 | return; 93 | } 94 | files = folderFilesList.get(folderIndex); 95 | fileIndex = 0; 96 | } 97 | 98 | fetchNextFile(); 99 | tmpBinString = bufferedReader.readLine(); 100 | } 101 | } 102 | 103 | @Override 104 | public boolean hasNext() { 105 | return folderIndex < folderFilesList.size(); 106 | } 107 | 108 | @Override 109 | public DefaultBytesMessage next() { 110 | DefaultBytesMessage message = DefaultBytesMessage.valueOf(tmpBinString); 111 | try { 112 | nextAndUpdateIteratorStates(); 113 | 114 | } catch (IOException e) { 115 | e.printStackTrace(); 116 | } 117 | return message; 118 | } 119 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4Compressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.nio.ByteBuffer; 18 | import java.util.Arrays; 19 | 20 | /** 21 | * LZ4 compressor. 22 | *

23 | * Instances of this class are thread-safe. 24 | */ 25 | public abstract class LZ4Compressor { 26 | 27 | /** Return the maximum compressed length for an input of size length. */ 28 | @SuppressWarnings("static-method") 29 | public final int maxCompressedLength(int length) { 30 | return LZ4Utils.maxCompressedLength(length); 31 | } 32 | 33 | /** 34 | * Compress src[srcOff:srcOff+srcLen] into 35 | * dest[destOff:destOff+destLen] and return the compressed 36 | * length. 37 | * 38 | * This method will throw a {@link LZ4Exception} if this compressor is unable 39 | * to compress the input into less than maxDestLen bytes. To 40 | * prevent this exception to be thrown, you should make sure that 41 | * maxDestLen >= maxCompressedLength(srcLen). 42 | * 43 | * @throws LZ4Exception if maxDestLen is too small 44 | * @return the compressed size 45 | */ 46 | public abstract int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen); 47 | 48 | /** 49 | * Compress src[srcOff:srcOff+srcLen] into 50 | * dest[destOff:destOff+destLen] and return the compressed 51 | * length. 52 | * 53 | * This method will throw a {@link LZ4Exception} if this compressor is unable 54 | * to compress the input into less than maxDestLen bytes. To 55 | * prevent this exception to be thrown, you should make sure that 56 | * maxDestLen >= maxCompressedLength(srcLen). 57 | * 58 | * {@link ByteBuffer} positions remain unchanged. 59 | * 60 | * @throws LZ4Exception if maxDestLen is too small 61 | * @return the compressed size 62 | */ 63 | public abstract int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen); 64 | 65 | /** 66 | * Convenience method, equivalent to calling 67 | * {@link #compress(byte[], int, int, byte[], int, int) compress(src, srcOff, srcLen, dest, destOff, dest.length - destOff)}. 68 | */ 69 | public final int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff) { 70 | return compress(src, srcOff, srcLen, dest, destOff, dest.length - destOff); 71 | } 72 | 73 | /** 74 | * Convenience method, equivalent to calling 75 | * {@link #compress(byte[], int, int, byte[], int) compress(src, 0, src.length, dest, 0)}. 76 | */ 77 | public final int compress(byte[] src, byte[] dest) { 78 | return compress(src, 0, src.length, dest, 0); 79 | } 80 | 81 | /** 82 | * Convenience method which returns src[srcOff:srcOff+srcLen] 83 | * compressed. 84 | *

Warning: this method has an 85 | * important overhead due to the fact that it needs to allocate a buffer to 86 | * compress into, and then needs to resize this buffer to the actual 87 | * compressed length.

88 | *

Here is how this method is implemented:

89 | *
 90 |    * final int maxCompressedLength = maxCompressedLength(srcLen);
 91 |    * final byte[] compressed = new byte[maxCompressedLength];
 92 |    * final int compressedLength = compress(src, srcOff, srcLen, compressed, 0);
 93 |    * return Arrays.copyOf(compressed, compressedLength);
 94 |    * 
95 | */ 96 | public final byte[] compress(byte[] src, int srcOff, int srcLen) { 97 | final int maxCompressedLength = maxCompressedLength(srcLen); 98 | final byte[] compressed = new byte[maxCompressedLength]; 99 | final int compressedLength = compress(src, srcOff, srcLen, compressed, 0); 100 | return Arrays.copyOf(compressed, compressedLength); 101 | } 102 | 103 | /** 104 | * Convenience method, equivalent to calling 105 | * {@link #compress(byte[], int, int) compress(src, 0, src.length)}. 106 | */ 107 | public final byte[] compress(byte[] src) { 108 | return compress(src, 0, src.length); 109 | } 110 | 111 | /** 112 | * Compress src into dest. Calling this method 113 | * will update the positions of both {@link ByteBuffer}s. 114 | */ 115 | public final void compress(ByteBuffer src, ByteBuffer dest) { 116 | final int cpLen = compress(src, src.position(), src.remaining(), dest, dest.position(), dest.remaining()); 117 | src.position(src.limit()); 118 | dest.position(dest.position() + cpLen); 119 | } 120 | 121 | @Override 122 | public String toString() { 123 | return getClass().getSimpleName(); 124 | } 125 | 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4Constants.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | enum LZ4Constants { 19 | ; 20 | 21 | static final int DEFAULT_COMPRESSION_LEVEL = 8+1; 22 | static final int MAX_COMPRESSION_LEVEL = 16+1; 23 | 24 | static final int MEMORY_USAGE = 14; 25 | static final int NOT_COMPRESSIBLE_DETECTION_LEVEL = 6; 26 | 27 | static final int MIN_MATCH = 4; 28 | 29 | static final int HASH_LOG = MEMORY_USAGE - 2; 30 | static final int HASH_TABLE_SIZE = 1 << HASH_LOG; 31 | 32 | static final int SKIP_STRENGTH = Math.max(NOT_COMPRESSIBLE_DETECTION_LEVEL, 2); 33 | static final int COPY_LENGTH = 8; 34 | static final int LAST_LITERALS = 5; 35 | static final int MF_LIMIT = COPY_LENGTH + MIN_MATCH; 36 | static final int MIN_LENGTH = MF_LIMIT + 1; 37 | 38 | static final int MAX_DISTANCE = 1 << 16; 39 | 40 | static final int ML_BITS = 4; 41 | static final int ML_MASK = (1 << ML_BITS) - 1; 42 | static final int RUN_BITS = 8 - ML_BITS; 43 | static final int RUN_MASK = (1 << RUN_BITS) - 1; 44 | 45 | static final int LZ4_64K_LIMIT = (1 << 16) + (MF_LIMIT - 1); 46 | static final int HASH_LOG_64K = HASH_LOG + 1; 47 | static final int HASH_TABLE_SIZE_64K = 1 << HASH_LOG_64K; 48 | 49 | static final int HASH_LOG_HC = 15; 50 | static final int HASH_TABLE_SIZE_HC = 1 << HASH_LOG_HC; 51 | static final int OPTIMAL_ML = ML_MASK - 1 + MIN_MATCH; 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4Decompressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * @deprecated Use {@link LZ4FastDecompressor} instead. 19 | */ 20 | @Deprecated 21 | public interface LZ4Decompressor { 22 | 23 | int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen); 24 | 25 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4Exception.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * LZ4 compression or decompression error. 19 | */ 20 | public class LZ4Exception extends RuntimeException { 21 | 22 | private static final long serialVersionUID = 1L; 23 | 24 | public LZ4Exception(String msg, Throwable t) { 25 | super(msg, t); 26 | } 27 | 28 | public LZ4Exception(String msg) { 29 | super(msg); 30 | } 31 | 32 | public LZ4Exception() { 33 | super(); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4FastDecompressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | /* 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * LZ4 decompressor that requires the size of the original input to be known. 21 | * Use {@link LZ4SafeDecompressor} if you only know the size of the 22 | * compressed stream. 23 | *

24 | * Instances of this class are thread-safe. 25 | */ 26 | public abstract class LZ4FastDecompressor implements LZ4Decompressor { 27 | 28 | /** Decompress src[srcOff:] into dest[destOff:destOff+destLen] 29 | * and return the number of bytes read from src. 30 | * destLen must be exactly the size of the decompressed data. 31 | * 32 | * @param destLen the exact size of the original input 33 | * @return the number of bytes read to restore the original input 34 | */ 35 | public abstract int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen); 36 | 37 | /** Decompress src[srcOff:] into dest[destOff:destOff+destLen] 38 | * and return the number of bytes read from src. 39 | * destLen must be exactly the size of the decompressed data. 40 | * The positions and limits of the {@link ByteBuffer}s remain unchanged. 41 | * 42 | * @param destLen the exact size of the original input 43 | * @return the number of bytes read to restore the original input 44 | */ 45 | public abstract int decompress(ByteBuffer src, int srcOff, ByteBuffer dest, int destOff, int destLen); 46 | 47 | /** 48 | * Convenience method, equivalent to calling 49 | * {@link #decompress(byte[], int, byte[], int, int) decompress(src, 0, dest, 0, destLen)}. 50 | */ 51 | public final int decompress(byte[] src, byte[] dest, int destLen) { 52 | return decompress(src, 0, dest, 0, destLen); 53 | } 54 | 55 | /** 56 | * Convenience method, equivalent to calling 57 | * {@link #decompress(byte[], byte[], int) decompress(src, dest, dest.length)}. 58 | */ 59 | public final int decompress(byte[] src, byte[] dest) { 60 | return decompress(src, dest, dest.length); 61 | } 62 | 63 | /** 64 | * Convenience method which returns src[srcOff:?] 65 | * decompressed. 66 | *

Warning: this method has an 67 | * important overhead due to the fact that it needs to allocate a buffer to 68 | * decompress into.

69 | *

Here is how this method is implemented:

70 | *
 71 |    * final byte[] decompressed = new byte[destLen];
 72 |    * decompress(src, srcOff, decompressed, 0, destLen);
 73 |    * return decompressed;
 74 |    * 
75 | */ 76 | public final byte[] decompress(byte[] src, int srcOff, int destLen) { 77 | final byte[] decompressed = new byte[destLen]; 78 | decompress(src, srcOff, decompressed, 0, destLen); 79 | return decompressed; 80 | } 81 | 82 | /** 83 | * Convenience method, equivalent to calling 84 | * {@link #decompress(byte[], int, int) decompress(src, 0, destLen)}. 85 | */ 86 | public final byte[] decompress(byte[] src, int destLen) { 87 | return decompress(src, 0, destLen); 88 | } 89 | 90 | /** 91 | * Decompress src into dest. dest's 92 | * {@link ByteBuffer#remaining()} must be exactly the size of the decompressed 93 | * data. This method moves the positions of the buffers. 94 | */ 95 | public final void decompress(ByteBuffer src, ByteBuffer dest) { 96 | final int read = decompress(src, src.position(), dest, dest.position(), dest.remaining()); 97 | dest.position(dest.limit()); 98 | src.position(src.position() + read); 99 | } 100 | 101 | @Override 102 | public String toString() { 103 | return getClass().getSimpleName(); 104 | } 105 | 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4HCJNICompressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import io.openmessaging.demo.net.jpountz.util.ByteBufferUtils; 18 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import static io.openmessaging.demo.net.jpountz.lz4.LZ4Constants.DEFAULT_COMPRESSION_LEVEL; 23 | 24 | /** 25 | * High compression {@link LZ4Compressor}s implemented with JNI bindings to the 26 | * original C implementation of LZ4. 27 | */ 28 | final class LZ4HCJNICompressor extends LZ4Compressor { 29 | 30 | public static final LZ4HCJNICompressor INSTANCE = new LZ4HCJNICompressor(); 31 | private static LZ4Compressor SAFE_INSTANCE; 32 | 33 | private final int compressionLevel; 34 | 35 | LZ4HCJNICompressor() { this(DEFAULT_COMPRESSION_LEVEL); } 36 | LZ4HCJNICompressor(int compressionLevel) { 37 | this.compressionLevel = compressionLevel; 38 | } 39 | 40 | @Override 41 | public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) { 42 | SafeUtils.checkRange(src, srcOff, srcLen); 43 | SafeUtils.checkRange(dest, destOff, maxDestLen); 44 | final int result = LZ4JNI.LZ4_compressHC(src, null, srcOff, srcLen, dest, null, destOff, maxDestLen, compressionLevel); 45 | if (result <= 0) { 46 | throw new LZ4Exception(); 47 | } 48 | return result; 49 | } 50 | 51 | @Override 52 | public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) { 53 | ByteBufferUtils.checkNotReadOnly(dest); 54 | ByteBufferUtils.checkRange(src, srcOff, srcLen); 55 | ByteBufferUtils.checkRange(dest, destOff, maxDestLen); 56 | 57 | if ((src.hasArray() || src.isDirect()) && (dest.hasArray() || dest.isDirect())) { 58 | byte[] srcArr = null, destArr = null; 59 | ByteBuffer srcBuf = null, destBuf = null; 60 | if (src.hasArray()) { 61 | srcArr = src.array(); 62 | srcOff += src.arrayOffset(); 63 | } else { 64 | assert src.isDirect(); 65 | srcBuf = src; 66 | } 67 | if (dest.hasArray()) { 68 | destArr = dest.array(); 69 | destOff += dest.arrayOffset(); 70 | } else { 71 | assert dest.isDirect(); 72 | destBuf = dest; 73 | } 74 | 75 | final int result = LZ4JNI.LZ4_compressHC(srcArr, srcBuf, srcOff, srcLen, destArr, destBuf, destOff, maxDestLen, compressionLevel); 76 | if (result <= 0) { 77 | throw new LZ4Exception(); 78 | } 79 | return result; 80 | } else { 81 | LZ4Compressor safeInstance = SAFE_INSTANCE; 82 | if (safeInstance == null) { 83 | safeInstance = SAFE_INSTANCE = LZ4Factory.safeInstance().highCompressor(compressionLevel); 84 | } 85 | return safeInstance.compress(src, srcOff, srcLen, dest, destOff, maxDestLen); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4JNI.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import io.openmessaging.demo.net.jpountz.util.Native; 18 | 19 | import java.nio.ByteBuffer; 20 | 21 | 22 | /** 23 | * JNI bindings to the original C implementation of LZ4. 24 | */ 25 | enum LZ4JNI { 26 | ; 27 | 28 | static { 29 | Native.load(); 30 | init(); 31 | } 32 | 33 | static native void init(); 34 | static native int LZ4_compress_limitedOutput(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen); 35 | static native int LZ4_compressHC(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen, int compressionLevel); 36 | static native int LZ4_decompress_fast(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, byte[] destArray, ByteBuffer destBuffer, int destOff, int destLen); 37 | static native int LZ4_decompress_safe(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen); 38 | static native int LZ4_compressBound(int len); 39 | 40 | } 41 | 42 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4JNICompressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import io.openmessaging.demo.net.jpountz.util.ByteBufferUtils; 18 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import static io.openmessaging.demo.net.jpountz.util.SafeUtils.checkRange; 23 | 24 | /** 25 | * Fast {@link LZ4FastCompressor}s implemented with JNI bindings to the original C 26 | * implementation of LZ4. 27 | */ 28 | final class LZ4JNICompressor extends LZ4Compressor { 29 | 30 | public static final LZ4Compressor INSTANCE = new LZ4JNICompressor(); 31 | private static LZ4Compressor SAFE_INSTANCE; 32 | 33 | @Override 34 | public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) { 35 | SafeUtils.checkRange(src, srcOff, srcLen); 36 | SafeUtils.checkRange(dest, destOff, maxDestLen); 37 | final int result = LZ4JNI.LZ4_compress_limitedOutput(src, null, srcOff, srcLen, dest, null, destOff, maxDestLen); 38 | if (result <= 0) { 39 | throw new LZ4Exception("maxDestLen is too small"); 40 | } 41 | return result; 42 | } 43 | 44 | @Override 45 | public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) { 46 | ByteBufferUtils.checkNotReadOnly(dest); 47 | ByteBufferUtils.checkRange(src, srcOff, srcLen); 48 | ByteBufferUtils.checkRange(dest, destOff, maxDestLen); 49 | 50 | if ((src.hasArray() || src.isDirect()) && (dest.hasArray() || dest.isDirect())) { 51 | byte[] srcArr = null, destArr = null; 52 | ByteBuffer srcBuf = null, destBuf = null; 53 | if (src.hasArray()) { 54 | srcArr = src.array(); 55 | srcOff += src.arrayOffset(); 56 | } else { 57 | assert src.isDirect(); 58 | srcBuf = src; 59 | } 60 | if (dest.hasArray()) { 61 | destArr = dest.array(); 62 | destOff += dest.arrayOffset(); 63 | } else { 64 | assert dest.isDirect(); 65 | destBuf = dest; 66 | } 67 | 68 | final int result = LZ4JNI.LZ4_compress_limitedOutput(srcArr, srcBuf, srcOff, srcLen, destArr, destBuf, destOff, maxDestLen); 69 | if (result <= 0) { 70 | throw new LZ4Exception("maxDestLen is too small"); 71 | } 72 | return result; 73 | } else { 74 | LZ4Compressor safeInstance = SAFE_INSTANCE; 75 | if (safeInstance == null) { 76 | safeInstance = SAFE_INSTANCE = LZ4Factory.safeInstance().fastCompressor(); 77 | } 78 | return safeInstance.compress(src, srcOff, srcLen, dest, destOff, maxDestLen); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4JNIFastDecompressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | import io.openmessaging.demo.net.jpountz.util.ByteBufferUtils; 19 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | 24 | /** 25 | * {@link LZ4FastDecompressor} implemented with JNI bindings to the original C 26 | * implementation of LZ4. 27 | */ 28 | final class LZ4JNIFastDecompressor extends LZ4FastDecompressor { 29 | 30 | public static final LZ4JNIFastDecompressor INSTANCE = new LZ4JNIFastDecompressor(); 31 | private static LZ4FastDecompressor SAFE_INSTANCE; 32 | 33 | @Override 34 | public final int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen) { 35 | SafeUtils.checkRange(src, srcOff); 36 | SafeUtils.checkRange(dest, destOff, destLen); 37 | final int result = LZ4JNI.LZ4_decompress_fast(src, null, srcOff, dest, null, destOff, destLen); 38 | if (result < 0) { 39 | throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer"); 40 | } 41 | return result; 42 | } 43 | 44 | @Override 45 | public int decompress(ByteBuffer src, int srcOff, ByteBuffer dest, int destOff, int destLen) { 46 | ByteBufferUtils.checkNotReadOnly(dest); 47 | ByteBufferUtils.checkRange(src, srcOff); 48 | ByteBufferUtils.checkRange(dest, destOff, destLen); 49 | 50 | if ((src.hasArray() || src.isDirect()) && (dest.hasArray() || dest.isDirect())) { 51 | byte[] srcArr = null, destArr = null; 52 | ByteBuffer srcBuf = null, destBuf = null; 53 | if (src.hasArray()) { 54 | srcArr = src.array(); 55 | srcOff += src.arrayOffset(); 56 | } else { 57 | assert src.isDirect(); 58 | srcBuf = src; 59 | } 60 | if (dest.hasArray()) { 61 | destArr = dest.array(); 62 | destOff += dest.arrayOffset(); 63 | } else { 64 | assert dest.isDirect(); 65 | destBuf = dest; 66 | } 67 | 68 | final int result = LZ4JNI.LZ4_decompress_fast(srcArr, srcBuf, srcOff, destArr, destBuf, destOff, destLen); 69 | if (result < 0) { 70 | throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer"); 71 | } 72 | return result; 73 | } else { 74 | LZ4FastDecompressor safeInstance = SAFE_INSTANCE; 75 | if (safeInstance == null) { 76 | safeInstance = SAFE_INSTANCE = LZ4Factory.safeInstance().fastDecompressor(); 77 | } 78 | return safeInstance.decompress(src, srcOff, dest, destOff, destLen); 79 | } 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4JNISafeDecompressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | import io.openmessaging.demo.net.jpountz.util.ByteBufferUtils; 19 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | /** 24 | * {@link LZ4SafeDecompressor} implemented with JNI bindings to the original C 25 | * implementation of LZ4. 26 | */ 27 | final class LZ4JNISafeDecompressor extends LZ4SafeDecompressor { 28 | 29 | public static final LZ4JNISafeDecompressor INSTANCE = new LZ4JNISafeDecompressor(); 30 | private static LZ4SafeDecompressor SAFE_INSTANCE; 31 | 32 | @Override 33 | public final int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) { 34 | SafeUtils.checkRange(src, srcOff, srcLen); 35 | SafeUtils.checkRange(dest, destOff, maxDestLen); 36 | final int result = LZ4JNI.LZ4_decompress_safe(src, null, srcOff, srcLen, dest, null, destOff, maxDestLen); 37 | if (result < 0) { 38 | throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer"); 39 | } 40 | return result; 41 | } 42 | 43 | @Override 44 | public int decompress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) { 45 | ByteBufferUtils.checkNotReadOnly(dest); 46 | ByteBufferUtils.checkRange(src, srcOff, srcLen); 47 | ByteBufferUtils.checkRange(dest, destOff, maxDestLen); 48 | 49 | if ((src.hasArray() || src.isDirect()) && (dest.hasArray() || dest.isDirect())) { 50 | byte[] srcArr = null, destArr = null; 51 | ByteBuffer srcBuf = null, destBuf = null; 52 | if (src.hasArray()) { 53 | srcArr = src.array(); 54 | srcOff += src.arrayOffset(); 55 | } else { 56 | assert src.isDirect(); 57 | srcBuf = src; 58 | } 59 | if (dest.hasArray()) { 60 | destArr = dest.array(); 61 | destOff += dest.arrayOffset(); 62 | } else { 63 | assert dest.isDirect(); 64 | destBuf = dest; 65 | } 66 | 67 | final int result = LZ4JNI.LZ4_decompress_safe(srcArr, srcBuf, srcOff, srcLen, destArr, destBuf, destOff, maxDestLen); 68 | if (result < 0) { 69 | throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer"); 70 | } 71 | return result; 72 | } else { 73 | LZ4SafeDecompressor safeInstance = SAFE_INSTANCE; 74 | if (safeInstance == null) { 75 | safeInstance = SAFE_INSTANCE = LZ4Factory.safeInstance().safeDecompressor(); 76 | } 77 | return safeInstance.decompress(src, srcOff, srcLen, dest, destOff, maxDestLen); 78 | } 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4SafeDecompressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.nio.ByteBuffer; 18 | import java.util.Arrays; 19 | 20 | /** 21 | * LZ4 decompressor that requires the size of the compressed data to be known. 22 | *

23 | * Implementations of this class are usually a little slower than those of 24 | * {@link LZ4FastDecompressor} but do not require the size of the original data to 25 | * be known. 26 | */ 27 | public abstract class LZ4SafeDecompressor implements LZ4UnknownSizeDecompressor { 28 | 29 | /** 30 | * Decompress src[srcOff:srcLen] into 31 | * dest[destOff:destOff+maxDestLen] and returns the number of 32 | * decompressed bytes written into dest. 33 | * 34 | * @param srcLen the exact size of the compressed stream 35 | * @return the original input size 36 | * @throws LZ4Exception if maxDestLen is too small 37 | */ 38 | public abstract int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen); 39 | 40 | /** 41 | * Uncompress src[srcOff:srcLen] into 42 | * dest[destOff:destOff+maxDestLen] and returns the number of 43 | * decompressed bytes written into dest. 44 | * 45 | * @param srcLen the exact size of the compressed stream 46 | * @return the original input size 47 | * @throws LZ4Exception if maxDestLen is too small 48 | */ 49 | public abstract int decompress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen); 50 | 51 | /** 52 | * Convenience method, equivalent to calling 53 | * {@link #decompress(byte[], int, int, byte[], int, int) decompress(src, srcOff, srcLen, dest, destOff, dest.length - destOff)}. 54 | */ 55 | public final int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff) { 56 | return decompress(src, srcOff, srcLen, dest, destOff, dest.length - destOff); 57 | } 58 | 59 | /** 60 | * Convenience method, equivalent to calling 61 | * {@link #decompress(byte[], int, int, byte[], int) decompress(src, 0, src.length, dest, 0)} 62 | */ 63 | public final int decompress(byte[] src, byte[] dest) { 64 | return decompress(src, 0, src.length, dest, 0); 65 | } 66 | 67 | /** 68 | * Convenience method which returns src[srcOff:srcOff+srcLen] 69 | * decompressed. 70 | *

Warning: this method has an 71 | * important overhead due to the fact that it needs to allocate a buffer to 72 | * decompress into, and then needs to resize this buffer to the actual 73 | * decompressed length.

74 | *

Here is how this method is implemented:

75 | *
 76 |    * byte[] decompressed = new byte[maxDestLen];
 77 |    * final int decompressedLength = decompress(src, srcOff, srcLen, decompressed, 0, maxDestLen);
 78 |    * if (decompressedLength != decompressed.length) {
 79 |    *   decompressed = Arrays.copyOf(decompressed, decompressedLength);
 80 |    * }
 81 |    * return decompressed;
 82 |    * 
83 | */ 84 | public final byte[] decompress(byte[] src, int srcOff, int srcLen, int maxDestLen) { 85 | byte[] decompressed = new byte[maxDestLen]; 86 | final int decompressedLength = decompress(src, srcOff, srcLen, decompressed, 0, maxDestLen); 87 | if (decompressedLength != decompressed.length) { 88 | decompressed = Arrays.copyOf(decompressed, decompressedLength); 89 | } 90 | return decompressed; 91 | } 92 | 93 | /** 94 | * Convenience method, equivalent to calling 95 | * {@link #decompress(byte[], int, int, int) decompress(src, 0, src.length, maxDestLen)}. 96 | */ 97 | public final byte[] decompress(byte[] src, int maxDestLen) { 98 | return decompress(src, 0, src.length, maxDestLen); 99 | } 100 | 101 | /** 102 | * Decompress src into dest. src's 103 | * {@link ByteBuffer#remaining()} must be exactly the size of the compressed 104 | * data. This method moves the positions of the buffers. 105 | */ 106 | public final void decompress(ByteBuffer src, ByteBuffer dest) { 107 | final int decompressed = decompress(src, src.position(), src.remaining(), dest, dest.position(), dest.remaining()); 108 | src.position(src.limit()); 109 | dest.position(dest.position() + decompressed); 110 | } 111 | 112 | @Override 113 | public String toString() { 114 | return getClass().getSimpleName(); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4UnknownSizeDecompressor.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * @deprecated Use {@link LZ4SafeDecompressor} instead. 19 | */ 20 | @Deprecated 21 | public interface LZ4UnknownSizeDecompressor { 22 | 23 | int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen); 24 | 25 | int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff); 26 | 27 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/LZ4Utils.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.lz4; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | enum LZ4Utils { 18 | ; 19 | 20 | private static final int MAX_INPUT_SIZE = 0x7E000000; 21 | 22 | static int maxCompressedLength(int length) { 23 | if (length < 0) { 24 | throw new IllegalArgumentException("length must be >= 0, got " + length); 25 | } else if (length >= MAX_INPUT_SIZE) { 26 | throw new IllegalArgumentException("length must be < " + MAX_INPUT_SIZE); 27 | } 28 | return length + length / 255 + 16; 29 | } 30 | 31 | static int hash(int i) { 32 | return (i * -1640531535) >>> ((LZ4Constants.MIN_MATCH * 8) - LZ4Constants.HASH_LOG); 33 | } 34 | 35 | static int hash64k(int i) { 36 | return (i * -1640531535) >>> ((LZ4Constants.MIN_MATCH * 8) - LZ4Constants.HASH_LOG_64K); 37 | } 38 | 39 | static int hashHC(int i) { 40 | return (i * -1640531535) >>> ((LZ4Constants.MIN_MATCH * 8) - LZ4Constants.HASH_LOG_HC); 41 | } 42 | 43 | static class Match { 44 | int start, ref, len; 45 | 46 | void fix(int correction) { 47 | start += correction; 48 | ref += correction; 49 | len -= correction; 50 | } 51 | 52 | int end() { 53 | return start + len; 54 | } 55 | } 56 | 57 | static void copyTo(Match m1, Match m2) { 58 | m2.len = m1.len; 59 | m2.start = m1.start; 60 | m2.ref = m1.ref; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/lz4/package.html: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 |

LZ4 compression. The entry point of the API is the 21 | {@link net.jpountz.lz4.LZ4Factory} class, which gives access to 22 | {@link net.io.openmessaging.demo.net.jpountz.lz4.LZ4Compressor compressors} and 23 | {@link net.io.openmessaging.demo.net.jpountz.lz4.LZ4SafeDecompressor decompressors}.

24 | 25 | 26 |

Sample usage:

27 | 28 |
29 |     LZ4Factory factory = LZ4Factory.fastestInstance();
30 | 
31 |     byte[] data = "12345345234572".getBytes("UTF-8");
32 |     final int decompressedLength = data.length;
33 | 
34 |     // compress data
35 |     LZ4Compressor compressor = factory.fastCompressor();
36 |     int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
37 |     byte[] compressed = new byte[maxCompressedLength];
38 |     int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
39 | 
40 |     // decompress data
41 |     // - method 1: when the decompressed length is known
42 |     LZ4FastDecompressor decompressor = factory.fastDecompressor();
43 |     byte[] restored = new byte[decompressedLength];
44 |     int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
45 |     // compressedLength == compressedLength2
46 | 
47 |     // - method 2: when the compressed length is known (a little slower)
48 |     // the destination buffer needs to be over-sized
49 |     LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
50 |     int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
51 |     // decompressedLength == decompressedLength2
52 | 
53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/util/ByteBufferUtils.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.util; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.nio.ByteOrder; 5 | import java.nio.ReadOnlyBufferException; 6 | 7 | public enum ByteBufferUtils { 8 | ; 9 | 10 | public static void checkRange(ByteBuffer buf, int off, int len) { 11 | SafeUtils.checkLength(len); 12 | if (len > 0) { 13 | checkRange(buf, off); 14 | checkRange(buf, off + len - 1); 15 | } 16 | } 17 | 18 | public static void checkRange(ByteBuffer buf, int off) { 19 | if (off < 0 || off >= buf.capacity()) { 20 | throw new ArrayIndexOutOfBoundsException(off); 21 | } 22 | } 23 | 24 | public static ByteBuffer inLittleEndianOrder(ByteBuffer buf) { 25 | if (buf.order().equals(ByteOrder.LITTLE_ENDIAN)) { 26 | return buf; 27 | } else { 28 | return buf.duplicate().order(ByteOrder.LITTLE_ENDIAN); 29 | } 30 | } 31 | 32 | public static ByteBuffer inNativeByteOrder(ByteBuffer buf) { 33 | if (buf.order().equals(Utils.NATIVE_BYTE_ORDER)) { 34 | return buf; 35 | } else { 36 | return buf.duplicate().order(Utils.NATIVE_BYTE_ORDER); 37 | } 38 | } 39 | 40 | public static byte readByte(ByteBuffer buf, int i) { 41 | return buf.get(i); 42 | } 43 | 44 | public static void writeInt(ByteBuffer buf, int i, int v) { 45 | assert buf.order() == Utils.NATIVE_BYTE_ORDER; 46 | buf.putInt(i, v); 47 | } 48 | 49 | public static int readInt(ByteBuffer buf, int i) { 50 | assert buf.order() == Utils.NATIVE_BYTE_ORDER; 51 | return buf.getInt(i); 52 | } 53 | 54 | public static int readIntLE(ByteBuffer buf, int i) { 55 | assert buf.order() == ByteOrder.LITTLE_ENDIAN; 56 | return buf.getInt(i); 57 | } 58 | 59 | public static void writeLong(ByteBuffer buf, int i, long v) { 60 | assert buf.order() == Utils.NATIVE_BYTE_ORDER; 61 | buf.putLong(i, v); 62 | } 63 | 64 | public static long readLong(ByteBuffer buf, int i) { 65 | assert buf.order() == Utils.NATIVE_BYTE_ORDER; 66 | return buf.getLong(i); 67 | } 68 | 69 | public static long readLongLE(ByteBuffer buf, int i) { 70 | assert buf.order() == ByteOrder.LITTLE_ENDIAN; 71 | return buf.getLong(i); 72 | } 73 | 74 | public static void writeByte(ByteBuffer dest, int off, int i) { 75 | dest.put(off, (byte) i); 76 | } 77 | 78 | public static void writeShortLE(ByteBuffer dest, int off, int i) { 79 | dest.put(off, (byte) i); 80 | dest.put(off + 1, (byte) (i >>> 8)); 81 | } 82 | 83 | public static void checkNotReadOnly(ByteBuffer buffer) { 84 | if (buffer.isReadOnly()) { 85 | throw new ReadOnlyBufferException(); 86 | } 87 | } 88 | 89 | public static int readShortLE(ByteBuffer buf, int i) { 90 | return (buf.get(i) & 0xFF) | ((buf.get(i+1) & 0xFF) << 8); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/util/Native.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.util; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.io.File; 18 | import java.io.FileOutputStream; 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | 22 | /** FOR INTERNAL USE ONLY */ 23 | public enum Native { 24 | ; 25 | 26 | private enum OS { 27 | // Even on Windows, the default compiler from cpptasks (gcc) uses .so as a shared lib extension 28 | WINDOWS("win32", "so"), LINUX("linux", "so"), MAC("darwin", "dylib"), SOLARIS("solaris", "so"); 29 | public final String name, libExtension; 30 | 31 | private OS(String name, String libExtension) { 32 | this.name = name; 33 | this.libExtension = libExtension; 34 | } 35 | } 36 | 37 | private static String arch() { 38 | return System.getProperty("os.arch"); 39 | } 40 | 41 | private static OS os() { 42 | String osName = System.getProperty("os.name"); 43 | if (osName.contains("Linux")) { 44 | return OS.LINUX; 45 | } else if (osName.contains("Mac")) { 46 | return OS.MAC; 47 | } else if (osName.contains("Windows")) { 48 | return OS.WINDOWS; 49 | } else if (osName.contains("Solaris") || osName.contains("SunOS")) { 50 | return OS.SOLARIS; 51 | } else { 52 | throw new UnsupportedOperationException("Unsupported operating system: " 53 | + osName); 54 | } 55 | } 56 | 57 | private static String resourceName() { 58 | OS os = os(); 59 | String packagePrefix = Native.class.getPackage().getName().replace('.', '/'); 60 | 61 | return "/" + packagePrefix + "/" + os.name + "/" + arch() + "/liblz4-java." + os.libExtension; 62 | } 63 | 64 | private static boolean loaded = false; 65 | 66 | public static synchronized boolean isLoaded() { 67 | return loaded; 68 | } 69 | 70 | public static synchronized void load() { 71 | if (loaded) { 72 | return; 73 | } 74 | 75 | // Try to load lz4-java (liblz4-java.so on Linux) from the java.library.path. 76 | try { 77 | System.loadLibrary("lz4-java"); 78 | loaded = true; 79 | return; 80 | } catch (UnsatisfiedLinkError ex) { 81 | // Doesn't exist, so proceed to loading bundled library. 82 | } 83 | 84 | String resourceName = resourceName(); 85 | InputStream is = Native.class.getResourceAsStream(resourceName); 86 | if (is == null) { 87 | throw new UnsupportedOperationException("Unsupported OS/arch, cannot find " + resourceName + ". Please try building from source."); 88 | } 89 | File tempLib; 90 | try { 91 | tempLib = File.createTempFile("liblz4-java", "." + os().libExtension); 92 | // copy to tempLib 93 | FileOutputStream out = new FileOutputStream(tempLib); 94 | try { 95 | byte[] buf = new byte[4096]; 96 | while (true) { 97 | int read = is.read(buf); 98 | if (read == -1) { 99 | break; 100 | } 101 | out.write(buf, 0, read); 102 | } 103 | try { 104 | out.close(); 105 | out = null; 106 | } catch (IOException e) { 107 | // ignore 108 | } 109 | System.load(tempLib.getAbsolutePath()); 110 | loaded = true; 111 | } finally { 112 | try { 113 | if (out != null) { 114 | out.close(); 115 | } 116 | } catch (IOException e) { 117 | // ignore 118 | } 119 | if (tempLib != null && tempLib.exists()) { 120 | if (!loaded) { 121 | tempLib.delete(); 122 | } else { 123 | // try to delete on exit, does it work on Windows? 124 | tempLib.deleteOnExit(); 125 | } 126 | } 127 | } 128 | } catch (IOException e) { 129 | throw new ExceptionInInitializerError("Cannot unpack liblz4-java"); 130 | } 131 | } 132 | 133 | } 134 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/util/SafeUtils.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.util; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.nio.ByteOrder; 18 | 19 | public enum SafeUtils { 20 | ; 21 | 22 | public static void checkRange(byte[] buf, int off) { 23 | if (off < 0 || off >= buf.length) { 24 | throw new ArrayIndexOutOfBoundsException(off); 25 | } 26 | } 27 | 28 | public static void checkRange(byte[] buf, int off, int len) { 29 | checkLength(len); 30 | if (len > 0) { 31 | checkRange(buf, off); 32 | checkRange(buf, off + len - 1); 33 | } 34 | } 35 | 36 | public static void checkLength(int len) { 37 | if (len < 0) { 38 | throw new IllegalArgumentException("lengths must be >= 0"); 39 | } 40 | } 41 | 42 | public static byte readByte(byte[] buf, int i) { 43 | return buf[i]; 44 | } 45 | 46 | public static int readIntBE(byte[] buf, int i) { 47 | return ((buf[i] & 0xFF) << 24) | ((buf[i+1] & 0xFF) << 16) | ((buf[i+2] & 0xFF) << 8) | (buf[i+3] & 0xFF); 48 | } 49 | 50 | public static int readIntLE(byte[] buf, int i) { 51 | return (buf[i] & 0xFF) | ((buf[i+1] & 0xFF) << 8) | ((buf[i+2] & 0xFF) << 16) | ((buf[i+3] & 0xFF) << 24); 52 | } 53 | 54 | public static int readInt(byte[] buf, int i) { 55 | if (Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { 56 | return readIntBE(buf, i); 57 | } else { 58 | return readIntLE(buf, i); 59 | } 60 | } 61 | 62 | public static long readLongLE(byte[] buf, int i) { 63 | return (buf[i] & 0xFFL) | ((buf[i+1] & 0xFFL) << 8) | ((buf[i+2] & 0xFFL) << 16) | ((buf[i+3] & 0xFFL) << 24) 64 | | ((buf[i+4] & 0xFFL) << 32) | ((buf[i+5] & 0xFFL) << 40) | ((buf[i+6] & 0xFFL) << 48) | ((buf[i+7] & 0xFFL) << 56); 65 | } 66 | 67 | public static void writeShortLE(byte[] buf, int off, int v) { 68 | buf[off++] = (byte) v; 69 | buf[off++] = (byte) (v >>> 8); 70 | } 71 | 72 | public static void writeInt(int[] buf, int off, int v) { 73 | buf[off] = v; 74 | } 75 | 76 | public static int readInt(int[] buf, int off) { 77 | return buf[off]; 78 | } 79 | 80 | public static void writeByte(byte[] dest, int off, int i) { 81 | dest[off] = (byte) i; 82 | } 83 | 84 | public static void writeShort(short[] buf, int off, int v) { 85 | buf[off] = (short) v; 86 | } 87 | 88 | public static int readShortLE(byte[] buf, int i) { 89 | return (buf[i] & 0xFF) | ((buf[i+1] & 0xFF) << 8); 90 | } 91 | 92 | public static int readShort(short[] buf, int off) { 93 | return buf[off] & 0xFFFF; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/util/UnsafeUtils.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.util; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import sun.misc.Unsafe; 18 | 19 | import java.lang.reflect.Field; 20 | import java.nio.ByteOrder; 21 | 22 | public enum UnsafeUtils { 23 | ; 24 | 25 | private static final Unsafe UNSAFE; 26 | private static final long BYTE_ARRAY_OFFSET; 27 | private static final int BYTE_ARRAY_SCALE; 28 | private static final long INT_ARRAY_OFFSET; 29 | private static final int INT_ARRAY_SCALE; 30 | private static final long SHORT_ARRAY_OFFSET; 31 | private static final int SHORT_ARRAY_SCALE; 32 | 33 | static { 34 | try { 35 | Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe"); 36 | theUnsafe.setAccessible(true); 37 | UNSAFE = (Unsafe) theUnsafe.get(null); 38 | BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); 39 | BYTE_ARRAY_SCALE = UNSAFE.arrayIndexScale(byte[].class); 40 | INT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(int[].class); 41 | INT_ARRAY_SCALE = UNSAFE.arrayIndexScale(int[].class); 42 | SHORT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(short[].class); 43 | SHORT_ARRAY_SCALE = UNSAFE.arrayIndexScale(short[].class); 44 | } catch (IllegalAccessException e) { 45 | throw new ExceptionInInitializerError("Cannot access Unsafe"); 46 | } catch (NoSuchFieldException e) { 47 | throw new ExceptionInInitializerError("Cannot access Unsafe"); 48 | } catch (SecurityException e) { 49 | throw new ExceptionInInitializerError("Cannot access Unsafe"); 50 | } 51 | } 52 | 53 | public static void checkRange(byte[] buf, int off) { 54 | SafeUtils.checkRange(buf, off); 55 | } 56 | 57 | public static void checkRange(byte[] buf, int off, int len) { 58 | SafeUtils.checkRange(buf, off, len); 59 | } 60 | 61 | public static void checkLength(int len) { 62 | SafeUtils.checkLength(len); 63 | } 64 | 65 | public static byte readByte(byte[] src, int srcOff) { 66 | return UNSAFE.getByte(src, BYTE_ARRAY_OFFSET + BYTE_ARRAY_SCALE * srcOff); 67 | } 68 | 69 | public static void writeByte(byte[] src, int srcOff, byte value) { 70 | UNSAFE.putByte(src, BYTE_ARRAY_OFFSET + BYTE_ARRAY_SCALE * srcOff, (byte) value); 71 | } 72 | 73 | public static void writeByte(byte[] src, int srcOff, int value) { 74 | writeByte(src, srcOff, (byte) value); 75 | } 76 | 77 | public static long readLong(byte[] src, int srcOff) { 78 | return UNSAFE.getLong(src, BYTE_ARRAY_OFFSET + srcOff); 79 | } 80 | 81 | public static long readLongLE(byte[] src, int srcOff) { 82 | long i = readLong(src, srcOff); 83 | if (Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { 84 | i = Long.reverseBytes(i); 85 | } 86 | return i; 87 | } 88 | 89 | public static void writeLong(byte[] dest, int destOff, long value) { 90 | UNSAFE.putLong(dest, BYTE_ARRAY_OFFSET + destOff, value); 91 | } 92 | 93 | public static int readInt(byte[] src, int srcOff) { 94 | return UNSAFE.getInt(src, BYTE_ARRAY_OFFSET + srcOff); 95 | } 96 | 97 | public static int readIntLE(byte[] src, int srcOff) { 98 | int i = readInt(src, srcOff); 99 | if (Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { 100 | i = Integer.reverseBytes(i); 101 | } 102 | return i; 103 | } 104 | 105 | public static void writeInt(byte[] dest, int destOff, int value) { 106 | UNSAFE.putInt(dest, BYTE_ARRAY_OFFSET + destOff, value); 107 | } 108 | 109 | public static short readShort(byte[] src, int srcOff) { 110 | return UNSAFE.getShort(src, BYTE_ARRAY_OFFSET + srcOff); 111 | } 112 | 113 | public static int readShortLE(byte[] src, int srcOff) { 114 | short s = readShort(src, srcOff); 115 | if (Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { 116 | s = Short.reverseBytes(s); 117 | } 118 | return s & 0xFFFF; 119 | } 120 | 121 | public static void writeShort(byte[] dest, int destOff, short value) { 122 | UNSAFE.putShort(dest, BYTE_ARRAY_OFFSET + destOff, value); 123 | } 124 | 125 | public static void writeShortLE(byte[] buf, int off, int v) { 126 | writeByte(buf, off, (byte) v); 127 | writeByte(buf, off + 1, (byte) (v >>> 8)); 128 | } 129 | 130 | public static int readInt(int[] src, int srcOff) { 131 | return UNSAFE.getInt(src, INT_ARRAY_OFFSET + INT_ARRAY_SCALE * srcOff); 132 | } 133 | 134 | public static void writeInt(int[] dest, int destOff, int value) { 135 | UNSAFE.putInt(dest, INT_ARRAY_OFFSET + INT_ARRAY_SCALE * destOff, value); 136 | } 137 | 138 | public static int readShort(short[] src, int srcOff) { 139 | return UNSAFE.getShort(src, SHORT_ARRAY_OFFSET + SHORT_ARRAY_SCALE * srcOff) & 0xFFFF; 140 | } 141 | 142 | public static void writeShort(short[] dest, int destOff, int value) { 143 | UNSAFE.putShort(dest, SHORT_ARRAY_OFFSET + SHORT_ARRAY_SCALE * destOff, (short) value); 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/util/Utils.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.util; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.nio.ByteOrder; 18 | 19 | public enum Utils { 20 | ; 21 | 22 | public static final ByteOrder NATIVE_BYTE_ORDER = ByteOrder.nativeOrder(); 23 | 24 | private static final boolean unalignedAccessAllowed; 25 | static { 26 | String arch = System.getProperty("os.arch"); 27 | unalignedAccessAllowed = arch.equals("i386") || arch.equals("x86") 28 | || arch.equals("amd64") || arch.equals("x86_64") 29 | || arch.equals("aarch64") || arch.equals("ppc64le"); 30 | } 31 | 32 | public static boolean isUnalignedAccessAllowed() { 33 | return unalignedAccessAllowed; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/util/package.html: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 |

Utility classes.

21 | 22 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/AbstractStreamingXXHash32Java.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.PRIME1; 18 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.PRIME2; 19 | 20 | abstract class AbstractStreamingXXHash32Java extends StreamingXXHash32 { 21 | 22 | int v1, v2, v3, v4, memSize; 23 | long totalLen; 24 | final byte[] memory; 25 | 26 | AbstractStreamingXXHash32Java(int seed) { 27 | super(seed); 28 | memory = new byte[16]; 29 | reset(); 30 | } 31 | 32 | @Override 33 | public void reset() { 34 | v1 = seed + PRIME1 + PRIME2; 35 | v2 = seed + PRIME2; 36 | v3 = seed + 0; 37 | v4 = seed - PRIME1; 38 | totalLen = 0; 39 | memSize = 0; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/AbstractStreamingXXHash64Java.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.PRIME64_1; 18 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.PRIME64_2; 19 | 20 | abstract class AbstractStreamingXXHash64Java extends StreamingXXHash64 { 21 | 22 | int memSize; 23 | long v1, v2, v3, v4; 24 | long totalLen; 25 | final byte[] memory; 26 | 27 | AbstractStreamingXXHash64Java(long seed) { 28 | super(seed); 29 | memory = new byte[32]; 30 | reset(); 31 | } 32 | 33 | @Override 34 | public void reset() { 35 | v1 = seed + PRIME64_1 + PRIME64_2; 36 | v2 = seed + PRIME64_2; 37 | v3 = seed + 0; 38 | v4 = seed - PRIME64_1; 39 | totalLen = 0; 40 | memSize = 0; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/StreamingXXHash32.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | import java.util.zip.Checksum; 4 | 5 | /* 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | /** 21 | * Streaming interface for {@link XXHash32}. 22 | *

23 | * This API is compatible with the {@link XXHash32 block API} and the following 24 | * code samples are equivalent: 25 | *

 26 |  *   int hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, int seed) {
 27 |  *     return xxhashFactory.hash32().hash(buf, off, len, seed);
 28 |  *   }
 29 |  * 
30 | *
 31 |  *   int hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, int seed) {
 32 |  *     StreamingXXHash32 sh32 = xxhashFactory.newStreamingHash32(seed);
 33 |  *     sh32.update(buf, off, len);
 34 |  *     return sh32.getValue();
 35 |  *   }
 36 |  * 
37 | *

38 | * Instances of this class are not thread-safe. 39 | */ 40 | public abstract class StreamingXXHash32 { 41 | 42 | interface Factory { 43 | 44 | StreamingXXHash32 newStreamingHash(int seed); 45 | 46 | } 47 | 48 | final int seed; 49 | 50 | StreamingXXHash32(int seed) { 51 | this.seed = seed; 52 | } 53 | 54 | /** 55 | * Get the value of the checksum. 56 | */ 57 | public abstract int getValue(); 58 | 59 | /** 60 | * Update the value of the hash with buf[off:off+len]. 61 | */ 62 | public abstract void update(byte[] buf, int off, int len); 63 | 64 | /** 65 | * Reset this instance to the state it had right after instantiation. The 66 | * seed remains unchanged. 67 | */ 68 | public abstract void reset(); 69 | 70 | @Override 71 | public String toString() { 72 | return getClass().getSimpleName() + "(seed=" + seed + ")"; 73 | } 74 | 75 | /** 76 | * Return a {@link Checksum} view of this instance. Modifications to the view 77 | * will modify this instance too and vice-versa. 78 | */ 79 | public final Checksum asChecksum() { 80 | return new Checksum() { 81 | 82 | @Override 83 | public long getValue() { 84 | return StreamingXXHash32.this.getValue() & 0xFFFFFFFL; 85 | } 86 | 87 | @Override 88 | public void reset() { 89 | StreamingXXHash32.this.reset(); 90 | } 91 | 92 | @Override 93 | public void update(int b) { 94 | StreamingXXHash32.this.update(new byte[] {(byte) b}, 0, 1); 95 | } 96 | 97 | @Override 98 | public void update(byte[] b, int off, int len) { 99 | StreamingXXHash32.this.update(b, off, len); 100 | } 101 | 102 | @Override 103 | public String toString() { 104 | return StreamingXXHash32.this.toString(); 105 | } 106 | 107 | }; 108 | } 109 | 110 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/StreamingXXHash32JNI.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | final class StreamingXXHash32JNI extends StreamingXXHash32 { 19 | 20 | static class Factory implements StreamingXXHash32.Factory { 21 | 22 | public static final StreamingXXHash32.Factory INSTANCE = new Factory(); 23 | 24 | @Override 25 | public StreamingXXHash32 newStreamingHash(int seed) { 26 | return new StreamingXXHash32JNI(seed); 27 | } 28 | 29 | } 30 | 31 | private long state; 32 | 33 | StreamingXXHash32JNI(int seed) { 34 | super(seed); 35 | state = XXHashJNI.XXH32_init(seed); 36 | } 37 | 38 | private void checkState() { 39 | if (state == 0) { 40 | throw new AssertionError("Already finalized"); 41 | } 42 | } 43 | 44 | @Override 45 | public void reset() { 46 | checkState(); 47 | XXHashJNI.XXH32_free(state); 48 | state = XXHashJNI.XXH32_init(seed); 49 | } 50 | 51 | @Override 52 | public int getValue() { 53 | checkState(); 54 | return XXHashJNI.XXH32_digest(state); 55 | } 56 | 57 | @Override 58 | public void update(byte[] bytes, int off, int len) { 59 | checkState(); 60 | XXHashJNI.XXH32_update(state, bytes, off, len); 61 | } 62 | 63 | @Override 64 | protected void finalize() throws Throwable { 65 | super.finalize(); 66 | // free memory 67 | XXHashJNI.XXH32_free(state); 68 | state = 0; 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/StreamingXXHash32JavaSafe.java: -------------------------------------------------------------------------------- 1 | // Auto-generated: DO NOT EDIT 2 | 3 | package io.openmessaging.demo.net.jpountz.xxhash; 4 | 5 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 6 | 7 | import static java.lang.Integer.rotateLeft; 8 | 9 | /** 10 | * Streaming xxhash. 11 | */ 12 | final class StreamingXXHash32JavaSafe extends AbstractStreamingXXHash32Java { 13 | 14 | static class Factory implements StreamingXXHash32.Factory { 15 | 16 | public static final StreamingXXHash32.Factory INSTANCE = new Factory(); 17 | 18 | @Override 19 | public StreamingXXHash32 newStreamingHash(int seed) { 20 | return new StreamingXXHash32JavaSafe(seed); 21 | } 22 | 23 | } 24 | 25 | StreamingXXHash32JavaSafe(int seed) { 26 | super(seed); 27 | } 28 | 29 | @Override 30 | public int getValue() { 31 | int h32; 32 | if (totalLen >= 16) { 33 | h32 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); 34 | } else { 35 | h32 = seed + XXHashConstants.PRIME5; 36 | } 37 | 38 | h32 += totalLen; 39 | 40 | int off = 0; 41 | while (off <= memSize - 4) { 42 | h32 += SafeUtils.readIntLE(memory, off) * XXHashConstants.PRIME3; 43 | h32 = rotateLeft(h32, 17) * XXHashConstants.PRIME4; 44 | off += 4; 45 | } 46 | 47 | while (off < memSize) { 48 | h32 += (SafeUtils.readByte(memory, off) & 0xFF) * XXHashConstants.PRIME5; 49 | h32 = rotateLeft(h32, 11) * XXHashConstants.PRIME1; 50 | ++off; 51 | } 52 | 53 | h32 ^= h32 >>> 15; 54 | h32 *= XXHashConstants.PRIME2; 55 | h32 ^= h32 >>> 13; 56 | h32 *= XXHashConstants.PRIME3; 57 | h32 ^= h32 >>> 16; 58 | 59 | return h32; 60 | } 61 | 62 | @Override 63 | public void update(byte[] buf, int off, int len) { 64 | SafeUtils.checkRange(buf, off, len); 65 | 66 | totalLen += len; 67 | 68 | if (memSize + len < 16) { // fill in tmp buffer 69 | System.arraycopy(buf, off, memory, memSize, len); 70 | memSize += len; 71 | return; 72 | } 73 | 74 | final int end = off + len; 75 | 76 | if (memSize > 0) { // data left from previous update 77 | System.arraycopy(buf, off, memory, memSize, 16 - memSize); 78 | 79 | v1 += SafeUtils.readIntLE(memory, 0) * XXHashConstants.PRIME2; 80 | v1 = rotateLeft(v1, 13); 81 | v1 *= XXHashConstants.PRIME1; 82 | 83 | v2 += SafeUtils.readIntLE(memory, 4) * XXHashConstants.PRIME2; 84 | v2 = rotateLeft(v2, 13); 85 | v2 *= XXHashConstants.PRIME1; 86 | 87 | v3 += SafeUtils.readIntLE(memory, 8) * XXHashConstants.PRIME2; 88 | v3 = rotateLeft(v3, 13); 89 | v3 *= XXHashConstants.PRIME1; 90 | 91 | v4 += SafeUtils.readIntLE(memory, 12) * XXHashConstants.PRIME2; 92 | v4 = rotateLeft(v4, 13); 93 | v4 *= XXHashConstants.PRIME1; 94 | 95 | off += 16 - memSize; 96 | memSize = 0; 97 | } 98 | 99 | { 100 | final int limit = end - 16; 101 | int v1 = this.v1; 102 | int v2 = this.v2; 103 | int v3 = this.v3; 104 | int v4 = this.v4; 105 | 106 | while (off <= limit) { 107 | v1 += SafeUtils.readIntLE(buf, off) * XXHashConstants.PRIME2; 108 | v1 = rotateLeft(v1, 13); 109 | v1 *= XXHashConstants.PRIME1; 110 | off += 4; 111 | 112 | v2 += SafeUtils.readIntLE(buf, off) * XXHashConstants.PRIME2; 113 | v2 = rotateLeft(v2, 13); 114 | v2 *= XXHashConstants.PRIME1; 115 | off += 4; 116 | 117 | v3 += SafeUtils.readIntLE(buf, off) * XXHashConstants.PRIME2; 118 | v3 = rotateLeft(v3, 13); 119 | v3 *= XXHashConstants.PRIME1; 120 | off += 4; 121 | 122 | v4 += SafeUtils.readIntLE(buf, off) * XXHashConstants.PRIME2; 123 | v4 = rotateLeft(v4, 13); 124 | v4 *= XXHashConstants.PRIME1; 125 | off += 4; 126 | } 127 | 128 | this.v1 = v1; 129 | this.v2 = v2; 130 | this.v3 = v3; 131 | this.v4 = v4; 132 | } 133 | 134 | if (off < end) { 135 | System.arraycopy(buf, off, memory, 0, end - off); 136 | memSize = end - off; 137 | } 138 | } 139 | 140 | } 141 | 142 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/StreamingXXHash32JavaUnsafe.java: -------------------------------------------------------------------------------- 1 | // Auto-generated: DO NOT EDIT 2 | 3 | package io.openmessaging.demo.net.jpountz.xxhash; 4 | 5 | import static java.lang.Integer.rotateLeft; 6 | import static io.openmessaging.demo.net.jpountz.util.SafeUtils.checkRange; 7 | import static io.openmessaging.demo.net.jpountz.util.UnsafeUtils.*; 8 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.*; 9 | 10 | /** 11 | * Streaming xxhash. 12 | */ 13 | public final class StreamingXXHash32JavaUnsafe extends AbstractStreamingXXHash32Java { 14 | 15 | static class Factory implements StreamingXXHash32.Factory { 16 | 17 | public static final StreamingXXHash32.Factory INSTANCE = new Factory(); 18 | 19 | @Override 20 | public StreamingXXHash32 newStreamingHash(int seed) { 21 | return new StreamingXXHash32JavaUnsafe(seed); 22 | } 23 | 24 | } 25 | 26 | public StreamingXXHash32JavaUnsafe(int seed) { 27 | super(seed); 28 | } 29 | 30 | @Override 31 | public int getValue() { 32 | int h32; 33 | if (totalLen >= 16) { 34 | h32 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); 35 | } else { 36 | h32 = seed + PRIME5; 37 | } 38 | 39 | h32 += totalLen; 40 | 41 | int off = 0; 42 | while (off <= memSize - 4) { 43 | h32 += readIntLE(memory, off) * PRIME3; 44 | h32 = rotateLeft(h32, 17) * PRIME4; 45 | off += 4; 46 | } 47 | 48 | while (off < memSize) { 49 | h32 += (readByte(memory, off) & 0xFF) * PRIME5; 50 | h32 = rotateLeft(h32, 11) * PRIME1; 51 | ++off; 52 | } 53 | 54 | h32 ^= h32 >>> 15; 55 | h32 *= PRIME2; 56 | h32 ^= h32 >>> 13; 57 | h32 *= PRIME3; 58 | h32 ^= h32 >>> 16; 59 | 60 | return h32; 61 | } 62 | 63 | @Override 64 | public void update(byte[] buf, int off, int len) { 65 | checkRange(buf, off, len); 66 | 67 | totalLen += len; 68 | 69 | if (memSize + len < 16) { // fill in tmp buffer 70 | System.arraycopy(buf, off, memory, memSize, len); 71 | memSize += len; 72 | return; 73 | } 74 | 75 | final int end = off + len; 76 | 77 | if (memSize > 0) { // data left from previous update 78 | System.arraycopy(buf, off, memory, memSize, 16 - memSize); 79 | 80 | v1 += readIntLE(memory, 0) * PRIME2; 81 | v1 = rotateLeft(v1, 13); 82 | v1 *= PRIME1; 83 | 84 | v2 += readIntLE(memory, 4) * PRIME2; 85 | v2 = rotateLeft(v2, 13); 86 | v2 *= PRIME1; 87 | 88 | v3 += readIntLE(memory, 8) * PRIME2; 89 | v3 = rotateLeft(v3, 13); 90 | v3 *= PRIME1; 91 | 92 | v4 += readIntLE(memory, 12) * PRIME2; 93 | v4 = rotateLeft(v4, 13); 94 | v4 *= PRIME1; 95 | 96 | off += 16 - memSize; 97 | memSize = 0; 98 | } 99 | 100 | { 101 | final int limit = end - 16; 102 | int v1 = this.v1; 103 | int v2 = this.v2; 104 | int v3 = this.v3; 105 | int v4 = this.v4; 106 | 107 | while (off <= limit) { 108 | v1 += readIntLE(buf, off) * PRIME2; 109 | v1 = rotateLeft(v1, 13); 110 | v1 *= PRIME1; 111 | off += 4; 112 | 113 | v2 += readIntLE(buf, off) * PRIME2; 114 | v2 = rotateLeft(v2, 13); 115 | v2 *= PRIME1; 116 | off += 4; 117 | 118 | v3 += readIntLE(buf, off) * PRIME2; 119 | v3 = rotateLeft(v3, 13); 120 | v3 *= PRIME1; 121 | off += 4; 122 | 123 | v4 += readIntLE(buf, off) * PRIME2; 124 | v4 = rotateLeft(v4, 13); 125 | v4 *= PRIME1; 126 | off += 4; 127 | } 128 | 129 | this.v1 = v1; 130 | this.v2 = v2; 131 | this.v3 = v3; 132 | this.v4 = v4; 133 | } 134 | 135 | if (off < end) { 136 | System.arraycopy(buf, off, memory, 0, end - off); 137 | memSize = end - off; 138 | } 139 | } 140 | 141 | } 142 | 143 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/StreamingXXHash64.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | import java.util.zip.Checksum; 4 | 5 | /* 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | /** 21 | * Streaming interface for {@link XXHash64}. 22 | *

23 | * This API is compatible with the {@link XXHash64 block API} and the following 24 | * code samples are equivalent: 25 | *

 26 |  *   long hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, long seed) {
 27 |  *     return xxhashFactory.hash64().hash(buf, off, len, seed);
 28 |  *   }
 29 |  * 
30 | *
 31 |  *   long hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, long seed) {
 32 |  *     StreamingXXHash64 sh64 = xxhashFactory.newStreamingHash64(seed);
 33 |  *     sh64.update(buf, off, len);
 34 |  *     return sh64.getValue();
 35 |  *   }
 36 |  * 
37 | *

38 | * Instances of this class are not thread-safe. 39 | */ 40 | public abstract class StreamingXXHash64 { 41 | 42 | interface Factory { 43 | 44 | StreamingXXHash64 newStreamingHash(long seed); 45 | 46 | } 47 | 48 | final long seed; 49 | 50 | StreamingXXHash64(long seed) { 51 | this.seed = seed; 52 | } 53 | 54 | /** 55 | * Get the value of the checksum. 56 | */ 57 | public abstract long getValue(); 58 | 59 | /** 60 | * Update the value of the hash with buf[off:off+len]. 61 | */ 62 | public abstract void update(byte[] buf, int off, int len); 63 | 64 | /** 65 | * Reset this instance to the state it had right after instantiation. The 66 | * seed remains unchanged. 67 | */ 68 | public abstract void reset(); 69 | 70 | @Override 71 | public String toString() { 72 | return getClass().getSimpleName() + "(seed=" + seed + ")"; 73 | } 74 | 75 | /** 76 | * Return a {@link Checksum} view of this instance. Modifications to the view 77 | * will modify this instance too and vice-versa. 78 | */ 79 | public final Checksum asChecksum() { 80 | return new Checksum() { 81 | 82 | @Override 83 | public long getValue() { 84 | return StreamingXXHash64.this.getValue(); 85 | } 86 | 87 | @Override 88 | public void reset() { 89 | StreamingXXHash64.this.reset(); 90 | } 91 | 92 | @Override 93 | public void update(int b) { 94 | StreamingXXHash64.this.update(new byte[] {(byte) b}, 0, 1); 95 | } 96 | 97 | @Override 98 | public void update(byte[] b, int off, int len) { 99 | StreamingXXHash64.this.update(b, off, len); 100 | } 101 | 102 | @Override 103 | public String toString() { 104 | return StreamingXXHash64.this.toString(); 105 | } 106 | 107 | }; 108 | } 109 | 110 | } -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/StreamingXXHash64JNI.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | final class StreamingXXHash64JNI extends StreamingXXHash64 { 19 | 20 | static class Factory implements StreamingXXHash64.Factory { 21 | 22 | public static final StreamingXXHash64.Factory INSTANCE = new Factory(); 23 | 24 | @Override 25 | public StreamingXXHash64 newStreamingHash(long seed) { 26 | return new StreamingXXHash64JNI(seed); 27 | } 28 | 29 | } 30 | 31 | private long state; 32 | 33 | StreamingXXHash64JNI(long seed) { 34 | super(seed); 35 | state = XXHashJNI.XXH64_init(seed); 36 | } 37 | 38 | private void checkState() { 39 | if (state == 0) { 40 | throw new AssertionError("Already finalized"); 41 | } 42 | } 43 | 44 | @Override 45 | public void reset() { 46 | checkState(); 47 | XXHashJNI.XXH64_free(state); 48 | state = XXHashJNI.XXH64_init(seed); 49 | } 50 | 51 | @Override 52 | public long getValue() { 53 | checkState(); 54 | return XXHashJNI.XXH64_digest(state); 55 | } 56 | 57 | @Override 58 | public void update(byte[] bytes, int off, int len) { 59 | checkState(); 60 | XXHashJNI.XXH64_update(state, bytes, off, len); 61 | } 62 | 63 | @Override 64 | protected void finalize() throws Throwable { 65 | super.finalize(); 66 | // free memory 67 | XXHashJNI.XXH64_free(state); 68 | state = 0; 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/StreamingXXHash64JavaSafe.java: -------------------------------------------------------------------------------- 1 | // Auto-generated: DO NOT EDIT 2 | 3 | package io.openmessaging.demo.net.jpountz.xxhash; 4 | 5 | import static java.lang.Long.rotateLeft; 6 | import static io.openmessaging.demo.net.jpountz.util.SafeUtils.*; 7 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.*; 8 | 9 | /** 10 | * Streaming xxhash. 11 | */ 12 | final class StreamingXXHash64JavaSafe extends AbstractStreamingXXHash64Java { 13 | 14 | static class Factory implements StreamingXXHash64.Factory { 15 | 16 | public static final StreamingXXHash64.Factory INSTANCE = new Factory(); 17 | 18 | @Override 19 | public StreamingXXHash64 newStreamingHash(long seed) { 20 | return new StreamingXXHash64JavaSafe(seed); 21 | } 22 | 23 | } 24 | 25 | StreamingXXHash64JavaSafe(long seed) { 26 | super(seed); 27 | } 28 | 29 | @Override 30 | public long getValue() { 31 | long h64; 32 | if (totalLen >= 32) { 33 | long v1 = this.v1; 34 | long v2 = this.v2; 35 | long v3 = this.v3; 36 | long v4 = this.v4; 37 | 38 | h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); 39 | 40 | v1 *= PRIME64_2; v1 = rotateLeft(v1, 31); v1 *= PRIME64_1; h64 ^= v1; 41 | h64 = h64*PRIME64_1 + PRIME64_4; 42 | 43 | v2 *= PRIME64_2; v2 = rotateLeft(v2, 31); v2 *= PRIME64_1; h64 ^= v2; 44 | h64 = h64*PRIME64_1 + PRIME64_4; 45 | 46 | v3 *= PRIME64_2; v3 = rotateLeft(v3, 31); v3 *= PRIME64_1; h64 ^= v3; 47 | h64 = h64*PRIME64_1 + PRIME64_4; 48 | 49 | v4 *= PRIME64_2; v4 = rotateLeft(v4, 31); v4 *= PRIME64_1; h64 ^= v4; 50 | h64 = h64*PRIME64_1 + PRIME64_4; 51 | } else { 52 | h64 = seed + PRIME64_5; 53 | } 54 | 55 | h64 += totalLen; 56 | 57 | int off = 0; 58 | while (off <= memSize - 8) { 59 | long k1 = readLongLE(memory, off); 60 | k1 *= PRIME64_2; k1 = rotateLeft(k1, 31); k1 *= PRIME64_1; h64 ^= k1; 61 | h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; 62 | off += 8; 63 | } 64 | 65 | if (off <= memSize - 4) { 66 | h64 ^= (readIntLE(memory, off) & 0xFFFFFFFFL) * PRIME64_1; 67 | h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; 68 | off += 4; 69 | } 70 | 71 | while (off < memSize) { 72 | h64 ^= (memory[off] & 0xFF) * PRIME64_5; 73 | h64 = rotateLeft(h64, 11) * PRIME64_1; 74 | ++off; 75 | } 76 | 77 | h64 ^= h64 >>> 33; 78 | h64 *= PRIME64_2; 79 | h64 ^= h64 >>> 29; 80 | h64 *= PRIME64_3; 81 | h64 ^= h64 >>> 32; 82 | 83 | return h64; 84 | } 85 | 86 | @Override 87 | public void update(byte[] buf, int off, int len) { 88 | checkRange(buf, off, len); 89 | 90 | totalLen += len; 91 | 92 | if (memSize + len < 32) { // fill in tmp buffer 93 | System.arraycopy(buf, off, memory, memSize, len); 94 | memSize += len; 95 | return; 96 | } 97 | 98 | final int end = off + len; 99 | 100 | if (memSize > 0) { // data left from previous update 101 | System.arraycopy(buf, off, memory, memSize, 32 - memSize); 102 | 103 | v1 += readLongLE(memory, 0) * PRIME64_2; 104 | v1 = rotateLeft(v1, 31); 105 | v1 *= PRIME64_1; 106 | 107 | v2 += readLongLE(memory, 8) * PRIME64_2; 108 | v2 = rotateLeft(v2, 31); 109 | v2 *= PRIME64_1; 110 | 111 | v3 += readLongLE(memory, 16) * PRIME64_2; 112 | v3 = rotateLeft(v3, 31); 113 | v3 *= PRIME64_1; 114 | 115 | v4 += readLongLE(memory, 24) * PRIME64_2; 116 | v4 = rotateLeft(v4, 31); 117 | v4 *= PRIME64_1; 118 | 119 | off += 32 - memSize; 120 | memSize = 0; 121 | } 122 | 123 | { 124 | final int limit = end - 32; 125 | long v1 = this.v1; 126 | long v2 = this.v2; 127 | long v3 = this.v3; 128 | long v4 = this.v4; 129 | 130 | while (off <= limit) { 131 | v1 += readLongLE(buf, off) * PRIME64_2; 132 | v1 = rotateLeft(v1, 31); 133 | v1 *= PRIME64_1; 134 | off += 8; 135 | 136 | v2 += readLongLE(buf, off) * PRIME64_2; 137 | v2 = rotateLeft(v2, 31); 138 | v2 *= PRIME64_1; 139 | off += 8; 140 | 141 | v3 += readLongLE(buf, off) * PRIME64_2; 142 | v3 = rotateLeft(v3, 31); 143 | v3 *= PRIME64_1; 144 | off += 8; 145 | 146 | v4 += readLongLE(buf, off) * PRIME64_2; 147 | v4 = rotateLeft(v4, 31); 148 | v4 *= PRIME64_1; 149 | off += 8; 150 | } 151 | 152 | this.v1 = v1; 153 | this.v2 = v2; 154 | this.v3 = v3; 155 | this.v4 = v4; 156 | } 157 | 158 | if (off < end) { 159 | System.arraycopy(buf, off, memory, 0, end - off); 160 | memSize = end - off; 161 | } 162 | } 163 | 164 | } 165 | 166 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/StreamingXXHash64JavaUnsafe.java: -------------------------------------------------------------------------------- 1 | // Auto-generated: DO NOT EDIT 2 | 3 | package io.openmessaging.demo.net.jpountz.xxhash; 4 | 5 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 6 | 7 | import static java.lang.Long.rotateLeft; 8 | import static io.openmessaging.demo.net.jpountz.util.SafeUtils.checkRange; 9 | import static io.openmessaging.demo.net.jpountz.util.UnsafeUtils.*; 10 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.*; 11 | 12 | /** 13 | * Streaming xxhash. 14 | */ 15 | final class StreamingXXHash64JavaUnsafe extends AbstractStreamingXXHash64Java { 16 | 17 | static class Factory implements StreamingXXHash64.Factory { 18 | 19 | public static final StreamingXXHash64.Factory INSTANCE = new Factory(); 20 | 21 | @Override 22 | public StreamingXXHash64 newStreamingHash(long seed) { 23 | return new StreamingXXHash64JavaUnsafe(seed); 24 | } 25 | 26 | } 27 | 28 | StreamingXXHash64JavaUnsafe(long seed) { 29 | super(seed); 30 | } 31 | 32 | @Override 33 | public long getValue() { 34 | long h64; 35 | if (totalLen >= 32) { 36 | long v1 = this.v1; 37 | long v2 = this.v2; 38 | long v3 = this.v3; 39 | long v4 = this.v4; 40 | 41 | h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); 42 | 43 | v1 *= PRIME64_2; v1 = rotateLeft(v1, 31); v1 *= PRIME64_1; h64 ^= v1; 44 | h64 = h64*PRIME64_1 + PRIME64_4; 45 | 46 | v2 *= PRIME64_2; v2 = rotateLeft(v2, 31); v2 *= PRIME64_1; h64 ^= v2; 47 | h64 = h64*PRIME64_1 + PRIME64_4; 48 | 49 | v3 *= PRIME64_2; v3 = rotateLeft(v3, 31); v3 *= PRIME64_1; h64 ^= v3; 50 | h64 = h64*PRIME64_1 + PRIME64_4; 51 | 52 | v4 *= PRIME64_2; v4 = rotateLeft(v4, 31); v4 *= PRIME64_1; h64 ^= v4; 53 | h64 = h64*PRIME64_1 + PRIME64_4; 54 | } else { 55 | h64 = seed + PRIME64_5; 56 | } 57 | 58 | h64 += totalLen; 59 | 60 | int off = 0; 61 | while (off <= memSize - 8) { 62 | long k1 = readLongLE(memory, off); 63 | k1 *= PRIME64_2; k1 = rotateLeft(k1, 31); k1 *= PRIME64_1; h64 ^= k1; 64 | h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; 65 | off += 8; 66 | } 67 | 68 | if (off <= memSize - 4) { 69 | h64 ^= (readIntLE(memory, off) & 0xFFFFFFFFL) * PRIME64_1; 70 | h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; 71 | off += 4; 72 | } 73 | 74 | while (off < memSize) { 75 | h64 ^= (memory[off] & 0xFF) * PRIME64_5; 76 | h64 = rotateLeft(h64, 11) * PRIME64_1; 77 | ++off; 78 | } 79 | 80 | h64 ^= h64 >>> 33; 81 | h64 *= PRIME64_2; 82 | h64 ^= h64 >>> 29; 83 | h64 *= PRIME64_3; 84 | h64 ^= h64 >>> 32; 85 | 86 | return h64; 87 | } 88 | 89 | @Override 90 | public void update(byte[] buf, int off, int len) { 91 | SafeUtils.checkRange(buf, off, len); 92 | 93 | totalLen += len; 94 | 95 | if (memSize + len < 32) { // fill in tmp buffer 96 | System.arraycopy(buf, off, memory, memSize, len); 97 | memSize += len; 98 | return; 99 | } 100 | 101 | final int end = off + len; 102 | 103 | if (memSize > 0) { // data left from previous update 104 | System.arraycopy(buf, off, memory, memSize, 32 - memSize); 105 | 106 | v1 += readLongLE(memory, 0) * PRIME64_2; 107 | v1 = rotateLeft(v1, 31); 108 | v1 *= PRIME64_1; 109 | 110 | v2 += readLongLE(memory, 8) * PRIME64_2; 111 | v2 = rotateLeft(v2, 31); 112 | v2 *= PRIME64_1; 113 | 114 | v3 += readLongLE(memory, 16) * PRIME64_2; 115 | v3 = rotateLeft(v3, 31); 116 | v3 *= PRIME64_1; 117 | 118 | v4 += readLongLE(memory, 24) * PRIME64_2; 119 | v4 = rotateLeft(v4, 31); 120 | v4 *= PRIME64_1; 121 | 122 | off += 32 - memSize; 123 | memSize = 0; 124 | } 125 | 126 | { 127 | final int limit = end - 32; 128 | long v1 = this.v1; 129 | long v2 = this.v2; 130 | long v3 = this.v3; 131 | long v4 = this.v4; 132 | 133 | while (off <= limit) { 134 | v1 += readLongLE(buf, off) * PRIME64_2; 135 | v1 = rotateLeft(v1, 31); 136 | v1 *= PRIME64_1; 137 | off += 8; 138 | 139 | v2 += readLongLE(buf, off) * PRIME64_2; 140 | v2 = rotateLeft(v2, 31); 141 | v2 *= PRIME64_1; 142 | off += 8; 143 | 144 | v3 += readLongLE(buf, off) * PRIME64_2; 145 | v3 = rotateLeft(v3, 31); 146 | v3 *= PRIME64_1; 147 | off += 8; 148 | 149 | v4 += readLongLE(buf, off) * PRIME64_2; 150 | v4 = rotateLeft(v4, 31); 151 | v4 *= PRIME64_1; 152 | off += 8; 153 | } 154 | 155 | this.v1 = v1; 156 | this.v2 = v2; 157 | this.v3 = v3; 158 | this.v4 = v4; 159 | } 160 | 161 | if (off < end) { 162 | System.arraycopy(buf, off, memory, 0, end - off); 163 | memSize = end - off; 164 | } 165 | } 166 | 167 | } 168 | 169 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/XXHash32.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | /* 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * A 32-bits hash. 21 | *

22 | * Instances of this class are thread-safe. 23 | */ 24 | public abstract class XXHash32 { 25 | 26 | /** 27 | * Compute the 32-bits hash of buf[off:off+len] using seed 28 | * seed. 29 | */ 30 | public abstract int hash(byte[] buf, int off, int len, int seed); 31 | 32 | /** 33 | * Compute the hash of the given slice of the {@link ByteBuffer}. 34 | * {@link ByteBuffer#position() position} and {@link ByteBuffer#limit() limit} 35 | * are not modified. 36 | */ 37 | public abstract int hash(ByteBuffer buf, int off, int len, int seed); 38 | 39 | /** 40 | * Compute the hash of the given {@link ByteBuffer}. The 41 | * {@link ByteBuffer#position() position} is moved in order to reflect bytes 42 | * which have been read. 43 | */ 44 | public final int hash(ByteBuffer buf, int seed) { 45 | final int hash = hash(buf, buf.position(), buf.remaining(), seed); 46 | buf.position(buf.limit()); 47 | return hash; 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return getClass().getSimpleName(); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/XXHash32JNI.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import io.openmessaging.demo.net.jpountz.util.ByteBufferUtils; 18 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import static io.openmessaging.demo.net.jpountz.util.SafeUtils.checkRange; 23 | 24 | final class XXHash32JNI extends XXHash32 { 25 | 26 | public static final XXHash32 INSTANCE = new XXHash32JNI(); 27 | private static XXHash32 SAFE_INSTANCE; 28 | 29 | @Override 30 | public int hash(byte[] buf, int off, int len, int seed) { 31 | SafeUtils.checkRange(buf, off, len); 32 | return XXHashJNI.XXH32(buf, off, len, seed); 33 | } 34 | 35 | @Override 36 | public int hash(ByteBuffer buf, int off, int len, int seed) { 37 | if (buf.isDirect()) { 38 | ByteBufferUtils.checkRange(buf, off, len); 39 | return XXHashJNI.XXH32BB(buf, off, len, seed); 40 | } else if (buf.hasArray()) { 41 | return hash(buf.array(), off + buf.arrayOffset(), len, seed); 42 | } else { 43 | XXHash32 safeInstance = SAFE_INSTANCE; 44 | if (safeInstance == null) { 45 | safeInstance = SAFE_INSTANCE = XXHashFactory.safeInstance().hash32(); 46 | } 47 | return safeInstance.hash(buf, off, len, seed); 48 | } 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/XXHash32JavaSafe.java: -------------------------------------------------------------------------------- 1 | // Auto-generated: DO NOT EDIT 2 | 3 | package io.openmessaging.demo.net.jpountz.xxhash; 4 | 5 | import io.openmessaging.demo.net.jpountz.util.ByteBufferUtils; 6 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 7 | 8 | import java.nio.ByteBuffer; 9 | 10 | import static java.lang.Integer.rotateLeft; 11 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.*; 12 | 13 | /** 14 | * {@link XXHash32} implementation. 15 | */ 16 | final class XXHash32JavaSafe extends XXHash32 { 17 | 18 | public static final XXHash32 INSTANCE = new XXHash32JavaSafe(); 19 | 20 | @Override 21 | public int hash(byte[] buf, int off, int len, int seed) { 22 | 23 | SafeUtils.checkRange(buf, off, len); 24 | 25 | final int end = off + len; 26 | int h32; 27 | 28 | if (len >= 16) { 29 | final int limit = end - 16; 30 | int v1 = seed + PRIME1 + PRIME2; 31 | int v2 = seed + PRIME2; 32 | int v3 = seed + 0; 33 | int v4 = seed - PRIME1; 34 | do { 35 | v1 += SafeUtils.readIntLE(buf, off) * PRIME2; 36 | v1 = rotateLeft(v1, 13); 37 | v1 *= PRIME1; 38 | off += 4; 39 | 40 | v2 += SafeUtils.readIntLE(buf, off) * PRIME2; 41 | v2 = rotateLeft(v2, 13); 42 | v2 *= PRIME1; 43 | off += 4; 44 | 45 | v3 += SafeUtils.readIntLE(buf, off) * PRIME2; 46 | v3 = rotateLeft(v3, 13); 47 | v3 *= PRIME1; 48 | off += 4; 49 | 50 | v4 += SafeUtils.readIntLE(buf, off) * PRIME2; 51 | v4 = rotateLeft(v4, 13); 52 | v4 *= PRIME1; 53 | off += 4; 54 | } while (off <= limit); 55 | 56 | h32 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); 57 | } else { 58 | h32 = seed + PRIME5; 59 | } 60 | 61 | h32 += len; 62 | 63 | while (off <= end - 4) { 64 | h32 += SafeUtils.readIntLE(buf, off) * PRIME3; 65 | h32 = rotateLeft(h32, 17) * PRIME4; 66 | off += 4; 67 | } 68 | 69 | while (off < end) { 70 | h32 += (SafeUtils.readByte(buf, off) & 0xFF) * PRIME5; 71 | h32 = rotateLeft(h32, 11) * PRIME1; 72 | ++off; 73 | } 74 | 75 | h32 ^= h32 >>> 15; 76 | h32 *= PRIME2; 77 | h32 ^= h32 >>> 13; 78 | h32 *= PRIME3; 79 | h32 ^= h32 >>> 16; 80 | 81 | return h32; 82 | } 83 | 84 | @Override 85 | public int hash(ByteBuffer buf, int off, int len, int seed) { 86 | 87 | if (buf.hasArray()) { 88 | return hash(buf.array(), off + buf.arrayOffset(), len, seed); 89 | } 90 | ByteBufferUtils.checkRange(buf, off, len); 91 | buf = ByteBufferUtils.inLittleEndianOrder(buf); 92 | 93 | final int end = off + len; 94 | int h32; 95 | 96 | if (len >= 16) { 97 | final int limit = end - 16; 98 | int v1 = seed + PRIME1 + PRIME2; 99 | int v2 = seed + PRIME2; 100 | int v3 = seed + 0; 101 | int v4 = seed - PRIME1; 102 | do { 103 | v1 += ByteBufferUtils.readIntLE(buf, off) * PRIME2; 104 | v1 = rotateLeft(v1, 13); 105 | v1 *= PRIME1; 106 | off += 4; 107 | 108 | v2 += ByteBufferUtils.readIntLE(buf, off) * PRIME2; 109 | v2 = rotateLeft(v2, 13); 110 | v2 *= PRIME1; 111 | off += 4; 112 | 113 | v3 += ByteBufferUtils.readIntLE(buf, off) * PRIME2; 114 | v3 = rotateLeft(v3, 13); 115 | v3 *= PRIME1; 116 | off += 4; 117 | 118 | v4 += ByteBufferUtils.readIntLE(buf, off) * PRIME2; 119 | v4 = rotateLeft(v4, 13); 120 | v4 *= PRIME1; 121 | off += 4; 122 | } while (off <= limit); 123 | 124 | h32 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); 125 | } else { 126 | h32 = seed + PRIME5; 127 | } 128 | 129 | h32 += len; 130 | 131 | while (off <= end - 4) { 132 | h32 += ByteBufferUtils.readIntLE(buf, off) * PRIME3; 133 | h32 = rotateLeft(h32, 17) * PRIME4; 134 | off += 4; 135 | } 136 | 137 | while (off < end) { 138 | h32 += (ByteBufferUtils.readByte(buf, off) & 0xFF) * PRIME5; 139 | h32 = rotateLeft(h32, 11) * PRIME1; 140 | ++off; 141 | } 142 | 143 | h32 ^= h32 >>> 15; 144 | h32 *= PRIME2; 145 | h32 ^= h32 >>> 13; 146 | h32 *= PRIME3; 147 | h32 ^= h32 >>> 16; 148 | 149 | return h32; 150 | } 151 | 152 | 153 | } 154 | 155 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/XXHash32JavaUnsafe.java: -------------------------------------------------------------------------------- 1 | // Auto-generated: DO NOT EDIT 2 | 3 | package io.openmessaging.demo.net.jpountz.xxhash; 4 | 5 | import io.openmessaging.demo.net.jpountz.util.ByteBufferUtils; 6 | import io.openmessaging.demo.net.jpountz.util.UnsafeUtils; 7 | 8 | import java.nio.ByteBuffer; 9 | 10 | import static java.lang.Integer.rotateLeft; 11 | import static io.openmessaging.demo.net.jpountz.xxhash.XXHashConstants.*; 12 | 13 | /** 14 | * {@link XXHash32} implementation. 15 | */ 16 | final class XXHash32JavaUnsafe extends XXHash32 { 17 | 18 | public static final XXHash32 INSTANCE = new XXHash32JavaUnsafe(); 19 | 20 | @Override 21 | public int hash(byte[] buf, int off, int len, int seed) { 22 | 23 | UnsafeUtils.checkRange(buf, off, len); 24 | 25 | final int end = off + len; 26 | int h32; 27 | 28 | if (len >= 16) { 29 | final int limit = end - 16; 30 | int v1 = seed + PRIME1 + PRIME2; 31 | int v2 = seed + PRIME2; 32 | int v3 = seed + 0; 33 | int v4 = seed - PRIME1; 34 | do { 35 | v1 += UnsafeUtils.readIntLE(buf, off) * PRIME2; 36 | v1 = rotateLeft(v1, 13); 37 | v1 *= PRIME1; 38 | off += 4; 39 | 40 | v2 += UnsafeUtils.readIntLE(buf, off) * PRIME2; 41 | v2 = rotateLeft(v2, 13); 42 | v2 *= PRIME1; 43 | off += 4; 44 | 45 | v3 += UnsafeUtils.readIntLE(buf, off) * PRIME2; 46 | v3 = rotateLeft(v3, 13); 47 | v3 *= PRIME1; 48 | off += 4; 49 | 50 | v4 += UnsafeUtils.readIntLE(buf, off) * PRIME2; 51 | v4 = rotateLeft(v4, 13); 52 | v4 *= PRIME1; 53 | off += 4; 54 | } while (off <= limit); 55 | 56 | h32 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); 57 | } else { 58 | h32 = seed + PRIME5; 59 | } 60 | 61 | h32 += len; 62 | 63 | while (off <= end - 4) { 64 | h32 += UnsafeUtils.readIntLE(buf, off) * PRIME3; 65 | h32 = rotateLeft(h32, 17) * PRIME4; 66 | off += 4; 67 | } 68 | 69 | while (off < end) { 70 | h32 += (UnsafeUtils.readByte(buf, off) & 0xFF) * PRIME5; 71 | h32 = rotateLeft(h32, 11) * PRIME1; 72 | ++off; 73 | } 74 | 75 | h32 ^= h32 >>> 15; 76 | h32 *= PRIME2; 77 | h32 ^= h32 >>> 13; 78 | h32 *= PRIME3; 79 | h32 ^= h32 >>> 16; 80 | 81 | return h32; 82 | } 83 | 84 | @Override 85 | public int hash(ByteBuffer buf, int off, int len, int seed) { 86 | 87 | if (buf.hasArray()) { 88 | return hash(buf.array(), off + buf.arrayOffset(), len, seed); 89 | } 90 | ByteBufferUtils.checkRange(buf, off, len); 91 | buf = ByteBufferUtils.inLittleEndianOrder(buf); 92 | 93 | final int end = off + len; 94 | int h32; 95 | 96 | if (len >= 16) { 97 | final int limit = end - 16; 98 | int v1 = seed + PRIME1 + PRIME2; 99 | int v2 = seed + PRIME2; 100 | int v3 = seed + 0; 101 | int v4 = seed - PRIME1; 102 | do { 103 | v1 += ByteBufferUtils.readIntLE(buf, off) * PRIME2; 104 | v1 = rotateLeft(v1, 13); 105 | v1 *= PRIME1; 106 | off += 4; 107 | 108 | v2 += ByteBufferUtils.readIntLE(buf, off) * PRIME2; 109 | v2 = rotateLeft(v2, 13); 110 | v2 *= PRIME1; 111 | off += 4; 112 | 113 | v3 += ByteBufferUtils.readIntLE(buf, off) * PRIME2; 114 | v3 = rotateLeft(v3, 13); 115 | v3 *= PRIME1; 116 | off += 4; 117 | 118 | v4 += ByteBufferUtils.readIntLE(buf, off) * PRIME2; 119 | v4 = rotateLeft(v4, 13); 120 | v4 *= PRIME1; 121 | off += 4; 122 | } while (off <= limit); 123 | 124 | h32 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); 125 | } else { 126 | h32 = seed + PRIME5; 127 | } 128 | 129 | h32 += len; 130 | 131 | while (off <= end - 4) { 132 | h32 += ByteBufferUtils.readIntLE(buf, off) * PRIME3; 133 | h32 = rotateLeft(h32, 17) * PRIME4; 134 | off += 4; 135 | } 136 | 137 | while (off < end) { 138 | h32 += (ByteBufferUtils.readByte(buf, off) & 0xFF) * PRIME5; 139 | h32 = rotateLeft(h32, 11) * PRIME1; 140 | ++off; 141 | } 142 | 143 | h32 ^= h32 >>> 15; 144 | h32 *= PRIME2; 145 | h32 ^= h32 >>> 13; 146 | h32 *= PRIME3; 147 | h32 ^= h32 >>> 16; 148 | 149 | return h32; 150 | } 151 | 152 | 153 | } 154 | 155 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/XXHash64.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | /* 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * A 64-bits hash. 21 | *

22 | * Instances of this class are thread-safe. 23 | */ 24 | public abstract class XXHash64 { 25 | 26 | /** 27 | * Compute the 64-bits hash of buf[off:off+len] using seed 28 | * seed. 29 | */ 30 | public abstract long hash(byte[] buf, int off, int len, long seed); 31 | 32 | /** 33 | * Compute the hash of the given slice of the {@link ByteBuffer}. 34 | * {@link ByteBuffer#position() position} and {@link ByteBuffer#limit() limit} 35 | * are not modified. 36 | */ 37 | public abstract long hash(ByteBuffer buf, int off, int len, long seed); 38 | 39 | /** 40 | * Compute the hash of the given {@link ByteBuffer}. The 41 | * {@link ByteBuffer#position() position} is moved in order to reflect bytes 42 | * which have been read. 43 | */ 44 | public final long hash(ByteBuffer buf, long seed) { 45 | final long hash = hash(buf, buf.position(), buf.remaining(), seed); 46 | buf.position(buf.limit()); 47 | return hash; 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return getClass().getSimpleName(); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/XXHash64JNI.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import io.openmessaging.demo.net.jpountz.util.ByteBufferUtils; 18 | import io.openmessaging.demo.net.jpountz.util.SafeUtils; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import static io.openmessaging.demo.net.jpountz.util.SafeUtils.checkRange; 23 | 24 | final class XXHash64JNI extends XXHash64 { 25 | 26 | public static final XXHash64 INSTANCE = new XXHash64JNI(); 27 | private static XXHash64 SAFE_INSTANCE; 28 | 29 | @Override 30 | public long hash(byte[] buf, int off, int len, long seed) { 31 | SafeUtils.checkRange(buf, off, len); 32 | return XXHashJNI.XXH64(buf, off, len, seed); 33 | } 34 | 35 | @Override 36 | public long hash(ByteBuffer buf, int off, int len, long seed) { 37 | if (buf.isDirect()) { 38 | ByteBufferUtils.checkRange(buf, off, len); 39 | return XXHashJNI.XXH64BB(buf, off, len, seed); 40 | } else if (buf.hasArray()) { 41 | return hash(buf.array(), off + buf.arrayOffset(), len, seed); 42 | } else { 43 | XXHash64 safeInstance = SAFE_INSTANCE; 44 | if (safeInstance == null) { 45 | safeInstance = SAFE_INSTANCE = XXHashFactory.safeInstance().hash64(); 46 | } 47 | return safeInstance.hash(buf, off, len, seed); 48 | } 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/XXHashConstants.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | enum XXHashConstants { 18 | ; 19 | 20 | static final int PRIME1 = -1640531535; 21 | static final int PRIME2 = -2048144777; 22 | static final int PRIME3 = -1028477379; 23 | static final int PRIME4 = 668265263; 24 | static final int PRIME5 = 374761393; 25 | 26 | static final long PRIME64_1 = -7046029288634856825L; //11400714785074694791 27 | static final long PRIME64_2 = -4417276706812531889L; //14029467366897019727 28 | static final long PRIME64_3 = 1609587929392839161L; 29 | static final long PRIME64_4 = -8796714831421723037L; //9650029242287828579 30 | static final long PRIME64_5 = 2870177450012600261L; 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/XXHashJNI.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.net.jpountz.xxhash; 2 | 3 | /* 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import io.openmessaging.demo.net.jpountz.util.Native; 18 | 19 | import java.nio.ByteBuffer; 20 | 21 | enum XXHashJNI { 22 | ; 23 | 24 | static { 25 | Native.load(); 26 | init(); 27 | } 28 | 29 | private static native void init(); 30 | static native int XXH32(byte[] input, int offset, int len, int seed); 31 | static native int XXH32BB(ByteBuffer input, int offset, int len, int seed); 32 | static native long XXH32_init(int seed); 33 | static native void XXH32_update(long state, byte[] input, int offset, int len); 34 | static native int XXH32_digest(long state); 35 | static native void XXH32_free(long state); 36 | 37 | static native long XXH64(byte[] input, int offset, int len, long seed); 38 | static native long XXH64BB(ByteBuffer input, int offset, int len, long seed); 39 | static native long XXH64_init(long seed); 40 | static native void XXH64_update(long state, byte[] input, int offset, int len); 41 | static native long XXH64_digest(long state); 42 | static native void XXH64_free(long state); 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/net/jpountz/xxhash/package.html: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 |

xxhash hashing. This package supports both block hashing via 21 | {@link net.jpountz.xxhash.XXHash32} and streaming hashing via 22 | {@link net.jpountz.xxhash.StreamingXXHash32}. Have a look at 23 | {@link net.jpountz.xxhash.XXHashFactory} to know how to get instances of these 24 | interfaces.

25 | 26 |

Streaming hashing is a little slower but doesn't require to load the whole 27 | stream into memory.

28 | 29 |

Sample block usage:

30 | 31 |
32 |     XXHashFactory factory = XXHashFactory.fastestInstance();
33 | 
34 |     byte[] data = "12345345234572".getBytes("UTF-8");
35 | 
36 |     XXHash32 hash32 = factory.hash32();
37 |     int seed = 0x9747b28c; // used to initialize the hash value, use whatever
38 |                            // value you want, but always the same
39 |     int hash = hash32.hash(data, 0, data.length, seed);
40 | 
41 | 42 |

Sample streaming usage:

43 | 44 |
45 |     XXHashFactory factory = XXHashFactory.fastestInstance();
46 | 
47 |     byte[] data = "12345345234572".getBytes("UTF-8");
48 |     ByteArrayInputStream in = new ByteArrayInputStream(data);
49 | 
50 |     int seed = 0x9747b28c; // used to initialize the hash value, use whatever
51 |                            // value you want, but always the same
52 |     StreamingXXHash32 hash32 = factory.newStreamingHash32(seed);
53 |     byte[] buf = new byte[8]; // for real-world usage, use a larger buffer, like 8192 bytes
54 |     for (;;) {
55 |       int read = in.read(buf);
56 |       if (read == -1) {
57 |         break;
58 |       }
59 |       hash32.update(buf, 0, read);
60 |     }
61 |     int hash = hash32.getValue();
62 | 
63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/demo/unitTest/MessageStringTest.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.demo.unitTest; 2 | 3 | import io.openmessaging.demo.DefaultBytesMessage; 4 | 5 | /** 6 | * Created by yche on 6/4/17. 7 | */ 8 | public class MessageStringTest { 9 | public static void main(String[] args) { 10 | DefaultBytesMessage defaultBytesMessage = new DefaultBytesMessage(new String("Hello").getBytes()); 11 | defaultBytesMessage.putHeaders("h1", "pro1_val"); 12 | defaultBytesMessage.putHeaders("h2", 1); 13 | defaultBytesMessage.putHeaders("head2", 1.0); 14 | defaultBytesMessage.putProperties("pro2", 1); 15 | defaultBytesMessage.putProperties("pro3", 2L); 16 | 17 | System.out.println(defaultBytesMessage.toString()); 18 | DefaultBytesMessage another = DefaultBytesMessage.valueOf(defaultBytesMessage.toString()); 19 | System.out.println(another.toString()); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/exception/OMSException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging.exception; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | * @version OMS 1.0 23 | * @since OMS 1.0 24 | */ 25 | public class OMSException extends Exception { 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/exception/OMSNotSupportedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging.exception; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | * @version OMS 1.0 23 | * @since OMS 1.0 24 | */ 25 | public class OMSNotSupportedException extends OMSRuntimeException { 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/exception/OMSResourceNotExistException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging.exception; 19 | 20 | /** 21 | * The {@code OMSResourceNotExistException} occurs if requested resource is not exist. 22 | * 23 | * @author vintagewang@apache.org 24 | * @version OMS 1.0 25 | * @since OMS 1.0 26 | */ 27 | public class OMSResourceNotExistException extends OMSException { 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/exception/OMSRuntimeException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging.exception; 19 | 20 | /** 21 | * @author vintagewang@apache.org 22 | * @version OMS 1.0 23 | * @since OMS 1.0 24 | */ 25 | public class OMSRuntimeException extends RuntimeException { 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/filter/FilterChain.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging.filter; 19 | 20 | import io.openmessaging.Filters; 21 | 22 | /** 23 | * @author vintagewang@apache.org 24 | * @version OMS 1.0 25 | * @since OMS 1.0 26 | */ 27 | public interface FilterChain { 28 | FilterChain addFirst(Filters filters); 29 | 30 | FilterChain addLast(Filters filters); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/internal/DefaultKeyValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging.internal; 19 | 20 | import io.openmessaging.KeyValue; 21 | 22 | import java.util.Set; 23 | 24 | /** 25 | * WARN: The current interface prohibits direct access by the end user 26 | * 27 | * @version OMS 1.0 28 | * @since OMS 1.0 29 | */ 30 | public class DefaultKeyValue implements KeyValue { 31 | @Override 32 | public KeyValue put(String key, int value) { 33 | return null; 34 | } 35 | 36 | @Override 37 | public KeyValue put(String key, long value) { 38 | return null; 39 | } 40 | 41 | @Override 42 | public KeyValue put(String key, double value) { 43 | return null; 44 | } 45 | 46 | @Override 47 | public KeyValue put(String key, String value) { 48 | return null; 49 | } 50 | 51 | @Override 52 | public int getInt(String key) { 53 | return 0; 54 | } 55 | 56 | @Override 57 | public long getLong(String key) { 58 | return 0; 59 | } 60 | 61 | @Override 62 | public double getDouble(String key) { 63 | return 0; 64 | } 65 | 66 | @Override 67 | public String getString(String key) { 68 | return null; 69 | } 70 | 71 | @Override 72 | public Set keySet() { 73 | return null; 74 | } 75 | 76 | @Override 77 | public boolean containsKey(String key) { 78 | return false; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/observer/Observer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.openmessaging.observer; 19 | 20 | import java.util.Observable; 21 | 22 | /** 23 | * @author vintagewang@apache.org 24 | * @version OMS 1.0 25 | * @since OMS 1.0 26 | */ 27 | public interface Observer { 28 | /** 29 | * Notifies the Observer that the {@link Observable} has finished sending push-based notifications. 30 | *

31 | * The {@link Observable} will not call this method if it calls {@link #onError}. 32 | */ 33 | void onCompleted(); 34 | 35 | /** 36 | * Notifies the Observer that the {@link Observable} has experienced an error condition. 37 | *

38 | * If the {@link Observable} calls this method, it will not thereafter call 39 | * {@link #onCompleted}. 40 | * 41 | * @param e the exception encountered by the Observable 42 | */ 43 | void onError(Throwable e); 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/official/tester/ProducerTester.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.official.tester; 2 | 3 | public class ProducerTester { 4 | 5 | 6 | public static void main(String[] args) { 7 | System.out.println("This is demo tester just for check"); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/tester/Constants.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.tester; 2 | 3 | public class Constants { 4 | public final static String STORE_PATH = System.getProperty("store.path", "/tmp/test"); 5 | public final static int PRO_NUM = Integer.valueOf(System.getProperty("pro.num", "10")); 6 | public final static int CON_NUM = Integer.valueOf(System.getProperty("con.num", "10")); 7 | public final static String PRO_PRE = System.getProperty("pro.pre", "PRODUCER_"); 8 | public final static int PRO_MAX = Integer.valueOf(System.getProperty("pro.max", "4000000")); 9 | public final static String CON_PRE = System.getProperty("con.pre", "CONSUMER_"); 10 | public final static String TOPIC_PRE = System.getProperty("topic.pre", "TOPIC_"); 11 | public final static String QUEUE_PRE = System.getProperty("topic.pre", "QUEUE_"); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/tester/ConsumerTester.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.tester; 2 | 3 | import io.openmessaging.BytesMessage; 4 | import io.openmessaging.KeyValue; 5 | import io.openmessaging.MessageHeader; 6 | import io.openmessaging.PullConsumer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.Collections; 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.concurrent.atomic.AtomicInteger; 15 | 16 | public class ConsumerTester { 17 | 18 | static Logger logger = LoggerFactory.getLogger(ConsumerTester.class); 19 | //0表示默认; 20 | static AtomicInteger state = new AtomicInteger(0); 21 | static String errorMessage = ""; 22 | 23 | public static void main(String[] args) throws Exception { 24 | Thread[] ts = new Thread[Constants.CON_NUM]; 25 | for (int i = 0; i < ts.length; i++) { 26 | ts[i] = new ConsumerTask(Constants.QUEUE_PRE + i, 27 | Collections.singletonList(Constants.TOPIC_PRE + i)); 28 | } 29 | long start = System.currentTimeMillis(); 30 | for (int i = 0; i < ts.length; i++) { 31 | ts[i].start(); 32 | } 33 | for (int i = 0; i < ts.length; i++) { 34 | ts[i].join(); 35 | } 36 | int pullNum = 0; 37 | for (int i = 0; i < ts.length; i++) { 38 | pullNum += ((ConsumerTask) ts[i]).getPullNum(); 39 | } 40 | long end = System.currentTimeMillis(); 41 | logger.info("Consumer Finished, Cost {} ms, Num {}", end - start, pullNum); 42 | } 43 | 44 | static class ConsumerTask extends Thread { 45 | String queue; 46 | List topics; 47 | PullConsumer consumer; 48 | int pullNum; 49 | Map> offsets = new HashMap<>(); 50 | 51 | public ConsumerTask(String queue, List topics) { 52 | this.queue = queue; 53 | this.topics = topics; 54 | init(); 55 | } 56 | 57 | public void init() { 58 | //init consumer 59 | try { 60 | Class kvClass = Class.forName("io.openmessaging.demo.DefaultKeyValue"); 61 | KeyValue keyValue = (KeyValue) kvClass.newInstance(); 62 | keyValue.put("STORE_PATH", Constants.STORE_PATH); 63 | Class consumerClass = Class.forName("io.openmessaging.demo.DefaultPullConsumer"); 64 | consumer = (PullConsumer) consumerClass.getConstructor(new Class[]{KeyValue.class}).newInstance(new Object[]{keyValue}); 65 | if (consumer == null) { 66 | throw new InstantiationException("Init Producer Failed"); 67 | } 68 | consumer.attachQueue(queue, topics); 69 | } catch (Exception e) { 70 | logger.error("please check the package name and class name:", e); 71 | } 72 | //init offsets 73 | offsets.put(queue, new HashMap<>()); 74 | for (String topic : topics) { 75 | offsets.put(topic, new HashMap<>()); 76 | } 77 | for (Map map : offsets.values()) { 78 | for (int i = 0; i < Constants.PRO_NUM; i++) { 79 | map.put(Constants.PRO_PRE + i, 0); 80 | } 81 | } 82 | } 83 | 84 | @Override 85 | public void run() { 86 | while (true) { 87 | try { 88 | BytesMessage message = (BytesMessage) consumer.poll(); 89 | if (message == null) { 90 | break; 91 | } 92 | String queueOrTopic; 93 | if (message.headers().getString(MessageHeader.QUEUE) != null) { 94 | queueOrTopic = message.headers().getString(MessageHeader.QUEUE); 95 | } else { 96 | queueOrTopic = message.headers().getString(MessageHeader.TOPIC); 97 | } 98 | if (queueOrTopic == null || queueOrTopic.length() == 0) { 99 | throw new Exception("Queue or Topic name is empty"); 100 | } 101 | String body = new String(message.getBody()); 102 | int index = body.lastIndexOf("_"); 103 | String producer = body.substring(0, index); 104 | int offset = Integer.parseInt(body.substring(index + 1)); 105 | if (offset != offsets.get(queueOrTopic).get(producer)) { 106 | logger.error("Offset not equal expected:{} actual:{} producer:{} queueOrTopic:{}", 107 | offsets.get(producer), offset, producer, queueOrTopic); 108 | break; 109 | } else { 110 | offsets.get(queueOrTopic).put(producer, offset + 1); 111 | } 112 | pullNum++; 113 | } catch (Exception e) { 114 | logger.error("Error occurred in the consuming process", e); 115 | break; 116 | } 117 | } 118 | } 119 | 120 | public int getPullNum() { 121 | return pullNum; 122 | } 123 | 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/tester/ProducerAndConsumerTest.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.tester; 2 | 3 | public class ProducerAndConsumerTest { 4 | 5 | 6 | public static void main(String[] args) throws Exception { 7 | ProducerTester.main(null); 8 | ConsumerTester.main(null); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/io/openmessaging/tester/ProducerTester.java: -------------------------------------------------------------------------------- 1 | package io.openmessaging.tester; 2 | 3 | import io.openmessaging.KeyValue; 4 | import io.openmessaging.Message; 5 | import io.openmessaging.Producer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | import java.util.Random; 12 | import java.util.concurrent.atomic.AtomicInteger; 13 | 14 | public class ProducerTester { 15 | 16 | static Logger logger = LoggerFactory.getLogger(ProducerTester.class); 17 | //0表示默认; 18 | static AtomicInteger state = new AtomicInteger(0); 19 | static String errorMessage = ""; 20 | 21 | public static void main(String[] args) throws Exception { 22 | Thread[] ts = new Thread[Constants.PRO_NUM]; 23 | for (int i = 0; i < ts.length; i++) { 24 | ts[i] = new ProducerTask(Constants.PRO_PRE + i); 25 | } 26 | long start = System.currentTimeMillis(); 27 | for (int i = 0; i < ts.length; i++) { 28 | ts[i].start(); 29 | } 30 | for (int i = 0; i < ts.length; i++) { 31 | ts[i].join(); 32 | } 33 | long end = System.currentTimeMillis(); 34 | logger.info("Produce Finished, Cost {} ms", end - start); 35 | } 36 | 37 | static class ProducerTask extends Thread { 38 | String label = Thread.currentThread().getName(); 39 | Random random = new Random(); 40 | Producer producer = null; 41 | int sendNum = 0; 42 | Map offsets = new HashMap<>(); 43 | 44 | public ProducerTask(String label) { 45 | this.label = label; 46 | init(); 47 | } 48 | 49 | public void init() { 50 | //init producer 51 | try { 52 | Class kvClass = Class.forName("io.openmessaging.demo.DefaultKeyValue"); 53 | KeyValue keyValue = (KeyValue) kvClass.newInstance(); 54 | keyValue.put("STORE_PATH", Constants.STORE_PATH); 55 | Class producerClass = Class.forName("io.openmessaging.demo.DefaultProducer"); 56 | producer = (Producer) producerClass.getConstructor(new Class[]{KeyValue.class}).newInstance(new Object[]{keyValue}); 57 | if (producer == null) { 58 | throw new InstantiationException("Init Producer Failed"); 59 | } 60 | } catch (Exception e) { 61 | logger.error("please check the package name and class name:", e); 62 | } 63 | //init offsets 64 | for (int i = 0; i < 10; i++) { 65 | offsets.put("TOPIC_" + i, 0); 66 | offsets.put("QUEUE_" + i, 0); 67 | } 68 | 69 | } 70 | 71 | @Override 72 | public void run() { 73 | while (true) { 74 | try { 75 | String queueOrTopic; 76 | if (sendNum % 10 == 0) { 77 | queueOrTopic = "QUEUE_" + random.nextInt(10); 78 | } else { 79 | queueOrTopic = "TOPIC_" + random.nextInt(10); 80 | } 81 | Message message = producer.createBytesMessageToQueue(queueOrTopic, (label + "_" + offsets.get(queueOrTopic)).getBytes()); 82 | logger.debug("queueOrTopic:{} offset:{}", queueOrTopic, label + "_" + offsets.get(queueOrTopic)); 83 | offsets.put(queueOrTopic, offsets.get(queueOrTopic) + 1); 84 | producer.send(message); 85 | sendNum++; 86 | if (sendNum >= Constants.PRO_MAX) { 87 | producer.flush(); 88 | break; 89 | } 90 | } catch (Exception e) { 91 | logger.error("Error occurred in the sending process", e); 92 | break; 93 | } 94 | } 95 | } 96 | 97 | 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | log4j.rootLogger=INFO, STDOUT 17 | 18 | log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender 19 | log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout 20 | log4j.appender.STDOUT.layout.ConversionPattern=[%d] %p %m (%c)%n 21 | 22 | -------------------------------------------------------------------------------- /src/main/resources/package.xml: -------------------------------------------------------------------------------- 1 | 5 | package 6 | 7 | dir 8 | 9 | true 10 | 11 | 12 | src/main/resources 13 | conf 14 | false 15 | 16 | 17 | 18 | 19 | lib 20 | runtime 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/test/java/ProducerAndConsumerTest.java: -------------------------------------------------------------------------------- 1 | import io.openmessaging.tester.ConsumerTester; 2 | import io.openmessaging.tester.ProducerTester; 3 | 4 | public class ProducerAndConsumerTest { 5 | 6 | 7 | public static void main(String[] args) throws Exception { 8 | //new io.openmessaging.demo.unitTest.MessageDumpTester().multiThreadsWriter(); 9 | ProducerTester.main(null); 10 | ConsumerTester.main(null); 11 | //new io.openmessaging.demo.unitTest.MessageDumpTester().multiThreadsReader(); 12 | } 13 | } 14 | --------------------------------------------------------------------------------