├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── User_Guide.md ├── cacti.md ├── config ├── config.properties └── log4j.properties ├── ini ├── install.md ├── pom.xml ├── run.bat ├── run.sh └── src ├── main ├── assemble │ └── package.xml └── java │ └── com │ ├── google │ └── code │ │ └── fqueue │ │ ├── FQueue.java │ │ ├── FSQueue.java │ │ ├── exception │ │ ├── ConfigException.java │ │ ├── FileEOFException.java │ │ └── FileFormatException.java │ │ ├── log │ │ ├── FileRunner.java │ │ ├── LogEntity.java │ │ └── LogIndex.java │ │ ├── memcached │ │ ├── StartNewQueue.java │ │ ├── StartServer.java │ │ └── storage │ │ │ ├── FSStorage.java │ │ │ └── QueueClient.java │ │ └── util │ │ ├── Config.java │ │ ├── JVMMonitor.java │ │ └── MappedByteBufferUtil.java │ └── thimbleware │ └── jmemcached │ ├── AbstractCache.java │ ├── Cache.java │ ├── CacheElement.java │ ├── CacheImpl.java │ ├── LocalCacheElement.java │ ├── MemCacheDaemon.java │ ├── StatsCounter.java │ ├── protocol │ ├── Command.java │ ├── CommandMessage.java │ ├── MemcachedCommandHandler.java │ ├── ResponseMessage.java │ ├── SessionStatus.java │ ├── binary │ │ ├── MemcachedBinaryCommandDecoder.java │ │ ├── MemcachedBinaryPipelineFactory.java │ │ └── MemcachedBinaryResponseEncoder.java │ ├── exceptions │ │ ├── ClientException.java │ │ ├── DatabaseException.java │ │ ├── IncorrectlyTerminatedPayloadException.java │ │ ├── InvalidProtocolStateException.java │ │ ├── MalformedCommandException.java │ │ └── UnknownCommandException.java │ └── text │ │ ├── MemcachedCommandDecoder.java │ │ ├── MemcachedFrameDecoder.java │ │ ├── MemcachedPipelineFactory.java │ │ └── MemcachedResponseEncoder.java │ └── storage │ ├── CacheStorage.java │ ├── bytebuffer │ ├── ByteBufferBlockStore.java │ └── Region.java │ ├── hash │ └── SizedItem.java │ └── mmap │ └── MemoryMappedBlockStore.java └── test └── java └── com └── google └── code └── fqueue ├── DatabaseExceptionTest.java ├── FSQueueTest.java ├── memcached └── TestFqueueServer.java └── util └── TestJVMMonitor.java /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | ****************************************** 17 | FQueue Change log 18 | ***************************************** 19 | 0.1-beta 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #FQueue介绍 2 | #labels 3 | ### Introduction 4 | 5 | FQueue是一个高性能、基于磁盘持久存储的队列消息系统。兼容memcached协议，能用memcached的语言都可以良好的与它通信。 6 | FQueue为你提供一个不需要特别优化，高性能的一个消息系统。 7 | [ppt介绍](http://www.slideshare.net/slideshow/embed_code/10345815) 8 | 9 | 10 | ### 特性 11 | 12 | * 基于磁盘持久化存储。 13 | * 支持memcached协议。 14 | * 支持多队列，密码验证功能。 15 | * 高性能，能达到数十万qps。 16 | * 低内存消耗。100-300M内存即可工作得很好。 17 | * 高效率IO读写算法，IO效率高。 18 | * 纯JAVA代码。支持进程内JVM级别的直接调用。 19 | * 在不需要强顺序的场景下，支持多机负载均衡。 20 | ### 不支持 21 | * 不支持topic方式的订阅功能。 22 | * 不支持主从复制。 23 | ### 使用 24 | 下载压缩包，解压后，chmode 755 run.sh 25 | ./run.sh start 即可启动，默认监听12000端口 26 | 测试使用 27 | ``` 28 | //memcached协议入队 29 | memcache.set("key_abc",0,"message1");//key为队列名，abc为密码，在conf/config.properties中配置 30 | memcache.set("key_abc",0,"message2"); 31 | //获取队列的数据 32 | memcache.get("key_abc");//取回的应该是message1 33 | memcache.get("key_abc");//取回的应该是message2 34 | 35 | ``` 36 | ### Fqueue有多快 37 | #### 进程内 38 | Fqueue的底层存储非常高效。下面做个单线程测试（JAVA）： 39 | ``` 40 | public static void main(String[] args) throws Exception { 41 | FQueue fQueue = new FQueue("/home/q/db/"); 42 | StringBuilder sb = new StringBuilder(); 43 | int length = Integer.parseInt(args[0]); 44 | for (int i = 0; i < length; i++) { 45 | sb.append("a"); 46 | } 47 | byte[] data = sb.toString().getBytes(); 48 | fQueue.add(data);// 预热一下 49 | long start = System.currentTimeMillis(); 50 | for (int i = 0; i < 100000000; i++) { 51 | fQueue.add(data); 52 | } 53 | System.out.println(100000000.0 / ((System.currentTimeMillis() - start) / 1000) + "qps"); 54 | fQueue.close(); 55 | } 56 | ``` 57 | 运行后输出： 58 | ``` 59 | #每次写入10字节 60 | ~/memcachedbench-0.1.0]# ./test.sh 10 61 | 9090909qps 62 | #每次写入1024字节 63 | ~/memcachedbench-0.1.0]# ./test.sh 1024 64 | 196078qps 65 | ``` 66 | #### 服务模式 67 | 多个客户端往Server端每次写入10byte的数据 68 | 用以下php脚本测试速度： 69 | ``` 70 | $mem=new Memcache(); 71 | $mem->connect("host",12000); 72 | $start=microtime(true); 73 | for($i=0;$i<30;$i++){ 74 | $count=$mem->get("size|bbs|pass"); 75 | echo microtime(true)."\t".$count."\r\n"; 76 | sleep(1); 77 | } 78 | ``` 79 | 输出： 80 | ``` 81 | 1315645933.5273 288426996 82 | 1315645934.5343 288750720 83 | 1315645935.5414 289080492 84 | 1315645936.5483 289412664 85 | 1315645937.5547 289727366 86 | 1315645938.5618 290053230 87 | 1315645939.5679 290380550 88 | 1315645940.5758 290697886 89 | 1315645941.5816 291025822 90 | 1315645942.5888 291349510 91 | 1315645943.5948 291671034 92 | 1315645944.6027 292005258 93 | 1315645945.6099 292336265 94 | 1315645946.6176 292663838 95 | 1315645947.6282 292988327 96 | 1315645948.6347 293317381 97 | 1315645949.6428 293624922 98 | 1315645950.6492 293944354 99 | 1315645951.6554 294269555 100 | 1315645952.6632 294595757 101 | 1315645953.6697 294924215 102 | 1315645954.677 295253636 103 | 1315645955.6829 295593660 104 | 1315645956.69 295927374 105 | 1315645957.6973 296254742 106 | 1315645958.7045 296585110 107 | 1315645959.7109 296916046 108 | 1315645960.7171 297246379 109 | 1315645961.7228 297577807 110 | 1315645962.7297 297913268 111 | ``` 112 | 对应的每秒写入 113 | ``` 114 | 323724 115 | 329772 116 | 332172 117 | 314702 118 | 325864 119 | 327320 120 | 317336 121 | 327936 122 | 323688 123 | 321524 124 | 334224 125 | 331007 126 | 327573 127 | 324489 128 | 329054 129 | 307541 130 | 319432 131 | 325201 132 | 326202 133 | 328458 134 | 329421 135 | 340024 136 | 333714 137 | 327368 138 | 330368 139 | 330936 140 | 330333 141 | 331428 142 | 335461 143 | 144 | ``` 145 | 146 | 147 | 148 | 149 | 150 | -------------------------------------------------------------------------------- /User_Guide.md: -------------------------------------------------------------------------------- 1 | # 协议介绍 2 | FQueue的协议基于Memcached协议。 3 | 注意：目前Fqueue会自动忽略flag参数，也就是说不支持客户端的压缩、自动序列化和反序列化。 4 | ## 入队 5 | ``` 6 | add queuename_password[_其他任意字符] flags exptime \r\n 7 | \r\n 8 | ``` 9 | 或者： 10 | ``` 11 | set queuename_password[_其他任意字符] flags exptime \r\n 12 | \r\n 13 | ``` 14 | [_其他任意字符]是可选的，对于最终的存储并不会有任何影响，在FQueue内部会自动忽略。比如： 15 | ``` 16 | add queuename_password_123333 flags exptime \r\n 17 | \r\n 18 | ``` 19 | 通过使用增加后缀的方式，可以实现client根据key做hash，实现分布式。 20 | flags exptime参数在0.1版本中会被忽略。即不支持memcached的过期，自动序列化和反序列化，压缩。 21 | ## 出队 22 | ``` 23 | get queuename_password\r\n 24 | ``` 25 | ## 获取队列大小 26 | 获取队列大小只需队列名，无需密码： 27 | ``` 28 | get size|queuename\r\n 29 | ``` 30 | ## 清空 31 | 清空某个队列的所有数据： 32 | ``` 33 | get clear|queuename|password\r\n 34 | ``` 35 | ## 重新加载权限配置 36 | 重新加载配置文件设置的权限信息： 37 | ``` 38 | get reload|queuename|password\r\n 39 | ``` 40 | 只需要任何一个queuename,password即可。在运行期，可以方便增加新的队列或者更改密码。 41 | ## JVM监控信息 42 | 获取可以监控JVM的监控信息 43 | ``` 44 | get monitor|items\r\n 45 | ``` 46 | items选项可以是 47 | ``` 48 | fileDescriptor,tomcat,load,allThreadsCount,peakThreadCount,daemonThreadCount,totalStartedThreadCount,deadLockCount,heapMemory,noHeapMemory,memory,classCount,GCTime,memoryPoolCollectionUsage,memoryPoolUsage,memoryPoolPeakUsage 49 | ``` 50 | ## stats 51 | 与memcached同 52 | ``` 53 | stats\r\n 54 | ``` 55 | 比如会输出： 56 | ``` 57 | stats 58 | STAT bytes_written 80513657 59 | STAT connection_structures 0 60 | STAT bytes 0 61 | STAT total_items 0 62 | STAT total_connections 8986 63 | STAT rusage_system 0.0 64 | STAT rusage_user 21 65 | STAT uptime 89923 66 | STAT current_bytes 0 67 | STAT pid 14 68 | STAT get_hits 693687 69 | STAT curr_items 0 70 | STAT free_bytes 300950568 71 | STAT version 0.1 72 | STAT cmd_get 719106 73 | STAT time 1313230151 74 | STAT cmd_set 686142 75 | STAT threads 16 76 | STAT limit_maxbytes 0 77 | STAT bytes_read 57971278 78 | STAT curr_connections 37 79 | STAT system_load 0.67 80 | STAT get_misses 25419 81 | END 82 | ``` 83 | # 使用 84 | ## PHP使用 85 | ```` 86 | $mem=new Memcache(); 87 | $mem->connect('127.0.0.1',12000); 88 | $mem->add('queuename_password',$message,0,0); 89 | $msg=$mem->get('queuename_password'); 90 | echo $msg; 91 | ``` 92 | ## JAVA使用 93 | ### JAVA进程内使用（性能最高） 94 | Fqueue的底层FSQueue可以直接在java应用中，作为嵌入式的持久化队列使用 95 | # 关于性能 96 | 97 | # 监控 98 | 99 | # 高可用设计 100 | -------------------------------------------------------------------------------- /cacti.md: -------------------------------------------------------------------------------- 1 | # cacti监控演示 2 | 3 | 通过FQueue提供的监控接口，cacti的监控截图演示： 4 | 5 | ![cacti](https://cloud.githubusercontent.com/assets/3821439/7227495/b4097eae-e783-11e4-8056-ac5b1404beb0.jpg) 6 | -------------------------------------------------------------------------------- /config/config.properties: -------------------------------------------------------------------------------- 1 | port=12000 2 | path=db 3 | logsize=40 4 | authorization=key|abc@@bbs|pass 5 | -------------------------------------------------------------------------------- /config/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO,stdout 2 | 3 | 4 | log4j.appender.stdout =org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout =org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern =[%d{yyyy-MM-dd HH:mm:ss} %5p %C{1}:%L] %m%n 7 | 8 | # LOGFILE is set to be a File appender using a PatternLayout. 9 | log4j.appender.LOGFILE=org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.LOGFILE.File=logs/log.log 11 | log4j.appender.LOGFILE.DatePattern=yyyy-MM-dd'.log' 12 | log4j.appender.LOGFILE.Append=true 13 | log4j.appender.LOGFILE.Threshold=ERROR 14 | log4j.appender.LOGFILE.encoding=UTF-8 15 | log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayout 16 | log4j.appender.LOGFILE.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss} %5p %C{1}:%L] %m%n 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /ini: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | mvn clean eclipse:eclipse -DdownloadSources=true 3 | -------------------------------------------------------------------------------- /install.md: -------------------------------------------------------------------------------- 1 | # 部署 2 | ## JAVA环境 3 | 安装jdk6 4 | ## 部署Fqueue 5 | * 直接从google code下载压缩包，解压后./run.sh start 即可运行 6 | * svn check out出源代码，用maven 的mvn package即可打包出上一步使用的压缩包 7 | # 配置 8 | Fqueue的配置文件位于conf/config.properties,默认配置： 9 | ``` 10 | port=12000 11 | path=db 12 | logsize=40 13 | authorization=key|abc@@bbs|pass 14 | ``` 15 | * port Fqueue的启动端口 16 | * path Fqueue的数据在磁盘上的存储目录 17 | * logsize Fqueue在存储数据到磁盘上时，每个文件的最大大小（单位MB）,logsize不要设置太大，一般100M以内吧。推荐用40左右的大小。 18 | * authorization 权限配置信息格式 19 | ``` 20 | 队列1|队列1的密码@@队列2|队列2的密码@@…… 21 | ``` 22 | 注意：队列名和密码不能包含“@”、"|"、,"_"和中文字符。 23 | # VM启动配置 24 | 关于JVM的启动配置可以在./run.sh中修改 25 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.google.code 5 | fqueue 6 | 0.1.3-release 7 | jar 8 | fqueue 9 | http://code.google.com/p/fqueue/ 10 | 11 | 12 | sunli1223 13 | sunli 14 | sunli1223@gmail.com 15 | 16 | 17 | 18 | 19 | 20 | maven-assembly-plugin 21 | 2.2 22 | 23 | 24 | false 25 | 26 | src/main/assemble/package.xml 27 | 28 | 29 | 30 | 31 | 32 | make-assembly 33 | package 34 | 35 | single 36 | 37 | 38 | 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-compiler-plugin 43 | 2.3.2 44 | 45 | 1.6 46 | 1.6 47 | utf-8 48 | 49 | 50 | 51 | 52 | 53 | 54 | UTF-8 55 | 1.6.1 56 | 57 | 58 | 59 | repository.jboss.org 60 | http://repository.jboss.org/nexus/content/groups/public/ 61 | 62 | false 63 | 64 | 65 | 66 | 67 | 68 | junit 69 | junit 70 | 3.8.2 71 | test 72 | 73 | 74 | com.googlecode.xmemcached 75 | xmemcached 76 | 1.3.3 77 | test 78 | 79 | 80 | commons-lang 81 | commons-lang 82 | 2.5 83 | 84 | 85 | commons-logging 86 | commons-logging 87 | 1.1.1 88 | 89 | 90 | log4j 91 | log4j 92 | 1.2.16 93 | 94 | 95 | org.slf4j 96 | slf4j-log4j12 97 | ${slf4j.version} 98 | 99 | 100 | org.slf4j 101 | slf4j-api 102 | ${slf4j.version} 103 | 104 | 105 | org.jboss.netty 106 | netty 107 | 3.2.4.Final 108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /run.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setLocal EnableDelayedExpansion 3 | set CLASSPATH=.;config/ 4 | for %%a in (lib/*.jar) do ( 5 | set CLASSPATH=!CLASSPATH!;lib/%%a 6 | ) 7 | set CLASSPATH=!CLASSPATH! 8 | rem echo %CLASSPATH% 9 | java -classpath %CLASSPATH% com.google.code.fqueue.memcached.StartServer -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ulimit -SHn 51200 3 | dir=`dirname $0` 4 | pidfile=pid 5 | cd $dir 6 | CP=.:config/ 7 | for file in lib/*; 8 | do CP=${CP}:$file; 9 | done 10 | retval=0 11 | # start the server 12 | start(){ 13 | printf 'Starting the server of FQueue\n' 14 | if [ -f "$pidfile" ] ; then 15 | pid=`cat "$pidfile"` 16 | printf 'Existing process: %d\n' "$pid" 17 | retval=1 18 | else 19 | java -Xms300M \ 20 | -Xmx300M \ 21 | -XX:+UseParallelGC \ 22 | -XX:+AggressiveOpts \ 23 | -XX:+UseFastAccessorMethods \ 24 | -XX:+HeapDumpOnOutOfMemoryError \ 25 | -verbose:gc \ 26 | -XX:+PrintGCDetails \ 27 | -XX:+PrintGCTimeStamps \ 28 | -Xloggc:logs/gc`date +%Y%m%d%H%M%S`.log \ 29 | -cp $CP com.google.code.fqueue.memcached.StartServer >>logs/log.log & 30 | echo $! >"$pidfile" 31 | if [ "$?" -eq 0 ] ; then 32 | printf 'Done\n' 33 | else 34 | printf 'The server could not started\n' 35 | retval=1 36 | fi 37 | fi 38 | } 39 | # stop the server 40 | stop(){ 41 | printf 'Stopping the server of FQueue\n' 42 | if [ -f "$pidfile" ] ; then 43 | pid=`cat "$pidfile"` 44 | printf "Sending the terminal signal to the process: %s\n" "$pid" 45 | PROCESSPID=`ps -ef|awk '{print $2}'|grep -w "$pid"` 46 | if [ $PROCESSPID -ne "$pid" ] ; then 47 | rm -f "$pidfile"; 48 | printf 'Done\n' 49 | fi 50 | kill -TERM "$pid" 51 | c=0 52 | while true ; do 53 | sleep 0.1 54 | PROCESSPID=`ps -ef|awk '{print $2}'|grep -w "$pid"` 55 | if [ $PROCESSPID ] && [ $PROCESSPID -eq "$pid" ] ; then 56 | c=`expr $c + 1` 57 | if [ "$c" -ge 100 ] ; then 58 | printf 'Hanging process: %d\n' "$pid" 59 | retval=1 60 | break 61 | fi 62 | else 63 | printf 'Done\n' 64 | rm -f "$pidfile"; 65 | break 66 | fi 67 | done 68 | else 69 | printf 'No process found\n' 70 | retval=1 71 | fi 72 | } 73 | # dispatch the command 74 | case "$1" in 75 | start) 76 | start 77 | ;; 78 | stop) 79 | stop 80 | ;; 81 | restart) 82 | stop 83 | start 84 | ;; 85 | hup) 86 | hup 87 | ;; 88 | *) 89 | printf 'Usage: %s {start|stop|restart}\n' 90 | exit 1 91 | ;; 92 | esac 93 | 94 | 95 | # exit 96 | exit "$retval" 97 | 98 | 99 | 100 | # END OF FILE 101 | -------------------------------------------------------------------------------- /src/main/assemble/package.xml: -------------------------------------------------------------------------------- 1 | 3 | package 4 | 5 | tar.gz 6 | 7 | true 8 | 9 | 10 | run.bat 11 | / 12 | run.bat 13 | 0755 14 | 15 | 16 | run.sh 17 | / 18 | run.sh 19 | 0755 20 | 21 | 22 | 23 | 24 | src/main/logs 25 | logs 26 | 27 | 28 | config 29 | config 30 | 31 | 32 | src/main/db 33 | db 34 | 35 | 36 | 37 | 38 | lib 39 | runtime 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/FQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue; 17 | 18 | import java.io.IOException; 19 | import java.util.AbstractQueue; 20 | import java.util.Iterator; 21 | import java.util.Queue; 22 | import java.util.concurrent.locks.Lock; 23 | import java.util.concurrent.locks.ReentrantReadWriteLock; 24 | 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import com.google.code.fqueue.exception.FileFormatException; 29 | 30 | /** 31 | * 基于文件系统的持久化队列 32 | * 33 | * @author sunli 34 | * @date 2010-8-13 35 | * @version $Id$ 36 | */ 37 | public class FQueue extends AbstractQueue implements Queue, 38 | java.io.Serializable { 39 | private static final long serialVersionUID = -5960741434564940154L; 40 | private FSQueue fsQueue = null; 41 | final Logger log = LoggerFactory.getLogger(FQueue.class); 42 | private Lock lock = new ReentrantReadWriteLock().writeLock(); 43 | 44 | public FQueue(String path) throws Exception { 45 | fsQueue = new FSQueue(path, 1024 * 1024 * 300); 46 | } 47 | 48 | public FQueue(String path, int logsize) throws Exception { 49 | fsQueue = new FSQueue(path, logsize); 50 | } 51 | 52 | @Override 53 | public Iterator iterator() { 54 | throw new UnsupportedOperationException("iterator Unsupported now"); 55 | } 56 | 57 | @Override 58 | public int size() { 59 | return fsQueue.getQueuSize(); 60 | } 61 | 62 | @Override 63 | public boolean offer(byte[] e) { 64 | try { 65 | lock.lock(); 66 | fsQueue.add(e); 67 | return true; 68 | } catch (IOException e1) { 69 | e1.printStackTrace(); 70 | } catch (FileFormatException e1) { 71 | e1.printStackTrace(); 72 | } finally { 73 | lock.unlock(); 74 | } 75 | return false; 76 | } 77 | 78 | @Override 79 | public byte[] peek() { 80 | throw new UnsupportedOperationException("peek Unsupported now"); 81 | } 82 | 83 | @Override 84 | public byte[] poll() { 85 | try { 86 | lock.lock(); 87 | return fsQueue.readNextAndRemove(); 88 | } catch (IOException e) { 89 | log.error(e.getMessage(), e); 90 | return null; 91 | } catch (FileFormatException e) { 92 | log.error(e.getMessage(), e); 93 | return null; 94 | } finally { 95 | lock.unlock(); 96 | } 97 | } 98 | 99 | public void close() { 100 | if (fsQueue != null) { 101 | fsQueue.close(); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/FSQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.util.concurrent.ExecutorService; 21 | import java.util.concurrent.Executors; 22 | 23 | import org.apache.commons.logging.Log; 24 | import org.apache.commons.logging.LogFactory; 25 | 26 | import com.google.code.fqueue.exception.FileEOFException; 27 | import com.google.code.fqueue.exception.FileFormatException; 28 | import com.google.code.fqueue.log.FileRunner; 29 | import com.google.code.fqueue.log.LogEntity; 30 | import com.google.code.fqueue.log.LogIndex; 31 | 32 | /** 33 | * 完成基于文件的先进先出的读写功能 34 | * 35 | * @author sunli 36 | * @date 2010-8-13 37 | * @version $Id$ 38 | */ 39 | public class FSQueue { 40 | private static final Log log = LogFactory.getLog(FSQueue.class); 41 | public static final String filePrefix = "fqueue"; 42 | private int fileLimitLength = 1024 * 1024 * 100; 43 | private static final String dbName = "icqueue.db"; 44 | private static final String fileSeparator = System.getProperty("file.separator"); 45 | private String path = null; 46 | private final ExecutorService executor = Executors.newSingleThreadExecutor(); 47 | 48 | private FileRunner deleteFileRunner; 49 | /** 50 | * 文件操作实例 51 | */ 52 | private LogIndex db = null; 53 | private LogEntity writerHandle = null; 54 | private LogEntity readerHandle = null; 55 | /** 56 | * 文件操作位置信息 57 | */ 58 | private int readerIndex = -1; 59 | private int writerIndex = -1; 60 | 61 | public FSQueue(String path) throws Exception { 62 | this(path, 1024 * 1024 * 150); 63 | } 64 | 65 | /** 66 | * 在指定的目录中，以fileLimitLength为单个数据文件的最大大小限制初始化队列存储 67 | * 68 | * @param dir 69 | * 队列数据存储的路径 70 | * @param fileLimitLength 71 | * 单个数据文件的大小，不能超过2G 72 | * @throws Exception 73 | */ 74 | public FSQueue(String dir, int fileLimitLength) throws Exception { 75 | this.fileLimitLength = fileLimitLength; 76 | File fileDir = new File(dir); 77 | if (fileDir.exists() == false && fileDir.isDirectory() == false) { 78 | if (fileDir.mkdirs() == false) { 79 | throw new IOException("create dir error"); 80 | } 81 | } 82 | path = fileDir.getAbsolutePath(); 83 | // 打开db 84 | db = new LogIndex(path + fileSeparator + dbName); 85 | writerIndex = db.getWriterIndex(); 86 | readerIndex = db.getReaderIndex(); 87 | writerHandle = createLogEntity(path + fileSeparator + filePrefix + "data_" + writerIndex + ".idb", db, 88 | writerIndex); 89 | if (readerIndex == writerIndex) { 90 | readerHandle = writerHandle; 91 | } else { 92 | readerHandle = createLogEntity(path + fileSeparator + filePrefix + "data_" + readerIndex + ".idb", db, 93 | readerIndex); 94 | 95 | } 96 | deleteFileRunner = new FileRunner(path + fileSeparator + filePrefix + "data_", fileLimitLength); 97 | executor.execute(deleteFileRunner); 98 | } 99 | 100 | /** 101 | * 创建或者获取一个数据读写实例 102 | * 103 | * @param dbpath 104 | * @param db 105 | * @param fileNumber 106 | * @return 107 | * @throws IOException 108 | * @throws FileFormatException 109 | */ 110 | private LogEntity createLogEntity(String dbpath, LogIndex db, int fileNumber) throws IOException, 111 | FileFormatException { 112 | return new LogEntity(dbpath, db, fileNumber, this.fileLimitLength); 113 | } 114 | 115 | /** 116 | * 一个文件的数据写入达到fileLimitLength的时候，滚动到下一个文件实例 117 | * 118 | * @throws IOException 119 | * @throws FileFormatException 120 | */ 121 | private void rotateNextLogWriter() throws IOException, FileFormatException { 122 | writerIndex = writerIndex + 1; 123 | writerHandle.putNextFile(writerIndex); 124 | if (readerHandle != writerHandle) { 125 | writerHandle.close(); 126 | } 127 | db.putWriterIndex(writerIndex); 128 | writerHandle = createLogEntity(path + fileSeparator + filePrefix + "data_" + writerIndex + ".idb", db, 129 | writerIndex); 130 | } 131 | 132 | /** 133 | * 向队列存储添加一个字符串 134 | * 135 | * @param message 136 | * message 137 | * @throws IOException 138 | * @throws FileFormatException 139 | */ 140 | public void add(String message) throws IOException, FileFormatException { 141 | add(message.getBytes()); 142 | } 143 | 144 | /** 145 | * 向队列存储添加一个byte数组 146 | * 147 | * @param message 148 | * @throws IOException 149 | * @throws FileFormatException 150 | */ 151 | public void add(byte[] message) throws IOException, FileFormatException { 152 | short status = writerHandle.write(message); 153 | if (status == LogEntity.WRITEFULL) { 154 | rotateNextLogWriter(); 155 | status = writerHandle.write(message); 156 | } 157 | if (status == LogEntity.WRITESUCCESS) { 158 | db.incrementSize(); 159 | } 160 | 161 | } 162 | /** 163 | * 从队列存储中取出最先入队的数据，并移除它 164 | * @return 165 | * @throws IOException 166 | * @throws FileFormatException 167 | */ 168 | public byte[] readNextAndRemove() throws IOException, FileFormatException { 169 | byte[] b = null; 170 | try { 171 | b = readerHandle.readNextAndRemove(); 172 | } catch (FileEOFException e) { 173 | int deleteNum = readerHandle.getCurrentFileNumber(); 174 | int nextfile = readerHandle.getNextFile(); 175 | readerHandle.close(); 176 | FileRunner.addDeleteFile(path + fileSeparator + filePrefix + "data_" + deleteNum + ".idb"); 177 | // 更新下一次读取的位置和索引 178 | db.putReaderPosition(LogEntity.messageStartPosition); 179 | db.putReaderIndex(nextfile); 180 | if (writerHandle.getCurrentFileNumber() == nextfile) { 181 | readerHandle = writerHandle; 182 | } else { 183 | readerHandle = createLogEntity(path + fileSeparator + filePrefix + "data_" + nextfile + ".idb", db, 184 | nextfile); 185 | } 186 | try { 187 | b = readerHandle.readNextAndRemove(); 188 | } catch (FileEOFException e1) { 189 | log.error("read new log file FileEOFException error occurred",e1); 190 | } 191 | } 192 | if (b != null) { 193 | db.decrementSize(); 194 | } 195 | return b; 196 | } 197 | 198 | public void close() { 199 | readerHandle.close(); 200 | writerHandle.close(); 201 | deleteFileRunner.exit(); 202 | executor.shutdown(); 203 | } 204 | 205 | public int getQueuSize() { 206 | return db.getSize(); 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/exception/ConfigException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.exception; 17 | /** 18 | *@author sunli 19 | *@date 2011-5-18 20 | *@version $Id$ 21 | */ 22 | public class ConfigException extends RuntimeException { 23 | /** 24 | * 25 | */ 26 | private static final long serialVersionUID = -5680376999772385539L; 27 | public ConfigException() { 28 | } 29 | 30 | public ConfigException(String message) { 31 | super(message); 32 | } 33 | 34 | public ConfigException(Throwable cause) { 35 | super(cause); 36 | } 37 | 38 | public ConfigException(String message, Throwable cause) { 39 | super(message, cause); 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/exception/FileEOFException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.exception; 17 | /** 18 | *@author sunli 19 | *@date 2011-5-18 20 | *@version $Id$ 21 | */ 22 | public class FileEOFException extends Exception { 23 | 24 | private static final long serialVersionUID = 4701796168682302255L; 25 | 26 | public FileEOFException() { 27 | super(); 28 | } 29 | public FileEOFException(String message) { 30 | super(message); 31 | } 32 | 33 | public FileEOFException(String message, Throwable cause) { 34 | super(message, cause); 35 | } 36 | 37 | public FileEOFException(Throwable cause) { 38 | super(cause); 39 | } 40 | @Override 41 | public Throwable fillInStackTrace() { 42 | return this; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/exception/FileFormatException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.exception; 17 | /** 18 | *@author sunli 19 | *@date 2011-5-18 20 | *@version $Id$ 21 | */ 22 | public class FileFormatException extends Exception { 23 | 24 | 25 | /** 26 | * 27 | */ 28 | private static final long serialVersionUID = 6950322066714479555L; 29 | 30 | /** 31 | * Constructs an {@code FileFormatException} with {@code null} as its error 32 | * detail message. 33 | */ 34 | public FileFormatException() { 35 | super(); 36 | } 37 | 38 | public FileFormatException(String message) { 39 | super(message); 40 | } 41 | 42 | public FileFormatException(String message, Throwable cause) { 43 | super(message, cause); 44 | } 45 | 46 | public FileFormatException(Throwable cause) { 47 | super(cause); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/log/FileRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.log; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.io.RandomAccessFile; 21 | import java.nio.MappedByteBuffer; 22 | import java.nio.channels.FileChannel; 23 | import java.nio.channels.FileChannel.MapMode; 24 | import java.util.Queue; 25 | import java.util.concurrent.ConcurrentLinkedQueue; 26 | 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import com.google.code.fqueue.util.MappedByteBufferUtil; 31 | 32 | /** 33 | * @author sunli 34 | * @date 2011-5-18 35 | * @version $Id$ 36 | */ 37 | public class FileRunner implements Runnable { 38 | private final Logger log = LoggerFactory.getLogger(FileRunner.class); 39 | // 删除队列 40 | private static final Queue deleteQueue = new ConcurrentLinkedQueue(); 41 | // 新创建队列 42 | private static final Queue createQueue = new ConcurrentLinkedQueue(); 43 | private String baseDir = null; 44 | private int fileLimitLength = 0; 45 | private volatile boolean keepRunning = true; 46 | 47 | public static void addDeleteFile(String path) { 48 | deleteQueue.add(path); 49 | } 50 | 51 | public static void addCreateFile(String path) { 52 | createQueue.add(path); 53 | } 54 | 55 | public FileRunner(String baseDir, int fileLimitLength) { 56 | this.baseDir = baseDir; 57 | this.fileLimitLength = fileLimitLength; 58 | } 59 | 60 | @Override 61 | public void run() { 62 | String filePath, fileNum; 63 | while (keepRunning) { 64 | filePath = deleteQueue.poll(); 65 | fileNum = createQueue.poll(); 66 | if (filePath == null && fileNum == null) { 67 | try { 68 | Thread.sleep(10); 69 | } catch (InterruptedException e) { 70 | log.error(e.getMessage(), e); 71 | } 72 | continue; 73 | } 74 | if (filePath != null) { 75 | File delFile = new File(filePath); 76 | delFile.delete(); 77 | } 78 | 79 | if (fileNum != null) { 80 | filePath = baseDir + fileNum + ".idb"; 81 | try { 82 | create(filePath); 83 | } catch (IOException e) { 84 | log.error("预创建数据文件失败", e); 85 | } 86 | } 87 | } 88 | 89 | } 90 | 91 | private boolean create(String path) throws IOException { 92 | File file = new File(path); 93 | if (file.exists() == false) { 94 | if (file.createNewFile() == false) { 95 | return false; 96 | } 97 | RandomAccessFile raFile = new RandomAccessFile(file, "rwd"); 98 | FileChannel fc = raFile.getChannel(); 99 | MappedByteBuffer mappedByteBuffer = fc.map(MapMode.READ_WRITE, 0, this.fileLimitLength); 100 | mappedByteBuffer.put(LogEntity.MAGIC.getBytes()); 101 | mappedByteBuffer.putInt(1);// 8 version 102 | mappedByteBuffer.putInt(-1);// 12next fileindex 103 | mappedByteBuffer.putInt(-2);// 16 104 | mappedByteBuffer.force(); 105 | MappedByteBufferUtil.clean(mappedByteBuffer); 106 | fc.close(); 107 | raFile.close(); 108 | return true; 109 | } else { 110 | return false; 111 | } 112 | } 113 | 114 | public void exit(){ 115 | keepRunning = false; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/log/LogEntity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.log; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.io.RandomAccessFile; 21 | import java.lang.reflect.Method; 22 | import java.nio.MappedByteBuffer; 23 | import java.nio.channels.FileChannel; 24 | import java.nio.channels.FileChannel.MapMode; 25 | import java.security.AccessController; 26 | import java.security.PrivilegedAction; 27 | import java.util.concurrent.ExecutorService; 28 | import java.util.concurrent.Executors; 29 | 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | import com.google.code.fqueue.exception.FileEOFException; 34 | import com.google.code.fqueue.exception.FileFormatException; 35 | /** 36 | *@author sunli 37 | *@date 2011-5-18 38 | *@version $Id$ 39 | */ 40 | public class LogEntity { 41 | private final Logger log = LoggerFactory.getLogger(LogEntity.class); 42 | public static final byte WRITESUCCESS = 1; 43 | public static final byte WRITEFAILURE = 2; 44 | public static final byte WRITEFULL = 3; 45 | public static final String MAGIC = "FQueuefs"; 46 | public static int messageStartPosition = 20; 47 | private final ExecutorService executor = Executors.newSingleThreadExecutor(); 48 | private File file; 49 | private RandomAccessFile raFile; 50 | private FileChannel fc; 51 | public MappedByteBuffer mappedByteBuffer; 52 | private int fileLimitLength = 1024 * 1024 * 40; 53 | 54 | private LogIndex db = null; 55 | /** 56 | * 文件操作位置信息 57 | */ 58 | private String magicString = null; 59 | private int version = -1; 60 | private int readerPosition = -1; 61 | private int writerPosition = -1; 62 | private int nextFile = -1; 63 | private int endPosition = -1; 64 | private int currentFileNumber = -1; 65 | 66 | public LogEntity(String path, LogIndex db, int fileNumber, 67 | int fileLimitLength) throws IOException, FileFormatException { 68 | this.currentFileNumber = fileNumber; 69 | this.fileLimitLength = fileLimitLength; 70 | this.db = db; 71 | file = new File(path); 72 | // 文件不存在，创建文件 73 | if (file.exists() == false) { 74 | createLogEntity(); 75 | FileRunner.addCreateFile(Integer.toString(fileNumber + 1)); 76 | } else { 77 | raFile = new RandomAccessFile(file, "rwd"); 78 | if (raFile.length() < LogEntity.messageStartPosition) { 79 | throw new FileFormatException("file format error"); 80 | } 81 | fc = raFile.getChannel(); 82 | mappedByteBuffer = fc.map(MapMode.READ_WRITE, 0, 83 | this.fileLimitLength); 84 | // magicString 85 | byte[] b = new byte[8]; 86 | mappedByteBuffer.get(b); 87 | magicString = new String(b); 88 | if (magicString.equals(MAGIC) == false) { 89 | throw new FileFormatException("file format error"); 90 | } 91 | // version 92 | version = mappedByteBuffer.getInt(); 93 | // nextfile 94 | nextFile = mappedByteBuffer.getInt(); 95 | endPosition = mappedByteBuffer.getInt(); 96 | // 未写满 97 | if (endPosition == -1) { 98 | this.writerPosition = db.getWriterPosition(); 99 | } else if (endPosition == -2) {// 预分配的文件 100 | this.writerPosition = LogEntity.messageStartPosition; 101 | db.putWriterPosition(this.writerPosition); 102 | mappedByteBuffer.position(16); 103 | mappedByteBuffer.putInt(-1); 104 | this.endPosition = -1; 105 | 106 | } else { 107 | this.writerPosition = endPosition; 108 | } 109 | if (db.getReaderIndex() == this.currentFileNumber) { 110 | this.readerPosition = db.getReaderPosition(); 111 | } else { 112 | this.readerPosition = LogEntity.messageStartPosition; 113 | } 114 | } 115 | executor.execute(new Sync()); 116 | 117 | } 118 | 119 | public class Sync implements Runnable { 120 | @Override 121 | public void run() { 122 | while (true) { 123 | if (mappedByteBuffer != null) { 124 | try { 125 | mappedByteBuffer.force(); 126 | } catch (Exception e) { 127 | break; 128 | } 129 | try { 130 | Thread.sleep(10); 131 | } catch (InterruptedException e) { 132 | break; 133 | } 134 | } else { 135 | break; 136 | } 137 | } 138 | 139 | } 140 | 141 | } 142 | 143 | public int getCurrentFileNumber() { 144 | return this.currentFileNumber; 145 | } 146 | 147 | public int getNextFile() { 148 | return this.nextFile; 149 | } 150 | 151 | private boolean createLogEntity() throws IOException { 152 | if (file.createNewFile() == false) { 153 | return false; 154 | } 155 | raFile = new RandomAccessFile(file, "rwd"); 156 | fc = raFile.getChannel(); 157 | mappedByteBuffer = fc.map(MapMode.READ_WRITE, 0, this.fileLimitLength); 158 | mappedByteBuffer.put(MAGIC.getBytes()); 159 | mappedByteBuffer.putInt(version);// 8 version 160 | mappedByteBuffer.putInt(nextFile);// 12next fileindex 161 | mappedByteBuffer.putInt(endPosition);// 16 162 | mappedByteBuffer.force(); 163 | this.magicString = MAGIC; 164 | this.writerPosition = LogEntity.messageStartPosition; 165 | this.readerPosition = LogEntity.messageStartPosition; 166 | db.putWriterPosition(this.writerPosition); 167 | return true; 168 | } 169 | 170 | /** 171 | * 记录写位置 172 | * 173 | * @param pos 174 | */ 175 | private void putWriterPosition(int pos) { 176 | db.putWriterPosition(pos); 177 | } 178 | 179 | private void putReaderPosition(int pos) { 180 | db.putReaderPosition(pos); 181 | } 182 | 183 | /** 184 | * write next File number id. 185 | * 186 | * @param number 187 | */ 188 | public void putNextFile(int number) { 189 | mappedByteBuffer.position(12); 190 | mappedByteBuffer.putInt(number); 191 | this.nextFile = number; 192 | } 193 | 194 | public boolean isFull(int increment) { 195 | // confirm if the file is full 196 | if (this.fileLimitLength < this.writerPosition + increment) { 197 | return true; 198 | } 199 | return false; 200 | } 201 | 202 | public byte write(byte[] log) { 203 | int increment = log.length + 4; 204 | if (isFull(increment)) { 205 | mappedByteBuffer.position(16); 206 | mappedByteBuffer.putInt(this.writerPosition); 207 | this.endPosition = this.writerPosition; 208 | return WRITEFULL; 209 | } 210 | mappedByteBuffer.position(this.writerPosition); 211 | mappedByteBuffer.putInt(log.length); 212 | mappedByteBuffer.put(log); 213 | this.writerPosition += increment; 214 | putWriterPosition(this.writerPosition); 215 | return WRITESUCCESS; 216 | } 217 | 218 | public byte[] readNextAndRemove() throws FileEOFException { 219 | if (this.endPosition != -1 && this.readerPosition >= this.endPosition) { 220 | throw new FileEOFException("file eof"); 221 | } 222 | // readerPosition must be less than writerPosition 223 | if (this.readerPosition >= this.writerPosition) { 224 | return null; 225 | } 226 | mappedByteBuffer.position(this.readerPosition); 227 | int length = mappedByteBuffer.getInt(); 228 | byte[] b = new byte[length]; 229 | this.readerPosition += length + 4; 230 | mappedByteBuffer.get(b); 231 | putReaderPosition(this.readerPosition); 232 | return b; 233 | } 234 | 235 | public void close() { 236 | try { 237 | if(mappedByteBuffer==null){ 238 | return; 239 | } 240 | mappedByteBuffer.force(); 241 | AccessController.doPrivileged(new PrivilegedAction() { 242 | public Object run() { 243 | try { 244 | Method getCleanerMethod = mappedByteBuffer.getClass() 245 | .getMethod("cleaner", new Class[0]); 246 | getCleanerMethod.setAccessible(true); 247 | sun.misc.Cleaner cleaner = (sun.misc.Cleaner) getCleanerMethod 248 | .invoke(mappedByteBuffer, new Object[0]); 249 | cleaner.clean(); 250 | } catch (Exception e) { 251 | e.printStackTrace(); 252 | } 253 | return null; 254 | } 255 | }); 256 | mappedByteBuffer = null; 257 | executor.shutdown(); 258 | fc.close(); 259 | raFile.close(); 260 | } catch (IOException e) { 261 | log.error("close logentity file error:", e); 262 | } 263 | } 264 | 265 | public String headerInfo() { 266 | StringBuilder sb = new StringBuilder(); 267 | sb.append(" magicString:"); 268 | sb.append(magicString); 269 | sb.append(" version:"); 270 | sb.append(version); 271 | sb.append(" readerPosition:"); 272 | sb.append(readerPosition); 273 | sb.append(" writerPosition:"); 274 | sb.append(writerPosition); 275 | sb.append(" nextFile:"); 276 | sb.append(nextFile); 277 | sb.append(" endPosition:"); 278 | sb.append(endPosition); 279 | sb.append(" currentFileNumber:"); 280 | sb.append(currentFileNumber); 281 | return sb.toString(); 282 | } 283 | 284 | } 285 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/log/LogIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.log; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.io.RandomAccessFile; 21 | import java.lang.reflect.Method; 22 | import java.nio.ByteBuffer; 23 | import java.nio.MappedByteBuffer; 24 | import java.nio.channels.FileChannel; 25 | import java.nio.channels.FileChannel.MapMode; 26 | import java.security.AccessController; 27 | import java.security.PrivilegedAction; 28 | import java.util.concurrent.atomic.AtomicInteger; 29 | 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | import com.google.code.fqueue.exception.FileFormatException; 34 | 35 | /** 36 | * 数据索引文件 37 | * 38 | * @author sunli 39 | * @date 2011-5-18 40 | * @version $Id$ 41 | */ 42 | public class LogIndex { 43 | final Logger log = LoggerFactory.getLogger(LogIndex.class); 44 | private final int dbFileLimitLength = 32; 45 | private RandomAccessFile dbRandFile = null; 46 | private FileChannel fc; 47 | private MappedByteBuffer mappedByteBuffer; 48 | 49 | /** 50 | * 文件操作位置信息 51 | */ 52 | private String magicString = null; 53 | private int version = -1; 54 | private int readerPosition = -1; 55 | private int writerPosition = -1; 56 | private int readerIndex = -1; 57 | private int writerIndex = -1; 58 | private AtomicInteger size = new AtomicInteger(); 59 | 60 | public LogIndex(String path) throws IOException, FileFormatException { 61 | File dbFile = new File(path); 62 | 63 | // 文件不存在，创建文件 64 | if (dbFile.exists() == false) { 65 | dbFile.createNewFile(); 66 | dbRandFile = new RandomAccessFile(dbFile, "rwd"); 67 | dbRandFile.write(LogEntity.MAGIC.getBytes());// magic 68 | dbRandFile.writeInt(1);// 8version 69 | dbRandFile.writeInt(LogEntity.messageStartPosition);// 12 reader 70 | // pos 71 | dbRandFile.writeInt(LogEntity.messageStartPosition); // 16write 72 | // pos 73 | dbRandFile.writeInt(1);// 20readerindex 74 | dbRandFile.writeInt(1);// 24writerindex 75 | dbRandFile.writeInt(0);// 28 size 76 | magicString = LogEntity.MAGIC; 77 | version = 1; 78 | readerPosition = LogEntity.messageStartPosition; 79 | writerPosition = LogEntity.messageStartPosition; 80 | readerIndex = 1; 81 | writerIndex = 1; 82 | } else { 83 | dbRandFile = new RandomAccessFile(dbFile, "rwd"); 84 | if (dbRandFile.length() < 32) { 85 | throw new FileFormatException("file format error"); 86 | } 87 | byte[] b = new byte[this.dbFileLimitLength]; 88 | dbRandFile.read(b); 89 | ByteBuffer buffer = ByteBuffer.wrap(b); 90 | b = new byte[LogEntity.MAGIC.getBytes().length]; 91 | buffer.get(b); 92 | magicString = new String(b); 93 | version = buffer.getInt(); 94 | readerPosition = buffer.getInt(); 95 | writerPosition = buffer.getInt(); 96 | readerIndex = buffer.getInt(); 97 | writerIndex = buffer.getInt(); 98 | size.set(buffer.getInt()); 99 | 100 | } 101 | fc = dbRandFile.getChannel(); 102 | mappedByteBuffer = fc.map(MapMode.READ_WRITE, 0, this.dbFileLimitLength); 103 | } 104 | 105 | /** 106 | * 记录写位置 107 | * 108 | * @param pos 109 | */ 110 | public void putWriterPosition(int pos) { 111 | mappedByteBuffer.position(16); 112 | mappedByteBuffer.putInt(pos); 113 | this.writerPosition = pos; 114 | } 115 | 116 | /** 117 | * 记录读取的位置 118 | * 119 | * @param pos 120 | */ 121 | public void putReaderPosition(int pos) { 122 | mappedByteBuffer.position(12); 123 | mappedByteBuffer.putInt(pos); 124 | this.readerPosition = pos; 125 | } 126 | 127 | /** 128 | * 记录写文件索引 129 | * 130 | * @param index 131 | */ 132 | public void putWriterIndex(int index) { 133 | mappedByteBuffer.position(24); 134 | mappedByteBuffer.putInt(index); 135 | this.writerIndex = index; 136 | } 137 | 138 | /** 139 | * 记录读取文件索引 140 | * 141 | * @param index 142 | */ 143 | public void putReaderIndex(int index) { 144 | mappedByteBuffer.position(20); 145 | mappedByteBuffer.putInt(index); 146 | this.readerIndex = index; 147 | } 148 | 149 | public void incrementSize() { 150 | int num = size.incrementAndGet(); 151 | mappedByteBuffer.position(28); 152 | mappedByteBuffer.putInt(num); 153 | } 154 | 155 | public void decrementSize() { 156 | int num = size.decrementAndGet(); 157 | mappedByteBuffer.position(28); 158 | mappedByteBuffer.putInt(num); 159 | } 160 | 161 | public String getMagicString() { 162 | return magicString; 163 | } 164 | 165 | public int getVersion() { 166 | return version; 167 | } 168 | 169 | public int getReaderPosition() { 170 | return readerPosition; 171 | } 172 | 173 | public int getWriterPosition() { 174 | return writerPosition; 175 | } 176 | 177 | public int getReaderIndex() { 178 | return readerIndex; 179 | } 180 | 181 | public int getWriterIndex() { 182 | return writerIndex; 183 | } 184 | 185 | public int getSize() { 186 | return size.get(); 187 | } 188 | 189 | /** 190 | * 关闭索引文件 191 | */ 192 | public void close() { 193 | try { 194 | mappedByteBuffer.force(); 195 | AccessController.doPrivileged(new PrivilegedAction() { 196 | public Object run() { 197 | try { 198 | Method getCleanerMethod = mappedByteBuffer.getClass().getMethod("cleaner", new Class[0]); 199 | getCleanerMethod.setAccessible(true); 200 | sun.misc.Cleaner cleaner = (sun.misc.Cleaner) getCleanerMethod.invoke(mappedByteBuffer, 201 | new Object[0]); 202 | cleaner.clean(); 203 | } catch (Exception e) { 204 | log.error("close logindexy file error:", e); 205 | } 206 | return null; 207 | } 208 | }); 209 | fc.close(); 210 | dbRandFile.close(); 211 | mappedByteBuffer = null; 212 | fc = null; 213 | dbRandFile = null; 214 | } catch (IOException e) { 215 | log.error("close logindex file error:", e); 216 | } 217 | } 218 | 219 | public String headerInfo() { 220 | StringBuilder sb = new StringBuilder(); 221 | sb.append(" magicString:"); 222 | sb.append(magicString); 223 | sb.append(" version:"); 224 | sb.append(version); 225 | sb.append(" readerPosition:"); 226 | sb.append(readerPosition); 227 | sb.append(" writerPosition:"); 228 | sb.append(writerPosition); 229 | sb.append(" size:"); 230 | sb.append(size); 231 | sb.append(" readerIndex:"); 232 | sb.append(readerIndex); 233 | sb.append(" writerIndex:"); 234 | sb.append(writerIndex); 235 | return sb.toString(); 236 | } 237 | 238 | } 239 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/memcached/StartNewQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.memcached; 17 | 18 | import java.net.InetSocketAddress; 19 | 20 | import org.apache.commons.logging.Log; 21 | import org.apache.commons.logging.LogFactory; 22 | 23 | import com.google.code.fqueue.memcached.storage.FSStorage; 24 | import com.thimbleware.jmemcached.CacheImpl; 25 | import com.thimbleware.jmemcached.LocalCacheElement; 26 | import com.thimbleware.jmemcached.MemCacheDaemon; 27 | import com.thimbleware.jmemcached.storage.CacheStorage; 28 | 29 | /** 30 | * @author sunli 31 | * @date 2011-5-18 32 | * @version $Id$ 33 | */ 34 | public class StartNewQueue { 35 | private static final Log log = LogFactory.getLog(StartNewQueue.class); 36 | 37 | public static void newQueueInstance(int port) { 38 | InetSocketAddress addr = new InetSocketAddress("0.0.0.0", port); 39 | int idle = -1; 40 | boolean verbose = false; 41 | MemCacheDaemon.memcachedVersion = "0.1"; 42 | final MemCacheDaemon daemon = new MemCacheDaemon(); 43 | CacheStorage storage = new FSStorage(); 44 | CacheImpl cacheImpl = new CacheImpl(storage); 45 | daemon.setCache(cacheImpl); 46 | daemon.setAddr(addr); 47 | daemon.setBinary(false); 48 | daemon.setIdleTime(idle); 49 | daemon.setVerbose(verbose); 50 | daemon.start(); 51 | log.info("\r\n\t FQueue instance started,port:" + port 52 | + " [version 0.1] \r\n\t\t\t Copyright (C) 2011 sunli"); 53 | Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { 54 | public void run() { 55 | if (daemon != null && daemon.isRunning()) 56 | daemon.stop(); 57 | log.info("shutdown server"); 58 | } 59 | })); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/memcached/StartServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.memcached; 17 | 18 | import org.apache.commons.logging.Log; 19 | import org.apache.commons.logging.LogFactory; 20 | import org.apache.log4j.PropertyConfigurator; 21 | 22 | import com.google.code.fqueue.util.Config; 23 | 24 | /** 25 | * @author sunli 26 | * @date 2011-5-18 27 | * @version $Id$ 28 | */ 29 | public class StartServer { 30 | private static final Log log = LogFactory.getLog(StartServer.class); 31 | static { 32 | PropertyConfigurator.configure("config/log4j.properties"); 33 | } 34 | 35 | /** 36 | * @param args 37 | */ 38 | public static void main(String[] args) { 39 | PropertyConfigurator.configureAndWatch("config/log4j.properties", 5000); 40 | StartNewQueue.newQueueInstance(Integer.parseInt(Config.getSetting("port"))); 41 | log.info("running at port " + Config.getSetting("port")); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/memcached/storage/FSStorage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.memcached.storage; 17 | 18 | import java.io.IOException; 19 | import java.util.AbstractQueue; 20 | import java.util.Map; 21 | import java.util.Set; 22 | import java.util.concurrent.ConcurrentHashMap; 23 | import java.util.concurrent.locks.ReentrantLock; 24 | 25 | import org.apache.commons.lang.StringUtils; 26 | import org.apache.commons.logging.Log; 27 | import org.apache.commons.logging.LogFactory; 28 | 29 | import com.google.code.fqueue.FQueue; 30 | import com.google.code.fqueue.exception.ConfigException; 31 | import com.google.code.fqueue.util.Config; 32 | import com.google.code.fqueue.util.JVMMonitor; 33 | import com.thimbleware.jmemcached.LocalCacheElement; 34 | import com.thimbleware.jmemcached.protocol.exceptions.ClientException; 35 | import com.thimbleware.jmemcached.protocol.exceptions.DatabaseException; 36 | import com.thimbleware.jmemcached.storage.CacheStorage; 37 | 38 | /** 39 | * Memcached的缓存存储实现，底层通过FQueue实现。 通过Memcached协议的key来实现二次协议 40 | * 41 | * @author sunli 42 | * @date 2011-5-11 43 | * @version $Id$ 44 | */ 45 | public class FSStorage implements CacheStorage { 46 | /** 47 | * 存储所有的队列服务 48 | */ 49 | private Map> queuemMap = new ConcurrentHashMap>(); 50 | private final ReentrantLock lock = new ReentrantLock(); 51 | private final static Log log = LogFactory.getLog(FSStorage.class); 52 | /** 53 | * 每个队列服务的单个日至存储的大小限制 配置文件中的单位为M 54 | */ 55 | private final static int logSize = 1024 * 1024 * Integer.parseInt(Config.getSetting("logsize")); 56 | /** 57 | * 数据存储路径 58 | */ 59 | private final static String dbpath = Config.getSetting("path").trim(); 60 | /** 61 | * 安全验证map 62 | */ 63 | private static Map authorizationMap = new ConcurrentHashMap(20); 64 | static { 65 | loadAuthorization(); 66 | } 67 | 68 | private static void loadAuthorization() { 69 | String line = Config.getSetting("authorization"); 70 | if (line != null) { 71 | if (line.indexOf("_") != -1) { 72 | log.error("权限配置不能包含'_'符号"); 73 | } 74 | String[] group = line.split("@@"); 75 | for (int i = 0, groupLen = group.length; i < groupLen; i++) { 76 | String[] item = group[i].split("\\|"); 77 | if (item.length == 2) { 78 | authorizationMap.put(item[0], item[1]); 79 | } 80 | } 81 | } 82 | } 83 | 84 | /** 85 | * 重新加载安全验证数据，可用于在线动态增加队列，修改密码等操作 86 | */ 87 | private static void reloadAuthorization() { 88 | Config.reload(); 89 | loadAuthorization(); 90 | } 91 | 92 | @Override 93 | public int capacity() { 94 | return 0; 95 | } 96 | 97 | @Override 98 | public void clear() throws DatabaseException, Exception { 99 | } 100 | 101 | @Override 102 | public void close() throws IOException { 103 | for (String key : queuemMap.keySet()) { 104 | ((FQueue) queuemMap.get(key)).close(); 105 | } 106 | log.info("close queue"); 107 | } 108 | 109 | /** 110 | * get方式的二次协议实现 111 | * 112 | * @param keystring 113 | * @return 114 | * @throws ClientException 115 | */ 116 | private LocalCacheElement getProtocol(String keystring) throws ClientException { 117 | // 获取队列中元素的个数 118 | if (keystring.startsWith("size")) { 119 | try { 120 | // size|bbs 121 | // size操作无密码验证，只需要队列名称正确即可 122 | String[] clientInfo = QueueClient.parse(keystring, '|'); 123 | if (clientInfo.length < 2 || authorizationMap.containsKey(clientInfo[1]) == false) { 124 | return null; 125 | } 126 | AbstractQueue sizeQueue = getClientQueue(clientInfo[1]); 127 | if (sizeQueue == null) { 128 | return null; 129 | } 130 | int size = sizeQueue.size(); 131 | LocalCacheElement element = new LocalCacheElement(keystring, 0, 0, 0); 132 | element.setData(String.valueOf(size).getBytes()); 133 | return element; 134 | } catch (Exception e) { 135 | log.error("getsize " + keystring + "error", e); 136 | return null; 137 | } 138 | } 139 | // 清空队列中所有的元素 140 | if (keystring.startsWith("clear")) { 141 | try { 142 | // clear|bbs|pass 143 | String[] clientInfo = QueueClient.parse(keystring, '|'); 144 | if (clientInfo.length < 3 || valid(clientInfo[1], clientInfo[2]) == false) { 145 | throw new ClientException("Authorization error"); 146 | } 147 | AbstractQueue queue = getClientQueue(clientInfo[1]); 148 | queue.clear(); 149 | LocalCacheElement element = new LocalCacheElement(keystring, 0, 0, 0); 150 | element.setData(String.valueOf(queue.size()).getBytes()); 151 | return element; 152 | } catch (Exception e) { 153 | log.error("getsize " + keystring + "error", e); 154 | return null; 155 | } 156 | } 157 | // 重新加载权限信息 158 | if (keystring.startsWith("reload")) { 159 | try { 160 | // reload|bbs|pass 161 | String[] clientInfo = QueueClient.parse(keystring, '|'); 162 | if (clientInfo.length < 3 || valid(clientInfo[1], clientInfo[2]) == false) { 163 | throw new ClientException("Authorization error"); 164 | } 165 | reloadAuthorization(); 166 | LocalCacheElement element = new LocalCacheElement(keystring, 0, 0, 0); 167 | element.setData("reloadAuthorization".getBytes()); 168 | return element; 169 | } catch (ConfigException e) { 170 | log.error(e.getMessage(), e); 171 | } catch (Exception e) { 172 | log.error("reloadAuthorization error", e); 173 | return null; 174 | } 175 | } 176 | // 重新加载权限信息 177 | if (keystring.startsWith("monitor")) { 178 | try { 179 | // size|bbs 180 | // size操作无密码验证，只需要队列名称正确即可 181 | String[] clientInfo = QueueClient.parse(keystring, '|'); 182 | if (clientInfo.length < 2) { 183 | return null; 184 | } 185 | String items = clientInfo[1]; 186 | StringBuilder stats = new StringBuilder(); 187 | if (items != null) { 188 | String[] itemList = StringUtils.split(items, ","); 189 | for (int i = 0, len = itemList.length; i < len; i++) { 190 | if (i > 0) { 191 | stats.append("\r\n"); 192 | } 193 | String data = JVMMonitor.getMonitorStats(itemList[i]); 194 | stats.append(data); 195 | } 196 | } else { 197 | stats.append("need items"); 198 | } 199 | 200 | LocalCacheElement element = new LocalCacheElement(keystring, 0, 0, 0); 201 | element.setData(stats.toString().getBytes()); 202 | return element; 203 | } catch (Exception e) { 204 | log.error("monitor " + keystring + "error", e); 205 | return null; 206 | } 207 | } 208 | throw new ClientException(keystring + " command Unsupported now"); 209 | } 210 | 211 | @Override 212 | public LocalCacheElement get(String keystring) { 213 | try { 214 | if (keystring.indexOf("_") == -1) { 215 | return getProtocol(keystring); 216 | } 217 | String[] clientInfo = QueueClient.parseWithCache(keystring); 218 | if (valid(clientInfo[0], clientInfo[1])) { 219 | byte[] data; 220 | data = getClientQueue(clientInfo[0]).poll(); 221 | if (data != null) { 222 | LocalCacheElement element = new LocalCacheElement(keystring, 0, 0, 0); 223 | element.setData(data); 224 | return element; 225 | } else { 226 | log.info("queue empty"); 227 | } 228 | } else { 229 | log.error("unvalid " + keystring); 230 | throw new ClientException("Authorization error"); 231 | } 232 | } catch (Exception e) { 233 | log.error("get queue " + keystring + " error", e); 234 | return null; 235 | } 236 | return null; 237 | } 238 | 239 | @Override 240 | public long getMemoryCapacity() { 241 | return 0; 242 | } 243 | 244 | @Override 245 | public long getMemoryUsed() { 246 | return 0; 247 | } 248 | 249 | @Override 250 | public Set keySet() { 251 | return null; 252 | } 253 | 254 | @Override 255 | public LocalCacheElement put(String keystring, LocalCacheElement e) throws DatabaseException, Exception { 256 | String[] clientInfo = QueueClient.parseWithCache(keystring); 257 | if (valid(clientInfo[0], clientInfo[1])) {// 先进行密码验证 258 | getClientQueue(clientInfo[0]).add(e.getData()); 259 | return null; 260 | } else { 261 | throw new ClientException("Authorization error"); 262 | } 263 | } 264 | 265 | @Override 266 | public LocalCacheElement putIfAbsent(String keystring, LocalCacheElement e) throws DatabaseException, Exception { 267 | String[] clientInfo = QueueClient.parseWithCache(keystring); 268 | if (valid(clientInfo[0], clientInfo[1])) {// 先进行密码验证 269 | getClientQueue(clientInfo[0]).add(e.getData()); 270 | return null; 271 | } else { 272 | throw new ClientException("Authorization error"); 273 | } 274 | 275 | } 276 | 277 | @Override 278 | public LocalCacheElement remove(String key) throws DatabaseException, Exception { 279 | return null; 280 | } 281 | 282 | @Override 283 | public boolean replace(String keystring, LocalCacheElement old, LocalCacheElement prepend) 284 | throws DatabaseException, Exception { 285 | return false; 286 | } 287 | 288 | @Override 289 | public LocalCacheElement replace(String key, LocalCacheElement placeHolder) throws DatabaseException, Exception { 290 | return null; 291 | } 292 | 293 | @Override 294 | public long size() { 295 | return 0; 296 | } 297 | 298 | private boolean valid(String appid, String pwd) { 299 | return pwd.equals(authorizationMap.get(appid)); 300 | } 301 | 302 | /** 303 | * 获取指定名称的队列存储实例 如果不存存在，根据create参数决定是否创建 304 | * 305 | * @param name 306 | * @return 307 | * @throws Exception 308 | */ 309 | private AbstractQueue getClientQueue(String name, boolean create) throws Exception { 310 | AbstractQueue queue = queuemMap.get(name); 311 | if (queue == null) { 312 | if (create == true) { 313 | lock.lock(); 314 | try { 315 | queue = queuemMap.get(name); 316 | if (queue == null) { 317 | queue = new FQueue(dbpath + "/" + name, logSize); 318 | queuemMap.put(name, queue); 319 | } 320 | } finally { 321 | lock.unlock(); 322 | } 323 | } 324 | } 325 | return queue; 326 | } 327 | 328 | /** 329 | * 获取或者创建指定名称的队列存储实例 330 | * 331 | * @param name 332 | * @return 333 | * @throws Exception 334 | */ 335 | private AbstractQueue getClientQueue(String name) throws Exception { 336 | return getClientQueue(name, true); 337 | } 338 | } 339 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/memcached/storage/QueueClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.memcached.storage; 17 | 18 | import java.util.LinkedList; 19 | import java.util.Map; 20 | import java.util.concurrent.ConcurrentSkipListMap; 21 | 22 | /** 23 | * @author sunli 24 | * @date 2011-5-23 25 | * @version $Id$ 26 | */ 27 | public class QueueClient { 28 | public static String[] parse(String keyString) { 29 | return parse(keyString, '_'); 30 | } 31 | 32 | /** 33 | * 按照"_"进行切分 34 | * 35 | * @param keyString 36 | * @return 37 | */ 38 | public static String[] parseWithCache(String keyString) { 39 | return parse(keyString, '_'); 40 | } 41 | 42 | public static String[] parse(String keyString, Character separator) { 43 | LinkedList list = new LinkedList(); 44 | int start = 0; 45 | for (int i = 0, len = keyString.length(); i < len; i++) { 46 | if (keyString.charAt(i) == separator) { 47 | list.add(keyString.substring(start, i)); 48 | start = i + 1; 49 | } 50 | if (i == len - 1) { 51 | list.add(keyString.substring(start)); 52 | } 53 | } 54 | String[] result = new String[list.size()]; 55 | return list.toArray(result); 56 | 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/util/Config.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.util; 17 | 18 | import java.io.File; 19 | import java.io.FileInputStream; 20 | import java.io.FileNotFoundException; 21 | import java.io.IOException; 22 | import java.util.Enumeration; 23 | import java.util.Map; 24 | import java.util.Properties; 25 | import java.util.concurrent.ConcurrentHashMap; 26 | 27 | import org.apache.commons.logging.Log; 28 | import org.apache.commons.logging.LogFactory; 29 | 30 | import com.google.code.fqueue.exception.ConfigException; 31 | 32 | /** 33 | * @author sunli 34 | * @date 2010-8-12 35 | * @version $Id$ 36 | */ 37 | public class Config { 38 | private final static Log log = LogFactory.getLog(Config.class); 39 | // map 存储配置信息 40 | private static Map setting = new ConcurrentHashMap(); 41 | // 调用初始化操作 42 | static { 43 | iniSetting(); 44 | } 45 | 46 | public Config() { 47 | 48 | } 49 | 50 | /** 51 | * 初始化加载配置文件 默认加载配置路径config/config.properties或者config.properties 52 | * 53 | * @throws FileNotFoundException 54 | */ 55 | public static synchronized void iniSetting() { 56 | File file; 57 | file = new File("config.properties"); 58 | if (!file.exists()) { 59 | iniSetting("config/config.properties"); 60 | } else { 61 | iniSetting("config.properties"); 62 | } 63 | } 64 | 65 | /** 66 | * 初始化加载配置文件 67 | * 68 | * @param path 69 | * 加载路径 70 | * @throws FileNotFoundException 71 | */ 72 | public static synchronized void iniSetting(String path) { 73 | File file; 74 | file = new File(path); 75 | FileInputStream in = null; 76 | try { 77 | in = new FileInputStream(file); 78 | Properties p = new Properties(); 79 | p.load(in); 80 | // 遍历配置文件加入到Map中进行缓存 81 | Enumeration item = p.propertyNames(); 82 | while (item.hasMoreElements()) { 83 | String key = (String) item.nextElement(); 84 | setting.put(key, p.getProperty(key)); 85 | } 86 | in.close(); 87 | } catch (FileNotFoundException e) { 88 | log.error("config file not found at" + file.getAbsolutePath()); 89 | throw new ConfigException("FileNotFoundException", e); 90 | } catch (IOException e) { 91 | log.error("config file not found at" + file.getAbsolutePath()); 92 | throw new ConfigException("IOException", e); 93 | } catch (Exception e) { 94 | throw new ConfigException("Exception", e); 95 | } 96 | } 97 | 98 | public static void reload() { 99 | try { 100 | iniSetting(); 101 | } catch (ConfigException e) { 102 | throw new ConfigException(e.getMessage(), e); 103 | } 104 | } 105 | 106 | /** 107 | * 获取配置文件的某个键值的配置信息 108 | * 109 | * @param key 110 | * 键 111 | * @return 值 112 | */ 113 | public static String getSetting(String key) { 114 | return setting.get(key); 115 | } 116 | 117 | /** 118 | * 设置配置文件的数据 119 | * 120 | * @param key 121 | * @param value 122 | */ 123 | public static void setSetting(String key, String value) { 124 | setting.put(key, value); 125 | } 126 | 127 | public static int getInt(String key) { 128 | return Integer.parseInt(setting.get(key)); 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/com/google/code/fqueue/util/MappedByteBufferUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.code.fqueue.util; 18 | 19 | import java.lang.reflect.Method; 20 | import java.security.AccessController; 21 | import java.security.PrivilegedAction; 22 | 23 | /** 24 | * @author sunli 25 | */ 26 | public class MappedByteBufferUtil { 27 | public static void clean(final Object buffer) { 28 | AccessController.doPrivileged(new PrivilegedAction() { 29 | public Object run() { 30 | try { 31 | Method getCleanerMethod = buffer.getClass().getMethod("cleaner", new Class[0]); 32 | getCleanerMethod.setAccessible(true); 33 | sun.misc.Cleaner cleaner = (sun.misc.Cleaner) getCleanerMethod.invoke(buffer, new Object[0]); 34 | cleaner.clean(); 35 | } catch (Exception e) { 36 | e.printStackTrace(); 37 | } 38 | return null; 39 | } 40 | }); 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/AbstractCache.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached; 2 | 3 | import java.lang.management.ManagementFactory; 4 | import java.lang.management.OperatingSystemMXBean; 5 | import java.util.HashMap; 6 | import java.util.HashSet; 7 | import java.util.Map; 8 | import java.util.Set; 9 | import java.util.concurrent.TimeUnit; 10 | import java.util.concurrent.atomic.AtomicInteger; 11 | import java.util.concurrent.atomic.AtomicLong; 12 | 13 | import org.jboss.netty.channel.ChannelHandlerContext; 14 | 15 | import com.thimbleware.jmemcached.protocol.exceptions.DatabaseException; 16 | /** 17 | * 在jmemcached的基础之上完善了stats信息 18 | * 19 | * 20 | */ 21 | 22 | 23 | /** 24 | * Abstract implementation of a cache handler for the memcache daemon; provides 25 | * some convenience methods and a general framework for implementation 26 | */ 27 | public abstract class AbstractCache implements Cache { 28 | 29 | protected final AtomicLong started = new AtomicLong(); 30 | 31 | protected final AtomicLong getCmds = new AtomicLong(); 32 | protected final AtomicLong setCmds = new AtomicLong(); 33 | protected final AtomicLong getHits = new AtomicLong(); 34 | protected final AtomicLong getMisses = new AtomicLong(); 35 | protected final AtomicLong casCounter = new AtomicLong(1); 36 | protected final OperatingSystemMXBean bean = ManagementFactory.getOperatingSystemMXBean(); 37 | 38 | public AbstractCache() { 39 | initStats(); 40 | } 41 | 42 | /** Get JVM CPU time in milliseconds */ 43 | public long getJVMCpuTime() { 44 | if (!(bean instanceof com.sun.management.OperatingSystemMXBean)) 45 | return 0L; 46 | return ((com.sun.management.OperatingSystemMXBean) bean).getProcessCpuTime(); 47 | } 48 | 49 | public double getJVMLoad() { 50 | if (!(bean instanceof com.sun.management.OperatingSystemMXBean)) 51 | return 0L; 52 | return ((com.sun.management.OperatingSystemMXBean) bean).getSystemLoadAverage(); 53 | } 54 | 55 | public int getAllthreadsCount() { 56 | ThreadGroup group = Thread.currentThread().getThreadGroup(); 57 | ThreadGroup topGroup = group; 58 | while (group != null) { 59 | topGroup = group; 60 | group = group.getParent(); 61 | } 62 | return topGroup.activeCount(); 63 | } 64 | 65 | /** 66 | * @return the current time in seconds (from epoch), used for expiries, etc. 67 | */ 68 | public static int Now() { 69 | return (int) (System.currentTimeMillis() / 1000); 70 | } 71 | 72 | public abstract Set keys(); 73 | 74 | public abstract long getCurrentItems(); 75 | 76 | public abstract long getLimitMaxBytes(); 77 | 78 | public abstract long getCurrentBytes(); 79 | 80 | public final long getGetCmds() { 81 | return getCmds.get(); 82 | } 83 | 84 | public final long getSetCmds() { 85 | return setCmds.get(); 86 | } 87 | 88 | public final long getGetHits() { 89 | return getHits.get(); 90 | } 91 | 92 | public final long getGetMisses() { 93 | return getMisses.get(); 94 | } 95 | 96 | /** 97 | * Return runtime statistics 98 | * 99 | * @param arg 100 | * additional arguments to the stats command 101 | * @return the full command response 102 | */ 103 | public final Map> stat(String arg, ChannelHandlerContext channelHandlerContext) { 104 | Map> result = new HashMap>(); 105 | // stats we know 106 | multiSet(result, "version", MemCacheDaemon.memcachedVersion); 107 | multiSet(result, "cmd_get", java.lang.String.valueOf(getGetCmds())); 108 | multiSet(result, "cmd_set", java.lang.String.valueOf(getSetCmds())); 109 | multiSet(result, "get_hits", java.lang.String.valueOf(getGetHits())); 110 | multiSet(result, "get_misses", java.lang.String.valueOf(getGetMisses())); 111 | multiSet(result, "time", java.lang.String.valueOf(java.lang.String.valueOf(Now()))); 112 | multiSet(result, "uptime", java.lang.String.valueOf(Now() - this.started.longValue())); 113 | multiSet(result, "curr_items", java.lang.String.valueOf(this.getCurrentItems())); 114 | multiSet(result, "total_items", java.lang.String.valueOf(this.getCurrentItems())); 115 | multiSet(result, "limit_maxbytes", java.lang.String.valueOf(this.getLimitMaxBytes())); 116 | multiSet(result, "current_bytes", java.lang.String.valueOf(this.getCurrentBytes())); 117 | multiSet(result, "bytes", java.lang.String.valueOf(this.getCurrentBytes())); 118 | multiSet(result, "free_bytes", java.lang.String.valueOf(Runtime.getRuntime().freeMemory())); 119 | 120 | // Not really the same thing precisely, but meaningful nonetheless. 121 | // potentially this should be renamed 122 | multiSet(result, "pid", java.lang.String.valueOf(Thread.currentThread().getId())); 123 | 124 | // stuff we know nothing about; gets faked only because some clients 125 | // expect this 126 | multiSet(result, "rusage_user", String.valueOf(TimeUnit.NANOSECONDS.toSeconds(getJVMCpuTime()))); 127 | multiSet(result, "rusage_system", "0.0"); 128 | multiSet(result, "connection_structures", "0"); 129 | multiSet(result, "curr_connections", String.valueOf(StatsCounter.curr_conns.longValue())); 130 | multiSet(result, "total_connections", String.valueOf(StatsCounter.total_conns.longValue())); 131 | 132 | // TODO we could collect these stats 133 | multiSet(result, "bytes_read", String.valueOf(StatsCounter.bytes_read.longValue())); 134 | multiSet(result, "bytes_written", String.valueOf(StatsCounter.bytes_written.longValue())); 135 | multiSet(result, "system_load", String.valueOf(getJVMLoad())); 136 | multiSet(result, "threads", String.valueOf(getAllthreadsCount())); 137 | return result; 138 | } 139 | 140 | private void multiSet(Map> map, String key, String val) { 141 | Set cur = map.get(key); 142 | if (cur == null) { 143 | cur = new HashSet(); 144 | } 145 | cur.add(val); 146 | map.put(key, cur); 147 | } 148 | 149 | /** 150 | * Initialize all statistic counters 151 | */ 152 | protected void initStats() { 153 | started.set(System.currentTimeMillis() / 1000); 154 | getCmds.set(0); 155 | setCmds.set(0); 156 | getHits.set(0); 157 | getMisses.set(0); 158 | 159 | } 160 | 161 | public abstract void asyncEventPing() throws DatabaseException, Exception; 162 | } 163 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/Cache.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | import java.util.Set; 6 | 7 | import org.jboss.netty.channel.ChannelHandlerContext; 8 | 9 | import com.thimbleware.jmemcached.protocol.exceptions.DatabaseException; 10 | 11 | 12 | /** 13 | */ 14 | public interface Cache { 15 | /** 16 | * Enum defining response statuses from set/add type commands 17 | */ 18 | public enum StoreResponse { 19 | STORED, NOT_STORED, EXISTS, NOT_FOUND 20 | } 21 | 22 | /** 23 | * Enum defining responses statuses from removal commands 24 | */ 25 | public enum DeleteResponse { 26 | DELETED, NOT_FOUND 27 | } 28 | 29 | /** 30 | * Handle the deletion of an item from the cache. 31 | * 32 | * @param key 33 | * the key for the item 34 | * @param time 35 | * an amount of time to block this entry in the cache for further 36 | * writes 37 | * @return the message response 38 | */ 39 | DeleteResponse delete(String key, int time) throws DatabaseException, Exception; 40 | 41 | /** 42 | * Add an element to the cache 43 | * 44 | * @param e 45 | * the element to add 46 | * @return the store response code 47 | */ 48 | StoreResponse add(CACHE_ELEMENT e) throws DatabaseException, Exception; 49 | 50 | /** 51 | * Replace an element in the cache 52 | * 53 | * @param e 54 | * the element to replace 55 | * @return the store response code 56 | */ 57 | StoreResponse replace(CACHE_ELEMENT e) throws DatabaseException, Exception; 58 | 59 | /** 60 | * Append bytes to the end of an element in the cache 61 | * 62 | * @param element 63 | * the element to append 64 | * @return the store response code 65 | */ 66 | StoreResponse append(CACHE_ELEMENT element) throws DatabaseException, Exception; 67 | 68 | /** 69 | * Prepend bytes to the end of an element in the cache 70 | * 71 | * @param element 72 | * the element to append 73 | * @return the store response code 74 | */ 75 | StoreResponse prepend(CACHE_ELEMENT element) throws DatabaseException, Exception; 76 | 77 | /** 78 | * Set an element in the cache 79 | * 80 | * @param e 81 | * the element to set 82 | * @return the store response code 83 | */ 84 | StoreResponse set(CACHE_ELEMENT e) throws DatabaseException, Exception; 85 | 86 | /** 87 | * Set an element in the cache but only if the element has not been touched 88 | * since the last 'gets' 89 | * 90 | * @param cas_key 91 | * the cas key returned by the last gets 92 | * @param e 93 | * the element to set 94 | * @return the store response code 95 | */ 96 | StoreResponse cas(Long cas_key, CACHE_ELEMENT e) throws DatabaseException, Exception; 97 | 98 | /** 99 | * Increment/decremen t an (integer) element in the cache 100 | * 101 | * @param key 102 | * the key to increment 103 | * @param mod 104 | * the amount to add to the value 105 | * @return the message response 106 | */ 107 | Integer get_add(String key, int mod) throws DatabaseException, Exception; 108 | 109 | /** 110 | * Get element(s) from the cache 111 | * 112 | * @param keys 113 | * the key for the element to lookup 114 | * @return the element, or 'null' in case of cache miss. 115 | */ 116 | CACHE_ELEMENT[] get(String... keys); 117 | 118 | /** 119 | * Flush all cache entries 120 | * 121 | * @return command response 122 | */ 123 | boolean flush_all() throws DatabaseException, Exception; 124 | 125 | /** 126 | * Flush all cache entries with a timestamp after a given expiration time 127 | * 128 | * @param expire 129 | * the flush time in seconds 130 | * @return command response 131 | */ 132 | boolean flush_all(int expire) throws DatabaseException, Exception; 133 | 134 | /** 135 | * Close the cache, freeing all resources on which it depends. 136 | * 137 | * @throws IOException 138 | */ 139 | void close() throws IOException; 140 | 141 | /** 142 | * @return all keys currently held in the cache 143 | */ 144 | Set keys(); 145 | 146 | /** 147 | * @return the # of items in the cache 148 | */ 149 | long getCurrentItems(); 150 | 151 | /** 152 | * @return the maximum size of the cache (in bytes) 153 | */ 154 | long getLimitMaxBytes(); 155 | 156 | /** 157 | * @return the current cache usage (in bytes) 158 | */ 159 | long getCurrentBytes(); 160 | 161 | /** 162 | * @return the number of get commands executed 163 | */ 164 | long getGetCmds(); 165 | 166 | /** 167 | * @return the number of set commands executed 168 | */ 169 | long getSetCmds(); 170 | 171 | /** 172 | * @return the number of get hits 173 | */ 174 | long getGetHits(); 175 | 176 | /** 177 | * @return the number of stats 178 | */ 179 | long getGetMisses(); 180 | 181 | /** 182 | * Retrieve stats about the cache. If an argument is specified, a specific 183 | * category of stats is requested. 184 | * 185 | * @param arg 186 | * a specific extended stat sub-category 187 | * @return a map of stats 188 | */ 189 | Map> stat(String arg, ChannelHandlerContext channelHandlerContext); 190 | 191 | /** 192 | * Called periodically by the network event loop to process any pending 193 | * events. (such as delete queues, etc.) 194 | */ 195 | void asyncEventPing() throws DatabaseException, Exception; 196 | 197 | } 198 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/CacheElement.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached; 2 | 3 | import com.thimbleware.jmemcached.storage.hash.SizedItem; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | */ 9 | public interface CacheElement extends Serializable, SizedItem { 10 | int THIRTY_DAYS = 60 * 60 * 24 * 30; 11 | 12 | int size(); 13 | 14 | int hashCode(); 15 | 16 | int getExpire(); 17 | 18 | int getFlags(); 19 | 20 | byte[] getData(); 21 | 22 | void setData(byte[] data); 23 | 24 | String getKeystring(); 25 | 26 | long getCasUnique(); 27 | 28 | void setCasUnique(long casUnique); 29 | 30 | boolean isBlocked(); 31 | 32 | void block(long blockedUntil); 33 | 34 | long getBlockedUntil(); 35 | 36 | LocalCacheElement append(LocalCacheElement element); 37 | 38 | LocalCacheElement prepend(LocalCacheElement element); 39 | 40 | LocalCacheElement.IncrDecrResult add(int mod); 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/CacheImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 ThimbleWare Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.thimbleware.jmemcached; 17 | 18 | import java.io.IOException; 19 | import java.util.Set; 20 | import java.util.concurrent.DelayQueue; 21 | import java.util.concurrent.Delayed; 22 | import java.util.concurrent.TimeUnit; 23 | import java.util.concurrent.locks.Lock; 24 | import java.util.concurrent.locks.ReadWriteLock; 25 | import java.util.concurrent.locks.ReentrantReadWriteLock; 26 | 27 | import com.thimbleware.jmemcached.protocol.exceptions.DatabaseException; 28 | import com.thimbleware.jmemcached.storage.CacheStorage; 29 | 30 | /** 31 | * Default implementation of the cache handler, supporting local memory cache 32 | * elements. 33 | */ 34 | public final class CacheImpl extends AbstractCache implements Cache { 35 | 36 | final CacheStorage storage; 37 | final DelayQueue deleteQueue; 38 | final boolean isReadOnly = false; 39 | final boolean forbiddenflush = false; 40 | // record lock 41 | private final int lockNum = 100; 42 | private Lock[] writeLocks = new Lock[lockNum]; 43 | private Lock[] readLocks = new Lock[lockNum]; 44 | 45 | /** 46 | * @inheritDoc 47 | */ 48 | public CacheImpl(CacheStorage storage) { 49 | super(); 50 | this.storage = storage; 51 | deleteQueue = new DelayQueue(); 52 | for (int i = 0; i < lockNum; i++) { 53 | ReadWriteLock lock = new ReentrantReadWriteLock(); 54 | writeLocks[i] = lock.writeLock(); 55 | readLocks[i] = lock.readLock(); 56 | } 57 | } 58 | 59 | private void checkWritePermission() throws Exception { 60 | if (isReadOnly) { 61 | throw new Exception("slave write is forbidden"); 62 | } 63 | } 64 | 65 | /** 66 | * @throws Exception 67 | * @throws DatabaseException 68 | * @throws Exception 69 | * @throws DatabaseException 70 | * @inheritDoc 71 | */ 72 | public DeleteResponse delete(String key, int time) throws DatabaseException, Exception { 73 | checkWritePermission(); 74 | boolean removed = false; 75 | // delayed remove 76 | if (time != 0) { 77 | // block the element and schedule a delete; replace its entry with a 78 | // blocked element 79 | LocalCacheElement placeHolder = new LocalCacheElement(key, 0, 0, 0L); 80 | placeHolder.setData(new byte[] {}); 81 | placeHolder.block(Now() + (long) time); 82 | 83 | storage.replace(key, placeHolder); 84 | 85 | // this must go on a queue for processing later... 86 | deleteQueue.add(new DelayedMCElement(placeHolder)); 87 | } else 88 | removed = storage.remove(key) != null; 89 | 90 | if (removed) 91 | return DeleteResponse.DELETED; 92 | else 93 | return DeleteResponse.NOT_FOUND; 94 | 95 | } 96 | 97 | /** 98 | * @throws Exception 99 | * @throws DatabaseException 100 | * @inheritDoc 101 | */ 102 | public StoreResponse add(LocalCacheElement e) throws DatabaseException, Exception { 103 | checkWritePermission(); 104 | return storage.putIfAbsent(e.getKeystring(), e) == null ? StoreResponse.STORED : StoreResponse.NOT_STORED; 105 | } 106 | 107 | /** 108 | * @throws Exception 109 | * @throws DatabaseException 110 | * @inheritDoc 111 | */ 112 | public StoreResponse replace(LocalCacheElement e) throws DatabaseException, Exception { 113 | checkWritePermission(); 114 | return storage.replace(e.getKeystring(), e) != null ? StoreResponse.STORED : StoreResponse.NOT_STORED; 115 | } 116 | 117 | /** 118 | * @throws Exception 119 | * @throws DatabaseException 120 | * @inheritDoc 121 | */ 122 | public StoreResponse append(LocalCacheElement element) throws DatabaseException, Exception { 123 | checkWritePermission(); 124 | int index = hash(element.getKeystring().hashCode()) & (lockNum - 1); 125 | writeLocks[index].lock(); 126 | try { 127 | LocalCacheElement old = storage.get(element.getKeystring()); 128 | if (old == null || isBlocked(old) || isExpired(old)) { 129 | getMisses.incrementAndGet(); 130 | return StoreResponse.NOT_FOUND; 131 | } else { 132 | return storage.replace(old.getKeystring(), old, old.append(element)) ? StoreResponse.STORED : StoreResponse.NOT_STORED; 133 | } 134 | 135 | } finally { 136 | writeLocks[index].unlock(); 137 | } 138 | } 139 | 140 | /** 141 | * @throws Exception 142 | * @throws DatabaseException 143 | * @inheritDoc 144 | */ 145 | public StoreResponse prepend(LocalCacheElement element) throws DatabaseException, Exception { 146 | checkWritePermission(); 147 | int index = hash(element.getKeystring().hashCode()) & (lockNum - 1); 148 | writeLocks[index].lock(); 149 | try { 150 | LocalCacheElement old = storage.get(element.getKeystring()); 151 | if (old == null || isBlocked(old) || isExpired(old)) { 152 | getMisses.incrementAndGet(); 153 | return StoreResponse.NOT_FOUND; 154 | } else { 155 | return storage.replace(old.getKeystring(), old, old.prepend(element)) ? StoreResponse.STORED : StoreResponse.NOT_STORED; 156 | } 157 | } finally { 158 | writeLocks[index].unlock(); 159 | } 160 | } 161 | 162 | /** 163 | * @throws Exception 164 | * @throws DatabaseException 165 | * @inheritDoc 166 | */ 167 | public StoreResponse set(LocalCacheElement e) throws DatabaseException, Exception { 168 | checkWritePermission(); 169 | setCmds.incrementAndGet();// update stats 170 | 171 | e.setCasUnique(casCounter.getAndIncrement()); 172 | 173 | storage.put(e.getKeystring(), e); 174 | 175 | return StoreResponse.STORED; 176 | } 177 | 178 | /** 179 | * @throws Exception 180 | * @throws DatabaseException 181 | * @inheritDoc 182 | */ 183 | public StoreResponse cas(Long cas_key, LocalCacheElement e) throws DatabaseException, Exception { 184 | checkWritePermission(); 185 | int index = hash(e.getKeystring().hashCode()) & (lockNum - 1); 186 | writeLocks[index].lock(); 187 | try { 188 | // have to get the element 189 | LocalCacheElement element = storage.get(e.getKeystring()); 190 | if (element == null || isBlocked(element)) { 191 | getMisses.incrementAndGet(); 192 | return StoreResponse.NOT_FOUND; 193 | } 194 | 195 | if (element.getCasUnique() == cas_key) { 196 | // casUnique matches, now set the element 197 | if (storage.replace(e.getKeystring(), element, e)) 198 | return StoreResponse.STORED; 199 | else { 200 | getMisses.incrementAndGet(); 201 | return StoreResponse.NOT_FOUND; 202 | } 203 | } else { 204 | // cas didn't match; someone else beat us to it 205 | return StoreResponse.EXISTS; 206 | } 207 | } finally { 208 | writeLocks[index].unlock(); 209 | } 210 | } 211 | 212 | static int hash(int h) { 213 | h ^= (h >>> 20) ^ (h >>> 12); 214 | return h ^ (h >>> 7) ^ (h >>> 4); 215 | } 216 | 217 | /** 218 | * @throws Exception 219 | * @throws DatabaseException 220 | * @inheritDoc 221 | */ 222 | public Integer get_add(String key, int mod) throws DatabaseException, Exception { 223 | checkWritePermission(); 224 | int index = hash(key.hashCode()) & (lockNum - 1); 225 | writeLocks[index].lock(); 226 | try { 227 | LocalCacheElement old = storage.get(key); 228 | if (old == null || isBlocked(old) || isExpired(old)) { 229 | getMisses.incrementAndGet(); 230 | return null; 231 | } else { 232 | LocalCacheElement.IncrDecrResult result = old.add(mod); 233 | return storage.replace(old.getKeystring(), old, result.replace) ? result.oldValue : null; 234 | } 235 | } finally { 236 | writeLocks[index].unlock(); 237 | } 238 | } 239 | 240 | protected boolean isBlocked(CacheElement e) { 241 | return e.isBlocked() && e.getBlockedUntil() > Now(); 242 | } 243 | 244 | protected boolean isExpired(CacheElement e) { 245 | return e.getExpire() != 0 && e.getExpire() < Now(); 246 | } 247 | 248 | /** 249 | * @inheritDoc 250 | */ 251 | public LocalCacheElement[] get(String... keys) { 252 | getCmds.incrementAndGet();// updates stats 253 | 254 | LocalCacheElement[] elements = new LocalCacheElement[keys.length]; 255 | int x = 0; 256 | int hits = 0; 257 | int misses = 0; 258 | for (String key : keys) { 259 | LocalCacheElement e = storage.get(key); 260 | if (e == null || isExpired(e) || e.isBlocked()) { 261 | misses++; 262 | 263 | elements[x] = null; 264 | } else { 265 | hits++; 266 | 267 | elements[x] = e; 268 | } 269 | x++; 270 | 271 | } 272 | getMisses.addAndGet(misses); 273 | getHits.addAndGet(hits); 274 | 275 | return elements; 276 | 277 | } 278 | 279 | /** 280 | * @throws Exception 281 | * @throws DatabaseException 282 | * @inheritDoc 283 | */ 284 | public boolean flush_all() throws DatabaseException, Exception { 285 | checkWritePermission(); 286 | if (!forbiddenflush) { 287 | return flush_all(0); 288 | } else { 289 | return false; 290 | } 291 | } 292 | 293 | /** 294 | * @throws Exception 295 | * @throws DatabaseException 296 | * @inheritDoc 297 | */ 298 | public boolean flush_all(int expire) throws DatabaseException, Exception { 299 | checkWritePermission(); 300 | // TODO implement this, it isn't right... but how to handle efficiently? 301 | // (don't want to linear scan entire cacheStorage) 302 | if (!forbiddenflush) { 303 | storage.clear(); 304 | return true; 305 | } else { 306 | return false; 307 | } 308 | } 309 | 310 | /** 311 | * @inheritDoc 312 | */ 313 | public void close() throws IOException { 314 | storage.close(); 315 | } 316 | 317 | /** 318 | * @inheritDoc 319 | */ 320 | @Override 321 | public Set keys() { 322 | return storage.keySet(); 323 | } 324 | 325 | /** 326 | * @inheritDoc 327 | */ 328 | @Override 329 | public long getCurrentItems() { 330 | return storage.size(); 331 | } 332 | 333 | /** 334 | * @inheritDoc 335 | */ 336 | @Override 337 | public long getLimitMaxBytes() { 338 | return storage.getMemoryCapacity(); 339 | } 340 | 341 | /** 342 | * @inheritDoc 343 | */ 344 | @Override 345 | public long getCurrentBytes() { 346 | return storage.getMemoryUsed(); 347 | } 348 | 349 | /** 350 | * @throws Exception 351 | * @throws DatabaseException 352 | * @inheritDoc 353 | */ 354 | @Override 355 | public void asyncEventPing() throws DatabaseException, Exception { 356 | DelayedMCElement toDelete = deleteQueue.poll(); 357 | if (toDelete != null) { 358 | storage.remove(toDelete.element.getKeystring()); 359 | } 360 | } 361 | 362 | /** 363 | * Delayed key blocks get processed occasionally. 364 | */ 365 | protected static class DelayedMCElement implements Delayed { 366 | private CacheElement element; 367 | 368 | public DelayedMCElement(CacheElement element) { 369 | this.element = element; 370 | } 371 | 372 | public long getDelay(TimeUnit timeUnit) { 373 | return timeUnit.convert(element.getBlockedUntil() - Now(), TimeUnit.MILLISECONDS); 374 | } 375 | 376 | public int compareTo(Delayed delayed) { 377 | if (!(delayed instanceof CacheImpl.DelayedMCElement)) 378 | return -1; 379 | else 380 | return element.getKeystring().compareTo(((DelayedMCElement) delayed).element.getKeystring()); 381 | } 382 | } 383 | } 384 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/LocalCacheElement.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 ThimbleWare Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.thimbleware.jmemcached; 17 | 18 | import java.io.Externalizable; 19 | import java.io.IOException; 20 | import java.io.ObjectInput; 21 | import java.io.ObjectOutput; 22 | import java.nio.ByteBuffer; 23 | import java.util.Arrays; 24 | 25 | import static java.lang.Integer.parseInt; 26 | import static java.lang.String.valueOf; 27 | 28 | /** 29 | * Represents information about a cache entry. 30 | */ 31 | public final class LocalCacheElement implements CacheElement, Externalizable { 32 | private int expire ; 33 | private int flags; 34 | private byte[] data; 35 | private String keystring; 36 | private long casUnique = 0L; 37 | private boolean blocked = false; 38 | private long blockedUntil; 39 | 40 | public LocalCacheElement() { 41 | } 42 | 43 | public LocalCacheElement(String keystring) { 44 | this.keystring = keystring; 45 | } 46 | 47 | public LocalCacheElement(String keystring, int flags, int expire, long casUnique) { 48 | this.keystring = keystring; 49 | this.flags = flags; 50 | this.expire = expire; 51 | this.casUnique = casUnique; 52 | } 53 | 54 | /** 55 | * @return the current time in seconds 56 | */ 57 | public static int Now() { 58 | return (int) (System.currentTimeMillis() / 1000); 59 | } 60 | 61 | public int size() { 62 | return getData().length; 63 | } 64 | 65 | public LocalCacheElement append(LocalCacheElement element) { 66 | int newLength = getData().length + element.getData().length; 67 | LocalCacheElement replace = new LocalCacheElement(getKeystring(), getFlags(), getExpire(), 0L); 68 | ByteBuffer b = ByteBuffer.allocateDirect(newLength); 69 | b.put(getData()); 70 | b.put(element.getData()); 71 | replace.setData(new byte[newLength]); 72 | b.flip(); 73 | b.get(replace.getData()); 74 | replace.setCasUnique(replace.getCasUnique() + 1); 75 | 76 | return replace; 77 | } 78 | 79 | public LocalCacheElement prepend(LocalCacheElement element) { 80 | int newLength = getData().length + element.getData().length; 81 | 82 | LocalCacheElement replace = new LocalCacheElement(getKeystring(), getFlags(), getExpire(), 0L); 83 | ByteBuffer b = ByteBuffer.allocateDirect(newLength); 84 | b.put(element.getData()); 85 | b.put(getData()); 86 | replace.setData(new byte[newLength]); 87 | b.flip(); 88 | b.get(replace.getData()); 89 | replace.setCasUnique(replace.getCasUnique() + 1); 90 | 91 | return replace; 92 | } 93 | 94 | public static class IncrDecrResult { 95 | int oldValue; 96 | LocalCacheElement replace; 97 | 98 | public IncrDecrResult(int oldValue, LocalCacheElement replace) { 99 | this.oldValue = oldValue; 100 | this.replace = replace; 101 | } 102 | } 103 | 104 | public IncrDecrResult add(int mod) { 105 | // TODO handle parse failure! 106 | int old_val = parseInt(new String(getData())) + mod; // change value 107 | if (old_val < 0) { 108 | old_val = 0; 109 | 110 | } // check for underflow 111 | 112 | byte[] newData = valueOf(old_val).getBytes(); 113 | 114 | LocalCacheElement replace = new LocalCacheElement(getKeystring(), getFlags(), getExpire(), 0L); 115 | replace.setData(newData); 116 | replace.setCasUnique(replace.getCasUnique() + 1); 117 | 118 | return new IncrDecrResult(old_val, replace); 119 | } 120 | 121 | @Override 122 | public boolean equals(Object o) { 123 | if (this == o) return true; 124 | if (o == null || getClass() != o.getClass()) return false; 125 | 126 | LocalCacheElement that = (LocalCacheElement) o; 127 | 128 | if (blocked != that.blocked) return false; 129 | if (blockedUntil != that.blockedUntil) return false; 130 | if (casUnique != that.casUnique) return false; 131 | if (expire != that.expire) return false; 132 | if (flags != that.flags) return false; 133 | if (!Arrays.equals(data, that.data)) return false; 134 | if (!keystring.equals(that.keystring)) return false; 135 | 136 | return true; 137 | } 138 | 139 | @Override 140 | public int hashCode() { 141 | int result = expire; 142 | result = 31 * result + flags; 143 | result = 31 * result + (data != null ? Arrays.hashCode(data) : 0); 144 | result = 31 * result + keystring.hashCode(); 145 | result = 31 * result + (int) (casUnique ^ (casUnique >>> 32)); 146 | result = 31 * result + (blocked ? 1 : 0); 147 | result = 31 * result + (int) (blockedUntil ^ (blockedUntil >>> 32)); 148 | return result; 149 | } 150 | 151 | public static LocalCacheElement key(String key) { 152 | return new LocalCacheElement(key); 153 | } 154 | 155 | public int getExpire() { 156 | return expire; 157 | } 158 | 159 | public int getFlags() { 160 | return flags; 161 | } 162 | 163 | public byte[] getData() { 164 | return data; 165 | } 166 | 167 | public String getKeystring() { 168 | return keystring; 169 | } 170 | 171 | public long getCasUnique() { 172 | return casUnique; 173 | } 174 | 175 | public boolean isBlocked() { 176 | return blocked; 177 | } 178 | 179 | public long getBlockedUntil() { 180 | return blockedUntil; 181 | } 182 | 183 | public void setCasUnique(long casUnique) { 184 | this.casUnique = casUnique; 185 | } 186 | 187 | public void block(long blockedUntil) { 188 | this.blocked = true; 189 | this.blockedUntil = blockedUntil; 190 | } 191 | 192 | 193 | public void setData(byte[] data) { 194 | this.data = data; 195 | } 196 | 197 | public void readExternal(ObjectInput in) throws IOException{ 198 | expire = in.readInt() ; 199 | flags = in.readInt(); 200 | 201 | final int length = in.readInt(); 202 | int readSize = 0; 203 | data = new byte[length]; 204 | while( readSize < length) 205 | readSize += in.read(data, readSize, length - readSize); 206 | 207 | keystring = in.readUTF(); 208 | casUnique = in.readLong(); 209 | blocked = in.readBoolean(); 210 | blockedUntil = in.readLong(); 211 | } 212 | 213 | public void writeExternal(ObjectOutput out) throws IOException { 214 | out.writeInt(expire) ; 215 | out.writeInt(flags); 216 | out.writeInt(data.length); 217 | out.write(data); 218 | out.writeUTF(keystring); 219 | out.writeLong(casUnique); 220 | out.writeBoolean(blocked); 221 | out.writeLong(blockedUntil); 222 | } 223 | } -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/MemCacheDaemon.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 ThimbleWare Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.thimbleware.jmemcached; 17 | 18 | import java.io.IOException; 19 | import java.net.InetSocketAddress; 20 | import java.util.concurrent.Executors; 21 | 22 | import org.jboss.netty.bootstrap.ServerBootstrap; 23 | import org.jboss.netty.channel.Channel; 24 | import org.jboss.netty.channel.ChannelPipelineFactory; 25 | import org.jboss.netty.channel.group.ChannelGroupFuture; 26 | import org.jboss.netty.channel.group.DefaultChannelGroup; 27 | import org.jboss.netty.channel.socket.ServerSocketChannelFactory; 28 | import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import com.thimbleware.jmemcached.protocol.binary.MemcachedBinaryPipelineFactory; 33 | import com.thimbleware.jmemcached.protocol.text.MemcachedPipelineFactory; 34 | 35 | /** 36 | * The actual daemon - responsible for the binding and configuration of the 37 | * network configuration. 38 | */ 39 | public class MemCacheDaemon { 40 | 41 | final Logger log = LoggerFactory.getLogger(MemCacheDaemon.class); 42 | 43 | public static String memcachedVersion = "0.9"; 44 | 45 | private int frameSize = 32768 * 1024; 46 | 47 | private boolean binary = false; 48 | private boolean verbose; 49 | private int idleTime; 50 | private InetSocketAddress addr; 51 | private Cache cache; 52 | 53 | private boolean running = false; 54 | private ServerSocketChannelFactory channelFactory; 55 | private DefaultChannelGroup allChannels; 56 | 57 | public MemCacheDaemon() { 58 | } 59 | 60 | public MemCacheDaemon(Cache cache) { 61 | this.cache = cache; 62 | } 63 | 64 | /** 65 | * Bind the network connection and start the network processing threads. 66 | */ 67 | public void start() { 68 | // TODO provide tweakable options here for passing in custom executors. 69 | channelFactory = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(), Executors 70 | .newCachedThreadPool()); 71 | 72 | allChannels = new DefaultChannelGroup("jmemcachedChannelGroup"); 73 | 74 | ServerBootstrap bootstrap = new ServerBootstrap(channelFactory); 75 | 76 | ChannelPipelineFactory pipelineFactory; 77 | if (binary) 78 | pipelineFactory = createMemcachedBinaryPipelineFactory(cache, memcachedVersion, verbose, idleTime, 79 | allChannels); 80 | else 81 | pipelineFactory = createMemcachedPipelineFactory(cache, memcachedVersion, verbose, idleTime, frameSize, 82 | allChannels); 83 | 84 | bootstrap.setOption("child.tcpNoDelay", true); 85 | bootstrap.setOption("child.keepAlive", true); 86 | bootstrap.setOption("child.receiveBufferSize", 1024 * 64); 87 | bootstrap.setPipelineFactory(pipelineFactory); 88 | 89 | Channel serverChannel = bootstrap.bind(addr); 90 | allChannels.add(serverChannel); 91 | 92 | log.info("Listening on " + String.valueOf(addr.getHostName()) + ":" + addr.getPort()); 93 | 94 | running = true; 95 | } 96 | 97 | protected ChannelPipelineFactory createMemcachedBinaryPipelineFactory(Cache cache, String memcachedVersion, 98 | boolean verbose, int idleTime, DefaultChannelGroup allChannels) { 99 | return new MemcachedBinaryPipelineFactory(cache, memcachedVersion, verbose, idleTime, allChannels); 100 | } 101 | 102 | protected ChannelPipelineFactory createMemcachedPipelineFactory(Cache cache, String memcachedVersion, 103 | boolean verbose, int idleTime, int receiveBufferSize, DefaultChannelGroup allChannels) { 104 | return new MemcachedPipelineFactory(cache, memcachedVersion, verbose, idleTime, receiveBufferSize, allChannels); 105 | } 106 | 107 | public void stop() { 108 | log.info("terminating daemon; closing all channels"); 109 | 110 | ChannelGroupFuture future = allChannels.close(); 111 | future.awaitUninterruptibly(); 112 | if (!future.isCompleteSuccess()) { 113 | throw new RuntimeException("failure to complete closing all network channels"); 114 | } 115 | log.info("channels closed, freeing cache storage"); 116 | try { 117 | cache.close(); 118 | } catch (IOException e) { 119 | throw new RuntimeException("exception while closing storage", e); 120 | } 121 | channelFactory.releaseExternalResources(); 122 | 123 | running = false; 124 | log.info("successfully shut down"); 125 | } 126 | 127 | public void setVerbose(boolean verbose) { 128 | this.verbose = verbose; 129 | } 130 | 131 | public void setIdleTime(int idleTime) { 132 | this.idleTime = idleTime; 133 | } 134 | 135 | public void setAddr(InetSocketAddress addr) { 136 | this.addr = addr; 137 | } 138 | 139 | public Cache getCache() { 140 | return cache; 141 | } 142 | 143 | public void setCache(Cache cache) { 144 | this.cache = cache; 145 | } 146 | 147 | public boolean isRunning() { 148 | return running; 149 | } 150 | 151 | public boolean isBinary() { 152 | return binary; 153 | } 154 | 155 | public void setBinary(boolean binary) { 156 | this.binary = binary; 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/StatsCounter.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached; 2 | 3 | import java.util.concurrent.atomic.AtomicInteger; 4 | import java.util.concurrent.atomic.AtomicLong; 5 | 6 | public class StatsCounter { 7 | public static final AtomicLong bytes_written = new AtomicLong(); 8 | public static final AtomicLong bytes_read = new AtomicLong(); 9 | public static final AtomicInteger curr_conns = new AtomicInteger(); 10 | public static final AtomicInteger total_conns = new AtomicInteger(); 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/Command.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 ThimbleWare Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.thimbleware.jmemcached.protocol; 17 | 18 | /** 19 | */ 20 | public enum Command { 21 | GET, GETS, APPEND, PREPEND, DELETE, DECR, 22 | INCR, REPLACE, ADD, SET, CAS, STATS, VERSION, 23 | QUIT, FLUSH_ALL 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/CommandMessage.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2008 ThimbleWare Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.thimbleware.jmemcached.protocol; 17 | 18 | import com.thimbleware.jmemcached.CacheElement; 19 | 20 | import java.io.Serializable; 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | /** 25 | * The payload object holding the parsed message. 26 | */ 27 | public final class CommandMessage implements Serializable { 28 | 29 | public static enum ErrorType { 30 | OK, ERROR, CLIENT_ERROR 31 | } 32 | 33 | public Command cmd; 34 | public CACHE_ELEMENT element; 35 | public List keys; 36 | public boolean noreply; 37 | public Long cas_key; 38 | public int time = 0; 39 | public ErrorType error = ErrorType.OK; 40 | public String errorString; 41 | public int opaque; 42 | public boolean addKeyToResponse = false; 43 | 44 | public Integer incrDefault; 45 | public int incrExpiry; 46 | public int incrAmount; 47 | 48 | private CommandMessage(Command cmd) { 49 | this.cmd = cmd; 50 | element = null; 51 | keys = new ArrayList(); 52 | } 53 | 54 | public static CommandMessage error(String errorString) { 55 | CommandMessage errcmd = new CommandMessage(null); 56 | errcmd.error = ErrorType.ERROR; 57 | errcmd.errorString = errorString; 58 | return errcmd; 59 | } 60 | 61 | public static CommandMessage clientError(String errorString) { 62 | CommandMessage errcmd = new CommandMessage(null); 63 | errcmd.error = ErrorType.CLIENT_ERROR; 64 | errcmd.errorString = errorString; 65 | return errcmd; 66 | } 67 | 68 | public static CommandMessage command(Command cmd) { 69 | return new CommandMessage(cmd); 70 | } 71 | } -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/ResponseMessage.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol; 2 | 3 | import com.thimbleware.jmemcached.Cache; 4 | import com.thimbleware.jmemcached.CacheElement; 5 | 6 | import java.io.Serializable; 7 | import java.util.Set; 8 | import java.util.Map; 9 | 10 | /** 11 | * Represents the response to a command. 12 | */ 13 | public final class ResponseMessage implements Serializable { 14 | 15 | public ResponseMessage(CommandMessage cmd) { 16 | this.cmd = cmd; 17 | } 18 | 19 | public CommandMessage cmd; 20 | public CACHE_ELEMENT[] elements; 21 | public Cache.StoreResponse response; 22 | public Map> stats; 23 | public String version; 24 | public Cache.DeleteResponse deleteResponse; 25 | public Integer incrDecrResponse; 26 | public boolean flushSuccess; 27 | 28 | public ResponseMessage withElements(CACHE_ELEMENT[] elements) { 29 | this.elements = elements; 30 | return this; 31 | } 32 | 33 | public ResponseMessage withResponse(Cache.StoreResponse response) { 34 | this.response = response; 35 | return this; 36 | } 37 | 38 | public ResponseMessage withDeleteResponse(Cache.DeleteResponse deleteResponse) { 39 | this.deleteResponse = deleteResponse; 40 | return this; 41 | } 42 | 43 | public ResponseMessage withIncrDecrResponse(Integer incrDecrResp) { 44 | this.incrDecrResponse = incrDecrResp; 45 | 46 | return this; 47 | } 48 | 49 | public ResponseMessage withStatResponse(Map> stats) { 50 | this.stats = stats; 51 | 52 | return this; 53 | } 54 | 55 | public ResponseMessage withFlushResponse(boolean success) { 56 | this.flushSuccess = success; 57 | 58 | return this; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/SessionStatus.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Class for holding the current session status. 7 | */ 8 | public final class SessionStatus implements Serializable { 9 | 10 | /** 11 | * Possible states that the current session is in. 12 | */ 13 | public static enum State { 14 | WAITING_FOR_DATA, 15 | READY, 16 | PROCESSING, 17 | PROCESSING_MULTILINE, 18 | } 19 | 20 | // the state the session is in 21 | public State state; 22 | 23 | // if we are waiting for more data, how much? 24 | public int bytesNeeded; 25 | 26 | // the current working command 27 | public CommandMessage cmd; 28 | 29 | 30 | public SessionStatus() { 31 | ready(); 32 | } 33 | 34 | public SessionStatus ready() { 35 | this.cmd = null; 36 | this.bytesNeeded = -1; 37 | this.state = State.READY; 38 | 39 | return this; 40 | } 41 | 42 | public SessionStatus processing() { 43 | this.state = State.PROCESSING; 44 | 45 | return this; 46 | } 47 | 48 | public SessionStatus processingMultiline() { 49 | this.state = State.PROCESSING_MULTILINE; 50 | 51 | return this; 52 | } 53 | 54 | public SessionStatus needMore(int size, CommandMessage cmd) { 55 | this.cmd = cmd; 56 | this.bytesNeeded = size; 57 | this.state = State.WAITING_FOR_DATA; 58 | 59 | return this; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/binary/MemcachedBinaryCommandDecoder.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.binary; 2 | 3 | import com.thimbleware.jmemcached.LocalCacheElement; 4 | import com.thimbleware.jmemcached.CacheElement; 5 | import com.thimbleware.jmemcached.protocol.Command; 6 | import com.thimbleware.jmemcached.protocol.CommandMessage; 7 | import com.thimbleware.jmemcached.protocol.exceptions.MalformedCommandException; 8 | import org.jboss.netty.buffer.ChannelBuffer; 9 | import org.jboss.netty.buffer.ChannelBuffers; 10 | import org.jboss.netty.channel.Channel; 11 | import org.jboss.netty.channel.ChannelHandler; 12 | import org.jboss.netty.channel.ChannelHandlerContext; 13 | import org.jboss.netty.channel.ChannelPipelineCoverage; 14 | import org.jboss.netty.handler.codec.frame.FrameDecoder; 15 | 16 | import java.nio.charset.Charset; 17 | import java.util.ArrayList; 18 | import java.nio.ByteOrder; 19 | 20 | /** 21 | */ 22 | @ChannelHandler.Sharable 23 | public class MemcachedBinaryCommandDecoder extends FrameDecoder { 24 | 25 | public static final Charset USASCII = Charset.forName("US-ASCII"); 26 | 27 | public static enum BinaryCommand { 28 | Get(0x00, Command.GET, false), 29 | Set(0x01, Command.SET, false), 30 | Add(0x02, Command.ADD, false), 31 | Replace(0x03, Command.REPLACE, false), 32 | Delete(0x04, Command.DELETE, false), 33 | Increment(0x05, Command.INCR, false), 34 | Decrement(0x06, Command.DECR, false), 35 | Quit(0x07, Command.QUIT, false), 36 | Flush(0x08, Command.FLUSH_ALL, false), 37 | GetQ(0x09, Command.GET, false), 38 | Noop(0x0A, null, false), 39 | Version(0x0B, Command.VERSION, false), 40 | GetK(0x0C, Command.GET, false, true), 41 | GetKQ(0x0D,Command.GET, true, true), 42 | Append(0x0E, Command.APPEND, false), 43 | Prepend(0x0F, Command.PREPEND, false), 44 | Stat(0x10, Command.STATS, false), 45 | SetQ(0x11, Command.SET, true), 46 | AddQ(0x12, Command.ADD, true), 47 | ReplaceQ(0x13, Command.REPLACE, true), 48 | DeleteQ(0x14, Command.DELETE, true), 49 | IncrementQ(0x15, Command.INCR, true), 50 | DecrementQ(0x16, Command.DECR, true), 51 | QuitQ(0x17, Command.QUIT, true), 52 | FlushQ(0x18, Command.FLUSH_ALL, true), 53 | AppendQ(0x19, Command.APPEND, true), 54 | PrependQ(0x1A, Command.PREPEND, true); 55 | 56 | public byte code; 57 | public Command correspondingCommand; 58 | public boolean noreply; 59 | public boolean addKeyToResponse = false; 60 | 61 | BinaryCommand(int code, Command correspondingCommand, boolean noreply) { 62 | this.code = (byte)code; 63 | this.correspondingCommand = correspondingCommand; 64 | this.noreply = noreply; 65 | } 66 | 67 | BinaryCommand(int code, Command correspondingCommand, boolean noreply, boolean addKeyToResponse) { 68 | this.code = (byte)code; 69 | this.correspondingCommand = correspondingCommand; 70 | this.noreply = noreply; 71 | this.addKeyToResponse = addKeyToResponse; 72 | } 73 | 74 | public static BinaryCommand forCommandMessage(CommandMessage msg) { 75 | for (BinaryCommand binaryCommand : values()) { 76 | if (binaryCommand.correspondingCommand == msg.cmd && binaryCommand.noreply == msg.noreply && binaryCommand.addKeyToResponse == msg.addKeyToResponse) { 77 | return binaryCommand; 78 | } 79 | } 80 | 81 | return null; 82 | } 83 | 84 | } 85 | 86 | protected Object decode(ChannelHandlerContext channelHandlerContext, Channel channel, ChannelBuffer channelBuffer) throws Exception { 87 | 88 | // need at least 24 bytes, to get header 89 | if (channelBuffer.readableBytes() < 24) return null; 90 | 91 | // get the header 92 | channelBuffer.markReaderIndex(); 93 | ChannelBuffer headerBuffer = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, 24); 94 | channelBuffer.readBytes(headerBuffer); 95 | 96 | short magic = headerBuffer.readUnsignedByte(); 97 | 98 | // magic should be 0x80 99 | if (magic != 0x80) { 100 | headerBuffer.resetReaderIndex(); 101 | 102 | throw new MalformedCommandException("binary request payload is invalid, magic byte incorrect"); 103 | } 104 | 105 | short opcode = headerBuffer.readUnsignedByte(); 106 | short keyLength = headerBuffer.readShort(); 107 | short extraLength = headerBuffer.readUnsignedByte(); 108 | short dataType = headerBuffer.readUnsignedByte(); // unused 109 | short reserved = headerBuffer.readShort(); // unused 110 | int totalBodyLength = headerBuffer.readInt(); 111 | int opaque = headerBuffer.readInt(); 112 | long cas = headerBuffer.readLong(); 113 | 114 | // we want the whole of totalBodyLength; otherwise, keep waiting. 115 | if (channelBuffer.readableBytes() < totalBodyLength) { 116 | channelBuffer.resetReaderIndex(); 117 | return null; 118 | } 119 | 120 | // This assumes correct order in the enum. If that ever changes, we will have to scan for 'code' field. 121 | BinaryCommand bcmd = BinaryCommand.values()[opcode]; 122 | 123 | Command cmdType = bcmd.correspondingCommand; 124 | CommandMessage cmdMessage = CommandMessage.command(cmdType); 125 | cmdMessage.noreply = bcmd.noreply; 126 | cmdMessage.cas_key = cas; 127 | cmdMessage.opaque = opaque; 128 | cmdMessage.addKeyToResponse = bcmd.addKeyToResponse; 129 | 130 | // get extras. could be empty. 131 | ChannelBuffer extrasBuffer = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, extraLength); 132 | channelBuffer.readBytes(extrasBuffer); 133 | 134 | // get the key if any 135 | if (keyLength != 0) { 136 | ChannelBuffer keyBuffer = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, keyLength); 137 | channelBuffer.readBytes(keyBuffer); 138 | 139 | ArrayList keys = new ArrayList(); 140 | String key = keyBuffer.toString(USASCII); 141 | keys.add(key); // TODO this or UTF-8? ISO-8859-1? 142 | 143 | cmdMessage.keys = keys; 144 | 145 | 146 | if (cmdType == Command.ADD || 147 | cmdType == Command.SET || 148 | cmdType == Command.REPLACE || 149 | cmdType == Command.APPEND || 150 | cmdType == Command.PREPEND) 151 | { 152 | // TODO these are backwards from the spec, but seem to be what spymemcached demands -- which has the mistake?! 153 | short expire = (short) (extrasBuffer.capacity() != 0 ? extrasBuffer.readUnsignedShort() : 0); 154 | short flags = (short) (extrasBuffer.capacity() != 0 ? extrasBuffer.readUnsignedShort() : 0); 155 | 156 | // the remainder of the message -- that is, totalLength - (keyLength + extraLength) should be the payload 157 | int size = totalBodyLength - keyLength - extraLength; 158 | 159 | cmdMessage.element = new LocalCacheElement(key, flags, expire != 0 && expire < CacheElement.THIRTY_DAYS ? LocalCacheElement.Now() + expire : expire, 0L); 160 | cmdMessage.element.setData(new byte[size]); 161 | channelBuffer.readBytes(cmdMessage.element.getData(), 0, size); 162 | } else if (cmdType == Command.INCR || cmdType == Command.DECR) { 163 | long initialValue = extrasBuffer.readUnsignedInt(); 164 | long amount = extrasBuffer.readUnsignedInt(); 165 | long expiration = extrasBuffer.readUnsignedInt(); 166 | 167 | cmdMessage.incrAmount = (int) amount; 168 | cmdMessage.incrDefault = (int) initialValue; 169 | cmdMessage.incrExpiry = (int) expiration; 170 | } 171 | } 172 | 173 | return cmdMessage; 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/binary/MemcachedBinaryPipelineFactory.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.binary; 2 | 3 | import com.thimbleware.jmemcached.Cache; 4 | import com.thimbleware.jmemcached.protocol.MemcachedCommandHandler; 5 | import org.jboss.netty.channel.ChannelPipeline; 6 | import org.jboss.netty.channel.ChannelPipelineFactory; 7 | import org.jboss.netty.channel.Channels; 8 | import org.jboss.netty.channel.group.DefaultChannelGroup; 9 | 10 | 11 | public class MemcachedBinaryPipelineFactory implements ChannelPipelineFactory { 12 | 13 | private Cache cache; 14 | private String version; 15 | private boolean verbose; 16 | private int idleTime; 17 | 18 | private DefaultChannelGroup channelGroup; 19 | 20 | private final MemcachedBinaryCommandDecoder decoder = new MemcachedBinaryCommandDecoder(); 21 | private final MemcachedCommandHandler memcachedCommandHandler; 22 | private final MemcachedBinaryResponseEncoder memcachedBinaryResponseEncoder = new MemcachedBinaryResponseEncoder(); 23 | 24 | public MemcachedBinaryPipelineFactory(Cache cache, String version, boolean verbose, int idleTime, DefaultChannelGroup channelGroup) { 25 | this.cache = cache; 26 | this.version = version; 27 | this.verbose = verbose; 28 | this.idleTime = idleTime; 29 | this.channelGroup = channelGroup; 30 | memcachedCommandHandler = new MemcachedCommandHandler(this.cache, this.version, this.verbose, this.idleTime, this.channelGroup); 31 | } 32 | 33 | public ChannelPipeline getPipeline() throws Exception { 34 | return Channels.pipeline( 35 | decoder, 36 | memcachedCommandHandler, 37 | memcachedBinaryResponseEncoder 38 | ); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/binary/MemcachedBinaryResponseEncoder.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.binary; 2 | 3 | import com.thimbleware.jmemcached.protocol.Command; 4 | import com.thimbleware.jmemcached.protocol.ResponseMessage; 5 | import com.thimbleware.jmemcached.protocol.exceptions.UnknownCommandException; 6 | import com.thimbleware.jmemcached.CacheElement; 7 | import org.jboss.netty.buffer.ChannelBuffer; 8 | import org.jboss.netty.buffer.ChannelBuffers; 9 | import org.jboss.netty.channel.*; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.nio.ByteOrder; 14 | import java.util.Set; 15 | import java.util.Map; 16 | import java.util.concurrent.ConcurrentHashMap; 17 | 18 | /** 19 | * 20 | */ 21 | // TODO refactor so this can be unit tested separate from netty? scalacheck? 22 | @ChannelHandler.Sharable 23 | public class MemcachedBinaryResponseEncoder extends SimpleChannelUpstreamHandler { 24 | 25 | private ConcurrentHashMap corkedBuffers = new ConcurrentHashMap(); 26 | 27 | final Logger logger = LoggerFactory.getLogger(MemcachedBinaryResponseEncoder.class); 28 | 29 | public static enum ResponseCode { 30 | OK(0x0000), 31 | KEYNF(0x0001), 32 | KEYEXISTS(0x0002), 33 | TOOLARGE(0x0003), 34 | INVARG(0x0004), 35 | NOT_STORED(0x0005), 36 | UNKNOWN(0x0081), 37 | OOM(0x00082); 38 | 39 | public short code; 40 | 41 | ResponseCode(int code) { 42 | this.code = (short)code; 43 | } 44 | } 45 | 46 | public ResponseCode getStatusCode(ResponseMessage command) { 47 | Command cmd = command.cmd.cmd; 48 | if (cmd == Command.GET || cmd == Command.GETS) { 49 | return ResponseCode.OK; 50 | } else if (cmd == Command.SET || cmd == Command.CAS || cmd == Command.ADD || cmd == Command.REPLACE || cmd == Command.APPEND || cmd == Command.PREPEND) { 51 | switch (command.response) { 52 | case EXISTS: 53 | return ResponseCode.KEYEXISTS; 54 | case NOT_FOUND: 55 | return ResponseCode.KEYNF; 56 | case NOT_STORED: 57 | return ResponseCode.NOT_STORED; 58 | case STORED: 59 | return ResponseCode.OK; 60 | } 61 | } else if (cmd == Command.INCR || cmd == Command.DECR) { 62 | return command.incrDecrResponse == null ? ResponseCode.KEYNF : ResponseCode.OK; 63 | } else if (cmd == Command.DELETE) { 64 | switch (command.deleteResponse) { 65 | case DELETED: 66 | return ResponseCode.OK; 67 | case NOT_FOUND: 68 | return ResponseCode.KEYNF; 69 | } 70 | } else if (cmd == Command.STATS) { 71 | return ResponseCode.OK; 72 | } else if (cmd == Command.VERSION) { 73 | return ResponseCode.OK; 74 | } else if (cmd == Command.FLUSH_ALL) { 75 | return ResponseCode.OK; 76 | } 77 | return ResponseCode.UNKNOWN; 78 | } 79 | 80 | 81 | 82 | public ChannelBuffer constructHeader(MemcachedBinaryCommandDecoder.BinaryCommand bcmd, ChannelBuffer extrasBuffer, ChannelBuffer keyBuffer, ChannelBuffer valueBuffer, short responseCode, int opaqueValue, long casUnique) { 83 | // take the ResponseMessage and turn it into a binary payload. 84 | ChannelBuffer header = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, 24); 85 | header.writeByte((byte)0x81); // magic 86 | header.writeByte(bcmd.code); // opcode 87 | short keyLength = (short) (keyBuffer != null ? keyBuffer.capacity() :0); 88 | 89 | header.writeShort(keyLength); 90 | int extrasLength = extrasBuffer != null ? extrasBuffer.capacity() : 0; 91 | header.writeByte((byte) extrasLength); // extra length = flags + expiry 92 | header.writeByte((byte)0); // data type unused 93 | header.writeShort(responseCode); // status code 94 | 95 | int dataLength = valueBuffer != null ? valueBuffer.capacity() : 0; 96 | header.writeInt(dataLength + keyLength + extrasLength); // data length 97 | header.writeInt(opaqueValue); // opaque 98 | 99 | header.writeLong(casUnique); 100 | 101 | return header; 102 | } 103 | 104 | /** 105 | * Handle exceptions in protocol processing. Exceptions are either client or internal errors. Report accordingly. 106 | * 107 | * @param ctx 108 | * @param e 109 | * @throws Exception 110 | */ 111 | @Override 112 | public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { 113 | try { 114 | throw e.getCause(); 115 | } catch (UnknownCommandException unknownCommand) { 116 | if (ctx.getChannel().isOpen()) 117 | ctx.getChannel().write(constructHeader(MemcachedBinaryCommandDecoder.BinaryCommand.Noop, null, null, null, (short)0x0081, 0, 0)); 118 | } catch (Throwable err) { 119 | logger.error("error", err); 120 | if (ctx.getChannel().isOpen()) 121 | ctx.getChannel().close(); 122 | } 123 | } 124 | 125 | @Override 126 | @SuppressWarnings("unchecked") 127 | public void messageReceived(ChannelHandlerContext channelHandlerContext, MessageEvent messageEvent) throws Exception { 128 | ResponseMessage command = (ResponseMessage) messageEvent.getMessage(); 129 | Object additional = messageEvent.getMessage(); 130 | 131 | MemcachedBinaryCommandDecoder.BinaryCommand bcmd = MemcachedBinaryCommandDecoder.BinaryCommand.forCommandMessage(command.cmd); 132 | 133 | // write extras == flags & expiry 134 | ChannelBuffer extrasBuffer = null; 135 | 136 | // write key if there is one 137 | ChannelBuffer keyBuffer = null; 138 | if (bcmd.addKeyToResponse && command.cmd.keys != null && command.cmd.keys.size() != 0) { 139 | keyBuffer = ChannelBuffers.wrappedBuffer(ByteOrder.BIG_ENDIAN, command.cmd.keys.get(0).getBytes()); 140 | } 141 | 142 | // write value if there is one 143 | ChannelBuffer valueBuffer = null; 144 | if (command.elements != null) { 145 | extrasBuffer = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, 4); 146 | CacheElement element = command.elements[0]; 147 | extrasBuffer.writeShort((short) (element != null ? element.getExpire() : 0)); 148 | extrasBuffer.writeShort((short) (element != null ? element.getFlags() : 0)); 149 | 150 | if ((command.cmd.cmd == Command.GET || command.cmd.cmd == Command.GETS)) { 151 | if (element != null) { 152 | valueBuffer = ChannelBuffers.wrappedBuffer(ByteOrder.BIG_ENDIAN, element.getData()); 153 | } else { 154 | valueBuffer = ChannelBuffers.buffer(0); 155 | } 156 | } else if (command.cmd.cmd == Command.INCR || command.cmd.cmd == Command.DECR) { 157 | valueBuffer = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, 8); 158 | valueBuffer.writeLong(command.incrDecrResponse); 159 | } 160 | } else if (command.cmd.cmd == Command.INCR || command.cmd.cmd == Command.DECR) { 161 | valueBuffer = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, 8); 162 | valueBuffer.writeLong(command.incrDecrResponse); 163 | } 164 | 165 | long casUnique = 0; 166 | if (command.elements != null && command.elements.length != 0 && command.elements[0] != null) { 167 | casUnique = command.elements[0].getCasUnique(); 168 | } 169 | 170 | // stats is special -- with it, we write N times, one for each stat, then an empty payload 171 | if (command.cmd.cmd == Command.STATS) { 172 | // first uncork any corked buffers 173 | if (corkedBuffers.containsKey(command.cmd.opaque)) uncork(command.cmd.opaque, messageEvent.getChannel()); 174 | 175 | for (Map.Entry> statsEntries : command.stats.entrySet()) { 176 | for (String stat : statsEntries.getValue()) { 177 | 178 | keyBuffer = ChannelBuffers.wrappedBuffer(ByteOrder.BIG_ENDIAN, statsEntries.getKey().getBytes(MemcachedBinaryCommandDecoder.USASCII)); 179 | valueBuffer = ChannelBuffers.wrappedBuffer(ByteOrder.BIG_ENDIAN, stat.getBytes(MemcachedBinaryCommandDecoder.USASCII)); 180 | 181 | ChannelBuffer headerBuffer = constructHeader(bcmd, extrasBuffer, keyBuffer, valueBuffer, getStatusCode(command).code, command.cmd.opaque, casUnique); 182 | 183 | writePayload(messageEvent, extrasBuffer, keyBuffer, valueBuffer, headerBuffer); 184 | } 185 | } 186 | 187 | keyBuffer = null; 188 | valueBuffer = null; 189 | 190 | ChannelBuffer headerBuffer = constructHeader(bcmd, extrasBuffer, keyBuffer, valueBuffer, getStatusCode(command).code, command.cmd.opaque, casUnique); 191 | 192 | writePayload(messageEvent, extrasBuffer, keyBuffer, valueBuffer, headerBuffer); 193 | 194 | } else { 195 | ChannelBuffer headerBuffer = constructHeader(bcmd, extrasBuffer, keyBuffer, valueBuffer, getStatusCode(command).code, command.cmd.opaque, casUnique); 196 | 197 | // write everything 198 | // is the command 'quiet?' if so, then we append to our 'corked' buffer until a non-corked command comes along 199 | if (bcmd.noreply) { 200 | int totalCapacity = headerBuffer.capacity() + (extrasBuffer != null ? extrasBuffer.capacity() : 0) 201 | + (keyBuffer != null ? keyBuffer.capacity() : 0) + (valueBuffer != null ? valueBuffer.capacity() : 0); 202 | 203 | ChannelBuffer corkedResponse = cork(command.cmd.opaque, totalCapacity); 204 | 205 | 206 | corkedResponse.writeBytes(headerBuffer); 207 | if (extrasBuffer != null) 208 | corkedResponse.writeBytes(extrasBuffer); 209 | if (keyBuffer != null) 210 | corkedResponse.writeBytes(keyBuffer); 211 | if (valueBuffer != null) 212 | corkedResponse.writeBytes(valueBuffer); 213 | } else { 214 | // first write out any corked responses 215 | if (corkedBuffers.containsKey(command.cmd.opaque)) uncork(command.cmd.opaque, messageEvent.getChannel()); 216 | 217 | 218 | writePayload(messageEvent, extrasBuffer, keyBuffer, valueBuffer, headerBuffer); 219 | } 220 | } 221 | } 222 | 223 | private ChannelBuffer cork(int opaque, int totalCapacity) { 224 | if (corkedBuffers.containsKey(opaque)) { 225 | ChannelBuffer corkedResponse = corkedBuffers.get(opaque); 226 | ChannelBuffer oldBuffer = corkedResponse; 227 | corkedResponse = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, totalCapacity + corkedResponse.capacity()); 228 | corkedResponse.writeBytes(oldBuffer); 229 | oldBuffer.clear(); 230 | 231 | corkedBuffers.remove(opaque); 232 | corkedBuffers.put(opaque, corkedResponse); 233 | return corkedResponse; 234 | } else { 235 | ChannelBuffer buffer = ChannelBuffers.buffer(ByteOrder.BIG_ENDIAN, totalCapacity); 236 | corkedBuffers.put(opaque, buffer); 237 | return buffer; 238 | } 239 | } 240 | 241 | private void uncork(int opaque, Channel channel) { 242 | ChannelBuffer corkedBuffer = corkedBuffers.get(opaque); 243 | assert corkedBuffer != null; 244 | channel.write(corkedBuffer); 245 | corkedBuffers.remove(opaque); 246 | } 247 | 248 | private void writePayload(MessageEvent messageEvent, ChannelBuffer extrasBuffer, ChannelBuffer keyBuffer, ChannelBuffer valueBuffer, ChannelBuffer headerBuffer) { 249 | if (messageEvent.getChannel().isOpen()) { 250 | messageEvent.getChannel().write(headerBuffer); 251 | if (extrasBuffer != null) 252 | messageEvent.getChannel().write(extrasBuffer); 253 | if (keyBuffer != null) 254 | messageEvent.getChannel().write(keyBuffer); 255 | if (valueBuffer != null) 256 | messageEvent.getChannel().write(valueBuffer); 257 | } 258 | } 259 | } 260 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/exceptions/ClientException.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.exceptions; 2 | 3 | /** 4 | */ 5 | public class ClientException extends Exception { 6 | 7 | /** 8 | * 9 | */ 10 | private static final long serialVersionUID = -7348226175632572914L; 11 | 12 | public ClientException() { 13 | } 14 | 15 | public ClientException(String s) { 16 | super(s); 17 | } 18 | 19 | public ClientException(String s, Throwable throwable) { 20 | super(s, throwable); 21 | } 22 | 23 | public ClientException(Throwable throwable) { 24 | super(throwable); 25 | } 26 | 27 | @Override 28 | public Throwable fillInStackTrace() { 29 | return this; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/exceptions/DatabaseException.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.exceptions; 2 | 3 | /** 4 | * @author sunli 5 | * @date 2010-8-23 6 | * @version $Id$ 7 | */ 8 | public class DatabaseException extends RuntimeException { 9 | 10 | @Override 11 | public Throwable fillInStackTrace() { 12 | return this; 13 | } 14 | 15 | /** 16 | * 17 | */ 18 | private static final long serialVersionUID = -7813092784916606140L; 19 | 20 | public DatabaseException(String message) { 21 | super(message); 22 | } 23 | 24 | public DatabaseException(Throwable cause) { 25 | super(cause); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/exceptions/IncorrectlyTerminatedPayloadException.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.exceptions; 2 | 3 | /** 4 | */ 5 | public class IncorrectlyTerminatedPayloadException extends ClientException { 6 | /** 7 | * 8 | */ 9 | private static final long serialVersionUID = -3832455564964217827L; 10 | 11 | public IncorrectlyTerminatedPayloadException() { 12 | } 13 | 14 | public IncorrectlyTerminatedPayloadException(String s) { 15 | super(s); 16 | } 17 | 18 | public IncorrectlyTerminatedPayloadException(String s, Throwable throwable) { 19 | super(s, throwable); 20 | } 21 | 22 | public IncorrectlyTerminatedPayloadException(Throwable throwable) { 23 | super(throwable); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/exceptions/InvalidProtocolStateException.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.exceptions; 2 | 3 | /** 4 | */ 5 | public class InvalidProtocolStateException extends Exception { 6 | /** 7 | * 8 | */ 9 | private static final long serialVersionUID = -2879556847197351720L; 10 | 11 | public InvalidProtocolStateException() { 12 | } 13 | 14 | public InvalidProtocolStateException(String s) { 15 | super(s); 16 | } 17 | 18 | public InvalidProtocolStateException(String s, Throwable throwable) { 19 | super(s, throwable); 20 | } 21 | 22 | public InvalidProtocolStateException(Throwable throwable) { 23 | super(throwable); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/exceptions/MalformedCommandException.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.exceptions; 2 | 3 | /** 4 | */ 5 | public class MalformedCommandException extends ClientException { 6 | /** 7 | * 8 | */ 9 | private static final long serialVersionUID = -514909313697495546L; 10 | 11 | public MalformedCommandException() { 12 | } 13 | 14 | public MalformedCommandException(String s) { 15 | super(s); 16 | } 17 | 18 | public MalformedCommandException(String s, Throwable throwable) { 19 | super(s, throwable); 20 | } 21 | 22 | public MalformedCommandException(Throwable throwable) { 23 | super(throwable); 24 | } 25 | } -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/exceptions/UnknownCommandException.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.exceptions; 2 | 3 | /** 4 | */ 5 | public class UnknownCommandException extends ClientException { 6 | 7 | /** 8 | * 9 | */ 10 | private static final long serialVersionUID = -1826699280038666904L; 11 | 12 | public UnknownCommandException() { 13 | } 14 | 15 | public UnknownCommandException(String s) { 16 | super(s); 17 | } 18 | 19 | public UnknownCommandException(String s, Throwable throwable) { 20 | super(s, throwable); 21 | } 22 | 23 | public UnknownCommandException(Throwable throwable) { 24 | super(throwable); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/text/MemcachedCommandDecoder.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.text; 2 | 3 | import static com.thimbleware.jmemcached.protocol.text.MemcachedPipelineFactory.USASCII; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | 9 | import org.jboss.netty.buffer.ChannelBuffer; 10 | import org.jboss.netty.channel.Channel; 11 | import org.jboss.netty.channel.ChannelHandlerContext; 12 | import org.jboss.netty.channel.Channels; 13 | import org.jboss.netty.channel.MessageEvent; 14 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 15 | 16 | import com.thimbleware.jmemcached.CacheElement; 17 | import com.thimbleware.jmemcached.LocalCacheElement; 18 | import com.thimbleware.jmemcached.protocol.Command; 19 | import com.thimbleware.jmemcached.protocol.CommandMessage; 20 | import com.thimbleware.jmemcached.protocol.SessionStatus; 21 | import com.thimbleware.jmemcached.protocol.exceptions.InvalidProtocolStateException; 22 | import com.thimbleware.jmemcached.protocol.exceptions.MalformedCommandException; 23 | import com.thimbleware.jmemcached.protocol.exceptions.UnknownCommandException; 24 | 25 | /** 26 | * The MemcachedCommandDecoder is responsible for taking lines from the 27 | * MemcachedFrameDecoder and parsing them into CommandMessage instances for 28 | * handling by the MemcachedCommandHandler 29 | *

30 | * Protocol status is held in the SessionStatus instance which is shared between 31 | * each of the decoders in the pipeline. 32 | */ 33 | public final class MemcachedCommandDecoder extends SimpleChannelUpstreamHandler { 34 | 35 | private SessionStatus status; 36 | 37 | private static final String NOREPLY = "noreply"; 38 | public static final AtomicInteger bytes_read = new AtomicInteger(); 39 | public static final AtomicInteger bytes_written = new AtomicInteger(); 40 | private static final byte space = (byte) ' '; 41 | 42 | public MemcachedCommandDecoder(SessionStatus status) { 43 | this.status = status; 44 | } 45 | 46 | /** 47 | * Process an inbound string from the pipeline's downstream, and depending 48 | * on the state (waiting for data or processing commands), turn them into 49 | * the correct type of command. 50 | * 51 | * @param channelHandlerContext 52 | * @param messageEvent 53 | * @throws Exception 54 | */ 55 | @Override 56 | public void messageReceived(ChannelHandlerContext channelHandlerContext, MessageEvent messageEvent) 57 | throws Exception { 58 | ChannelBuffer in = (ChannelBuffer) messageEvent.getMessage(); 59 | 60 | try { 61 | // Because of the frame handler, we are assured that we are 62 | // receiving only complete lines or payloads. 63 | // Verify that we are in 'processing()' mode 64 | if (status.state == SessionStatus.State.PROCESSING) { 65 | // split into pieces 66 | List pieces = new ArrayList(6); 67 | int pos = in.bytesBefore(space); 68 | do { 69 | if (pos != -1) { 70 | pieces.add(in.toString(in.readerIndex(), pos, USASCII)); 71 | in.skipBytes(pos + 1); 72 | } 73 | } while ((pos = in.bytesBefore(space)) != -1); 74 | pieces.add(in.toString(USASCII)); 75 | 76 | processLine(pieces, messageEvent.getChannel(), channelHandlerContext); 77 | } else if (status.state == SessionStatus.State.PROCESSING_MULTILINE) { 78 | ChannelBuffer slice = in.copy(); 79 | byte[] payload = slice.array(); 80 | in.skipBytes(in.readableBytes()); 81 | continueSet(messageEvent.getChannel(), status, payload, channelHandlerContext); 82 | } else { 83 | throw new InvalidProtocolStateException("invalid protocol state"); 84 | } 85 | 86 | } finally { 87 | // Now indicate that we need more for this command by changing the 88 | // session status's state. 89 | // This instructs the frame decoder to start collecting data for us. 90 | // Note, we don't do this if we're waiting for data. 91 | if (status.state != SessionStatus.State.WAITING_FOR_DATA) 92 | status.ready(); 93 | } 94 | } 95 | 96 | /** 97 | * Process an individual complete protocol line and either passes the 98 | * command for processing by the session handler, or (in the case of 99 | * SET-type commands) partially parses the command and sets the session into 100 | * a state to wait for additional data. 101 | * 102 | * @param parts 103 | * the (originally space separated) parts of the command 104 | * @param channel 105 | * @param channelHandlerContext 106 | */ 107 | private void processLine(List parts, Channel channel, ChannelHandlerContext channelHandlerContext) 108 | throws UnknownCommandException, MalformedCommandException { 109 | final int numParts = parts.size(); 110 | 111 | // Turn the command into an enum for matching on 112 | Command cmdType; 113 | try { 114 | cmdType = Command.valueOf(parts.get(0).toUpperCase()); 115 | } catch (IllegalArgumentException e) { 116 | throw new UnknownCommandException("unknown command: " + parts.get(0).toLowerCase()); 117 | } 118 | 119 | // Produce the initial command message, for filling in later 120 | CommandMessage cmd = CommandMessage.command(cmdType); 121 | 122 | // TODO there is a certain amount of fudgery here related to common 123 | // things like 'noreply', etc. that could be refactored nicely 124 | 125 | // Dispatch on the type of command 126 | if (cmdType == Command.ADD || cmdType == Command.SET || cmdType == Command.REPLACE || cmdType == Command.CAS 127 | || cmdType == Command.APPEND || cmdType == Command.PREPEND) { 128 | 129 | // if we don't have all the parts, it's malformed 130 | if (numParts < 5) { 131 | throw new MalformedCommandException("invalid command length"); 132 | } 133 | 134 | // Fill in all the elements of the command 135 | int size = Integer.parseInt(parts.get(4)); 136 | int expire = Integer.parseInt(parts.get(3)); 137 | int flags = Integer.parseInt(parts.get(2)); 138 | cmd.element = new LocalCacheElement(parts.get(1), flags, 139 | expire != 0 && expire < CacheElement.THIRTY_DAYS ? LocalCacheElement.Now() + expire : expire, 0L); 140 | 141 | // look for cas and "noreply" elements 142 | if (numParts > 5) { 143 | int noreply = cmdType == Command.CAS ? 6 : 5; 144 | if (cmdType == Command.CAS) { 145 | cmd.cas_key = Long.valueOf(parts.get(5)); 146 | } 147 | 148 | if (numParts == noreply + 1 && parts.get(noreply).equalsIgnoreCase(NOREPLY)) 149 | cmd.noreply = true; 150 | } 151 | 152 | // Now indicate that we need more for this command by changing the 153 | // session status's state. 154 | // This instructs the frame decoder to start collecting data for us. 155 | status.needMore(size, cmd); 156 | } else if (cmdType == Command.GET || cmdType == Command.GETS || cmdType == Command.STATS 157 | || cmdType == Command.QUIT || cmdType == Command.VERSION) { 158 | 159 | // Get all the keys 160 | cmd.keys.addAll(parts.subList(1, numParts)); 161 | 162 | // Pass it on. 163 | Channels.fireMessageReceived(channelHandlerContext, cmd, channel.getRemoteAddress()); 164 | } else if (cmdType == Command.INCR || cmdType == Command.DECR) { 165 | 166 | // Malformed 167 | if (numParts < 2 || numParts > 3) 168 | throw new MalformedCommandException("invalid increment command"); 169 | 170 | cmd.keys.add(parts.get(1)); 171 | cmd.incrAmount = Integer.valueOf(parts.get(2)); 172 | 173 | if (numParts == 3 && parts.get(2).equalsIgnoreCase(NOREPLY)) { 174 | cmd.noreply = true; 175 | } 176 | 177 | Channels.fireMessageReceived(channelHandlerContext, cmd, channel.getRemoteAddress()); 178 | } else if (cmdType == Command.DELETE) { 179 | cmd.keys.add(parts.get(1)); 180 | 181 | if (numParts >= 2) { 182 | if (parts.get(numParts - 1).equalsIgnoreCase(NOREPLY)) { 183 | cmd.noreply = true; 184 | if (numParts == 4) 185 | cmd.time = Integer.valueOf(parts.get(2)); 186 | } else if (numParts == 3) 187 | cmd.time = Integer.valueOf(parts.get(2)); 188 | } 189 | Channels.fireMessageReceived(channelHandlerContext, cmd, channel.getRemoteAddress()); 190 | } else if (cmdType == Command.FLUSH_ALL) { 191 | if (numParts >= 1) { 192 | if (parts.get(numParts - 1).equalsIgnoreCase(NOREPLY)) { 193 | cmd.noreply = true; 194 | if (numParts == 3) 195 | cmd.time = Integer.valueOf(parts.get(1)); 196 | } else if (numParts == 2) 197 | cmd.time = Integer.valueOf(parts.get(1)); 198 | } 199 | Channels.fireMessageReceived(channelHandlerContext, cmd, channel.getRemoteAddress()); 200 | } else { 201 | throw new UnknownCommandException("unknown command: " + cmdType); 202 | } 203 | 204 | } 205 | 206 | /** 207 | * Handles the continuation of a SET/ADD/REPLACE command with the data it 208 | * was waiting for. 209 | * 210 | * @param state 211 | * the current session status (unused) 212 | * @param remainder 213 | * the bytes picked up 214 | * @param channelHandlerContext 215 | * @return the new status to set the session to 216 | */ 217 | private void continueSet(Channel channel, SessionStatus state, byte[] remainder, 218 | ChannelHandlerContext channelHandlerContext) { 219 | state.cmd.element.setData(remainder); 220 | Channels.fireMessageReceived(channelHandlerContext, state.cmd, channelHandlerContext.getChannel() 221 | .getRemoteAddress()); 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/text/MemcachedFrameDecoder.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.text; 2 | 3 | import org.jboss.netty.buffer.ChannelBuffer; 4 | import org.jboss.netty.buffer.ChannelBufferIndexFinder; 5 | import org.jboss.netty.channel.ChannelHandlerContext; 6 | import org.jboss.netty.handler.codec.frame.FrameDecoder; 7 | import org.jboss.netty.handler.codec.frame.TooLongFrameException; 8 | 9 | import com.thimbleware.jmemcached.StatsCounter; 10 | import com.thimbleware.jmemcached.protocol.SessionStatus; 11 | import com.thimbleware.jmemcached.protocol.exceptions.IncorrectlyTerminatedPayloadException; 12 | 13 | /** 14 | * The frame decoder is responsible for breaking the original stream up into a 15 | * series of lines. 16 | *

17 | * The code here is heavily based on Netty's DelimiterBasedFrameDecoder, but has 18 | * been modified because the memcached protocol has two states: 1) processing 19 | * CRLF delimited lines and 2) spooling results for SET/ADD 20 | */ 21 | public final class MemcachedFrameDecoder extends FrameDecoder { 22 | 23 | private final SessionStatus status; 24 | 25 | private final int maxFrameLength; 26 | 27 | private boolean discardingTooLongFrame; 28 | private long tooLongFrameLength; 29 | 30 | /** 31 | * Creates a new instance. 32 | * 33 | * @param status 34 | * session status instance for holding state of the session 35 | * @param maxFrameLength 36 | * the maximum length of the decoded frame. A 37 | * {@link org.jboss.netty.handler.codec.frame.TooLongFrameException} 38 | * is thrown if frame length is exceeded 39 | */ 40 | public MemcachedFrameDecoder(SessionStatus status, int maxFrameLength) { 41 | this.status = status; 42 | validateMaxFrameLength(maxFrameLength); 43 | this.maxFrameLength = maxFrameLength; 44 | } 45 | 46 | @Override 47 | protected Object decode(ChannelHandlerContext ctx, org.jboss.netty.channel.Channel channel, ChannelBuffer buffer) 48 | throws Exception { 49 | // check the state. if we're WAITING_FOR_DATA that means instead of 50 | // breaking into lines, we need N bytes 51 | // otherwise, we're waiting for input 52 | if (status.state == SessionStatus.State.WAITING_FOR_DATA) { 53 | if (buffer.readableBytes() < status.bytesNeeded + MemcachedResponseEncoder.CRLF.capacity()) 54 | return null; 55 | 56 | // verify delimiter matches at the right location 57 | ChannelBuffer dest = buffer.slice(status.bytesNeeded + buffer.readerIndex(), 2); 58 | StatsCounter.bytes_written.addAndGet(status.bytesNeeded); 59 | if (!dest.equals(MemcachedResponseEncoder.CRLF)) { 60 | // before we throw error... we're ready for the next command 61 | status.ready(); 62 | 63 | // error, no delimiter at end of payload 64 | throw new IncorrectlyTerminatedPayloadException("payload not terminated correctly"); 65 | } else { 66 | status.processingMultiline(); 67 | 68 | // There's enough bytes in the buffer and the delimiter is at 69 | // the end. Read it. 70 | ChannelBuffer result = buffer.slice(buffer.readerIndex(), status.bytesNeeded); 71 | buffer.skipBytes(status.bytesNeeded + MemcachedResponseEncoder.CRLF.capacity()); 72 | 73 | return result; 74 | } 75 | 76 | } else { 77 | int minFrameLength = Integer.MAX_VALUE; 78 | ChannelBuffer foundDelimiter = null; 79 | // command length 80 | int frameLength = buffer.bytesBefore(buffer.readerIndex(), buffer.readableBytes(), 81 | ChannelBufferIndexFinder.CRLF); 82 | if (frameLength >= 0 && frameLength < minFrameLength && buffer.readableBytes() >= frameLength + 2) { 83 | minFrameLength = frameLength; 84 | foundDelimiter = MemcachedResponseEncoder.CRLF; 85 | } 86 | 87 | if (foundDelimiter != null) { 88 | int minDelimLength = foundDelimiter.capacity(); 89 | 90 | if (discardingTooLongFrame) { 91 | // We've just finished discarding a very large frame. 92 | // Throw an exception and go back to the initial state. 93 | long tooLongFrameLength = this.tooLongFrameLength; 94 | this.tooLongFrameLength = 0L; 95 | discardingTooLongFrame = false; 96 | buffer.skipBytes(minFrameLength + minDelimLength); 97 | fail(tooLongFrameLength + minFrameLength + minDelimLength); 98 | } 99 | 100 | if (minFrameLength > maxFrameLength) { 101 | // Discard read frame. 102 | buffer.skipBytes(minFrameLength + minDelimLength); 103 | StatsCounter.bytes_written.addAndGet(minFrameLength + minDelimLength); 104 | fail(minFrameLength); 105 | } 106 | 107 | ChannelBuffer frame = buffer.slice(buffer.readerIndex(), minFrameLength); 108 | buffer.skipBytes(minFrameLength + minDelimLength); 109 | status.processing(); 110 | StatsCounter.bytes_written.addAndGet(minFrameLength + minDelimLength); 111 | return frame; 112 | } else { 113 | if (buffer.readableBytes() > maxFrameLength) { 114 | // Discard the content of the buffer until a delimiter is 115 | // found. 116 | tooLongFrameLength = buffer.readableBytes(); 117 | buffer.skipBytes(buffer.readableBytes()); 118 | discardingTooLongFrame = true; 119 | StatsCounter.bytes_written.addAndGet(tooLongFrameLength); 120 | } 121 | 122 | return null; 123 | } 124 | } 125 | } 126 | 127 | public static String byte2hex(byte bytes[]) { 128 | StringBuffer retString = new StringBuffer(); 129 | for (int i = 0; i < bytes.length; ++i) { 130 | retString.append(Integer.toHexString(0x0100 + (bytes[i] & 0x00FF)).substring(1).toUpperCase()); 131 | } 132 | return retString.toString(); 133 | } 134 | 135 | private void fail(long frameLength) throws TooLongFrameException { 136 | throw new TooLongFrameException("The frame length exceeds " + maxFrameLength + ": " + frameLength); 137 | } 138 | 139 | private static void validateMaxFrameLength(int maxFrameLength) { 140 | if (maxFrameLength <= 0) { 141 | throw new IllegalArgumentException("maxFrameLength must be a positive integer: " + maxFrameLength); 142 | } 143 | } 144 | 145 | } 146 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/text/MemcachedPipelineFactory.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.text; 2 | 3 | import java.nio.charset.Charset; 4 | 5 | import org.jboss.netty.channel.ChannelPipeline; 6 | import org.jboss.netty.channel.ChannelPipelineFactory; 7 | import org.jboss.netty.channel.Channels; 8 | import org.jboss.netty.channel.group.DefaultChannelGroup; 9 | 10 | import com.thimbleware.jmemcached.Cache; 11 | import com.thimbleware.jmemcached.protocol.MemcachedCommandHandler; 12 | import com.thimbleware.jmemcached.protocol.SessionStatus; 13 | 14 | /** 15 | */ 16 | public final class MemcachedPipelineFactory implements ChannelPipelineFactory { 17 | public static final Charset USASCII = Charset.forName("US-ASCII"); 18 | 19 | private Cache cache; 20 | private String version; 21 | private boolean verbose; 22 | private int idleTime; 23 | 24 | private int frameSize; 25 | private DefaultChannelGroup channelGroup; 26 | private final MemcachedResponseEncoder memcachedResponseEncoder = new MemcachedResponseEncoder(); 27 | 28 | private final MemcachedCommandHandler memcachedCommandHandler; 29 | 30 | public MemcachedPipelineFactory(Cache cache, String version, 31 | boolean verbose, int idleTime, int frameSize, 32 | DefaultChannelGroup channelGroup) { 33 | this.cache = cache; 34 | this.version = version; 35 | this.verbose = verbose; 36 | this.idleTime = idleTime; 37 | this.frameSize = frameSize; 38 | this.channelGroup = channelGroup; 39 | memcachedCommandHandler = new MemcachedCommandHandler(this.cache, 40 | this.version, this.verbose, this.idleTime, this.channelGroup); 41 | } 42 | 43 | public final ChannelPipeline getPipeline() throws Exception { 44 | SessionStatus status = new SessionStatus().ready(); 45 | 46 | return Channels.pipeline(new MemcachedFrameDecoder(status, frameSize), 47 | new MemcachedCommandDecoder(status), memcachedCommandHandler, 48 | memcachedResponseEncoder); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/protocol/text/MemcachedResponseEncoder.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.protocol.text; 2 | 3 | import static com.thimbleware.jmemcached.protocol.text.MemcachedPipelineFactory.USASCII; 4 | import static java.lang.String.valueOf; 5 | 6 | import java.io.IOException; 7 | import java.nio.channels.ClosedChannelException; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | import org.jboss.netty.buffer.ChannelBuffer; 12 | import org.jboss.netty.buffer.ChannelBuffers; 13 | import org.jboss.netty.channel.Channel; 14 | import org.jboss.netty.channel.ChannelHandlerContext; 15 | import org.jboss.netty.channel.Channels; 16 | import org.jboss.netty.channel.ExceptionEvent; 17 | import org.jboss.netty.channel.MessageEvent; 18 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import com.thimbleware.jmemcached.Cache; 23 | import com.thimbleware.jmemcached.CacheElement; 24 | import com.thimbleware.jmemcached.StatsCounter; 25 | import com.thimbleware.jmemcached.protocol.Command; 26 | import com.thimbleware.jmemcached.protocol.ResponseMessage; 27 | import com.thimbleware.jmemcached.protocol.exceptions.ClientException; 28 | 29 | /** 30 | * Response encoder for the memcached text protocol. Produces strings destined 31 | * for the StringEncoder 32 | */ 33 | public final class MemcachedResponseEncoder extends SimpleChannelUpstreamHandler { 34 | 35 | final Logger logger = LoggerFactory.getLogger(MemcachedResponseEncoder.class); 36 | 37 | public static final ChannelBuffer CRLF = ChannelBuffers.copiedBuffer("\r\n", USASCII); 38 | private static final ChannelBuffer VALUE = ChannelBuffers.copiedBuffer("VALUE ", USASCII); 39 | private static final ChannelBuffer EXISTS = ChannelBuffers.copiedBuffer("EXISTS\r\n", USASCII); 40 | private static final ChannelBuffer NOT_FOUND = ChannelBuffers.copiedBuffer("NOT_FOUND\r\n", USASCII); 41 | private static final ChannelBuffer NOT_STORED = ChannelBuffers.copiedBuffer("NOT_STORED\r\n", USASCII); 42 | private static final ChannelBuffer STORED = ChannelBuffers.copiedBuffer("STORED\r\n", USASCII); 43 | private static final ChannelBuffer DELETED = ChannelBuffers.copiedBuffer("DELETED\r\n", USASCII); 44 | private static final ChannelBuffer END = ChannelBuffers.copiedBuffer("END\r\n", USASCII); 45 | private static final ChannelBuffer OK = ChannelBuffers.copiedBuffer("OK\r\n", USASCII); 46 | private static final ChannelBuffer ERROR = ChannelBuffers.copiedBuffer("ERROR\r\n", USASCII); 47 | private static final ChannelBuffer CLIENT_ERROR = ChannelBuffers.copiedBuffer("CLIENT_ERROR", USASCII); 48 | 49 | /** 50 | * Handle exceptions in protocol processing. Exceptions are either client or 51 | * internal errors. Report accordingly. 52 | * 53 | * @param ctx 54 | * @param e 55 | * @throws Exception 56 | */ 57 | @Override 58 | public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { 59 | try { 60 | throw e.getCause(); 61 | } catch (ClientException ce) { 62 | if (ctx.getChannel().isOpen()) 63 | ctx.getChannel().write(ChannelBuffers.wrappedBuffer(CLIENT_ERROR.array(), ce.getMessage().getBytes(), CRLF.array())); 64 | } catch (ClosedChannelException e2) { 65 | logger.info("ClosedChannelException" + e.getChannel().getRemoteAddress()); 66 | if (ctx.getChannel().isOpen()) { 67 | ctx.getChannel().write(ERROR); 68 | } 69 | } catch (IOException e2) { 70 | StackTraceElement[] stackTraceElements = e2.getStackTrace(); 71 | for (int i = 0; i < stackTraceElements.length; i++) { 72 | if (stackTraceElements[i].getClassName().equals("sun.nio.ch.SocketDispatcher")) { 73 | logger.info("IOException:" + e.getChannel().getRemoteAddress()); 74 | if (ctx.getChannel().isOpen()) { 75 | ctx.getChannel().write(ERROR); 76 | } 77 | return; 78 | } 79 | } 80 | logger.error("error", e2); 81 | 82 | } catch (Throwable tr) { 83 | logger.error("error", tr); 84 | if (ctx.getChannel().isOpen()) { 85 | ctx.getChannel().write(ERROR); 86 | } 87 | } 88 | } 89 | 90 | @Override 91 | public void messageReceived(ChannelHandlerContext channelHandlerContext, MessageEvent messageEvent) 92 | throws Exception { 93 | ResponseMessage command = (ResponseMessage) messageEvent.getMessage(); 94 | 95 | Command cmd = command.cmd.cmd; 96 | 97 | Channel channel = messageEvent.getChannel(); 98 | 99 | if (cmd == Command.GET || cmd == Command.GETS) { 100 | CacheElement[] results = command.elements; 101 | int totalBytes = 0; 102 | for (CacheElement result : results) { 103 | if (result != null) { 104 | totalBytes += result.size() + 256; 105 | } 106 | } 107 | ChannelBuffer writeBuffer = ChannelBuffers.dynamicBuffer(totalBytes); 108 | 109 | for (CacheElement result : results) { 110 | if (result != null) { 111 | writeBuffer.writeBytes(VALUE.duplicate()); 112 | writeBuffer.writeBytes(ChannelBuffers.copiedBuffer(result.getKeystring(), USASCII)); 113 | writeBuffer.writeByte((byte) ' '); 114 | writeBuffer.writeBytes(ChannelBuffers.copiedBuffer(String.valueOf(result.getFlags()), USASCII)); 115 | writeBuffer.writeByte((byte) ' '); 116 | writeBuffer.writeBytes(ChannelBuffers 117 | .copiedBuffer(String.valueOf(result.getData().length), USASCII)); 118 | if (cmd == Command.GETS) { 119 | writeBuffer.writeByte((byte) ' '); 120 | writeBuffer.writeBytes(ChannelBuffers.copiedBuffer(String.valueOf(result.getCasUnique()), 121 | USASCII)); 122 | } 123 | writeBuffer.writeByte((byte) '\r'); 124 | writeBuffer.writeByte((byte) '\n'); 125 | writeBuffer.writeBytes(result.getData()); 126 | writeBuffer.writeByte((byte) '\r'); 127 | writeBuffer.writeByte((byte) '\n'); 128 | } 129 | } 130 | writeBuffer.writeBytes(END.duplicate()); 131 | StatsCounter.bytes_read.addAndGet(writeBuffer.writerIndex()); 132 | Channels.write(channel, writeBuffer); 133 | } else if (cmd == Command.SET || cmd == Command.CAS || cmd == Command.ADD || cmd == Command.REPLACE 134 | || cmd == Command.APPEND || cmd == Command.PREPEND) { 135 | 136 | if (!command.cmd.noreply) 137 | Channels.write(channel, storeResponse(command.response)); 138 | } else if (cmd == Command.INCR || cmd == Command.DECR) { 139 | if (!command.cmd.noreply) 140 | Channels.write(channel, incrDecrResponseString(command.incrDecrResponse)); 141 | 142 | } else if (cmd == Command.DELETE) { 143 | if (!command.cmd.noreply) 144 | Channels.write(channel, deleteResponseString(command.deleteResponse)); 145 | 146 | } else if (cmd == Command.STATS) { 147 | for (Map.Entry> stat : command.stats.entrySet()) { 148 | for (String statVal : stat.getValue()) { 149 | StringBuilder builder = new StringBuilder(); 150 | builder.append("STAT "); 151 | builder.append(stat.getKey()); 152 | builder.append(" "); 153 | builder.append(String.valueOf(statVal)); 154 | builder.append("\r\n"); 155 | Channels.write(channel, ChannelBuffers.copiedBuffer(builder.toString(), USASCII)); 156 | } 157 | } 158 | Channels.write(channel, END.duplicate()); 159 | 160 | } else if (cmd == Command.VERSION) { 161 | Channels.write(channel, ChannelBuffers.copiedBuffer("VERSION " + command.version + "\r\n", USASCII)); 162 | } else if (cmd == Command.QUIT) { 163 | Channels.disconnect(channel); 164 | } else if (cmd == Command.FLUSH_ALL) { 165 | if (!command.cmd.noreply) { 166 | ChannelBuffer ret = command.flushSuccess ? OK.duplicate() : ERROR.duplicate(); 167 | 168 | Channels.write(channel, ret); 169 | } 170 | } else { 171 | Channels.write(channel, ERROR.duplicate()); 172 | logger.error("error; unrecognized command: " + cmd); 173 | } 174 | 175 | } 176 | 177 | private ChannelBuffer deleteResponseString(Cache.DeleteResponse deleteResponse) { 178 | if (deleteResponse == Cache.DeleteResponse.DELETED) { 179 | StatsCounter.bytes_read.addAndGet(9); 180 | return DELETED.duplicate(); 181 | } else { 182 | StatsCounter.bytes_read.addAndGet(11); 183 | return NOT_FOUND.duplicate(); 184 | } 185 | } 186 | 187 | private ChannelBuffer incrDecrResponseString(Integer ret) { 188 | if (ret == null) { 189 | StatsCounter.bytes_read.addAndGet(11); 190 | return NOT_FOUND.duplicate(); 191 | } else { 192 | StatsCounter.bytes_read.addAndGet(11); 193 | ChannelBuffer buffer = ChannelBuffers.copiedBuffer(valueOf(ret) + "\r\n", USASCII); 194 | StatsCounter.bytes_read.addAndGet(buffer.writerIndex()); 195 | return buffer; 196 | } 197 | } 198 | 199 | /** 200 | * Find the string response message which is equivalent to a response to a 201 | * set/add/replace message in the cache 202 | * 203 | * @param storeResponse 204 | * the response code 205 | * @return the string to output on the network 206 | */ 207 | private ChannelBuffer storeResponse(Cache.StoreResponse storeResponse) { 208 | switch (storeResponse) { 209 | case EXISTS: 210 | StatsCounter.bytes_read.addAndGet(8); 211 | return EXISTS.duplicate(); 212 | case NOT_FOUND: 213 | StatsCounter.bytes_read.addAndGet(11); 214 | return NOT_FOUND.duplicate(); 215 | case NOT_STORED: 216 | StatsCounter.bytes_read.addAndGet(12); 217 | return NOT_STORED.duplicate(); 218 | case STORED: 219 | StatsCounter.bytes_read.addAndGet(8); 220 | return STORED.duplicate(); 221 | } 222 | throw new RuntimeException("unknown store response from cache: " + storeResponse); 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/storage/CacheStorage.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.storage; 2 | 3 | 4 | import java.io.IOException; 5 | import java.util.Set; 6 | 7 | import com.thimbleware.jmemcached.LocalCacheElement; 8 | import com.thimbleware.jmemcached.protocol.exceptions.DatabaseException; 9 | import com.thimbleware.jmemcached.storage.hash.SizedItem; 10 | 11 | /** 12 | * The interface for cache storage. Essentially a concurrent map but with 13 | * methods for investigating the heap state of the storage unit and with 14 | * additional support for explicit resource-cleanup (close()). 15 | */ 16 | public interface CacheStorage { 17 | /** 18 | * @return the capacity (in bytes) of the storage 19 | */ 20 | long getMemoryCapacity(); 21 | 22 | /** 23 | * @return the current usage (in bytes) of the storage 24 | */ 25 | long getMemoryUsed(); 26 | 27 | /** 28 | * @return the capacity (in # of items) of the storage 29 | */ 30 | int capacity(); 31 | 32 | /** 33 | * Close the storage unit, deallocating any resources it might be currently 34 | * holding. 35 | * 36 | * @throws java.io.IOException 37 | * thrown if IO faults occur anywhere during close. 38 | */ 39 | void close() throws IOException; 40 | 41 | void clear() throws DatabaseException, Exception; 42 | 43 | LocalCacheElement remove(String key) throws DatabaseException, Exception; 44 | 45 | LocalCacheElement putIfAbsent(String keystring, LocalCacheElement e) 46 | throws DatabaseException, Exception; 47 | 48 | LocalCacheElement get(String keystring); 49 | 50 | boolean replace(String keystring, LocalCacheElement old, 51 | LocalCacheElement prepend) throws DatabaseException, Exception; 52 | 53 | LocalCacheElement put(String keystring, LocalCacheElement e) 54 | throws DatabaseException, Exception; 55 | 56 | long size(); 57 | 58 | Set keySet(); 59 | 60 | LocalCacheElement replace(String key, LocalCacheElement placeHolder) 61 | throws DatabaseException, Exception; 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/storage/bytebuffer/ByteBufferBlockStore.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.storage.bytebuffer; 2 | 3 | import java.io.IOException; 4 | import java.nio.ByteBuffer; 5 | import java.util.*; 6 | 7 | /** 8 | * Memory mapped block storage mechanism with a free-list maintained by TreeMap 9 | * 10 | * Allows memory for storage to be mapped outside of the VM's main memory, and outside the purvey 11 | * of the GC. 12 | * 13 | * Should offer O(Log(N)) search and free of blocks. 14 | */ 15 | public class ByteBufferBlockStore { 16 | 17 | protected ByteBuffer storageBuffer; 18 | 19 | private long freeBytes; 20 | 21 | private long storeSizeBytes; 22 | private int blockSizeBytes; 23 | 24 | private SortedMap freeList; 25 | private Set currentRegions; 26 | 27 | /** 28 | * Exception thrown on inability to allocate a new block 29 | */ 30 | public static class BadAllocationException extends RuntimeException { 31 | public BadAllocationException(String s) { 32 | super(s); 33 | } 34 | } 35 | 36 | /** 37 | * Construct a new memory mapped block storage against a filename, with a certain size 38 | * and block size. 39 | * @param storageBuffer 40 | * @param blockSizeBytes the size of a block in the store 41 | * @throws java.io.IOException thrown on failure to open the store or map the file 42 | */ 43 | public ByteBufferBlockStore(ByteBuffer storageBuffer, int blockSizeBytes) throws IOException { 44 | this.storageBuffer = storageBuffer; 45 | this.blockSizeBytes = blockSizeBytes; 46 | initialize(storageBuffer.capacity(), blockSizeBytes); 47 | } 48 | 49 | /** 50 | * Constructor used only be subclasses, allowing them to provide their own buffer. 51 | */ 52 | protected ByteBufferBlockStore() { 53 | } 54 | 55 | protected void initialize(int storeSizeBytes, int blockSizeBytes) { 56 | // set region size used throughout 57 | this.blockSizeBytes = blockSizeBytes; 58 | 59 | // set the size of the store in bytes 60 | this.storeSizeBytes = storageBuffer.capacity(); 61 | 62 | // the number of free bytes starts out as the entire store 63 | freeBytes = storeSizeBytes; 64 | 65 | // clear the buffer 66 | storageBuffer.clear(); 67 | 68 | // the free list starts with one giant free region; we create a tree map as index, then set initial values 69 | // with one empty region. 70 | freeList = new TreeMap(); 71 | currentRegions = new HashSet(); 72 | 73 | clear(); 74 | } 75 | 76 | 77 | /** 78 | * Rounds up a requested size to the nearest block width. 79 | * @param size the requested size 80 | * @param blockSize the block size to use 81 | * @return the actual mount to use 82 | */ 83 | public static long roundUp( long size, long blockSize ) { 84 | return size - 1L + blockSize - (size - 1L) % blockSize; 85 | } 86 | 87 | /** 88 | * Close the store, destroying all data and closing the backing file 89 | * @throws java.io.IOException thrown on failure to close file 90 | */ 91 | public void close() throws IOException { 92 | // clear the region list 93 | clear(); 94 | 95 | // 96 | freeResources(); 97 | 98 | // null out the storage to allow the GC to get rid of it 99 | storageBuffer = null; 100 | } 101 | 102 | protected void freeResources() throws IOException { 103 | // noop 104 | } 105 | 106 | /** 107 | * Allocate a region in the block storage 108 | * @param desiredSize size (in bytes) desired for the region 109 | * @param data initial data to place in it 110 | * @return the region descriptor 111 | */ 112 | public Region alloc(int desiredSize, byte[] data) { 113 | final long desiredBlockSize = roundUp(desiredSize, blockSizeBytes); 114 | 115 | /** Find a free entry. headMap should give us O(log(N)) search 116 | * time. 117 | */ 118 | Iterator> entryIterator = freeList.tailMap(desiredBlockSize).entrySet().iterator(); 119 | 120 | if (!entryIterator.hasNext()) { 121 | /** No more room. 122 | * We now have three options - we can throw error, grow the store, or compact 123 | * the holes in the existing one. 124 | * 125 | * We usually want to grow (fast), and compact (slow) only if necessary 126 | * (i.e. some periodic interval 127 | * has been reached or a maximum store size constant hit.) 128 | * 129 | * Incremental compaction is a Nice To Have. 130 | * 131 | * TODO implement compaction routine; throw; in theory the cache on top of us should be removing elements before we fill 132 | */ 133 | // compact(); 134 | 135 | throw new BadAllocationException("unable to allocate room; all blocks consumed"); 136 | } else { 137 | Map.Entry freeEntry = entryIterator.next(); 138 | 139 | /** Don't let this region overlap a page boundary 140 | */ 141 | // PUT CODE HERE 142 | 143 | long position = freeEntry.getValue(); 144 | 145 | /** Size of the free block to be placed back on the free list 146 | */ 147 | long newFreeSize = freeEntry.getKey() - desiredBlockSize; 148 | 149 | /** Split the free entry and add the entry to the allocated list 150 | */ 151 | freeList.remove(freeEntry.getKey()); 152 | 153 | /** Add it back to the free list if needed 154 | */ 155 | if ( newFreeSize > 0L ) { 156 | freeList.put( newFreeSize, position + desiredBlockSize) ; 157 | } 158 | 159 | freeBytes -= desiredBlockSize; 160 | 161 | // get the buffer to it 162 | storageBuffer.rewind(); 163 | storageBuffer.position((int)position); 164 | storageBuffer.put(data, 0, desiredSize); 165 | 166 | Region region = new Region(desiredSize, desiredBlockSize, position); 167 | 168 | currentRegions.add(region); 169 | 170 | return region; 171 | } 172 | } 173 | 174 | public byte[] get(Region region) { 175 | byte[] result = new byte[region.size]; 176 | storageBuffer.position((int)region.offset); 177 | storageBuffer.get(result, 0, region.size); 178 | return result; 179 | } 180 | 181 | public void free(Region region) { 182 | freeList.put(region.physicalSize, region.offset); 183 | freeBytes += region.physicalSize; 184 | region.valid = false; 185 | currentRegions.remove(region); 186 | } 187 | 188 | public void clear() 189 | { 190 | // mark all blocks invalid 191 | clearRegions(); 192 | 193 | // say goodbye to the region list 194 | freeList.clear(); 195 | 196 | // Add a free entry for the entire thing at the start 197 | freeList.put(storeSizeBytes, 0L); 198 | 199 | // reset the # of free bytes back to the max size 200 | freeBytes = storeSizeBytes; 201 | } 202 | 203 | private void clearRegions() { 204 | // mark all blocks invalid 205 | for (Region currentRegion : currentRegions) { 206 | currentRegion.setValid(false); 207 | } 208 | 209 | // clear the list of blocks so we're not holding onto them 210 | currentRegions.clear(); 211 | } 212 | 213 | public long getStoreSizeBytes() { 214 | return storeSizeBytes; 215 | } 216 | 217 | public int getBlockSizeBytes() { 218 | return blockSizeBytes; 219 | } 220 | 221 | public long getFreeBytes() { 222 | return freeBytes; 223 | } 224 | 225 | 226 | } -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/storage/bytebuffer/Region.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.storage.bytebuffer; 2 | 3 | /** 4 | * Represents a number of allocated blocks in the store 5 | */ 6 | public final class Region { 7 | /** 8 | * Size in bytes of the requested area 9 | */ 10 | public final int size; 11 | 12 | /** 13 | * Actual size the data rounded up to the nearest block. 14 | */ 15 | public final long physicalSize; 16 | 17 | /** 18 | * Offset into the memory region 19 | */ 20 | final long offset; 21 | 22 | /** 23 | * Flag which is true if the region is valid and in use. 24 | * Set to false on free() 25 | */ 26 | public boolean valid = false; 27 | 28 | public Region(int size, long physicalSize, long offset) { 29 | this.size = size; 30 | this.physicalSize = physicalSize; 31 | this.offset = offset; 32 | this.valid = true; 33 | } 34 | 35 | public void setValid(boolean valid) { 36 | this.valid = valid; 37 | } 38 | 39 | @Override 40 | public boolean equals(Object o) { 41 | if (this == o) return true; 42 | if (!(o instanceof Region)) return false; 43 | 44 | Region region = (Region) o; 45 | 46 | if (physicalSize != region.physicalSize) return false; 47 | if (offset != region.offset) return false; 48 | if (size != region.size) return false; 49 | 50 | return true; 51 | } 52 | 53 | @Override 54 | public int hashCode() { 55 | int result = size; 56 | result = 31 * result + (int) (physicalSize ^ (physicalSize >>> 32)); 57 | result = 31 * result + (int) (offset ^ (offset >>> 32)); 58 | return result; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/storage/hash/SizedItem.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.storage.hash; 2 | 3 | /** 4 | */ 5 | public interface SizedItem { 6 | int size(); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/thimbleware/jmemcached/storage/mmap/MemoryMappedBlockStore.java: -------------------------------------------------------------------------------- 1 | package com.thimbleware.jmemcached.storage.mmap; 2 | 3 | import com.thimbleware.jmemcached.storage.bytebuffer.ByteBufferBlockStore; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | import java.io.RandomAccessFile; 8 | import java.nio.MappedByteBuffer; 9 | import static java.nio.channels.FileChannel.MapMode.PRIVATE; 10 | 11 | /** 12 | * Memory mapped block storage mechanism with a free-list maintained by TreeMap 13 | * 14 | * Allows memory for storage to be mapped outside of the VM's main memory, and outside the purvey 15 | * of the GC. 16 | * 17 | * Should offer O(Log(N)) search and free of blocks. 18 | */ 19 | public final class MemoryMappedBlockStore extends ByteBufferBlockStore { 20 | 21 | private RandomAccessFile fileStorage; 22 | private String fileName; 23 | 24 | /** 25 | * Construct a new memory mapped block storage against a filename, with a certain size 26 | * and block size. 27 | * @param maxBytes the number of bytes to allocate in the file 28 | * @param fileName the filename to use 29 | * @param blockSizeBytes the size of a block in the store 30 | * @throws java.io.IOException thrown on failure to open the store or map the file 31 | */ 32 | public MemoryMappedBlockStore(long maxBytes, String fileName, int blockSizeBytes) throws IOException { 33 | super(); 34 | storageBuffer = getMemoryMappedFileStorage(maxBytes, fileName); 35 | initialize(storageBuffer.capacity(), blockSizeBytes); 36 | } 37 | 38 | private MappedByteBuffer getMemoryMappedFileStorage(long maxBytes, String fileName) throws IOException { 39 | // open the file for read-write 40 | this.fileName = fileName; 41 | fileStorage = new RandomAccessFile(fileName, "rw"); 42 | fileStorage.seek(maxBytes); 43 | fileStorage.getChannel().map(PRIVATE, 0, maxBytes); 44 | 45 | return fileStorage.getChannel().map(PRIVATE, 0, maxBytes); 46 | } 47 | 48 | @Override 49 | protected void freeResources() throws IOException { 50 | super.freeResources(); 51 | 52 | // close the actual file 53 | fileStorage.close(); 54 | 55 | // delete the file; it is no longer of any use 56 | new File(fileName).delete(); 57 | 58 | fileStorage = null; 59 | } 60 | 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/com/google/code/fqueue/DatabaseExceptionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue; 17 | 18 | import com.thimbleware.jmemcached.protocol.exceptions.DatabaseException; 19 | 20 | import junit.framework.TestCase; 21 | 22 | public class DatabaseExceptionTest extends TestCase { 23 | public void testPerformance() { 24 | DatabaseException exception = new DatabaseException("password wrong"); 25 | exception.printStackTrace(); 26 | long start = System.currentTimeMillis(); 27 | for (int i = 0; i < 10000000; i++) { 28 | exception = new DatabaseException("password wrong"); 29 | 30 | } 31 | System.out.println("spend:" + (System.currentTimeMillis() - start) 32 | + "ms"); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/com/google/code/fqueue/FSQueueTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue; 17 | 18 | import java.util.LinkedList; 19 | import java.util.List; 20 | 21 | import junit.framework.TestCase; 22 | 23 | /** 24 | * @author sunli 25 | * @date 2010-8-13 26 | * @version $Id$ 27 | */ 28 | public class FSQueueTest extends TestCase { 29 | private static FQueue queue; 30 | static { 31 | try { 32 | queue = new FQueue("db"); 33 | queue.clear(); 34 | } catch (Exception e) { 35 | e.printStackTrace(); 36 | } 37 | 38 | } 39 | 40 | @Override 41 | protected void setUp() throws Exception { 42 | 43 | } 44 | 45 | @Override 46 | protected void tearDown() throws Exception { 47 | super.tearDown(); 48 | } 49 | 50 | public void tesssstCrash() { 51 | queue.offer("testqueueoffer".getBytes()); 52 | System.exit(9); 53 | try { 54 | Thread.sleep(100000); 55 | } catch (InterruptedException e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | 60 | public void testOffer() { 61 | queue.offer("testqueueoffer".getBytes()); 62 | assertEquals(new String(queue.poll()), "testqueueoffer"); 63 | } 64 | 65 | public void testPoll() { 66 | queue.offer("testqueuepoll".getBytes()); 67 | assertEquals(new String(queue.poll()), "testqueuepoll"); 68 | } 69 | 70 | public void testAdd() { 71 | queue.add("testqueueadd".getBytes()); 72 | assertEquals(new String(queue.poll()), "testqueueadd"); 73 | } 74 | 75 | public void testAll() { 76 | queue.add("test1".getBytes()); 77 | queue.add("test2".getBytes()); 78 | assertEquals(new String(queue.poll()), "test1"); 79 | queue.add("test3".getBytes()); 80 | queue.add("test4".getBytes()); 81 | assertEquals(new String(queue.poll()), "test2"); 82 | assertEquals(new String(queue.poll()), "test3"); 83 | System.out.println(new String(queue.poll())); 84 | StringBuffer sBuffer = new StringBuffer(1024); 85 | for (int i = 0; i < 1024; i++) { 86 | sBuffer.append("a"); 87 | } 88 | String string = sBuffer.toString(); 89 | assertEquals(0, queue.size()); 90 | for (int i = 0; i < 100000; i++) { 91 | byte[] b = (string + i).getBytes(); 92 | queue.offer(b); 93 | } 94 | assertEquals(100000, queue.size()); 95 | for (int i = 0; i < 100000; i++) { 96 | if (i == 85301) { 97 | System.out.println(i); 98 | } 99 | byte[] b = queue.poll(); 100 | if (b == null) { 101 | i--; 102 | System.out.println("null" + i); 103 | continue; 104 | } 105 | assertEquals(new String(b), (string + i)); 106 | } 107 | queue.add("123".getBytes()); 108 | queue.add("123".getBytes()); 109 | assertEquals(queue.size(), 2); 110 | queue.clear(); 111 | assertNull(queue.poll()); 112 | } 113 | 114 | public void testFqueueVSList() { 115 | String message = "1234567890"; 116 | byte[] bytes = message.getBytes(); 117 | long start = System.currentTimeMillis(); 118 | for (int i = 0; i < 1000000; i++) { 119 | queue.add(bytes); 120 | } 121 | System.out.println("Fqueue写入10字节10000000次:" + (System.currentTimeMillis() - start)); 122 | queue.clear(); 123 | List list = new LinkedList(); 124 | start = System.currentTimeMillis(); 125 | for (int i = 0; i < 1000000; i++) { 126 | list.add(bytes); 127 | } 128 | System.out.println("LinkedList写入10字节10000000次:" + (System.currentTimeMillis() - start)); 129 | } 130 | 131 | public void testPerformance() { 132 | StringBuffer sBuffer = new StringBuffer(1024); 133 | for (int i = 0; i < 1024; i++) { 134 | sBuffer.append("a"); 135 | } 136 | String string = sBuffer.toString(); 137 | System.out.println("Test write 1000000 times 1K data to queue"); 138 | long start = System.currentTimeMillis(); 139 | for (int i = 0; i < 1000000; i++) { 140 | byte[] b = (string + i).getBytes(); 141 | queue.offer(b); 142 | } 143 | System.out.println("spend time:" + (System.currentTimeMillis() - start) + "ms"); 144 | System.out.println("Test read 1000000 times 1K data from queue"); 145 | start = System.currentTimeMillis(); 146 | for (int i = 0; i < 1000000; i++) { 147 | 148 | byte[] b = queue.poll(); 149 | if (b == null) { 150 | i--; 151 | System.out.println("null" + i); 152 | continue; 153 | } 154 | } 155 | assertEquals(0, queue.size()); 156 | System.out.println("spend:" + (System.currentTimeMillis() - start) + "ms"); 157 | } 158 | 159 | public void testPerformance2() { 160 | StringBuffer sBuffer = new StringBuffer(1024); 161 | for (int i = 0; i < 10; i++) { 162 | sBuffer.append("a"); 163 | } 164 | String string = sBuffer.toString(); 165 | System.out.println("Test write 10000000 times 10 Bytes data to queue"); 166 | long start = System.currentTimeMillis(); 167 | for (int i = 0; i < 10000000; i++) { 168 | byte[] b = (string + i).getBytes(); 169 | queue.offer(b); 170 | } 171 | System.out.println("spend time:" + (System.currentTimeMillis() - start) + "ms"); 172 | System.out.println("Test read 10000000 times 10 bytes data from queue"); 173 | start = System.currentTimeMillis(); 174 | for (int i = 0; i < 10000000; i++) { 175 | byte[] b = queue.poll(); 176 | if (b == null) { 177 | i--; 178 | System.out.println("null" + i); 179 | continue; 180 | } 181 | } 182 | assertEquals(0, queue.size()); 183 | System.out.println("spend:" + (System.currentTimeMillis() - start) + "ms"); 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/test/java/com/google/code/fqueue/memcached/TestFqueueServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.code.fqueue.memcached; 18 | 19 | import java.io.File; 20 | import java.io.IOException; 21 | import java.util.concurrent.CountDownLatch; 22 | import java.util.concurrent.ExecutorService; 23 | import java.util.concurrent.Executors; 24 | import java.util.concurrent.TimeoutException; 25 | import java.util.concurrent.atomic.AtomicInteger; 26 | 27 | import junit.framework.TestCase; 28 | import net.rubyeye.xmemcached.MemcachedClient; 29 | import net.rubyeye.xmemcached.MemcachedClientBuilder; 30 | import net.rubyeye.xmemcached.XMemcachedClientBuilder; 31 | import net.rubyeye.xmemcached.exception.MemcachedException; 32 | import net.rubyeye.xmemcached.utils.AddrUtil; 33 | 34 | import org.apache.commons.logging.Log; 35 | import org.apache.commons.logging.LogFactory; 36 | import org.apache.log4j.PropertyConfigurator; 37 | 38 | import com.google.code.fqueue.util.Config; 39 | import com.google.code.yanf4j.core.impl.StandardSocketOption; 40 | 41 | /** 42 | * @author sunli 43 | */ 44 | public class TestFqueueServer extends TestCase { 45 | private final static Log log = LogFactory.getLog(TestFqueueServer.class); 46 | public static final AtomicInteger counter = new AtomicInteger(0); 47 | private static MemcachedClientBuilder builder = null; 48 | private static MemcachedClient client; 49 | private static String keyName = "key_abc"; 50 | static { 51 | PropertyConfigurator.configure("config/log4j.properties"); 52 | 53 | } 54 | 55 | public void deleteData() { 56 | File file = new File("dbtest"); 57 | file.delete(); 58 | } 59 | 60 | /** 61 | * @param name 62 | */ 63 | public TestFqueueServer(String name) { 64 | super(name); 65 | } 66 | 67 | /* 68 | * (non-Javadoc) 69 | * @see junit.framework.TestCase#setUp() 70 | */ 71 | protected void setUp() throws Exception { 72 | Config.setSetting("port", "12001"); 73 | Config.setSetting("path", "dbtest"); 74 | Config.setSetting("logsize", "40"); 75 | Config.setSetting("authorization", "key|abc@@bbs|pass"); 76 | StartNewQueue.newQueueInstance(Integer.parseInt(Config.getSetting("port"))); 77 | log.info("running at port " + Config.getSetting("port")); 78 | builder = new XMemcachedClientBuilder(AddrUtil.getAddresses("127.0.0.1:12001")); 79 | builder.setConnectionPoolSize(50); // set connection pool size to five 80 | 81 | try { 82 | client = builder.build(); 83 | client.setOptimizeGet(false); 84 | builder.setSocketOption(StandardSocketOption.SO_KEEPALIVE, true); 85 | builder.setSocketOption(StandardSocketOption.SO_RCVBUF, 64 * 1024); 86 | builder.setSocketOption(StandardSocketOption.SO_SNDBUF, 64 * 1024); 87 | builder.setSocketOption(StandardSocketOption.SO_REUSEADDR, true); 88 | builder.setSocketOption(StandardSocketOption.TCP_NODELAY, false); 89 | } catch (IOException e) { 90 | throw new RuntimeException(e); 91 | } 92 | client.get("clear|key|abc"); 93 | } 94 | 95 | /* 96 | * (non-Javadoc) 97 | * @see junit.framework.TestCase#tearDown() 98 | */ 99 | protected void tearDown() throws Exception { 100 | super.tearDown(); 101 | } 102 | 103 | private int getSize() throws TimeoutException, InterruptedException, MemcachedException { 104 | String value = client.get("size|key"); 105 | if (value != null) { 106 | return Integer.parseInt(value); 107 | } 108 | return -1; 109 | } 110 | 111 | public void authorization() throws TimeoutException, InterruptedException, IOException { 112 | try { 113 | client.get("xxxxxxxxxx"); 114 | } catch (MemcachedException e) { 115 | assertEquals("xxxxxxxxxx command Unsupported now", e.getMessage()); 116 | } 117 | try { 118 | client.get("xxxxxxxxxx_xxx"); 119 | } catch (MemcachedException e) { 120 | assertEquals("Authorization error", e.getMessage()); 121 | } 122 | } 123 | 124 | public void testOperation() throws InterruptedException, TimeoutException, MemcachedException, IOException { 125 | assertEquals(0, getSize()); 126 | client.set(keyName, 0, "12345"); 127 | assertEquals(1, getSize()); 128 | assertEquals("12345", client.get(keyName)); 129 | assertEquals(0, getSize()); 130 | client.set(keyName + "_" + System.currentTimeMillis(), 0, "12345"); 131 | assertEquals(1, getSize()); 132 | assertEquals("12345", client.get(keyName + "_" + System.currentTimeMillis())); 133 | log.info("push 10000 items"); 134 | long start = System.currentTimeMillis(); 135 | // 测试顺序写入10000个数据，再按顺序取出来，是否正确 136 | for (int i = 0; i < 10000; i++) { 137 | client.set(keyName, 0, "value_" + i); 138 | } 139 | log.info("push 10000 items :" + (System.currentTimeMillis() - start) + " ms"); 140 | assertEquals(10000, getSize()); 141 | log.info("poll 10000 items"); 142 | start = System.currentTimeMillis(); 143 | for (int i = 0; i < 10000; i++) { 144 | assertEquals("value_" + i, client.get(keyName)); 145 | } 146 | log.info("poll 10000 items :" + (System.currentTimeMillis() - start) + " ms"); 147 | assertEquals(0, getSize()); 148 | log.info("开始测试权限状态"); 149 | authorization(); 150 | log.info("开始测试多线程操作"); 151 | mutiThreadWrite(); 152 | mutiThreadGet(); 153 | assertEquals(0, getSize()); 154 | } 155 | 156 | public void mutiThreadWrite() throws InterruptedException, TimeoutException, MemcachedException { 157 | int threadCount = 8; 158 | ExecutorService pool = Executors.newFixedThreadPool(threadCount); 159 | CountDownLatch latch = new CountDownLatch(threadCount); 160 | // MemcachedBenchJob.test = tester; 161 | MemcachedTest[] muti = new MemcachedTest[threadCount]; 162 | for (int i = 0; i < threadCount; i++) { 163 | muti[i] = new MemcachedTest(latch); 164 | } 165 | log.info("start"); 166 | long start = System.currentTimeMillis(); 167 | for (int i = 0; i < threadCount; i++) { 168 | pool.execute(muti[i]); 169 | } 170 | latch.await(); 171 | long spend = System.currentTimeMillis() - start; 172 | 173 | log.info(threadCount + "threads写入次数:" + threadCount * 10000 + " spend:" + spend + " ms"); 174 | assertEquals(threadCount * 10000, getSize()); 175 | 176 | } 177 | 178 | public void mutiThreadGet() throws InterruptedException, TimeoutException, MemcachedException { 179 | int threadCount = 8; 180 | ExecutorService pool = Executors.newFixedThreadPool(threadCount); 181 | CountDownLatch latch = new CountDownLatch(threadCount); 182 | // MemcachedBenchJob.test = tester; 183 | MemcachedTestGet[] muti = new MemcachedTestGet[threadCount]; 184 | for (int i = 0; i < threadCount; i++) { 185 | muti[i] = new MemcachedTestGet(latch); 186 | } 187 | log.info("start"); 188 | long start = System.currentTimeMillis(); 189 | for (int i = 0; i < threadCount; i++) { 190 | pool.execute(muti[i]); 191 | } 192 | latch.await(); 193 | long spend = System.currentTimeMillis() - start; 194 | log.info(threadCount + "threads 获取次数:" + threadCount * 10000); 195 | assertEquals(0, getSize()); 196 | 197 | } 198 | 199 | public class MemcachedTest implements Runnable { 200 | 201 | public CountDownLatch latch; 202 | 203 | public MemcachedTest(CountDownLatch latch) { 204 | this.latch = latch; 205 | } 206 | 207 | /* 208 | * (non-Javadoc) 209 | * @see java.lang.Runnable#run() 210 | */ 211 | @Override 212 | public void run() { 213 | try { 214 | for (int i = 0; i < 10000; i++) { 215 | client.set(keyName, 0, String.valueOf(counter.incrementAndGet())); 216 | } 217 | latch.countDown(); 218 | } catch (TimeoutException e) { 219 | // TODO Auto-generated catch block 220 | e.printStackTrace(); 221 | } catch (InterruptedException e) { 222 | // TODO Auto-generated catch block 223 | e.printStackTrace(); 224 | } catch (MemcachedException e) { 225 | // TODO Auto-generated catch block 226 | e.printStackTrace(); 227 | } 228 | } 229 | 230 | } 231 | 232 | public class MemcachedTestGet implements Runnable { 233 | 234 | public CountDownLatch latch; 235 | 236 | public MemcachedTestGet(CountDownLatch latch) { 237 | this.latch = latch; 238 | } 239 | 240 | /* 241 | * (non-Javadoc) 242 | * @see java.lang.Runnable#run() 243 | */ 244 | @Override 245 | public void run() { 246 | try { 247 | for (int i = 0; i < 10000; i++) { 248 | client.get(keyName); 249 | } 250 | latch.countDown(); 251 | } catch (TimeoutException e) { 252 | // TODO Auto-generated catch block 253 | e.printStackTrace(); 254 | } catch (InterruptedException e) { 255 | // TODO Auto-generated catch block 256 | e.printStackTrace(); 257 | } catch (MemcachedException e) { 258 | // TODO Auto-generated catch block 259 | e.printStackTrace(); 260 | } 261 | } 262 | 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /src/test/java/com/google/code/fqueue/util/TestJVMMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 sunli [sunli1223@gmail.com][weibo.com@sunli1223] 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.google.code.fqueue.util; 17 | 18 | import org.apache.commons.lang.StringUtils; 19 | 20 | import junit.framework.TestCase; 21 | 22 | /** 23 | * @author sunli 24 | */ 25 | public class TestJVMMonitor extends TestCase { 26 | 27 | /** 28 | * @param name 29 | */ 30 | public TestJVMMonitor(String name) { 31 | super(name); 32 | } 33 | 34 | /* 35 | * (non-Javadoc) 36 | * @see junit.framework.TestCase#setUp() 37 | */ 38 | protected void setUp() throws Exception { 39 | super.setUp(); 40 | } 41 | 42 | /* 43 | * (non-Javadoc) 44 | * @see junit.framework.TestCase#tearDown() 45 | */ 46 | protected void tearDown() throws Exception { 47 | super.tearDown(); 48 | } 49 | 50 | public void testMonitor() { 51 | String items = "fileDescriptor,load,allThreadsCount,peakThreadCount,daemonThreadCount,totalStartedThreadCount,deadLockCount,heapMemory,noHeapMemory,memory,classCount,GCTime,memoryPoolCollectionUsage,memoryPoolUsage,memoryPoolPeakUsage"; 52 | // String items = "memoryPoolPeakUsage"; 53 | long start = System.currentTimeMillis(); 54 | if (items != null) { 55 | String[] itemList = StringUtils.split(items, ","); 56 | for (int i = 0, len = itemList.length; i < len; i++) { 57 | String data = JVMMonitor.getMonitorStats(itemList[i]); 58 | System.out.println(data); 59 | } 60 | } 61 | System.out.println("获取JVM监控信息耗时：" + (System.currentTimeMillis() - start)); 62 | } 63 | } 64 | --------------------------------------------------------------------------------