├── doc ├── CVS │ ├── Root │ ├── Repository │ └── Entries ├── PORTABILITY.txt ├── README.txt ├── TODO.txt ├── LICENSE.txt ├── HOWTO.txt └── CHANGELOG.txt ├── lib └── log4j.jar ├── .gitignore └── src └── com └── meetup └── memcached ├── LineInputStream.java ├── NestedIOException.java ├── ContextObjectInputStream.java ├── test ├── TestMemcached.java ├── MemcachedBench.java ├── MemcachedTest.java └── UnitTests.java ├── ErrorHandler.java ├── ByteBufArrayInputStream.java ├── Logger.java ├── NativeHandler.java ├── SockIOPool.java └── MemcachedClient.java /doc/CVS/Root: -------------------------------------------------------------------------------- 1 | :ext:cvs.dev.meetup.com:/usr/local/cvs 2 | -------------------------------------------------------------------------------- /doc/CVS/Repository: -------------------------------------------------------------------------------- 1 | chapstick_java/src/com/danga/MemCached/doc 2 | -------------------------------------------------------------------------------- /lib/log4j.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gwhalin/Memcached-Java-Client/HEAD/lib/log4j.jar -------------------------------------------------------------------------------- /doc/PORTABILITY.txt: -------------------------------------------------------------------------------- 1 | This file lists the portability status of this client. Please send me any 2 | additional information (greg@meetup.com). 3 | 4 | The newest release has only been tested on: 5 | 6 | Sun's JRE 5 and up on Linux/x86/amd64 7 | -------------------------------------------------------------------------------- /doc/CVS/Entries: -------------------------------------------------------------------------------- 1 | /LICENSE.txt/1.1/Mon Nov 29 18:11:59 2004// 2 | /PORTABILITY.txt/1.2/Wed Aug 4 16:28:46 2004// 3 | /README.txt/1.2/Wed Aug 4 16:28:46 2004// 4 | /TODO.txt/1.5/Mon Jan 17 19:03:01 2005// 5 | /CHANGELOG.txt/1.13/Sun Feb 6 17:22:59 2005// 6 | D 7 | -------------------------------------------------------------------------------- /doc/README.txt: -------------------------------------------------------------------------------- 1 | WARNING! 2 | 3 | This version is not compatible with versions prior to version 0.9.2 4 | Please see javadocs for examples on how to use. 5 | 6 | This release required Java 5 in order to work. If you would like to downport to 1.4 7 | please feel free. 8 | -------------------------------------------------------------------------------- /doc/TODO.txt: -------------------------------------------------------------------------------- 1 | To Do: 2 | 3 | - NIO support for write operations 4 | - removeMulti support 5 | - binary protocol support 6 | - support for new add methods (cas,append,prepend) 7 | - clean up Logging code (auto-detect log4j/logging/etc) 8 | - clean up and add more unit/performance tests 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # git-ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | *~ 6 | classes/ 7 | *.swp 8 | __db.* 9 | dist/ 10 | *.class 11 | scratch/ 12 | .cachedir/ 13 | memcached-release* 14 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/LineInputStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author greg whalin 17 | */ 18 | package com.meetup.memcached; 19 | 20 | import java.io.IOException; 21 | 22 | public interface LineInputStream { 23 | 24 | /** 25 | * Read everything up to the next end-of-line. Does 26 | * not include the end of line, though it is consumed 27 | * from the input. 28 | * @return All next up to the next end of line. 29 | */ 30 | public String readLine() throws IOException; 31 | 32 | /** 33 | * Read everything up to and including the end of line. 34 | */ 35 | public void clearEOL() throws IOException; 36 | 37 | /** 38 | * Read some bytes. 39 | * @param buf The buffer into which read. 40 | * @return The number of bytes actually read, or -1 if none could be read. 41 | */ 42 | public int read( byte[] buf ) throws IOException; 43 | } 44 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/NestedIOException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author Kevin A. Burton 17 | */ 18 | package com.meetup.memcached; 19 | 20 | import java.io.*; 21 | 22 | /** 23 | * Bridge class to provide nested Exceptions with IOException which has 24 | * constructors that don't take Throwables. 25 | * 26 | * @author Kevin Burton 27 | * @version 1.2 28 | */ 29 | public class NestedIOException extends IOException { 30 | 31 | /** 32 | * Create a new NestedIOException instance. 33 | * @param cause object of type throwable 34 | */ 35 | public NestedIOException( Throwable cause ) { 36 | super( cause.getMessage() ); 37 | super.initCause( cause ); 38 | } 39 | 40 | public NestedIOException( String message, Throwable cause ) { 41 | super( message ); 42 | initCause( cause ); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/ContextObjectInputStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * Adds the ability for the MemCached client to be initialized 17 | * with a custom class loader. This will allow for the 18 | * deserialization of classes that are not visible to the system 19 | * class loader. 20 | * 21 | * @author Vin Chawla 22 | */ 23 | package com.meetup.memcached; 24 | 25 | import java.util.*; 26 | import java.util.zip.*; 27 | import java.io.*; 28 | 29 | public class ContextObjectInputStream extends ObjectInputStream { 30 | 31 | ClassLoader mLoader; 32 | 33 | public ContextObjectInputStream( InputStream in, ClassLoader loader ) throws IOException, SecurityException { 34 | super( in ); 35 | mLoader = loader; 36 | } 37 | 38 | protected Class resolveClass( ObjectStreamClass v ) throws IOException, ClassNotFoundException { 39 | if ( mLoader == null ) 40 | return super.resolveClass( v ); 41 | else 42 | return Class.forName( v.getName(), true, mLoader ); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /doc/LICENSE.txt: -------------------------------------------------------------------------------- 1 | * Copyright (c) 2007, Greg Whain 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * * Redistributions of source code must retain the above copyright 7 | * notice, this list of conditions and the following disclaimer. 8 | * * Redistributions in binary form must reproduce the above copyright 9 | * notice, this list of conditions and the following disclaimer in the 10 | * documentation and/or other materials provided with the distribution. 11 | * * Neither the name of the nor the 12 | * names of its contributors may be used to endorse or promote products 13 | * derived from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY Greg Whalin ``AS IS'' AND ANY 16 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | * DISCLAIMED. IN NO EVENT SHALL Greg Whalin BE LIABLE FOR ANY 19 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/test/TestMemcached.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author greg whalin 17 | */ 18 | package com.meetup.memcached.test; 19 | 20 | import com.meetup.memcached.*; 21 | import org.apache.log4j.*; 22 | 23 | public class TestMemcached { 24 | public static void main(String[] args) { 25 | // memcached should be running on port 11211 but NOT on 11212 26 | 27 | BasicConfigurator.configure(); 28 | String[] servers = { "192.168.1.1:1624", "192.168.1.1:1625" }; 29 | SockIOPool pool = SockIOPool.getInstance(); 30 | pool.setServers( servers ); 31 | pool.setFailover( true ); 32 | pool.setInitConn( 10 ); 33 | pool.setMinConn( 5 ); 34 | pool.setMaxConn( 250 ); 35 | pool.setMaintSleep( 30 ); 36 | pool.setNagle( false ); 37 | pool.setSocketTO( 3000 ); 38 | pool.setAliveCheck( true ); 39 | pool.initialize(); 40 | 41 | MemcachedClient mcc = new MemcachedClient(); 42 | 43 | // turn off most memcached client logging: 44 | com.meetup.memcached.Logger.getLogger( MemcachedClient.class.getName() ).setLevel( com.meetup.memcached.Logger.LEVEL_WARN ); 45 | 46 | for ( int i = 0; i < 10; i++ ) { 47 | boolean success = mcc.set( "" + i, "Hello!" ); 48 | String result = (String)mcc.get( "" + i ); 49 | System.out.println( String.format( "set( %d ): %s", i, success ) ); 50 | System.out.println( String.format( "get( %d ): %s", i, result ) ); 51 | } 52 | 53 | System.out.println( "\n\t -- sleeping --\n" ); 54 | try { Thread.sleep( 10000 ); } catch ( Exception ex ) { } 55 | 56 | for ( int i = 0; i < 10; i++ ) { 57 | boolean success = mcc.set( "" + i, "Hello!" ); 58 | String result = (String)mcc.get( "" + i ); 59 | System.out.println( String.format( "set( %d ): %s", i, success ) ); 60 | System.out.println( String.format( "get( %d ): %s", i, result ) ); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/ErrorHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * This is an interface implemented by classes that want to receive callbacks 17 | * in the event of an error in {@link MemcachedClient}. The implementor can do 18 | * things like flush caches or perform additioonal logging. 19 | * 20 | * @author Dan Zivkovic 21 | */ 22 | package com.meetup.memcached; 23 | 24 | public interface ErrorHandler { 25 | 26 | /** 27 | * Called for errors thrown during initialization. 28 | */ 29 | public void handleErrorOnInit( final MemcachedClient client , 30 | final Throwable error ); 31 | 32 | /** 33 | * Called for errors thrown during {@link MemcachedClient#get(String)} and related methods. 34 | */ 35 | public void handleErrorOnGet( final MemcachedClient client , 36 | final Throwable error , 37 | final String cacheKey ); 38 | 39 | /** 40 | * Called for errors thrown during {@link MemcachedClient#getMulti(String)} and related methods. 41 | */ 42 | public void handleErrorOnGet( final MemcachedClient client , 43 | final Throwable error , 44 | final String[] cacheKeys ); 45 | 46 | /** 47 | * Called for errors thrown during {@link MemcachedClient#set(String,Object)} and related methods. 48 | */ 49 | public void handleErrorOnSet( final MemcachedClient client , 50 | final Throwable error , 51 | final String cacheKey ); 52 | 53 | /** 54 | * Called for errors thrown during {@link MemcachedClient#delete(String)} and related methods. 55 | */ 56 | public void handleErrorOnDelete( final MemcachedClient client , 57 | final Throwable error , 58 | final String cacheKey ); 59 | 60 | /** 61 | * Called for errors thrown during {@link MemcachedClient#flushAll()} and related methods. 62 | */ 63 | public void handleErrorOnFlush( final MemcachedClient client , 64 | final Throwable error ); 65 | 66 | /** 67 | * Called for errors thrown during {@link MemcachedClient#stats()} and related methods. 68 | */ 69 | public void handleErrorOnStats( final MemcachedClient client , 70 | final Throwable error ); 71 | 72 | } // interface 73 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/test/MemcachedBench.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author Greg Whalin 17 | */ 18 | package com.meetup.memcached.test; 19 | 20 | import com.meetup.memcached.*; 21 | import java.util.*; 22 | import org.apache.log4j.Level; 23 | import org.apache.log4j.Logger; 24 | import org.apache.log4j.BasicConfigurator; 25 | 26 | public class MemcachedBench { 27 | 28 | // logger 29 | private static Logger log = 30 | Logger.getLogger( MemcachedBench.class.getName() ); 31 | 32 | public static void main(String[] args) { 33 | 34 | BasicConfigurator.configure(); 35 | org.apache.log4j.Logger.getRootLogger().setLevel( Level.OFF ); 36 | 37 | int runs = Integer.parseInt(args[0]); 38 | int start = Integer.parseInt(args[1]); 39 | 40 | String[] serverlist = { "192.168.1.50:1624" }; 41 | 42 | // initialize the pool for memcache servers 43 | SockIOPool pool = SockIOPool.getInstance( "test" ); 44 | pool.setServers(serverlist); 45 | 46 | pool.setInitConn( 100 ); 47 | pool.setMinConn( 100 ); 48 | pool.setMaxConn( 500 ); 49 | pool.setMaintSleep( 20 ); 50 | 51 | pool.setNagle( false ); 52 | pool.initialize(); 53 | 54 | // get client instance 55 | MemcachedClient mc = new MemcachedClient( "test" ); 56 | mc.setCompressEnable( false ); 57 | 58 | String keyBase = "testKey"; 59 | String object = "This is a test of an object blah blah es, serialization does not seem to slow things down so much. The gzip compression is horrible horrible performance, so we only use it for very large objects. I have not done any heavy benchmarking recently"; 60 | 61 | long begin = System.currentTimeMillis(); 62 | for (int i = start; i < start+runs; i++) { 63 | mc.set(keyBase + i, object); 64 | } 65 | long end = System.currentTimeMillis(); 66 | long time = end - begin; 67 | System.out.println(runs + " sets: " + time + "ms"); 68 | 69 | begin = System.currentTimeMillis(); 70 | for (int i = start; i < start+runs; i++) { 71 | String str = (String) mc.get(keyBase + i); 72 | } 73 | end = System.currentTimeMillis(); 74 | time = end - begin; 75 | System.out.println(runs + " gets: " + time + "ms"); 76 | 77 | String[] keys = new String[ runs ]; 78 | int j = 0; 79 | for (int i = start; i < start+runs; i++) { 80 | keys[ j ] = keyBase + i; 81 | j++; 82 | } 83 | begin = System.currentTimeMillis(); 84 | Map vals = mc.getMulti( keys ); 85 | end = System.currentTimeMillis(); 86 | time = end - begin; 87 | System.out.println(runs + " getMulti: " + time + "ms"); 88 | 89 | begin = System.currentTimeMillis(); 90 | for (int i = start; i < start+runs; i++) { 91 | mc.delete( keyBase + i ); 92 | } 93 | end = System.currentTimeMillis(); 94 | time = end - begin; 95 | System.out.println(runs + " deletes: " + time + "ms"); 96 | 97 | SockIOPool.getInstance( "test" ).shutDown(); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/ByteBufArrayInputStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author greg whalin 17 | */ 18 | package com.meetup.memcached; 19 | 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | import java.nio.ByteBuffer; 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | public final class ByteBufArrayInputStream extends InputStream implements LineInputStream { 28 | private ByteBuffer[] bufs; 29 | private int currentBuf = 0; 30 | 31 | public ByteBufArrayInputStream( List bufs ) throws Exception { 32 | this( bufs.toArray( new ByteBuffer[] {} ) ); 33 | } 34 | 35 | public ByteBufArrayInputStream( ByteBuffer[] bufs ) throws Exception { 36 | if ( bufs == null || bufs.length == 0 ) 37 | throw new Exception( "buffer is empty" ); 38 | 39 | this.bufs = bufs; 40 | for ( ByteBuffer b : bufs ) 41 | b.flip(); 42 | } 43 | 44 | public int read() { 45 | do { 46 | if ( bufs[currentBuf].hasRemaining() ) 47 | return bufs[currentBuf].get(); 48 | currentBuf++; 49 | } 50 | while ( currentBuf < bufs.length ); 51 | 52 | currentBuf--; 53 | return -1; 54 | } 55 | 56 | public int read( byte[] buf ) { 57 | int len = buf.length; 58 | int bufPos = 0; 59 | do { 60 | if ( bufs[currentBuf].hasRemaining() ) { 61 | int n = Math.min( bufs[currentBuf].remaining(), len-bufPos ); 62 | bufs[currentBuf].get( buf, bufPos, n ); 63 | bufPos += n; 64 | } 65 | currentBuf++; 66 | } 67 | while ( currentBuf < bufs.length && bufPos < len ); 68 | 69 | currentBuf--; 70 | 71 | if ( bufPos > 0 || ( bufPos == 0 && len == 0 ) ) 72 | return bufPos; 73 | else 74 | return -1; 75 | } 76 | 77 | public String readLine() throws IOException { 78 | byte[] b = new byte[1]; 79 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 80 | boolean eol = false; 81 | 82 | while ( read( b, 0, 1 ) != -1 ) { 83 | if ( b[0] == 13 ) { 84 | eol = true; 85 | } 86 | else { 87 | if ( eol ) { 88 | if ( b[0] == 10 ) 89 | break; 90 | eol = false; 91 | } 92 | } 93 | 94 | // cast byte into char array 95 | bos.write( b, 0, 1 ); 96 | } 97 | 98 | if ( bos == null || bos.size() <= 0 ) { 99 | throw new IOException( "++++ Stream appears to be dead, so closing it down" ); 100 | } 101 | 102 | // else return the string 103 | return bos.toString().trim(); 104 | } 105 | 106 | public void clearEOL() throws IOException { 107 | byte[] b = new byte[1]; 108 | boolean eol = false; 109 | while ( read( b, 0, 1 ) != -1 ) { 110 | 111 | // only stop when we see 112 | // \r (13) followed by \n (10) 113 | if ( b[0] == 13 ) { 114 | eol = true; 115 | continue; 116 | } 117 | 118 | if ( eol ) { 119 | if ( b[0] == 10 ) 120 | break; 121 | eol = false; 122 | } 123 | } 124 | } 125 | 126 | public String toString() { 127 | StringBuilder sb = new StringBuilder( "ByteBufArrayIS: " ); 128 | sb.append( bufs.length ).append( " bufs of sizes: \n" ); 129 | 130 | for ( int i=0; i < bufs.length; i++ ) { 131 | sb.append( " " ) 132 | .append (i ).append( ": " ).append( bufs[i] ).append( "\n" ); 133 | } 134 | return sb.toString(); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /doc/HOWTO.txt: -------------------------------------------------------------------------------- 1 | Howto 2 | ===== 3 | 4 | Basic Example: 5 | ============== 6 | 7 | Lets say you have 3 servers. Server 1 and server 2 have 3GB of space 8 | and server 3 has 2GB of space for cache. Here is how I would set up 9 | my client. 10 | 11 | import com.danga.MemCached.*; 12 | public class MyClass { 13 | 14 | // create a static client as most installs only need 15 | // a single instance 16 | protected static MemCachedClient mcc = new MemCachedClient(); 17 | 18 | // set up connection pool once at class load 19 | static { 20 | 21 | // server list and weights 22 | String[] servers = 23 | { 24 | "server1.mydomain.com:1624", 25 | "server2.mydomain.com:1624", 26 | "server3.mydomain.com:1624" 27 | }; 28 | 29 | Integer[] weights = { 3, 3, 2 }; 30 | 31 | // grab an instance of our connection pool 32 | SockIOPool pool = SockIOPool.getInstance(); 33 | 34 | // set the servers and the weights 35 | pool.setServers( servers ); 36 | pool.setWeights( weights ); 37 | 38 | // set some basic pool settings 39 | // 5 initial, 5 min, and 250 max conns 40 | // and set the max idle time for a conn 41 | // to 6 hours 42 | pool.setInitConn( 5 ); 43 | pool.setMinConn( 5 ); 44 | pool.setMaxConn( 250 ); 45 | pool.setMaxIdle( 1000 * 60 * 60 * 6 ); 46 | 47 | // set the sleep for the maint thread 48 | // it will wake up every x seconds and 49 | // maintain the pool size 50 | pool.setMaintSleep( 30 ); 51 | 52 | // set some TCP settings 53 | // disable nagle 54 | // set the read timeout to 3 secs 55 | // and don't set a connect timeout 56 | pool.setNagle( false ); 57 | pool.setSocketTO( 3000 ); 58 | pool.setSocketConnectTO( 0 ); 59 | 60 | // initialize the connection pool 61 | pool.initialize(); 62 | 63 | 64 | // lets set some compression on for the client 65 | // compress anything larger than 64k 66 | mcc.setCompressEnable( true ); 67 | mcc.setCompressThreshold( 64 * 1024 ); 68 | } 69 | 70 | // from here on down, you can call any of the client calls 71 | public static void examples() { 72 | mcc.set( "foo", "This is a test String" ); 73 | String bar = mcc.get( "foo" ); 74 | } 75 | } 76 | 77 | Multi-client Example: 78 | ===================== 79 | 80 | If you need to support multiple clients (i.e. Java, PHP, Perl, etc.) 81 | you need to make a few changes when you are setting things up: 82 | 83 | // use a compatible hashing algorithm 84 | pool.setHashingAlg( SockIOPool.NEW_COMPAT_HASH ); 85 | 86 | // store primitives as strings 87 | // the java client serializes primitives 88 | // 89 | // note: this will not help you when it comes to 90 | // storing non primitives 91 | mcc.setPrimitiveAsString( true ); 92 | 93 | // don't url encode keys 94 | // by default the java client url encodes keys 95 | // to sanitize them so they will always work on the server 96 | // however, other clients do not do this 97 | mcc.setSanitizeKeys( false ); 98 | 99 | 100 | Failover/Failback Notes: 101 | ======================== 102 | 103 | By default the java client will failover to a new server when a server 104 | dies. It will also failback to the original if it detects that the 105 | server comes back (it checks the server in a falling off pattern). 106 | 107 | If you want to disable this (useful if you have flapping servers), 108 | there are two settings to handle this. 109 | 110 | pool.setFailover( false ); 111 | pool.setFailback( false ); 112 | 113 | 114 | Serialization: 115 | ============== 116 | 117 | For java "native types", which include: 118 | 119 | Boolean 120 | Byte 121 | String 122 | Character 123 | StringBuffer 124 | StringBuilder 125 | Short 126 | Long 127 | Double 128 | Float 129 | Date 130 | Integer 131 | 132 | The client will by default *NOT* use java serialization, and instead 133 | will serialize using the primitive values to save space. You can 134 | override this by using the mcc.setPrimitiveAsString( true ), which 135 | will use the toString representation of the object. 136 | 137 | For other java objects, you need to make sure the class implements 138 | Serializable in order to be able to be stored in the cache. 139 | 140 | I would also reccomend that if possible, classes should instead 141 | implement Externalizable as opposed to Serializable. This allows the 142 | author of the class to define how objects of that class should 143 | serialize. In practice at Meetup.com, we saw a 60% reduction in the size 144 | of our serialized objects by doing this. This means less data to eat up 145 | cache space and less data to transfer over the network. 146 | 147 | Other: 148 | ====== 149 | See the java docs. 150 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/test/MemcachedTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author Greg Whalin 17 | */ 18 | package com.meetup.memcached.test; 19 | 20 | import com.meetup.memcached.*; 21 | import java.util.*; 22 | 23 | public class MemcachedTest { 24 | 25 | // store results from threads 26 | private static Hashtable threadInfo = 27 | new Hashtable(); 28 | 29 | /** 30 | * This runs through some simple tests of the MemcacheClient. 31 | * 32 | * Command line args: 33 | * args[0] = number of threads to spawn 34 | * args[1] = number of runs per thread 35 | * args[2] = size of object to store 36 | * 37 | * @param args the command line arguments 38 | */ 39 | public static void main(String[] args) { 40 | 41 | String[] serverlist = { "cache1.int.meetup.com:12345", "cache0.int.meetup.com:12345" }; 42 | 43 | // initialize the pool for memcache servers 44 | SockIOPool pool = SockIOPool.getInstance(); 45 | pool.setServers( serverlist ); 46 | 47 | pool.setInitConn(5); 48 | pool.setMinConn(5); 49 | pool.setMaxConn(50); 50 | pool.setMaintSleep(30); 51 | 52 | pool.setNagle(false); 53 | pool.initialize(); 54 | 55 | int threads = Integer.parseInt(args[0]); 56 | int runs = Integer.parseInt(args[1]); 57 | int size = 1024 * Integer.parseInt(args[2]); // how many kilobytes 58 | 59 | // get object to store 60 | int[] obj = new int[size]; 61 | for (int i = 0; i < size; i++) { 62 | obj[i] = i; 63 | } 64 | 65 | String[] keys = new String[size]; 66 | for (int i = 0; i < size; i++) { 67 | keys[i] = "test_key" + i; 68 | } 69 | 70 | for (int i = 0; i < threads; i++) { 71 | bench b = new bench(runs, i, obj, keys); 72 | b.start(); 73 | } 74 | 75 | int i = 0; 76 | while (i < threads) { 77 | if (threadInfo.containsKey(new Integer(i))) { 78 | System.out.println( threadInfo.get( new Integer( i ) ) ); 79 | i++; 80 | } 81 | else { 82 | try { 83 | Thread.sleep(1000); 84 | } 85 | catch (InterruptedException e) { 86 | e.printStackTrace(); 87 | } 88 | } 89 | } 90 | 91 | pool.shutDown(); 92 | System.exit(1); 93 | } 94 | 95 | /** 96 | * Test code per thread. 97 | */ 98 | private static class bench extends Thread { 99 | private int runs; 100 | private int threadNum; 101 | private int[] object; 102 | private String[] keys; 103 | private int size; 104 | 105 | public bench(int runs, int threadNum, int[] object, String[] keys) { 106 | this.runs = runs; 107 | this.threadNum = threadNum; 108 | this.object = object; 109 | this.keys = keys; 110 | this.size = object.length; 111 | } 112 | 113 | public void run() { 114 | 115 | StringBuilder result = new StringBuilder(); 116 | 117 | // get client instance 118 | MemcachedClient mc = new MemcachedClient(); 119 | mc.setCompressEnable(false); 120 | mc.setCompressThreshold(0); 121 | 122 | // time deletes 123 | long start = System.currentTimeMillis(); 124 | for (int i = 0; i < runs; i++) { 125 | mc.delete(keys[i]); 126 | } 127 | long elapse = System.currentTimeMillis() - start; 128 | float avg = (float) elapse / runs; 129 | result.append("\nthread " + threadNum + ": runs: " + runs + " deletes of obj " + (size/1024) + "KB -- avg time per req " + avg + " ms (total: " + elapse + " ms)"); 130 | 131 | // time stores 132 | start = System.currentTimeMillis(); 133 | for (int i = 0; i < runs; i++) { 134 | mc.set(keys[i], object); 135 | } 136 | elapse = System.currentTimeMillis() - start; 137 | avg = (float) elapse / runs; 138 | result.append("\nthread " + threadNum + ": runs: " + runs + " stores of obj " + (size/1024) + "KB -- avg time per req " + avg + " ms (total: " + elapse + " ms)"); 139 | 140 | start = System.currentTimeMillis(); 141 | for (int i = 0; i < runs; i++) { 142 | mc.get(keys[i]); 143 | } 144 | elapse = System.currentTimeMillis() - start; 145 | avg = (float) elapse / runs; 146 | result.append("\nthread " + threadNum + ": runs: " + runs + " gets of obj " + (size/1024) + "KB -- avg time per req " + avg + " ms (total: " + elapse + " ms)"); 147 | 148 | threadInfo.put(new Integer(threadNum), result); 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/Logger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author Greg Whalin 17 | */ 18 | package com.meetup.memcached; 19 | 20 | import java.util.*; 21 | 22 | /** 23 | * This is a generic logger class for use in logging. 24 | * 25 | * This can easily be swapped out for any other logging package in the main code. 26 | * For now, this is just a quick and dirty logger which will allow you to specify 27 | * log levels, but only wraps system.out.println. 28 | * 29 | * @author Greg Whalin 30 | * @version 1.5 31 | */ 32 | public class Logger { 33 | 34 | public static final int LEVEL_DEBUG = 0; 35 | public static final int LEVEL_INFO = 1; 36 | public static final int LEVEL_WARN = 2; 37 | public static final int LEVEL_ERROR = 3; 38 | public static final int LEVEL_FATAL = 4; 39 | 40 | private static Map loggers = 41 | new HashMap(); 42 | 43 | private String name; 44 | private int level; 45 | private boolean initialized = false; 46 | 47 | public void setLevel( int level ) { this.level = level; } 48 | public int getLevel() { return this.level; } 49 | 50 | protected Logger( String name, int level ) { 51 | this.name = name; 52 | this.level = level; 53 | this.initialized = true; 54 | } 55 | 56 | protected Logger( String name ) { 57 | this.name = name; 58 | this.level = LEVEL_INFO; 59 | this.initialized = true; 60 | } 61 | 62 | /** 63 | * Gets a Logger obj for given name and level. 64 | * 65 | * @param name 66 | * @param level 67 | * @return 68 | */ 69 | public static synchronized Logger getLogger( String name, int level ) { 70 | Logger log = getLogger( name ); 71 | if ( log.getLevel() != level ) 72 | log.setLevel( level ); 73 | 74 | return log; 75 | } 76 | 77 | /** 78 | * Gets a Logger obj for given name 79 | * and sets default level. 80 | * 81 | * @param name 82 | * @return 83 | */ 84 | public static synchronized Logger getLogger( String name ) { 85 | 86 | Logger log = null; 87 | if ( loggers.containsKey( name ) ) { 88 | log = loggers.get( name ); 89 | } 90 | else { 91 | log = new Logger( name ); 92 | loggers.put( name, log ); 93 | } 94 | 95 | return log; 96 | } 97 | 98 | /** 99 | * logs mesg to std out and prints stack trace if exception passed in 100 | * 101 | * @param mesg 102 | * @param ex 103 | */ 104 | private void log( String mesg, Throwable ex ) { 105 | System.out.println( name + " " + new Date() + " - " + mesg ); 106 | if ( ex != null ) 107 | ex.printStackTrace( System.out ); 108 | } 109 | 110 | /** 111 | * logs a debug mesg 112 | * 113 | * @param mesg 114 | * @param ex 115 | */ 116 | public void debug( String mesg, Throwable ex ) { 117 | if ( this.level > LEVEL_DEBUG ) 118 | return; 119 | 120 | log( mesg, ex ); 121 | } 122 | 123 | public void debug( String mesg ) { 124 | debug( mesg, null ); 125 | } 126 | 127 | public boolean isDebugEnabled() { 128 | return this.level <= LEVEL_DEBUG; 129 | } 130 | 131 | /** 132 | * logs info mesg 133 | * 134 | * @param mesg 135 | * @param ex 136 | */ 137 | public void info( String mesg, Throwable ex ) { 138 | if ( this.level > LEVEL_INFO ) 139 | return; 140 | 141 | log( mesg, ex ); 142 | } 143 | 144 | public void info( String mesg ) { 145 | info( mesg, null ); 146 | } 147 | 148 | public boolean isInfoEnabled() { 149 | return this.level <= LEVEL_INFO; 150 | } 151 | 152 | /** 153 | * logs warn mesg 154 | * 155 | * @param mesg 156 | * @param ex 157 | */ 158 | public void warn( String mesg, Throwable ex ) { 159 | if ( this.level > LEVEL_WARN ) 160 | return; 161 | 162 | log( mesg, ex ); 163 | } 164 | 165 | public void warn( String mesg ) { 166 | warn( mesg, null ); 167 | } 168 | 169 | /** 170 | * logs error mesg 171 | * 172 | * @param mesg 173 | * @param ex 174 | */ 175 | public void error( String mesg, Throwable ex ) { 176 | if ( this.level > LEVEL_ERROR ) 177 | return; 178 | 179 | log( mesg, ex ); 180 | } 181 | 182 | public void error( String mesg ) { 183 | error( mesg, null ); 184 | } 185 | 186 | /** 187 | * logs fatal mesg 188 | * 189 | * @param mesg 190 | * @param ex 191 | */ 192 | public void fatal( String mesg, Throwable ex ) { 193 | if ( this.level > LEVEL_FATAL ) 194 | return; 195 | 196 | log( mesg, ex ); 197 | } 198 | 199 | public void fatal( String mesg ) { 200 | fatal( mesg, null ); 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /doc/CHANGELOG.txt: -------------------------------------------------------------------------------- 1 | 28 Dec 2008 (gwhalin,pinkerton) 2 | -- released as 2.0 3 | -- much reduced lock contention 4 | -- NIO getMulti support 5 | -- NIO get support 6 | -- consistant hashing algorithm support 7 | -- much improved support for serialization of primitives 8 | -- extensive unit tests added 9 | -- many bugs fixed 10 | 11 | 08 Jul 2007 (gwhalin) 12 | -- add support for ErrorHandler hook (submitted by Dan Zivkovic @ Apple) 13 | -- ability to disable URLEncoding keys 14 | 15 | 08 Apr 2007 (gwhalin) 16 | -- added addOrIncr/addOrDecr methods 17 | -- internal map changed to avoid possible key collision in internal data 18 | structures 19 | 20 | 28 Feb 2007 (gwhalin) 21 | -- released as 1.5.1 22 | -- fix rehashing bug when all servers are dead 23 | -- fix javadoc generation problem 24 | 25 | 16 Dec 2006 (whalin) 26 | -- released as 1.5 27 | -- urlencode keys 28 | 29 | 21 Nov 2006 (whalin) 30 | -- some more fixes for the failover code 31 | -- add support for optional healthcheck on checkout (disabled by default) 32 | 33 | 10 Sep 2006 (whalin) 34 | -- some changes to lessen lock contention 35 | -- fixed a long lasting bug w/ rehashing for server failover 36 | -- add a failback flag to allow failover but no failback 37 | 38 | 08 Feb 2006 (whalin) 39 | -- switched to using java 5 syntax (generics,stringbuilder) 40 | 41 | 25 Nov 2005 (whalin, chawla, watts) 42 | -- released as 1.3.2 43 | -- allow passing of classloader for use in deserialization (chawla) 44 | -- additional param checking (bug reported by young) 45 | -- additional stats methods (watts) 46 | 47 | 09 Sep 2005 (whalin) 48 | -- fix keyExists for counters 49 | -- stop using jikes 50 | -- add code to pool to try to recover from a hung connection 51 | (should never happen) 52 | 53 | 07 Apr 2005 (whalin) 54 | -- fixed getCounter 55 | -- added method to check for existance of key 56 | -- update to unit tests 57 | -- support multiple pools in SockIOPool 58 | -- fix failover rehashing to match perl algorithm 59 | 60 | 20 Mar 2005 (whalin) 61 | -- released as version 1.2.1 62 | -- make non-blocking connect optional 63 | -- better ant scripts for dev work and packaging 64 | -- moved into dedicated subversion repo 65 | 66 | 19 Jan 2005 67 | -- bugfix to remove tmp array copy in String deoding (Popescu) 68 | 69 | 10 Jan 2005 70 | -- released as version 1.2 71 | -- spawn thread for initial socket connection so we can enforce a shorter 72 | timeout on connections (whalin) 73 | 74 | 03 Jan 2005 75 | -- add custom serialization for Date objects 76 | for greater space savings (whalin) 77 | 78 | 13 Dec 2004 79 | -- custom serialize primitive objects for greater 80 | space savings (burton) 81 | 82 | 12 Dec 2004 83 | -- set maint thread as a daemon thread (k. burton) 84 | 85 | 29 Nov 2004 86 | -- released as version 1.1 87 | -- switched to LGPL from GPL (whalin) 88 | 89 | 18 Sept 2004 90 | --- released as version 1.0 91 | -- built generic logging class so can remove dependancy on other loggers 92 | (still easy for client to use their own logger) (whalin) 93 | -- fixed bad case for maxcreate when poolmultiplier was greater than 94 | minConn (defaults to minConn now) (thanks to Daryn @ Spam Arrest) 95 | -- fixed bad loop logic when creating new connections when conns < 96 | minConn. (thanks to Daryn @ Spam Arrest) 97 | -- retagged as a release 1.0 98 | 99 | 27 July 2004 100 | -- released as version 0.9.8a 101 | -- change default hashing to native as it is faster (whalin) 102 | 103 | 26 July 2004 104 | -- released as version 0.9.8 105 | -- added flushAll method, which allows invalidating the entire cache on 106 | all servers (or subset). (whalin) 107 | -- added stats methods to pull back statistics from the caching servers 108 | (or subset). Data is pulled back in data structure for end user to 109 | format. (whalin) 110 | -- fixup to counter code (whalin) 111 | -- add in new hashing code based on CRC32 (whalin) 112 | 113 | 01 Jun 2004 114 | -- released as version 0.9.7 115 | -- clear pool for a given host when we detect it is dead. (whalin) 116 | -- changed failver code to rehash in a better manner (more efficient when 117 | a server dies) (russo) 118 | -- add optional custom hashing method, which should be compatable with 119 | other memcached clients (perl/php/python, etc), but will be slower 120 | Java's built in hashCode for String (due to String pooling). (whalin, 121 | russo) 122 | -- fixed Java to actually return modulus (instead of remainder) when 123 | selecting a bucket (russo) 124 | -- fixed a typo in the getNagle method (whalin) 125 | 126 | 26 May 2004 127 | -- released as version 0.9.6 128 | -- Fixed bug where reading from socket when server dies will cause tight 129 | loop. Also cleaned up error handling to throw exceptions in more 130 | failed situations. (whalin) 131 | 132 | 25 May 2004 133 | -- released as version 0.9.5 134 | -- Heavy cleanup of client code to better deal with error conditions, use 135 | java naming conventions, and remove some of the overloaded convenience 136 | methods. Also changed how client deals with pulling objects from 137 | cache when not serialized to be more type safe. 138 | -- Created connection pool to manage connections to various caching servers 139 | maintaining persistent connections. 140 | -- Changed SockIO to be internal static class to the pool. 141 | -- Added support for more socket options (nagle, timeout). 142 | -- Removed deprecated stream operations. 143 | -- Heavyily javadoc'd 144 | -- Added in useage of log4j (could use J2SE logger as well) 145 | -- NOTE: Not backwards compatible with earlier versions!!! 146 | 147 | 12 Oct 2003 148 | -- released as version 0.9.1 149 | -- Altered the SockIO helper class, so it no longer allows accessing 150 | the streams it contains directly, instead it has methods 151 | with identical signatures to the methods that were called 152 | on the streams... This makes the client code prettier. 153 | -- Changed looped non blocking read to blocking read, for getting 154 | items from the server. This probably reduces CPU usage in 155 | cases where the retrieval would block, and cleans up the 156 | code a bit. We're blocking on retrieval anyhow. 157 | -- Made get() not call get_multi(), and added single socket 158 | optimization. This parallels recent changes to the perl 159 | client 160 | -- Changed a few for loops to use iterators instead, since it's 161 | probably marginally more efficient, and it's probably 162 | better coding practice. 163 | -- Actually spell checked. :) 164 | 165 | 29 Sep 2003 166 | -- released as version 0.9.0 167 | -- Renumbered to reflect that it's not been realworld tested 168 | -- changed package to danga.com.MemCached (thanks) 169 | -- added dates to changelog 170 | -- added .txt to text files 171 | -- added to official memcached site :) 172 | 173 | 28 Sep 2003 174 | -- released as version 1.0 175 | -- Adjusted namespacing for SockIO, it shouldn't have been public; is now package level. 176 | As a bonus, this means I don't have to Javadoc it. :) 177 | -- Finished adding complete Javadoc to MemCachedClient. 178 | -- spellchecked 179 | -- added a couple versions of function variations that I missed. for example, some that 180 | didn't take an int directly as a hash value, and i missed a get_multi w/out hashes. 181 | -- removed java.net.Socket reference from MemCachedClient, SockIO has a new constructor which 182 | takes hostname and port number 183 | -- switched to three part version number 184 | 185 | 27 Sep 2003 186 | -- released as version 0.3 187 | -- Compression, for strings/stringified numbers, this is compatible w/ perl 188 | Serialized objects are incompatible w/ perl for obvious reasons. :) 189 | -- Added PORTABILITY file, to include information about using the client 190 | with various JVM's 191 | -- Updated string parsing to StreamTokenizer rather than regexp's in an 192 | effort to get sablevm to like the client 193 | 194 | 24 Sep 2003 195 | -- released as version 0.2 196 | -- Serialization works 197 | -- Possible BUG: Only the lower byte of the characters of keys are sent 198 | This is only a problem if the memcache server can handle 199 | unicode keys. (I haven't checked) 200 | -- Server Failures handled gracefully 201 | -- Partial Javadoc 202 | 203 | 23 Sep 2003 204 | -- Initial Release 0.1 205 | -- Storing and Retrieving numbers and strings works 206 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/test/UnitTests.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author Kevin Burton 17 | * @author greg whalin 18 | */ 19 | package com.meetup.memcached.test; 20 | 21 | import com.meetup.memcached.*; 22 | import java.util.*; 23 | import java.io.Serializable; 24 | 25 | import org.apache.log4j.Level; 26 | import org.apache.log4j.Logger; 27 | import org.apache.log4j.BasicConfigurator; 28 | 29 | public class UnitTests { 30 | 31 | // logger 32 | private static Logger log = 33 | Logger.getLogger( UnitTests.class.getName() ); 34 | 35 | public static MemcachedClient mc = null; 36 | 37 | public static void test1() { 38 | mc.set( "foo", Boolean.TRUE ); 39 | Boolean b = (Boolean)mc.get( "foo" ); 40 | assert b.booleanValue(); 41 | log.error( "+ store/retrieve Boolean type test passed" ); 42 | } 43 | 44 | public static void test2() { 45 | mc.set( "foo", new Integer( Integer.MAX_VALUE ) ); 46 | Integer i = (Integer)mc.get( "foo" ); 47 | assert i.intValue() == Integer.MAX_VALUE; 48 | log.error( "+ store/retrieve Integer type test passed" ); 49 | } 50 | 51 | public static void test3() { 52 | String input = "test of string encoding"; 53 | mc.set( "foo", input ); 54 | String s = (String)mc.get( "foo" ); 55 | assert s.equals( input ); 56 | log.error( "+ store/retrieve String type test passed" ); 57 | } 58 | 59 | public static void test4() { 60 | mc.set( "foo", new Character( 'z' ) ); 61 | Character c = (Character)mc.get( "foo" ); 62 | assert c.charValue() == 'z'; 63 | log.error( "+ store/retrieve Character type test passed" ); 64 | } 65 | 66 | public static void test5() { 67 | mc.set( "foo", new Byte( (byte)127 ) ); 68 | Byte b = (Byte)mc.get( "foo" ); 69 | assert b.byteValue() == 127; 70 | log.error( "+ store/retrieve Byte type test passed" ); 71 | } 72 | 73 | public static void test6() { 74 | mc.set( "foo", new StringBuffer( "hello" ) ); 75 | StringBuffer o = (StringBuffer)mc.get( "foo" ); 76 | assert o.toString().equals( "hello" ); 77 | log.error( "+ store/retrieve StringBuffer type test passed" ); 78 | } 79 | 80 | public static void test7() { 81 | mc.set( "foo", new Short( (short)100 ) ); 82 | Short o = (Short)mc.get( "foo" ); 83 | assert o.shortValue() == 100; 84 | log.error( "+ store/retrieve Short type test passed" ); 85 | } 86 | 87 | public static void test8() { 88 | mc.set( "foo", new Long( Long.MAX_VALUE ) ); 89 | Long o = (Long)mc.get( "foo" ); 90 | assert o.longValue() == Long.MAX_VALUE; 91 | log.error( "+ store/retrieve Long type test passed" ); 92 | } 93 | 94 | public static void test9() { 95 | mc.set( "foo", new Double( 1.1 ) ); 96 | Double o = (Double)mc.get( "foo" ); 97 | assert o.doubleValue() == 1.1; 98 | log.error( "+ store/retrieve Double type test passed" ); 99 | } 100 | 101 | public static void test10() { 102 | mc.set( "foo", new Float( 1.1f ) ); 103 | Float o = (Float)mc.get( "foo" ); 104 | assert o.floatValue() == 1.1f; 105 | log.error( "+ store/retrieve Float type test passed" ); 106 | } 107 | 108 | public static void test11() { 109 | mc.set( "foo", new Integer( 100 ), new Date( System.currentTimeMillis() )); 110 | try { Thread.sleep( 1000 ); } catch ( Exception ex ) { } 111 | assert mc.get( "foo" ) == null; 112 | log.error( "+ store/retrieve w/ expiration test passed" ); 113 | } 114 | 115 | public static void test12() { 116 | long i = 0; 117 | mc.storeCounter("foo", i); 118 | mc.incr("foo"); // foo now == 1 119 | mc.incr("foo", (long)5); // foo now == 6 120 | long j = mc.decr("foo", (long)2); // foo now == 4 121 | assert j == 4; 122 | assert j == mc.getCounter( "foo" ); 123 | log.error( "+ incr/decr test passed" ); 124 | } 125 | 126 | public static void test13() { 127 | Date d1 = new Date(); 128 | mc.set("foo", d1); 129 | Date d2 = (Date) mc.get("foo"); 130 | assert d1.equals( d2 ); 131 | log.error( "+ store/retrieve Date type test passed" ); 132 | } 133 | 134 | public static void test14() { 135 | assert !mc.keyExists( "foobar123" ); 136 | mc.set( "foobar123", new Integer( 100000) ); 137 | assert mc.keyExists( "foobar123" ); 138 | log.error( "+ store/retrieve test passed" ); 139 | 140 | assert !mc.keyExists( "counterTest123" ); 141 | mc.storeCounter( "counterTest123", 0 ); 142 | assert mc.keyExists( "counterTest123" ); 143 | log.error( "+ counter store test passed" ); 144 | } 145 | 146 | public static void test15() { 147 | 148 | Map stats = mc.statsItems(); 149 | assert stats != null; 150 | 151 | stats = mc.statsSlabs(); 152 | assert stats != null; 153 | 154 | log.error( "+ stats test passed" ); 155 | } 156 | 157 | public static void test16() { 158 | assert !mc.set( "foo", null ); 159 | log.error( "+ invalid data store [null] test passed" ); 160 | } 161 | 162 | public static void test17() { 163 | mc.set( "foo bar", Boolean.TRUE ); 164 | Boolean b = (Boolean)mc.get( "foo bar" ); 165 | assert b.booleanValue(); 166 | log.error( "+ store/retrieve Boolean type test passed" ); 167 | } 168 | 169 | public static void test18() { 170 | long i = 0; 171 | mc.addOrIncr( "foo" ); // foo now == 0 172 | mc.incr( "foo" ); // foo now == 1 173 | mc.incr( "foo", (long)5 ); // foo now == 6 174 | 175 | mc.addOrIncr( "foo" ); // foo now 7 176 | 177 | long j = mc.decr( "foo", (long)3 ); // foo now == 4 178 | assert j == 4; 179 | assert j == mc.getCounter( "foo" ); 180 | 181 | log.error( "+ incr/decr test passed" ); 182 | } 183 | 184 | public static void test19() { 185 | int max = 100; 186 | String[] keys = new String[ max ]; 187 | for ( int i=0; i results = mc.getMulti( keys ); 193 | for ( int i=0; i results = mc.getMulti( keys ); 217 | for ( int i=0; i results = mc.getMulti( allKeys ); 257 | 258 | assert allKeys.length == results.size(); 259 | for ( String key : setKeys ) { 260 | String val = (String)results.get( key ); 261 | assert key.equals( val ); 262 | } 263 | 264 | log.error( "+ getMulti w/ keys that don't exist test passed" ); 265 | } 266 | 267 | public static void runAlTests( MemcachedClient mc ) { 268 | test14(); 269 | for ( int t = 0; t < 2; t++ ) { 270 | mc.setCompressEnable( ( t&1 ) == 1 ); 271 | 272 | test1(); 273 | test2(); 274 | test3(); 275 | test4(); 276 | test5(); 277 | test6(); 278 | test7(); 279 | test8(); 280 | test9(); 281 | test10(); 282 | test11(); 283 | test12(); 284 | test13(); 285 | test15(); 286 | test16(); 287 | test17(); 288 | test21(); 289 | test22(); 290 | test23(); 291 | test24(); 292 | 293 | for ( int i = 0; i < 3; i++ ) 294 | test19(); 295 | 296 | test20( 8191, 1, 0 ); 297 | test20( 8192, 1, 0 ); 298 | test20( 8193, 1, 0 ); 299 | 300 | test20( 16384, 100, 0 ); 301 | test20( 17000, 128, 0 ); 302 | 303 | test20( 128*1024, 1023, 0 ); 304 | test20( 128*1024, 1023, 1 ); 305 | test20( 128*1024, 1024, 0 ); 306 | test20( 128*1024, 1024, 1 ); 307 | 308 | test20( 128*1024, 1023, 0 ); 309 | test20( 128*1024, 1023, 1 ); 310 | test20( 128*1024, 1024, 0 ); 311 | test20( 128*1024, 1024, 1 ); 312 | 313 | test20( 900*1024, 32*1024, 0 ); 314 | test20( 900*1024, 32*1024, 1 ); 315 | } 316 | 317 | } 318 | 319 | /** 320 | * This runs through some simple tests of the MemcacheClient. 321 | * 322 | * Command line args: 323 | * args[0] = number of threads to spawn 324 | * args[1] = number of runs per thread 325 | * args[2] = size of object to store 326 | * 327 | * @param args the command line arguments 328 | */ 329 | public static void main(String[] args) { 330 | 331 | BasicConfigurator.configure(); 332 | org.apache.log4j.Logger.getRootLogger().setLevel( Level.WARN ); 333 | 334 | if ( !UnitTests.class.desiredAssertionStatus() ) { 335 | System.err.println( "WARNING: assertions are disabled!" ); 336 | try { Thread.sleep( 3000 ); } catch ( InterruptedException e ) {} 337 | } 338 | 339 | String[] serverlist = { 340 | "192.168.1.50:1620", 341 | "192.168.1.50:1621", 342 | "192.168.1.50:1622", 343 | "192.168.1.50:1623", 344 | "192.168.1.50:1624", 345 | "192.168.1.50:1625", 346 | "192.168.1.50:1626", 347 | "192.168.1.50:1627", 348 | "192.168.1.50:1628", 349 | "192.168.1.50:1629" 350 | }; 351 | 352 | Integer[] weights = { 1, 1, 1, 1, 10, 5, 1, 1, 1, 3 }; 353 | 354 | if ( args.length > 0 ) 355 | serverlist = args; 356 | 357 | // initialize the pool for memcache servers 358 | SockIOPool pool = SockIOPool.getInstance( "test" ); 359 | pool.setServers( serverlist ); 360 | pool.setWeights( weights ); 361 | pool.setMaxConn( 250 ); 362 | pool.setNagle( false ); 363 | pool.setHashingAlg( SockIOPool.CONSISTENT_HASH ); 364 | pool.initialize(); 365 | 366 | mc = new MemcachedClient( "test" ); 367 | runAlTests( mc ); 368 | } 369 | 370 | /** 371 | * Class for testing serializing of objects. 372 | * 373 | * @author $Author: $ 374 | * @version $Revision: $ $Date: $ 375 | */ 376 | public static final class TestClass implements Serializable { 377 | 378 | private String field1; 379 | private String field2; 380 | private Integer field3; 381 | 382 | public TestClass( String field1, String field2, Integer field3 ) { 383 | this.field1 = field1; 384 | this.field2 = field2; 385 | this.field3 = field3; 386 | } 387 | 388 | public String getField1() { return this.field1; } 389 | public String getField2() { return this.field2; } 390 | public Integer getField3() { return this.field3; } 391 | 392 | public boolean equals( Object o ) { 393 | if ( this == o ) return true; 394 | if ( !( o instanceof TestClass ) ) return false; 395 | 396 | TestClass obj = (TestClass)o; 397 | 398 | return ( ( this.field1 == obj.getField1() || ( this.field1 != null && this.field1.equals( obj.getField1() ) ) ) 399 | && ( this.field2 == obj.getField2() || ( this.field2 != null && this.field2.equals( obj.getField2() ) ) ) 400 | && ( this.field3 == obj.getField3() || ( this.field3 != null && this.field3.equals( obj.getField3() ) ) ) ); 401 | } 402 | } 403 | } 404 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/NativeHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author Greg Whalin 17 | */ 18 | package com.meetup.memcached; 19 | 20 | import java.util.Date; 21 | import org.apache.log4j.Logger; 22 | 23 | /** 24 | * Handle encoding standard Java types directly which can result in significant 25 | * memory savings: 26 | * 27 | * Currently the Memcached driver for Java supports the setSerialize() option. 28 | * This can increase performance in some situations but has a few issues: 29 | * 30 | * Code that performs class casting will throw ClassCastExceptions when 31 | * setSerialize is enabled. For example: 32 | * 33 | * mc.set( "foo", new Integer( 1 ) ); Integer output = (Integer)mc.get("foo"); 34 | * 35 | * Will work just file when setSerialize is true but when its false will just throw 36 | * a ClassCastException. 37 | * 38 | * Also internally it doesn't support Boolean and since toString is called wastes a 39 | * lot of memory and causes additional performance issue. For example an Integer 40 | * can take anywhere from 1 byte to 10 bytes. 41 | * 42 | * Due to the way the memcached slab allocator works it seems like a LOT of wasted 43 | * memory to store primitive types as serialized objects (from a performance and 44 | * memory perspective). In our applications we have millions of small objects and 45 | * wasted memory would become a big problem. 46 | * 47 | * For example a Serialized Boolean takes 47 bytes which means it will fit into the 48 | * 64byte LRU. Using 1 byte means it will fit into the 8 byte LRU thus saving 8x 49 | * the memory. This also saves the CPU performance since we don't have to 50 | * serialize bytes back and forth and we can compute the byte[] value directly. 51 | * 52 | * One problem would be when the user calls get() because doing so would require 53 | * the app to know the type of the object stored as a bytearray inside memcached 54 | * (since the user will probably cast). 55 | * 56 | * If we assume the basic types are interned we could use the first byte as the 57 | * type with the remaining bytes as the value. Then on get() we could read the 58 | * first byte to determine the type and then construct the correct object for it. 59 | * This would prevent the ClassCastException I talked about above. 60 | * 61 | * We could remove the setSerialize() option and just assume that standard VM types 62 | * are always internd in this manner. 63 | * 64 | * mc.set( "foo", new Boolean.TRUE ); Boolean b = (Boolean)mc.get( "foo" ); 65 | * 66 | * And the type casts would work because internally we would create a new Boolean 67 | * to return back to the client. 68 | * 69 | * This would reduce memory footprint and allow for a virtual implementation of the 70 | * Externalizable interface which is much faster than Serialzation. 71 | * 72 | * Currently the memory improvements would be: 73 | * 74 | * java.lang.Boolean - 8x performance improvement (now just two bytes) 75 | * java.lang.Integer - 16x performance improvement (now just 5 bytes) 76 | * 77 | * Most of the other primitive types would benefit from this optimization. 78 | * java.lang.Character being another obvious example. 79 | * 80 | * I know it seems like I'm being really picky here but for our application I'd 81 | * save 1G of memory right off the bat. We'd go down from 1.152G of memory used 82 | * down to 144M of memory used which is much better IMO. 83 | * 84 | * http://java.sun.com/docs/books/tutorial/native1.1/integrating/types.html 85 | * 86 | * @author Kevin A. Burton 87 | * @author Greg Whalin 88 | */ 89 | public class NativeHandler { 90 | 91 | // logger 92 | private static Logger log = 93 | Logger.getLogger( NativeHandler.class.getName() ); 94 | 95 | /** 96 | * Detemine of object can be natively serialized by this class. 97 | * 98 | * @param value Object to test. 99 | * @return true/false 100 | */ 101 | public static boolean isHandled( Object value ) { 102 | 103 | return ( 104 | value instanceof Byte || 105 | value instanceof Boolean || 106 | value instanceof Integer || 107 | value instanceof Long || 108 | value instanceof Character || 109 | value instanceof String || 110 | value instanceof StringBuffer || 111 | value instanceof Float || 112 | value instanceof Short || 113 | value instanceof Double || 114 | value instanceof Date || 115 | value instanceof StringBuilder || 116 | value instanceof byte[] 117 | ) 118 | ? true 119 | : false; 120 | } 121 | 122 | /** 123 | * Returns the flag for marking the type of the byte array. 124 | * 125 | * @param value Object we are storing. 126 | * @return int marker 127 | */ 128 | public static int getMarkerFlag( Object value ) { 129 | 130 | if ( value instanceof Byte ) 131 | return MemcachedClient.MARKER_BYTE; 132 | 133 | if ( value instanceof Boolean ) 134 | return MemcachedClient.MARKER_BOOLEAN; 135 | 136 | if ( value instanceof Integer ) 137 | return MemcachedClient.MARKER_INTEGER; 138 | 139 | if ( value instanceof Long ) 140 | return MemcachedClient.MARKER_LONG; 141 | 142 | if ( value instanceof Character ) 143 | return MemcachedClient.MARKER_CHARACTER; 144 | 145 | if ( value instanceof String ) 146 | return MemcachedClient.MARKER_STRING; 147 | 148 | if ( value instanceof StringBuffer ) 149 | return MemcachedClient.MARKER_STRINGBUFFER; 150 | 151 | if ( value instanceof Float ) 152 | return MemcachedClient.MARKER_FLOAT; 153 | 154 | if ( value instanceof Short ) 155 | return MemcachedClient.MARKER_SHORT; 156 | 157 | if ( value instanceof Double ) 158 | return MemcachedClient.MARKER_DOUBLE; 159 | 160 | if ( value instanceof Date ) 161 | return MemcachedClient.MARKER_DATE; 162 | 163 | if ( value instanceof StringBuilder ) 164 | return MemcachedClient.MARKER_STRINGBUILDER; 165 | 166 | if ( value instanceof byte[] ) 167 | return MemcachedClient.MARKER_BYTEARR; 168 | 169 | return -1; 170 | } 171 | 172 | /** 173 | * Encodes supported types 174 | * 175 | * @param value Object to encode. 176 | * @return byte array 177 | * 178 | * @throws Exception If fail to encode. 179 | */ 180 | public static byte[] encode( Object value ) throws Exception { 181 | 182 | if ( value instanceof Byte ) 183 | return encode( (Byte)value ); 184 | 185 | if ( value instanceof Boolean ) 186 | return encode( (Boolean)value ); 187 | 188 | if ( value instanceof Integer ) 189 | return encode( ((Integer)value).intValue() ); 190 | 191 | if ( value instanceof Long ) 192 | return encode( ((Long)value).longValue() ); 193 | 194 | if ( value instanceof Character ) 195 | return encode( (Character)value ); 196 | 197 | if ( value instanceof String ) 198 | return encode( (String)value ); 199 | 200 | if ( value instanceof StringBuffer ) 201 | return encode( (StringBuffer)value ); 202 | 203 | if ( value instanceof Float ) 204 | return encode( ((Float)value).floatValue() ); 205 | 206 | if ( value instanceof Short ) 207 | return encode( (Short)value ); 208 | 209 | if ( value instanceof Double ) 210 | return encode( ((Double)value).doubleValue() ); 211 | 212 | if ( value instanceof Date ) 213 | return encode( (Date)value); 214 | 215 | if ( value instanceof StringBuilder ) 216 | return encode( (StringBuilder)value ); 217 | 218 | if ( value instanceof byte[] ) 219 | return encode( (byte[])value ); 220 | 221 | return null; 222 | } 223 | 224 | protected static byte[] encode( Byte value ) { 225 | byte[] b = new byte[1]; 226 | b[0] = value.byteValue(); 227 | return b; 228 | } 229 | 230 | protected static byte[] encode( Boolean value ) { 231 | byte[] b = new byte[1]; 232 | 233 | if ( value.booleanValue() ) 234 | b[0] = 1; 235 | else 236 | b[0] = 0; 237 | 238 | return b; 239 | } 240 | 241 | protected static byte[] encode( int value ) { 242 | return getBytes( value ); 243 | } 244 | 245 | protected static byte[] encode( long value ) throws Exception { 246 | return getBytes( value ); 247 | } 248 | 249 | protected static byte[] encode( Date value ) { 250 | return getBytes( value.getTime() ); 251 | } 252 | 253 | protected static byte[] encode( Character value ) { 254 | return encode( value.charValue() ); 255 | } 256 | 257 | protected static byte[] encode( String value ) throws Exception { 258 | return value.getBytes( "UTF-8" ); 259 | } 260 | 261 | protected static byte[] encode( StringBuffer value ) throws Exception { 262 | return encode( value.toString() ); 263 | } 264 | 265 | protected static byte[] encode( float value ) throws Exception { 266 | return encode( (int)Float.floatToIntBits( value ) ); 267 | } 268 | 269 | protected static byte[] encode( Short value ) throws Exception { 270 | return encode( (int)value.shortValue() ); 271 | } 272 | 273 | protected static byte[] encode( double value ) throws Exception { 274 | return encode( (long)Double.doubleToLongBits( value ) ); 275 | } 276 | 277 | protected static byte[] encode( StringBuilder value ) throws Exception { 278 | return encode( value.toString() ); 279 | } 280 | 281 | protected static byte[] encode( byte[] value ) { 282 | return value; 283 | } 284 | 285 | protected static byte[] getBytes( long value ) { 286 | byte[] b = new byte[8]; 287 | b[0] = (byte)((value >> 56) & 0xFF); 288 | b[1] = (byte)((value >> 48) & 0xFF); 289 | b[2] = (byte)((value >> 40) & 0xFF); 290 | b[3] = (byte)((value >> 32) & 0xFF); 291 | b[4] = (byte)((value >> 24) & 0xFF); 292 | b[5] = (byte)((value >> 16) & 0xFF); 293 | b[6] = (byte)((value >> 8) & 0xFF); 294 | b[7] = (byte)((value >> 0) & 0xFF); 295 | return b; 296 | } 297 | 298 | protected static byte[] getBytes( int value ) { 299 | byte[] b = new byte[4]; 300 | b[0] = (byte)((value >> 24) & 0xFF); 301 | b[1] = (byte)((value >> 16) & 0xFF); 302 | b[2] = (byte)((value >> 8) & 0xFF); 303 | b[3] = (byte)((value >> 0) & 0xFF); 304 | return b; 305 | } 306 | 307 | /** 308 | * Decodes byte array using memcache flag to determine type. 309 | * 310 | * @param b 311 | * @param marker 312 | * @return 313 | * @throws Exception 314 | */ 315 | public static Object decode( byte[] b, int flag ) throws Exception { 316 | 317 | if ( b.length < 1 ) 318 | return null; 319 | 320 | 321 | if ( ( flag & MemcachedClient.MARKER_BYTE ) == MemcachedClient.MARKER_BYTE ) 322 | return decodeByte( b ); 323 | 324 | if ( ( flag & MemcachedClient.MARKER_BOOLEAN ) == MemcachedClient.MARKER_BOOLEAN ) 325 | return decodeBoolean( b ); 326 | 327 | if ( ( flag & MemcachedClient.MARKER_INTEGER ) == MemcachedClient.MARKER_INTEGER ) 328 | return decodeInteger( b ); 329 | 330 | if ( ( flag & MemcachedClient.MARKER_LONG ) == MemcachedClient.MARKER_LONG ) 331 | return decodeLong( b ); 332 | 333 | if ( ( flag & MemcachedClient.MARKER_CHARACTER ) == MemcachedClient.MARKER_CHARACTER ) 334 | return decodeCharacter( b ); 335 | 336 | if ( ( flag & MemcachedClient.MARKER_STRING ) == MemcachedClient.MARKER_STRING ) 337 | return decodeString( b ); 338 | 339 | if ( ( flag & MemcachedClient.MARKER_STRINGBUFFER ) == MemcachedClient.MARKER_STRINGBUFFER ) 340 | return decodeStringBuffer( b ); 341 | 342 | if ( ( flag & MemcachedClient.MARKER_FLOAT ) == MemcachedClient.MARKER_FLOAT ) 343 | return decodeFloat( b ); 344 | 345 | if ( ( flag & MemcachedClient.MARKER_SHORT ) == MemcachedClient.MARKER_SHORT ) 346 | return decodeShort( b ); 347 | 348 | if ( ( flag & MemcachedClient.MARKER_DOUBLE ) == MemcachedClient.MARKER_DOUBLE ) 349 | return decodeDouble( b ); 350 | 351 | if ( ( flag & MemcachedClient.MARKER_DATE ) == MemcachedClient.MARKER_DATE ) 352 | return decodeDate( b ); 353 | 354 | if ( ( flag & MemcachedClient.MARKER_STRINGBUILDER ) == MemcachedClient.MARKER_STRINGBUILDER ) 355 | return decodeStringBuilder( b ); 356 | 357 | if ( ( flag & MemcachedClient.MARKER_BYTEARR ) == MemcachedClient.MARKER_BYTEARR ) 358 | return decodeByteArr( b ); 359 | 360 | return null; 361 | } 362 | 363 | // decode methods 364 | protected static Byte decodeByte( byte[] b ) { 365 | return new Byte( b[0] ); 366 | } 367 | 368 | protected static Boolean decodeBoolean( byte[] b ) { 369 | boolean value = b[0] == 1; 370 | return ( value ) ? Boolean.TRUE : Boolean.FALSE; 371 | } 372 | 373 | protected static Integer decodeInteger( byte[] b ) { 374 | return new Integer( toInt( b ) ); 375 | } 376 | 377 | protected static Long decodeLong( byte[] b ) throws Exception { 378 | return new Long( toLong( b ) ); 379 | } 380 | 381 | protected static Character decodeCharacter( byte[] b ) { 382 | return new Character( (char)decodeInteger( b ).intValue() ); 383 | } 384 | 385 | protected static String decodeString( byte[] b ) throws Exception { 386 | return new String( b, "UTF-8" ); 387 | } 388 | 389 | protected static StringBuffer decodeStringBuffer( byte[] b ) throws Exception { 390 | return new StringBuffer( decodeString( b ) ); 391 | } 392 | 393 | protected static Float decodeFloat( byte[] b ) throws Exception { 394 | Integer l = decodeInteger( b ); 395 | return new Float( Float.intBitsToFloat( l.intValue() ) ); 396 | } 397 | 398 | protected static Short decodeShort( byte[] b ) throws Exception { 399 | return new Short( (short)decodeInteger( b ).intValue() ); 400 | } 401 | 402 | protected static Double decodeDouble( byte[] b ) throws Exception { 403 | Long l = decodeLong( b ); 404 | return new Double( Double.longBitsToDouble( l.longValue() ) ); 405 | } 406 | 407 | protected static Date decodeDate( byte[] b ) { 408 | return new Date( toLong( b ) ); 409 | } 410 | 411 | protected static StringBuilder decodeStringBuilder( byte[] b ) throws Exception { 412 | return new StringBuilder( decodeString( b ) ); 413 | } 414 | 415 | protected static byte[] decodeByteArr( byte[] b ) { 416 | return b; 417 | } 418 | 419 | /** 420 | * This works by taking each of the bit patterns and converting them to 421 | * ints taking into account 2s complement and then adding them.. 422 | * 423 | * @param b 424 | * @return 425 | */ 426 | protected static int toInt( byte[] b ) { 427 | return (((((int) b[3]) & 0xFF) << 32) + 428 | ((((int) b[2]) & 0xFF) << 40) + 429 | ((((int) b[1]) & 0xFF) << 48) + 430 | ((((int) b[0]) & 0xFF) << 56)); 431 | } 432 | 433 | protected static long toLong( byte[] b ) { 434 | return ((((long) b[7]) & 0xFF) + 435 | ((((long) b[6]) & 0xFF) << 8) + 436 | ((((long) b[5]) & 0xFF) << 16) + 437 | ((((long) b[4]) & 0xFF) << 24) + 438 | ((((long) b[3]) & 0xFF) << 32) + 439 | ((((long) b[2]) & 0xFF) << 40) + 440 | ((((long) b[1]) & 0xFF) << 48) + 441 | ((((long) b[0]) & 0xFF) << 56)); 442 | } 443 | } 444 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/SockIOPool.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author greg whalin 17 | */ 18 | package com.meetup.memcached; 19 | 20 | import java.security.MessageDigest; 21 | import java.security.NoSuchAlgorithmException; 22 | 23 | // java.util 24 | import java.util.Map; 25 | import java.util.List; 26 | import java.util.Set; 27 | import java.util.Iterator; 28 | import java.util.ArrayList; 29 | import java.util.IdentityHashMap; 30 | import java.util.HashMap; 31 | import java.util.HashSet; 32 | import java.util.Date; 33 | import java.util.Arrays; 34 | import java.util.SortedMap; 35 | import java.util.TreeMap; 36 | 37 | import java.util.zip.*; 38 | import java.net.*; 39 | import java.io.*; 40 | import java.nio.*; 41 | import java.nio.channels.*; 42 | import java.util.concurrent.locks.ReentrantLock; 43 | import org.apache.log4j.Logger; 44 | 45 | /** 46 | * This class is a connection pool for maintaning a pool of persistent connections
47 | * to memcached servers. 48 | * 49 | * The pool must be initialized prior to use. This should typically be early on
50 | * in the lifecycle of the JVM instance.
51 | *
52 | *

An example of initializing using defaults:

53 | * 
  54 |  *
  55 |  *	static {
  56 |  *		String[] serverlist = { "cache0.server.com:12345", "cache1.server.com:12345" };
  57 |  *
  58 |  *		SockIOPool pool = SockIOPool.getInstance();
  59 |  *		pool.setServers(serverlist);
  60 |  *		pool.initialize();	
  61 |  *	}
  62 |  *
63 | *

An example of initializing using defaults and providing weights for servers:

64 | * 
  65 |  *	static {
  66 |  *		String[] serverlist = { "cache0.server.com:12345", "cache1.server.com:12345" };
  67 |  *		Integer[] weights   = { new Integer(5), new Integer(2) };
  68 |  *		
  69 |  *		SockIOPool pool = SockIOPool.getInstance();
  70 |  *		pool.setServers(serverlist);
  71 |  *		pool.setWeights(weights);	
  72 |  *		pool.initialize();	
  73 |  *	}
  74 |  *
75 | *

An example of initializing overriding defaults:

76 | * 
  77 |  *	static {
  78 |  *		String[] serverlist     = { "cache0.server.com:12345", "cache1.server.com:12345" };
  79 |  *		Integer[] weights       = { new Integer(5), new Integer(2) };	
  80 |  *		int initialConnections  = 10;
  81 |  *		int minSpareConnections = 5;
  82 |  *		int maxSpareConnections = 50;	
  83 |  *		long maxIdleTime        = 1000 * 60 * 30;	// 30 minutes
  84 |  *		long maxBusyTime        = 1000 * 60 * 5;	// 5 minutes
  85 |  *		long maintThreadSleep   = 1000 * 5;			// 5 seconds
  86 |  *		int	socketTimeOut       = 1000 * 3;			// 3 seconds to block on reads
  87 |  *		int	socketConnectTO     = 1000 * 3;			// 3 seconds to block on initial connections.  If 0, then will use blocking connect (default)
  88 |  *		boolean failover        = false;			// turn off auto-failover in event of server down	
  89 |  *		boolean nagleAlg        = false;			// turn off Nagle's algorithm on all sockets in pool	
  90 |  *		boolean aliveCheck      = false;			// disable health check of socket on checkout
  91 |  *
  92 |  *		SockIOPool pool = SockIOPool.getInstance();
  93 |  *		pool.setServers( serverlist );
  94 |  *		pool.setWeights( weights );	
  95 |  *		pool.setInitConn( initialConnections );
  96 |  *		pool.setMinConn( minSpareConnections );
  97 |  *		pool.setMaxConn( maxSpareConnections );
  98 |  *		pool.setMaxIdle( maxIdleTime );
  99 |  *		pool.setMaxBusyTime( maxBusyTime );
 100 |  *		pool.setMaintSleep( maintThreadSleep );
 101 |  *		pool.setSocketTO( socketTimeOut );
 102 |  *		pool.setNagle( nagleAlg );	
 103 |  *		pool.setHashingAlg( SockIOPool.NEW_COMPAT_HASH );
 104 |  *		pool.setAliveCheck( true );
 105 |  *		pool.initialize();	
 106 |  *	}
 107 |  *
108 | * The easiest manner in which to initialize the pool is to set the servers and rely on defaults as in the first example.
109 | * After pool is initialized, a client will request a SockIO object by calling getSock with the cache key
110 | * The client must always close the SockIO object when finished, which will return the connection back to the pool.
111 | *

An example of retrieving a SockIO object:

112 | * 
 113 |  *		SockIOPool.SockIO sock = SockIOPool.getInstance().getSock( key );
 114 |  *		try {
 115 |  *			sock.write( "version\r\n" );	
 116 |  *			sock.flush();	
 117 |  *			System.out.println( "Version: " + sock.readLine() );	
 118 |  *		}
 119 |  *		catch (IOException ioe) { System.out.println( "io exception thrown" ) };	
 120 |  *
 121 |  *		sock.close();	
 122 |  *
123 | * 124 | * @author greg whalin 125 | * @version 1.5 126 | */ 127 | public class SockIOPool { 128 | 129 | // logger 130 | private static Logger log = 131 | Logger.getLogger( SockIOPool.class.getName() ); 132 | 133 | // store instances of pools 134 | private static Map pools = 135 | new HashMap(); 136 | 137 | // avoid recurring construction 138 | private static ThreadLocal MD5 = new ThreadLocal() { 139 | @Override 140 | protected MessageDigest initialValue() { 141 | try { 142 | return MessageDigest.getInstance( "MD5" ); 143 | } 144 | catch ( NoSuchAlgorithmException e ) { 145 | log.error( "++++ no md5 algorithm found" ); 146 | throw new IllegalStateException( "++++ no md5 algorythm found"); 147 | } 148 | } 149 | }; 150 | 151 | // Constants 152 | private static final Integer ZERO = new Integer( 0 ); 153 | public static final int NATIVE_HASH = 0; // native String.hashCode(); 154 | public static final int OLD_COMPAT_HASH = 1; // original compatibility hashing algorithm (works with other clients) 155 | public static final int NEW_COMPAT_HASH = 2; // new CRC32 based compatibility hashing algorithm (works with other clients) 156 | public static final int CONSISTENT_HASH = 3; // MD5 Based -- Stops thrashing when a server added or removed 157 | 158 | public static final long MAX_RETRY_DELAY = 10 * 60 * 1000; // max of 10 minute delay for fall off 159 | 160 | // Pool data 161 | private MaintThread maintThread; 162 | private boolean initialized = false; 163 | private int maxCreate = 1; // this will be initialized by pool when the pool is initialized 164 | 165 | // initial, min and max pool sizes 166 | private int poolMultiplier = 3; 167 | private int initConn = 10; 168 | private int minConn = 5; 169 | private int maxConn = 100; 170 | private long maxIdle = 1000 * 60 * 5; // max idle time for avail sockets 171 | private long maxBusyTime = 1000 * 30; // max idle time for avail sockets 172 | private long maintSleep = 1000 * 30; // maintenance thread sleep time 173 | private int socketTO = 1000 * 3; // default timeout of socket reads 174 | private int socketConnectTO = 1000 * 3; // default timeout of socket connections 175 | private boolean aliveCheck = false; // default to not check each connection for being alive 176 | private boolean failover = true; // default to failover in event of cache server dead 177 | private boolean failback = true; // only used if failover is also set ... controls putting a dead server back into rotation 178 | private boolean nagle = false; // enable/disable Nagle's algorithm 179 | private int hashingAlg = NATIVE_HASH; // default to using the native hash as it is the fastest 180 | 181 | // locks 182 | private final ReentrantLock hostDeadLock = new ReentrantLock(); 183 | 184 | // list of all servers 185 | private String[] servers; 186 | private Integer[] weights; 187 | private Integer totalWeight = 0; 188 | 189 | private List buckets; 190 | private TreeMap consistentBuckets; 191 | 192 | // dead server map 193 | private Map hostDead; 194 | private Map hostDeadDur; 195 | 196 | // map to hold all available sockets 197 | // map to hold busy sockets 198 | // set to hold sockets to close 199 | private Map> availPool; 200 | private Map> busyPool; 201 | private Map deadPool; 202 | 203 | // empty constructor 204 | protected SockIOPool() { } 205 | 206 | /** 207 | * Factory to create/retrieve new pools given a unique poolName. 208 | * 209 | * @param poolName unique name of the pool 210 | * @return instance of SockIOPool 211 | */ 212 | public static synchronized SockIOPool getInstance( String poolName ) { 213 | if ( pools.containsKey( poolName ) ) 214 | return pools.get( poolName ); 215 | 216 | SockIOPool pool = new SockIOPool(); 217 | pools.put( poolName, pool ); 218 | 219 | return pool; 220 | } 221 | 222 | /** 223 | * Single argument version of factory used for back compat. 224 | * Simply creates a pool named "default". 225 | * 226 | * @return instance of SockIOPool 227 | */ 228 | public static SockIOPool getInstance() { 229 | return getInstance( "default" ); 230 | } 231 | 232 | /** 233 | * Sets the list of all cache servers. 234 | * 235 | * @param servers String array of servers [host:port] 236 | */ 237 | public void setServers( String[] servers ) { this.servers = servers; } 238 | 239 | /** 240 | * Returns the current list of all cache servers. 241 | * 242 | * @return String array of servers [host:port] 243 | */ 244 | public String[] getServers() { return this.servers; } 245 | 246 | /** 247 | * Sets the list of weights to apply to the server list. 248 | * 249 | * This is an int array with each element corresponding to an element
250 | * in the same position in the server String array. 251 | * 252 | * @param weights Integer array of weights 253 | */ 254 | public void setWeights( Integer[] weights ) { this.weights = weights; } 255 | 256 | /** 257 | * Returns the current list of weights. 258 | * 259 | * @return int array of weights 260 | */ 261 | public Integer[] getWeights() { return this.weights; } 262 | 263 | /** 264 | * Sets the initial number of connections per server in the available pool. 265 | * 266 | * @param initConn int number of connections 267 | */ 268 | public void setInitConn( int initConn ) { this.initConn = initConn; } 269 | 270 | /** 271 | * Returns the current setting for the initial number of connections per server in 272 | * the available pool. 273 | * 274 | * @return number of connections 275 | */ 276 | public int getInitConn() { return this.initConn; } 277 | 278 | /** 279 | * Sets the minimum number of spare connections to maintain in our available pool. 280 | * 281 | * @param minConn number of connections 282 | */ 283 | public void setMinConn( int minConn ) { this.minConn = minConn; } 284 | 285 | /** 286 | * Returns the minimum number of spare connections in available pool. 287 | * 288 | * @return number of connections 289 | */ 290 | public int getMinConn() { return this.minConn; } 291 | 292 | /** 293 | * Sets the maximum number of spare connections allowed in our available pool. 294 | * 295 | * @param maxConn number of connections 296 | */ 297 | public void setMaxConn( int maxConn ) { this.maxConn = maxConn; } 298 | 299 | /** 300 | * Returns the maximum number of spare connections allowed in available pool. 301 | * 302 | * @return number of connections 303 | */ 304 | public int getMaxConn() { return this.maxConn; } 305 | 306 | /** 307 | * Sets the max idle time for threads in the available pool. 308 | * 309 | * @param maxIdle idle time in ms 310 | */ 311 | public void setMaxIdle( long maxIdle ) { this.maxIdle = maxIdle; } 312 | 313 | /** 314 | * Returns the current max idle setting. 315 | * 316 | * @return max idle setting in ms 317 | */ 318 | public long getMaxIdle() { return this.maxIdle; } 319 | 320 | /** 321 | * Sets the max busy time for threads in the busy pool. 322 | * 323 | * @param maxBusyTime idle time in ms 324 | */ 325 | public void setMaxBusyTime( long maxBusyTime ) { this.maxBusyTime = maxBusyTime; } 326 | 327 | /** 328 | * Returns the current max busy setting. 329 | * 330 | * @return max busy setting in ms 331 | */ 332 | public long getMaxBusy() { return this.maxBusyTime; } 333 | 334 | /** 335 | * Set the sleep time between runs of the pool maintenance thread. 336 | * If set to 0, then the maint thread will not be started. 337 | * 338 | * @param maintSleep sleep time in ms 339 | */ 340 | public void setMaintSleep( long maintSleep ) { this.maintSleep = maintSleep; } 341 | 342 | /** 343 | * Returns the current maint thread sleep time. 344 | * 345 | * @return sleep time in ms 346 | */ 347 | public long getMaintSleep() { return this.maintSleep; } 348 | 349 | /** 350 | * Sets the socket timeout for reads. 351 | * 352 | * @param socketTO timeout in ms 353 | */ 354 | public void setSocketTO( int socketTO ) { this.socketTO = socketTO; } 355 | 356 | /** 357 | * Returns the socket timeout for reads. 358 | * 359 | * @return timeout in ms 360 | */ 361 | public int getSocketTO() { return this.socketTO; } 362 | 363 | /** 364 | * Sets the socket timeout for connect. 365 | * 366 | * @param socketConnectTO timeout in ms 367 | */ 368 | public void setSocketConnectTO( int socketConnectTO ) { this.socketConnectTO = socketConnectTO; } 369 | 370 | /** 371 | * Returns the socket timeout for connect. 372 | * 373 | * @return timeout in ms 374 | */ 375 | public int getSocketConnectTO() { return this.socketConnectTO; } 376 | 377 | /** 378 | * Sets the failover flag for the pool. 379 | * 380 | * If this flag is set to true, and a socket fails to connect,
381 | * the pool will attempt to return a socket from another server
382 | * if one exists. If set to false, then getting a socket
383 | * will return null if it fails to connect to the requested server. 384 | * 385 | * @param failover true/false 386 | */ 387 | public void setFailover( boolean failover ) { this.failover = failover; } 388 | 389 | /** 390 | * Returns current state of failover flag. 391 | * 392 | * @return true/false 393 | */ 394 | public boolean getFailover() { return this.failover; } 395 | 396 | /** 397 | * Sets the failback flag for the pool. 398 | * 399 | * If this is true and we have marked a host as dead, 400 | * will try to bring it back. If it is false, we will never 401 | * try to resurrect a dead host. 402 | * 403 | * @param failback true/false 404 | */ 405 | public void setFailback( boolean failback ) { this.failback = failback; } 406 | 407 | /** 408 | * Returns current state of failover flag. 409 | * 410 | * @return true/false 411 | */ 412 | public boolean getFailback() { return this.failback; } 413 | 414 | /** 415 | * Sets the aliveCheck flag for the pool. 416 | * 417 | * When true, this will attempt to talk to the server on 418 | * every connection checkout to make sure the connection is 419 | * still valid. This adds extra network chatter and thus is 420 | * defaulted off. May be useful if you want to ensure you do 421 | * not have any problems talking to the server on a dead connection. 422 | * 423 | * @param aliveCheck true/false 424 | */ 425 | public void setAliveCheck( boolean aliveCheck ) { this.aliveCheck = aliveCheck; } 426 | 427 | 428 | /** 429 | * Returns the current status of the aliveCheck flag. 430 | * 431 | * @return true / false 432 | */ 433 | public boolean getAliveCheck() { return this.aliveCheck; } 434 | 435 | /** 436 | * Sets the Nagle alg flag for the pool. 437 | * 438 | * If false, will turn off Nagle's algorithm on all sockets created. 439 | * 440 | * @param nagle true/false 441 | */ 442 | public void setNagle( boolean nagle ) { this.nagle = nagle; } 443 | 444 | /** 445 | * Returns current status of nagle flag 446 | * 447 | * @return true/false 448 | */ 449 | public boolean getNagle() { return this.nagle; } 450 | 451 | /** 452 | * Sets the hashing algorithm we will use. 453 | * 454 | * The types are as follows. 455 | * 456 | * SockIOPool.NATIVE_HASH (0) - native String.hashCode() - fast (cached) but not compatible with other clients 457 | * SockIOPool.OLD_COMPAT_HASH (1) - original compatibility hashing alg (works with other clients) 458 | * SockIOPool.NEW_COMPAT_HASH (2) - new CRC32 based compatibility hashing algorithm (fast and works with other clients) 459 | * 460 | * @param alg int value representing hashing algorithm 461 | */ 462 | public void setHashingAlg( int alg ) { this.hashingAlg = alg; } 463 | 464 | /** 465 | * Returns current status of customHash flag 466 | * 467 | * @return true/false 468 | */ 469 | public int getHashingAlg() { return this.hashingAlg; } 470 | 471 | /** 472 | * Internal private hashing method. 473 | * 474 | * This is the original hashing algorithm from other clients. 475 | * Found to be slow and have poor distribution. 476 | * 477 | * @param key String to hash 478 | * @return hashCode for this string using our own hashing algorithm 479 | */ 480 | private static long origCompatHashingAlg( String key ) { 481 | long hash = 0; 482 | char[] cArr = key.toCharArray(); 483 | 484 | for ( int i = 0; i < cArr.length; ++i ) { 485 | hash = (hash * 33) + cArr[i]; 486 | } 487 | 488 | return hash; 489 | } 490 | 491 | /** 492 | * Internal private hashing method. 493 | * 494 | * This is the new hashing algorithm from other clients. 495 | * Found to be fast and have very good distribution. 496 | * 497 | * UPDATE: This is dog slow under java 498 | * 499 | * @param key 500 | * @return 501 | */ 502 | private static long newCompatHashingAlg( String key ) { 503 | CRC32 checksum = new CRC32(); 504 | checksum.update( key.getBytes() ); 505 | long crc = checksum.getValue(); 506 | return (crc >> 16) & 0x7fff; 507 | } 508 | 509 | /** 510 | * Internal private hashing method. 511 | * 512 | * MD5 based hash algorithm for use in the consistent 513 | * hashing approach. 514 | * 515 | * @param key 516 | * @return 517 | */ 518 | private static long md5HashingAlg( String key ) { 519 | MessageDigest md5 = MD5.get(); 520 | md5.reset(); 521 | md5.update( key.getBytes() ); 522 | byte[] bKey = md5.digest(); 523 | long res = ((long)(bKey[3]&0xFF) << 24) | ((long)(bKey[2]&0xFF) << 16) | ((long)(bKey[1]&0xFF) << 8) | (long)(bKey[0]&0xFF); 524 | return res; 525 | } 526 | 527 | /** 528 | * Returns a bucket to check for a given key. 529 | * 530 | * @param key String key cache is stored under 531 | * @return int bucket 532 | */ 533 | private long getHash( String key, Integer hashCode ) { 534 | 535 | if ( hashCode != null ) { 536 | if ( hashingAlg == CONSISTENT_HASH ) 537 | return hashCode.longValue() & 0xffffffffL; 538 | else 539 | return hashCode.longValue(); 540 | } 541 | else { 542 | switch ( hashingAlg ) { 543 | case NATIVE_HASH: 544 | return (long)key.hashCode(); 545 | case OLD_COMPAT_HASH: 546 | return origCompatHashingAlg( key ); 547 | case NEW_COMPAT_HASH: 548 | return newCompatHashingAlg( key ); 549 | case CONSISTENT_HASH: 550 | return md5HashingAlg( key ); 551 | default: 552 | // use the native hash as a default 553 | hashingAlg = NATIVE_HASH; 554 | return (long)key.hashCode(); 555 | } 556 | } 557 | } 558 | 559 | private long getBucket( String key, Integer hashCode ) { 560 | long hc = getHash( key, hashCode ); 561 | 562 | if ( this.hashingAlg == CONSISTENT_HASH ) { 563 | return findPointFor( hc ); 564 | } 565 | else { 566 | long bucket = hc % buckets.size(); 567 | if ( bucket < 0 ) bucket *= -1; 568 | return bucket; 569 | } 570 | } 571 | 572 | /** 573 | * Gets the first available key equal or above the given one, if none found, 574 | * returns the first k in the bucket 575 | * @param k key 576 | * @return 577 | */ 578 | private Long findPointFor( Long hv ) { 579 | // this works in java 6, but still want to release support for java5 580 | //Long k = this.consistentBuckets.ceilingKey( hv ); 581 | //return ( k == null ) ? this.consistentBuckets.firstKey() : k; 582 | 583 | SortedMap tmap = 584 | this.consistentBuckets.tailMap( hv ); 585 | 586 | return ( tmap.isEmpty() ) ? this.consistentBuckets.firstKey() : tmap.firstKey(); 587 | } 588 | 589 | /** 590 | * Initializes the pool. 591 | */ 592 | public void initialize() { 593 | 594 | synchronized( this ) { 595 | 596 | // check to see if already initialized 597 | if ( initialized 598 | && ( buckets != null || consistentBuckets != null ) 599 | && ( availPool != null ) 600 | && ( busyPool != null ) ) { 601 | log.error( "++++ trying to initialize an already initialized pool" ); 602 | return; 603 | } 604 | 605 | // pools 606 | availPool = new HashMap>( servers.length * initConn ); 607 | busyPool = new HashMap>( servers.length * initConn ); 608 | deadPool = new IdentityHashMap(); 609 | 610 | hostDeadDur = new HashMap(); 611 | hostDead = new HashMap(); 612 | maxCreate = (poolMultiplier > minConn) ? minConn : minConn / poolMultiplier; // only create up to maxCreate connections at once 613 | 614 | if ( log.isDebugEnabled() ) { 615 | log.debug( "++++ initializing pool with following settings:" ); 616 | log.debug( "++++ initial size: " + initConn ); 617 | log.debug( "++++ min spare : " + minConn ); 618 | log.debug( "++++ max spare : " + maxConn ); 619 | } 620 | 621 | // if servers is not set, or it empty, then 622 | // throw a runtime exception 623 | if ( servers == null || servers.length <= 0 ) { 624 | log.error( "++++ trying to initialize with no servers" ); 625 | throw new IllegalStateException( "++++ trying to initialize with no servers" ); 626 | } 627 | 628 | // initalize our internal hashing structures 629 | if ( this.hashingAlg == CONSISTENT_HASH ) 630 | populateConsistentBuckets(); 631 | else 632 | populateBuckets(); 633 | 634 | // mark pool as initialized 635 | this.initialized = true; 636 | 637 | // start maint thread 638 | if ( this.maintSleep > 0 ) 639 | this.startMaintThread(); 640 | } 641 | } 642 | 643 | private void populateBuckets() { 644 | if ( log.isDebugEnabled() ) 645 | log.debug( "++++ initializing internal hashing structure for consistent hashing" ); 646 | 647 | // store buckets in tree map 648 | this.buckets = new ArrayList(); 649 | 650 | for ( int i = 0; i < servers.length; i++ ) { 651 | if ( this.weights != null && this.weights.length > i ) { 652 | for ( int k = 0; k < this.weights[i].intValue(); k++ ) { 653 | this.buckets.add( servers[i] ); 654 | if ( log.isDebugEnabled() ) 655 | log.debug( "++++ added " + servers[i] + " to server bucket" ); 656 | } 657 | } 658 | else { 659 | this.buckets.add( servers[i] ); 660 | if ( log.isDebugEnabled() ) 661 | log.debug( "++++ added " + servers[i] + " to server bucket" ); 662 | } 663 | 664 | // create initial connections 665 | if ( log.isDebugEnabled() ) 666 | log.debug( "+++ creating initial connections (" + initConn + ") for host: " + servers[i] ); 667 | 668 | for ( int j = 0; j < initConn; j++ ) { 669 | SockIO socket = createSocket( servers[i] ); 670 | if ( socket == null ) { 671 | log.error( "++++ failed to create connection to: " + servers[i] + " -- only " + j + " created." ); 672 | break; 673 | } 674 | 675 | addSocketToPool( availPool, servers[i], socket ); 676 | if ( log.isDebugEnabled() ) 677 | log.debug( "++++ created and added socket: " + socket.toString() + " for host " + servers[i] ); 678 | } 679 | } 680 | } 681 | 682 | private void populateConsistentBuckets() { 683 | if ( log.isDebugEnabled() ) 684 | log.debug( "++++ initializing internal hashing structure for consistent hashing" ); 685 | 686 | // store buckets in tree map 687 | this.consistentBuckets = new TreeMap(); 688 | 689 | MessageDigest md5 = MD5.get(); 690 | if ( this.totalWeight <= 0 && this.weights != null ) { 691 | for ( int i = 0; i < this.weights.length; i++ ) 692 | this.totalWeight += ( this.weights[i] == null ) ? 1 : this.weights[i]; 693 | } 694 | else if ( this.weights == null ) { 695 | this.totalWeight = this.servers.length; 696 | } 697 | 698 | for ( int i = 0; i < servers.length; i++ ) { 699 | int thisWeight = 1; 700 | if ( this.weights != null && this.weights[i] != null ) 701 | thisWeight = this.weights[i]; 702 | 703 | double factor = Math.floor( ((double)(40 * this.servers.length * thisWeight)) / (double)this.totalWeight ); 704 | 705 | for ( long j = 0; j < factor; j++ ) { 706 | byte[] d = md5.digest( ( servers[i] + "-" + j ).getBytes() ); 707 | for ( int h = 0 ; h < 4; h++ ) { 708 | Long k = 709 | ((long)(d[3+h*4]&0xFF) << 24) 710 | | ((long)(d[2+h*4]&0xFF) << 16) 711 | | ((long)(d[1+h*4]&0xFF) << 8) 712 | | ((long)(d[0+h*4]&0xFF)); 713 | 714 | consistentBuckets.put( k, servers[i] ); 715 | if ( log.isDebugEnabled() ) 716 | log.debug( "++++ added " + servers[i] + " to server bucket" ); 717 | } 718 | } 719 | 720 | // create initial connections 721 | if ( log.isDebugEnabled() ) 722 | log.debug( "+++ creating initial connections (" + initConn + ") for host: " + servers[i] ); 723 | 724 | for ( int j = 0; j < initConn; j++ ) { 725 | SockIO socket = createSocket( servers[i] ); 726 | if ( socket == null ) { 727 | log.error( "++++ failed to create connection to: " + servers[i] + " -- only " + j + " created." ); 728 | break; 729 | } 730 | 731 | addSocketToPool( availPool, servers[i], socket ); 732 | if ( log.isDebugEnabled() ) 733 | log.debug( "++++ created and added socket: " + socket.toString() + " for host " + servers[i] ); 734 | } 735 | } 736 | } 737 | 738 | /** 739 | * Returns state of pool. 740 | * 741 | * @return true if initialized. 742 | */ 743 | public boolean isInitialized() { 744 | return initialized; 745 | } 746 | 747 | /** 748 | * Creates a new SockIO obj for the given server. 749 | * 750 | * If server fails to connect, then return null and do not try
751 | * again until a duration has passed. This duration will grow
752 | * by doubling after each failed attempt to connect. 753 | * 754 | * @param host host:port to connect to 755 | * @return SockIO obj or null if failed to create 756 | */ 757 | protected SockIO createSocket( String host ) { 758 | 759 | SockIO socket = null; 760 | 761 | // if host is dead, then we don't need to try again 762 | // until the dead status has expired 763 | // we do not try to put back in if failback is off 764 | hostDeadLock.lock(); 765 | try { 766 | if ( failover && failback && hostDead.containsKey( host ) && hostDeadDur.containsKey( host ) ) { 767 | 768 | Date store = hostDead.get( host ); 769 | long expire = hostDeadDur.get( host ).longValue(); 770 | 771 | if ( (store.getTime() + expire) > System.currentTimeMillis() ) 772 | return null; 773 | } 774 | } 775 | finally { 776 | hostDeadLock.unlock(); 777 | } 778 | 779 | try { 780 | socket = new SockIO( this, host, this.socketTO, this.socketConnectTO, this.nagle ); 781 | 782 | if ( !socket.isConnected() ) { 783 | log.error( "++++ failed to get SockIO obj for: " + host + " -- new socket is not connected" ); 784 | deadPool.put( socket, ZERO ); 785 | socket = null; 786 | } 787 | } 788 | catch ( Exception ex ) { 789 | log.error( "++++ failed to get SockIO obj for: " + host ); 790 | log.error( ex.getMessage(), ex ); 791 | socket = null; 792 | } 793 | 794 | // if we failed to get socket, then mark 795 | // host dead for a duration which falls off 796 | hostDeadLock.lock(); 797 | try { 798 | if ( socket == null ) { 799 | Date now = new Date(); 800 | hostDead.put( host, now ); 801 | 802 | long expire = ( hostDeadDur.containsKey( host ) ) ? (((Long)hostDeadDur.get( host )).longValue() * 2) : 1000; 803 | 804 | if ( expire > MAX_RETRY_DELAY ) 805 | expire = MAX_RETRY_DELAY; 806 | 807 | hostDeadDur.put( host, new Long( expire ) ); 808 | if ( log.isDebugEnabled() ) 809 | log.debug( "++++ ignoring dead host: " + host + " for " + expire + " ms" ); 810 | 811 | // also clear all entries for this host from availPool 812 | clearHostFromPool( availPool, host ); 813 | } 814 | else { 815 | if ( log.isDebugEnabled() ) 816 | log.debug( "++++ created socket (" + socket.toString() + ") for host: " + host ); 817 | if ( hostDead.containsKey( host ) || hostDeadDur.containsKey( host ) ) { 818 | hostDead.remove( host ); 819 | hostDeadDur.remove( host ); 820 | } 821 | } 822 | } 823 | finally { 824 | hostDeadLock.unlock(); 825 | } 826 | 827 | return socket; 828 | } 829 | 830 | /** 831 | * @param key 832 | * @return 833 | */ 834 | public String getHost( String key ) { 835 | return getHost( key, null ); 836 | } 837 | 838 | /** 839 | * Gets the host that a particular key / hashcode resides on. 840 | * 841 | * @param key 842 | * @param hashcode 843 | * @return 844 | */ 845 | public String getHost( String key, Integer hashcode ) { 846 | SockIO socket = getSock( key, hashcode ); 847 | String host = socket.getHost(); 848 | socket.close(); 849 | return host; 850 | } 851 | 852 | /** 853 | * Returns appropriate SockIO object given 854 | * string cache key. 855 | * 856 | * @param key hashcode for cache key 857 | * @return SockIO obj connected to server 858 | */ 859 | public SockIO getSock( String key ) { 860 | return getSock( key, null ); 861 | } 862 | 863 | /** 864 | * Returns appropriate SockIO object given 865 | * string cache key and optional hashcode. 866 | * 867 | * Trys to get SockIO from pool. Fails over 868 | * to additional pools in event of server failure. 869 | * 870 | * @param key hashcode for cache key 871 | * @param hashCode if not null, then the int hashcode to use 872 | * @return SockIO obj connected to server 873 | */ 874 | public SockIO getSock( String key, Integer hashCode ) { 875 | 876 | if ( log.isDebugEnabled() ) 877 | log.debug( "cache socket pick " + key + " " + hashCode ); 878 | 879 | if ( !this.initialized ) { 880 | log.error( "attempting to get SockIO from uninitialized pool!" ); 881 | return null; 882 | } 883 | 884 | // if no servers return null 885 | if ( ( this.hashingAlg == CONSISTENT_HASH && consistentBuckets.size() == 0 ) 886 | || ( buckets != null && buckets.size() == 0 ) ) 887 | return null; 888 | 889 | // if only one server, return it 890 | if ( ( this.hashingAlg == CONSISTENT_HASH && consistentBuckets.size() == 1 ) 891 | || ( buckets != null && buckets.size() == 1 ) ) { 892 | 893 | SockIO sock = ( this.hashingAlg == CONSISTENT_HASH ) 894 | ? getConnection( consistentBuckets.get( consistentBuckets.firstKey() ) ) 895 | : getConnection( buckets.get( 0 ) ); 896 | 897 | if ( sock != null && sock.isConnected() ) { 898 | if ( aliveCheck ) { 899 | if ( !sock.isAlive() ) { 900 | sock.close(); 901 | try { sock.trueClose(); } catch ( IOException ioe ) { log.error( "failed to close dead socket" ); } 902 | sock = null; 903 | } 904 | } 905 | } 906 | else { 907 | if ( sock != null ) { 908 | deadPool.put( sock, ZERO ); 909 | sock = null; 910 | } 911 | } 912 | 913 | return sock; 914 | } 915 | 916 | // from here on, we are working w/ multiple servers 917 | // keep trying different servers until we find one 918 | // making sure we only try each server one time 919 | Set tryServers = new HashSet( Arrays.asList( servers ) ); 920 | 921 | // get initial bucket 922 | long bucket = getBucket( key, hashCode ); 923 | String server = ( this.hashingAlg == CONSISTENT_HASH ) 924 | ? consistentBuckets.get( bucket ) 925 | : buckets.get( (int)bucket ); 926 | 927 | while ( !tryServers.isEmpty() ) { 928 | 929 | // try to get socket from bucket 930 | SockIO sock = getConnection( server ); 931 | 932 | if ( log.isDebugEnabled() ) 933 | log.debug( "cache choose " + server + " for " + key ); 934 | 935 | if ( sock != null && sock.isConnected() ) { 936 | if ( aliveCheck ) { 937 | if ( sock.isAlive() ) { 938 | return sock; 939 | } 940 | else { 941 | sock.close(); 942 | try { sock.trueClose(); } catch ( IOException ioe ) { log.error( "failed to close dead socket" ); } 943 | sock = null; 944 | } 945 | } 946 | else { 947 | return sock; 948 | } 949 | } 950 | else { 951 | if ( sock != null ) { 952 | deadPool.put( sock, ZERO ); 953 | sock = null; 954 | } 955 | } 956 | 957 | // if we do not want to failover, then bail here 958 | if ( !failover ) 959 | return null; 960 | 961 | // log that we tried 962 | tryServers.remove( server ); 963 | 964 | if ( tryServers.isEmpty() ) 965 | break; 966 | 967 | // if we failed to get a socket from this server 968 | // then we try again by adding an incrementer to the 969 | // current key and then rehashing 970 | int rehashTries = 0; 971 | while ( !tryServers.contains( server ) ) { 972 | 973 | String newKey = String.format( "%s%s", rehashTries, key ); 974 | if ( log.isDebugEnabled() ) 975 | log.debug( "rehashing with: " + newKey ); 976 | 977 | bucket = getBucket( newKey, null ); 978 | server = ( this.hashingAlg == CONSISTENT_HASH ) 979 | ? consistentBuckets.get( bucket ) 980 | : buckets.get( (int)bucket ); 981 | 982 | rehashTries++; 983 | } 984 | } 985 | 986 | return null; 987 | } 988 | 989 | /** 990 | * Returns a SockIO object from the pool for the passed in host. 991 | * 992 | * Meant to be called from a more intelligent method
993 | * which handles choosing appropriate server
994 | * and failover. 995 | * 996 | * @param host host from which to retrieve object 997 | * @return SockIO object or null if fail to retrieve one 998 | */ 999 | public SockIO getConnection( String host ) { 1000 | 1001 | if ( !this.initialized ) { 1002 | log.error( "attempting to get SockIO from uninitialized pool!" ); 1003 | return null; 1004 | } 1005 | 1006 | if ( host == null ) 1007 | return null; 1008 | 1009 | synchronized( this ) { 1010 | 1011 | // if we have items in the pool 1012 | // then we can return it 1013 | if ( availPool != null && !availPool.isEmpty() ) { 1014 | 1015 | // take first connected socket 1016 | Map aSockets = availPool.get( host ); 1017 | 1018 | if ( aSockets != null && !aSockets.isEmpty() ) { 1019 | 1020 | for ( Iterator i = aSockets.keySet().iterator(); i.hasNext(); ) { 1021 | SockIO socket = i.next(); 1022 | 1023 | if ( socket.isConnected() ) { 1024 | if ( log.isDebugEnabled() ) 1025 | log.debug( "++++ moving socket for host (" + host + ") to busy pool ... socket: " + socket ); 1026 | 1027 | // remove from avail pool 1028 | i.remove(); 1029 | 1030 | // add to busy pool 1031 | addSocketToPool( busyPool, host, socket ); 1032 | 1033 | // return socket 1034 | return socket; 1035 | } 1036 | else { 1037 | // add to deadpool for later reaping 1038 | deadPool.put( socket, ZERO ); 1039 | 1040 | // remove from avail pool 1041 | i.remove(); 1042 | } 1043 | } 1044 | } 1045 | } 1046 | } 1047 | 1048 | // create one socket -- let the maint thread take care of creating more 1049 | SockIO socket = createSocket( host ); 1050 | if ( socket != null ) { 1051 | synchronized( this ) { 1052 | addSocketToPool( busyPool, host, socket ); 1053 | } 1054 | } 1055 | 1056 | return socket; 1057 | } 1058 | 1059 | /** 1060 | * Adds a socket to a given pool for the given host. 1061 | * THIS METHOD IS NOT THREADSAFE, SO BE CAREFUL WHEN USING! 1062 | * 1063 | * Internal utility method. 1064 | * 1065 | * @param pool pool to add to 1066 | * @param host host this socket is connected to 1067 | * @param socket socket to add 1068 | */ 1069 | protected void addSocketToPool( Map> pool, String host, SockIO socket ) { 1070 | 1071 | if ( pool.containsKey( host ) ) { 1072 | Map sockets = pool.get( host ); 1073 | 1074 | if ( sockets != null ) { 1075 | sockets.put( socket, new Long( System.currentTimeMillis() ) ); 1076 | return; 1077 | } 1078 | } 1079 | 1080 | Map sockets = 1081 | new IdentityHashMap(); 1082 | 1083 | sockets.put( socket, new Long( System.currentTimeMillis() ) ); 1084 | pool.put( host, sockets ); 1085 | } 1086 | 1087 | /** 1088 | * Removes a socket from specified pool for host. 1089 | * THIS METHOD IS NOT THREADSAFE, SO BE CAREFUL WHEN USING! 1090 | * 1091 | * Internal utility method. 1092 | * 1093 | * @param pool pool to remove from 1094 | * @param host host pool 1095 | * @param socket socket to remove 1096 | */ 1097 | protected void removeSocketFromPool( Map> pool, String host, SockIO socket ) { 1098 | if ( pool.containsKey( host ) ) { 1099 | Map sockets = pool.get( host ); 1100 | if ( sockets != null ) 1101 | sockets.remove( socket ); 1102 | } 1103 | } 1104 | 1105 | /** 1106 | * Closes and removes all sockets from specified pool for host. 1107 | * THIS METHOD IS NOT THREADSAFE, SO BE CAREFUL WHEN USING! 1108 | * 1109 | * Internal utility method. 1110 | * 1111 | * @param pool pool to clear 1112 | * @param host host to clear 1113 | */ 1114 | protected void clearHostFromPool( Map> pool, String host ) { 1115 | 1116 | if ( pool.containsKey( host ) ) { 1117 | Map sockets = pool.get( host ); 1118 | 1119 | if ( sockets != null && sockets.size() > 0 ) { 1120 | for ( Iterator i = sockets.keySet().iterator(); i.hasNext(); ) { 1121 | SockIO socket = i.next(); 1122 | try { 1123 | socket.trueClose(); 1124 | } 1125 | catch ( IOException ioe ) { 1126 | log.error( "++++ failed to close socket: " + ioe.getMessage() ); 1127 | } 1128 | 1129 | i.remove(); 1130 | socket = null; 1131 | } 1132 | } 1133 | } 1134 | } 1135 | 1136 | /** 1137 | * Checks a SockIO object in with the pool. 1138 | * 1139 | * This will remove SocketIO from busy pool, and optionally
1140 | * add to avail pool. 1141 | * 1142 | * @param socket socket to return 1143 | * @param addToAvail add to avail pool if true 1144 | */ 1145 | private void checkIn( SockIO socket, boolean addToAvail ) { 1146 | 1147 | String host = socket.getHost(); 1148 | if ( log.isDebugEnabled() ) 1149 | log.debug( "++++ calling check-in on socket: " + socket.toString() + " for host: " + host ); 1150 | 1151 | synchronized( this ) { 1152 | // remove from the busy pool 1153 | if ( log.isDebugEnabled() ) 1154 | log.debug( "++++ removing socket (" + socket.toString() + ") from busy pool for host: " + host ); 1155 | removeSocketFromPool( busyPool, host, socket ); 1156 | 1157 | if ( socket.isConnected() && addToAvail ) { 1158 | // add to avail pool 1159 | if ( log.isDebugEnabled() ) 1160 | log.debug( "++++ returning socket (" + socket.toString() + " to avail pool for host: " + host ); 1161 | addSocketToPool( availPool, host, socket ); 1162 | } 1163 | else { 1164 | deadPool.put( socket, ZERO ); 1165 | socket = null; 1166 | } 1167 | } 1168 | } 1169 | 1170 | /** 1171 | * Returns a socket to the avail pool. 1172 | * 1173 | * This is called from SockIO.close(). Calling this method
1174 | * directly without closing the SockIO object first
1175 | * will cause an IOException to be thrown. 1176 | * 1177 | * @param socket socket to return 1178 | */ 1179 | private void checkIn( SockIO socket ) { 1180 | checkIn( socket, true ); 1181 | } 1182 | 1183 | /** 1184 | * Closes all sockets in the passed in pool. 1185 | * 1186 | * Internal utility method. 1187 | * 1188 | * @param pool pool to close 1189 | */ 1190 | protected void closePool( Map> pool ) { 1191 | for ( Iterator i = pool.keySet().iterator(); i.hasNext(); ) { 1192 | String host = i.next(); 1193 | Map sockets = pool.get( host ); 1194 | 1195 | for ( Iterator j = sockets.keySet().iterator(); j.hasNext(); ) { 1196 | SockIO socket = j.next(); 1197 | 1198 | try { 1199 | socket.trueClose(); 1200 | } 1201 | catch ( IOException ioe ) { 1202 | log.error( "++++ failed to trueClose socket: " + socket.toString() + " for host: " + host ); 1203 | } 1204 | 1205 | j.remove(); 1206 | socket = null; 1207 | } 1208 | } 1209 | } 1210 | 1211 | /** 1212 | * Shuts down the pool. 1213 | * 1214 | * Cleanly closes all sockets.
1215 | * Stops the maint thread.
1216 | * Nulls out all internal maps
1217 | */ 1218 | public void shutDown() { 1219 | synchronized( this ) { 1220 | if ( log.isDebugEnabled() ) 1221 | log.debug( "++++ SockIOPool shutting down..." ); 1222 | 1223 | if ( maintThread != null && maintThread.isRunning() ) { 1224 | // stop the main thread 1225 | stopMaintThread(); 1226 | 1227 | // wait for the thread to finish 1228 | while ( maintThread.isRunning() ) { 1229 | if ( log.isDebugEnabled() ) 1230 | log.debug( "++++ waiting for main thread to finish run +++" ); 1231 | try { Thread.sleep( 500 ); } catch ( Exception ex ) { } 1232 | } 1233 | } 1234 | 1235 | if ( log.isDebugEnabled() ) 1236 | log.debug( "++++ closing all internal pools." ); 1237 | closePool( availPool ); 1238 | closePool( busyPool ); 1239 | availPool = null; 1240 | busyPool = null; 1241 | buckets = null; 1242 | consistentBuckets = null; 1243 | hostDeadDur = null; 1244 | hostDead = null; 1245 | maintThread = null; 1246 | initialized = false; 1247 | if ( log.isDebugEnabled() ) 1248 | log.debug( "++++ SockIOPool finished shutting down." ); 1249 | } 1250 | } 1251 | 1252 | /** 1253 | * Starts the maintenance thread. 1254 | * 1255 | * This thread will manage the size of the active pool
1256 | * as well as move any closed, but not checked in sockets
1257 | * back to the available pool. 1258 | */ 1259 | protected void startMaintThread() { 1260 | 1261 | if ( maintThread != null ) { 1262 | 1263 | if ( maintThread.isRunning() ) { 1264 | log.error( "main thread already running" ); 1265 | } 1266 | else { 1267 | maintThread.start(); 1268 | } 1269 | } 1270 | else { 1271 | maintThread = new MaintThread( this ); 1272 | maintThread.setInterval( this.maintSleep ); 1273 | maintThread.start(); 1274 | } 1275 | } 1276 | 1277 | /** 1278 | * Stops the maintenance thread. 1279 | */ 1280 | protected void stopMaintThread() { 1281 | if ( maintThread != null && maintThread.isRunning() ) 1282 | maintThread.stopThread(); 1283 | } 1284 | 1285 | /** 1286 | * Runs self maintenance on all internal pools. 1287 | * 1288 | * This is typically called by the maintenance thread to manage pool size. 1289 | */ 1290 | protected void selfMaint() { 1291 | if ( log.isDebugEnabled() ) 1292 | log.debug( "++++ Starting self maintenance...." ); 1293 | 1294 | // go through avail sockets and create sockets 1295 | // as needed to maintain pool settings 1296 | Map needSockets = 1297 | new HashMap(); 1298 | 1299 | synchronized( this ) { 1300 | // find out how many to create 1301 | for ( Iterator i = availPool.keySet().iterator(); i.hasNext(); ) { 1302 | String host = i.next(); 1303 | Map sockets = availPool.get( host ); 1304 | 1305 | if ( log.isDebugEnabled() ) 1306 | log.debug( "++++ Size of avail pool for host (" + host + ") = " + sockets.size() ); 1307 | 1308 | // if pool is too small (n < minSpare) 1309 | if ( sockets.size() < minConn ) { 1310 | // need to create new sockets 1311 | int need = minConn - sockets.size(); 1312 | needSockets.put( host, need ); 1313 | } 1314 | } 1315 | } 1316 | 1317 | // now create 1318 | Map> newSockets = 1319 | new HashMap>(); 1320 | 1321 | for ( String host : needSockets.keySet() ) { 1322 | Integer need = needSockets.get( host ); 1323 | 1324 | if ( log.isDebugEnabled() ) 1325 | log.debug( "++++ Need to create " + need + " new sockets for pool for host: " + host ); 1326 | 1327 | Set newSock = new HashSet( need ); 1328 | for ( int j = 0; j < need; j++ ) { 1329 | SockIO socket = createSocket( host ); 1330 | 1331 | if ( socket == null ) 1332 | break; 1333 | 1334 | newSock.add( socket ); 1335 | } 1336 | 1337 | newSockets.put( host, newSock ); 1338 | } 1339 | 1340 | // synchronize to add and remove to/from avail pool 1341 | // as well as clean up the busy pool (no point in releasing 1342 | // lock here as should be quick to pool adjust and no 1343 | // blocking ops here) 1344 | synchronized( this ) { 1345 | for ( String host : newSockets.keySet() ) { 1346 | Set sockets = newSockets.get( host ); 1347 | for ( SockIO socket : sockets ) 1348 | addSocketToPool( availPool, host, socket ); 1349 | } 1350 | 1351 | for ( Iterator i = availPool.keySet().iterator(); i.hasNext(); ) { 1352 | String host = i.next(); 1353 | Map sockets = availPool.get( host ); 1354 | if ( log.isDebugEnabled() ) 1355 | log.debug( "++++ Size of avail pool for host (" + host + ") = " + sockets.size() ); 1356 | 1357 | if ( sockets.size() > maxConn ) { 1358 | // need to close down some sockets 1359 | int diff = sockets.size() - maxConn; 1360 | int needToClose = (diff <= poolMultiplier) 1361 | ? diff 1362 | : (diff) / poolMultiplier; 1363 | 1364 | if ( log.isDebugEnabled() ) 1365 | log.debug( "++++ need to remove " + needToClose + " spare sockets for pool for host: " + host ); 1366 | 1367 | for ( Iterator j = sockets.keySet().iterator(); j.hasNext(); ) { 1368 | if ( needToClose <= 0 ) 1369 | break; 1370 | 1371 | // remove stale entries 1372 | SockIO socket = j.next(); 1373 | long expire = sockets.get( socket ).longValue(); 1374 | 1375 | // if past idle time 1376 | // then close socket 1377 | // and remove from pool 1378 | if ( (expire + maxIdle) < System.currentTimeMillis() ) { 1379 | if ( log.isDebugEnabled() ) 1380 | log.debug( "+++ removing stale entry from pool as it is past its idle timeout and pool is over max spare" ); 1381 | 1382 | // remove from the availPool 1383 | deadPool.put( socket, ZERO ); 1384 | j.remove(); 1385 | needToClose--; 1386 | } 1387 | } 1388 | } 1389 | } 1390 | 1391 | // go through busy sockets and destroy sockets 1392 | // as needed to maintain pool settings 1393 | for ( Iterator i = busyPool.keySet().iterator(); i.hasNext(); ) { 1394 | 1395 | String host = i.next(); 1396 | Map sockets = busyPool.get( host ); 1397 | 1398 | if ( log.isDebugEnabled() ) 1399 | log.debug( "++++ Size of busy pool for host (" + host + ") = " + sockets.size() ); 1400 | 1401 | // loop through all connections and check to see if we have any hung connections 1402 | for ( Iterator j = sockets.keySet().iterator(); j.hasNext(); ) { 1403 | // remove stale entries 1404 | SockIO socket = j.next(); 1405 | long hungTime = sockets.get( socket ).longValue(); 1406 | 1407 | // if past max busy time 1408 | // then close socket 1409 | // and remove from pool 1410 | if ( (hungTime + maxBusyTime) < System.currentTimeMillis() ) { 1411 | log.error( "+++ removing potentially hung connection from busy pool ... socket in pool for " + (System.currentTimeMillis() - hungTime) + "ms" ); 1412 | 1413 | // remove from the busy pool 1414 | deadPool.put( socket, ZERO ); 1415 | j.remove(); 1416 | } 1417 | } 1418 | } 1419 | } 1420 | 1421 | // finally clean out the deadPool 1422 | Set toClose; 1423 | synchronized( deadPool ) { 1424 | toClose = deadPool.keySet(); 1425 | deadPool = new IdentityHashMap(); 1426 | } 1427 | 1428 | for ( SockIO socket : toClose ) { 1429 | try { 1430 | socket.trueClose( false ); 1431 | } 1432 | catch ( Exception ex ) { 1433 | log.error( "++++ failed to close SockIO obj from deadPool" ); 1434 | log.error( ex.getMessage(), ex ); 1435 | } 1436 | 1437 | socket = null; 1438 | } 1439 | 1440 | if ( log.isDebugEnabled() ) 1441 | log.debug( "+++ ending self maintenance." ); 1442 | } 1443 | 1444 | /** 1445 | * Class which extends thread and handles maintenance of the pool. 1446 | * 1447 | * @author greg whalin 1448 | * @version 1.5 1449 | */ 1450 | protected static class MaintThread extends Thread { 1451 | 1452 | // logger 1453 | private static Logger log = 1454 | Logger.getLogger( MaintThread.class.getName() ); 1455 | 1456 | private SockIOPool pool; 1457 | private long interval = 1000 * 3; // every 3 seconds 1458 | private boolean stopThread = false; 1459 | private boolean running; 1460 | 1461 | protected MaintThread( SockIOPool pool ) { 1462 | this.pool = pool; 1463 | this.setDaemon( true ); 1464 | this.setName( "MaintThread" ); 1465 | } 1466 | 1467 | public void setInterval( long interval ) { this.interval = interval; } 1468 | 1469 | public boolean isRunning() { 1470 | return this.running; 1471 | } 1472 | 1473 | /** 1474 | * sets stop variable 1475 | * and interupts any wait 1476 | */ 1477 | public void stopThread() { 1478 | this.stopThread = true; 1479 | this.interrupt(); 1480 | } 1481 | 1482 | /** 1483 | * Start the thread. 1484 | */ 1485 | public void run() { 1486 | this.running = true; 1487 | 1488 | while ( !this.stopThread ) { 1489 | try { 1490 | Thread.sleep( interval ); 1491 | 1492 | // if pool is initialized, then 1493 | // run the maintenance method on itself 1494 | if ( pool.isInitialized() ) 1495 | pool.selfMaint(); 1496 | 1497 | } 1498 | catch ( Exception e ) { 1499 | break; 1500 | } 1501 | } 1502 | 1503 | this.running = false; 1504 | } 1505 | } 1506 | 1507 | /** 1508 | * MemCached client for Java, utility class for Socket IO. 1509 | * 1510 | * This class is a wrapper around a Socket and its streams. 1511 | * 1512 | * @author greg whalin 1513 | * @author Richard 'toast' Russo 1514 | * @version 1.5 1515 | */ 1516 | public static class SockIO implements LineInputStream { 1517 | 1518 | // logger 1519 | private static Logger log = 1520 | Logger.getLogger( SockIO.class.getName() ); 1521 | 1522 | // pool 1523 | private SockIOPool pool; 1524 | 1525 | // data 1526 | private String host; 1527 | private Socket sock; 1528 | 1529 | private DataInputStream in; 1530 | private BufferedOutputStream out; 1531 | 1532 | /** 1533 | * creates a new SockIO object wrapping a socket 1534 | * connection to host:port, and its input and output streams 1535 | * 1536 | * @param pool Pool this object is tied to 1537 | * @param host host to connect to 1538 | * @param port port to connect to 1539 | * @param timeout int ms to block on data for read 1540 | * @param connectTimeout timeout (in ms) for initial connection 1541 | * @param noDelay TCP NODELAY option? 1542 | * @throws IOException if an io error occurrs when creating socket 1543 | * @throws UnknownHostException if hostname is invalid 1544 | */ 1545 | public SockIO( SockIOPool pool, String host, int port, int timeout, int connectTimeout, boolean noDelay ) throws IOException, UnknownHostException { 1546 | 1547 | this.pool = pool; 1548 | 1549 | // get a socket channel 1550 | sock = getSocket( host, port, connectTimeout ); 1551 | 1552 | if ( timeout >= 0 ) 1553 | sock.setSoTimeout( timeout ); 1554 | 1555 | // testing only 1556 | sock.setTcpNoDelay( noDelay ); 1557 | 1558 | // wrap streams 1559 | in = new DataInputStream( new BufferedInputStream( sock.getInputStream() ) ); 1560 | out = new BufferedOutputStream( sock.getOutputStream() ); 1561 | 1562 | this.host = host + ":" + port; 1563 | } 1564 | 1565 | /** 1566 | * creates a new SockIO object wrapping a socket 1567 | * connection to host:port, and its input and output streams 1568 | * 1569 | * @param host hostname:port 1570 | * @param timeout read timeout value for connected socket 1571 | * @param connectTimeout timeout for initial connections 1572 | * @param noDelay TCP NODELAY option? 1573 | * @throws IOException if an io error occurrs when creating socket 1574 | * @throws UnknownHostException if hostname is invalid 1575 | */ 1576 | public SockIO( SockIOPool pool, String host, int timeout, int connectTimeout, boolean noDelay ) throws IOException, UnknownHostException { 1577 | 1578 | this.pool = pool; 1579 | 1580 | String[] ip = host.split(":"); 1581 | 1582 | // get socket: default is to use non-blocking connect 1583 | sock = getSocket( ip[ 0 ], Integer.parseInt( ip[ 1 ] ), connectTimeout ); 1584 | 1585 | if ( timeout >= 0 ) 1586 | this.sock.setSoTimeout( timeout ); 1587 | 1588 | // testing only 1589 | sock.setTcpNoDelay( noDelay ); 1590 | 1591 | // wrap streams 1592 | in = new DataInputStream( new BufferedInputStream( sock.getInputStream() ) ); 1593 | out = new BufferedOutputStream( sock.getOutputStream() ); 1594 | 1595 | this.host = host; 1596 | } 1597 | 1598 | /** 1599 | * Method which gets a connection from SocketChannel. 1600 | * 1601 | * @param host host to establish connection to 1602 | * @param port port on that host 1603 | * @param timeout connection timeout in ms 1604 | * 1605 | * @return connected socket 1606 | * @throws IOException if errors connecting or if connection times out 1607 | */ 1608 | protected static Socket getSocket( String host, int port, int timeout ) throws IOException { 1609 | SocketChannel sock = SocketChannel.open(); 1610 | sock.socket().connect( new InetSocketAddress( host, port ), timeout ); 1611 | return sock.socket(); 1612 | } 1613 | 1614 | /** 1615 | * Lets caller get access to underlying channel. 1616 | * 1617 | * @return the backing SocketChannel 1618 | */ 1619 | public SocketChannel getChannel() { return sock.getChannel(); } 1620 | 1621 | /** 1622 | * returns the host this socket is connected to 1623 | * 1624 | * @return String representation of host (hostname:port) 1625 | */ 1626 | public String getHost() { return this.host; } 1627 | 1628 | /** 1629 | * closes socket and all streams connected to it 1630 | * 1631 | * @throws IOException if fails to close streams or socket 1632 | */ 1633 | public void trueClose() throws IOException { 1634 | trueClose( true ); 1635 | } 1636 | 1637 | /** 1638 | * closes socket and all streams connected to it 1639 | * 1640 | * @throws IOException if fails to close streams or socket 1641 | */ 1642 | public void trueClose( boolean addToDeadPool ) throws IOException { 1643 | if ( log.isDebugEnabled() ) 1644 | log.debug( "++++ Closing socket for real: " + toString() ); 1645 | 1646 | boolean err = false; 1647 | StringBuilder errMsg = new StringBuilder(); 1648 | 1649 | if ( in != null ) { 1650 | try { 1651 | in.close(); 1652 | } 1653 | catch( IOException ioe ) { 1654 | log.error( "++++ error closing input stream for socket: " + toString() + " for host: " + getHost() ); 1655 | log.error( ioe.getMessage(), ioe ); 1656 | errMsg.append( "++++ error closing input stream for socket: " + toString() + " for host: " + getHost() + "\n" ); 1657 | errMsg.append( ioe.getMessage() ); 1658 | err = true; 1659 | } 1660 | } 1661 | 1662 | if ( out != null ) { 1663 | try { 1664 | out.close(); 1665 | } 1666 | catch ( IOException ioe ) { 1667 | log.error( "++++ error closing output stream for socket: " + toString() + " for host: " + getHost() ); 1668 | log.error( ioe.getMessage(), ioe ); 1669 | errMsg.append( "++++ error closing output stream for socket: " + toString() + " for host: " + getHost() + "\n" ); 1670 | errMsg.append( ioe.getMessage() ); 1671 | err = true; 1672 | } 1673 | } 1674 | 1675 | if ( sock != null ) { 1676 | try { 1677 | sock.close(); 1678 | } 1679 | catch ( IOException ioe ) { 1680 | log.error( "++++ error closing socket: " + toString() + " for host: " + getHost() ); 1681 | log.error( ioe.getMessage(), ioe ); 1682 | errMsg.append( "++++ error closing socket: " + toString() + " for host: " + getHost() + "\n" ); 1683 | errMsg.append( ioe.getMessage() ); 1684 | err = true; 1685 | } 1686 | } 1687 | 1688 | // check in to pool 1689 | if ( addToDeadPool && sock != null ) 1690 | pool.checkIn( this, false ); 1691 | 1692 | in = null; 1693 | out = null; 1694 | sock = null; 1695 | 1696 | if ( err ) 1697 | throw new IOException( errMsg.toString() ); 1698 | } 1699 | 1700 | /** 1701 | * sets closed flag and checks in to connection pool 1702 | * but does not close connections 1703 | */ 1704 | void close() { 1705 | // check in to pool 1706 | if ( log.isDebugEnabled() ) 1707 | log.debug("++++ marking socket (" + this.toString() + ") as closed and available to return to avail pool"); 1708 | pool.checkIn( this ); 1709 | } 1710 | 1711 | /** 1712 | * checks if the connection is open 1713 | * 1714 | * @return true if connected 1715 | */ 1716 | boolean isConnected() { 1717 | return ( sock != null && sock.isConnected() ); 1718 | } 1719 | 1720 | /* 1721 | * checks to see that the connection is still working 1722 | * 1723 | * @return true if still alive 1724 | */ 1725 | boolean isAlive() { 1726 | 1727 | if ( !isConnected() ) 1728 | return false; 1729 | 1730 | // try to talk to the server w/ a dumb query to ask its version 1731 | try { 1732 | this.write( "version\r\n".getBytes() ); 1733 | this.flush(); 1734 | String response = this.readLine(); 1735 | } 1736 | catch ( IOException ex ) { 1737 | return false; 1738 | } 1739 | 1740 | return true; 1741 | } 1742 | 1743 | /** 1744 | * reads a line 1745 | * intentionally not using the deprecated readLine method from DataInputStream 1746 | * 1747 | * @return String that was read in 1748 | * @throws IOException if io problems during read 1749 | */ 1750 | public String readLine() throws IOException { 1751 | if ( sock == null || !sock.isConnected() ) { 1752 | log.error( "++++ attempting to read from closed socket" ); 1753 | throw new IOException( "++++ attempting to read from closed socket" ); 1754 | } 1755 | 1756 | byte[] b = new byte[1]; 1757 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 1758 | boolean eol = false; 1759 | 1760 | while ( in.read( b, 0, 1 ) != -1 ) { 1761 | 1762 | if ( b[0] == 13 ) { 1763 | eol = true; 1764 | } 1765 | else { 1766 | if ( eol ) { 1767 | if ( b[0] == 10 ) 1768 | break; 1769 | 1770 | eol = false; 1771 | } 1772 | } 1773 | 1774 | // cast byte into char array 1775 | bos.write( b, 0, 1 ); 1776 | } 1777 | 1778 | if ( bos == null || bos.size() <= 0 ) { 1779 | throw new IOException( "++++ Stream appears to be dead, so closing it down" ); 1780 | } 1781 | 1782 | // else return the string 1783 | return bos.toString().trim(); 1784 | } 1785 | 1786 | /** 1787 | * reads up to end of line and returns nothing 1788 | * 1789 | * @throws IOException if io problems during read 1790 | */ 1791 | public void clearEOL() throws IOException { 1792 | if ( sock == null || !sock.isConnected() ) { 1793 | log.error( "++++ attempting to read from closed socket" ); 1794 | throw new IOException( "++++ attempting to read from closed socket" ); 1795 | } 1796 | 1797 | byte[] b = new byte[1]; 1798 | boolean eol = false; 1799 | while ( in.read( b, 0, 1 ) != -1 ) { 1800 | 1801 | // only stop when we see 1802 | // \r (13) followed by \n (10) 1803 | if ( b[0] == 13 ) { 1804 | eol = true; 1805 | continue; 1806 | } 1807 | 1808 | if ( eol ) { 1809 | if ( b[0] == 10 ) 1810 | break; 1811 | 1812 | eol = false; 1813 | } 1814 | } 1815 | } 1816 | 1817 | /** 1818 | * reads length bytes into the passed in byte array from dtream 1819 | * 1820 | * @param b byte array 1821 | * @throws IOException if io problems during read 1822 | */ 1823 | public int read( byte[] b ) throws IOException { 1824 | if ( sock == null || !sock.isConnected() ) { 1825 | log.error( "++++ attempting to read from closed socket" ); 1826 | throw new IOException( "++++ attempting to read from closed socket" ); 1827 | } 1828 | 1829 | int count = 0; 1830 | while ( count < b.length ) { 1831 | int cnt = in.read( b, count, (b.length - count) ); 1832 | count += cnt; 1833 | } 1834 | 1835 | return count; 1836 | } 1837 | 1838 | /** 1839 | * flushes output stream 1840 | * 1841 | * @throws IOException if io problems during read 1842 | */ 1843 | void flush() throws IOException { 1844 | if ( sock == null || !sock.isConnected() ) { 1845 | log.error( "++++ attempting to write to closed socket" ); 1846 | throw new IOException( "++++ attempting to write to closed socket" ); 1847 | } 1848 | out.flush(); 1849 | } 1850 | 1851 | /** 1852 | * writes a byte array to the output stream 1853 | * 1854 | * @param b byte array to write 1855 | * @throws IOException if an io error happens 1856 | */ 1857 | void write( byte[] b ) throws IOException { 1858 | if ( sock == null || !sock.isConnected() ) { 1859 | log.error( "++++ attempting to write to closed socket" ); 1860 | throw new IOException( "++++ attempting to write to closed socket" ); 1861 | } 1862 | out.write( b ); 1863 | } 1864 | 1865 | /** 1866 | * use the sockets hashcode for this object 1867 | * so we can key off of SockIOs 1868 | * 1869 | * @return int hashcode 1870 | */ 1871 | public int hashCode() { 1872 | return ( sock == null ) ? 0 : sock.hashCode(); 1873 | } 1874 | 1875 | /** 1876 | * returns the string representation of this socket 1877 | * 1878 | * @return string 1879 | */ 1880 | public String toString() { 1881 | return ( sock == null ) ? "" : sock.toString(); 1882 | } 1883 | 1884 | /** 1885 | * Hack to reap any leaking children. 1886 | */ 1887 | protected void finalize() throws Throwable { 1888 | try { 1889 | if ( sock != null ) { 1890 | log.error( "++++ closing potentially leaked socket in finalize" ); 1891 | sock.close(); 1892 | sock = null; 1893 | } 1894 | } 1895 | catch ( Throwable t ) { 1896 | log.error( t.getMessage(), t ); 1897 | } 1898 | finally { 1899 | super.finalize(); 1900 | } 1901 | } 1902 | } 1903 | } 1904 | -------------------------------------------------------------------------------- /src/com/meetup/memcached/MemcachedClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2008 Greg Whalin 3 | * All rights reserved. 4 | * 5 | * This library is free software; you can redistribute it and/or 6 | * modify it under the terms of the BSD license 7 | * 8 | * This library is distributed in the hope that it will be 9 | * useful, but WITHOUT ANY WARRANTY; without even the implied 10 | * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 11 | * PURPOSE. 12 | * 13 | * You should have received a copy of the BSD License along with this 14 | * library. 15 | * 16 | * @author Greg Whalin 17 | */ 18 | package com.meetup.memcached; 19 | 20 | import java.util.*; 21 | import java.util.zip.*; 22 | import java.nio.*; 23 | import java.net.InetAddress; 24 | import java.nio.charset.*; 25 | import java.nio.channels.*; 26 | import java.nio.channels.spi.*; 27 | import java.io.*; 28 | import java.net.URLEncoder; 29 | 30 | import org.apache.log4j.Logger; 31 | 32 | /** 33 | * This is a Memcached client for the Java platform available from 34 | * http://www.danga.com/memcached/. 35 | *
36 | * Supports setting, adding, replacing, deleting compressed/uncompressed and
37 | * serialized (can be stored as string if object is native class) objects to memcached.
38 | *
39 | * Now pulls SockIO objects from SockIOPool, which is a connection pool. The server failover
40 | * has also been moved into the SockIOPool class.
41 | * This pool needs to be initialized prior to the client working. See javadocs from SockIOPool.
42 | *
43 | * Some examples of use follow.
44 | *

To create cache client object and set params:

45 | * 
 
  46 |  *	MemcachedClient mc = new MemcachedClient();
  47 |  *
  48 |  *	// compression is enabled by default	
  49 |  *	mc.setCompressEnable(true);
  50 |  *
  51 |  *	// set compression threshhold to 4 KB (default: 15 KB)	
  52 |  *	mc.setCompressThreshold(4096);
  53 |  *
  54 |  *	// turn on storing primitive types as a string representation
  55 |  *	// Should not do this in most cases.	
  56 |  *	mc.setPrimitiveAsString(true);
  57 |  *
58 | *

To store an object:

59 | * 
  60 |  *	MemcachedClient mc = new MemcachedClient();
  61 |  *	String key   = "cacheKey1";	
  62 |  *	Object value = SomeClass.getObject();	
  63 |  *	mc.set(key, value);
  64 |  *
65 | *

To store an object using a custom server hashCode:

66 | * 
  67 |  *	MemcachedClient mc = new MemcachedClient();
  68 |  *	String key   = "cacheKey1";	
  69 |  *	Object value = SomeClass.getObject();	
  70 |  *	Integer hash = new Integer(45);	
  71 |  *	mc.set(key, value, hash);
  72 |  *
73 | * The set method shown above will always set the object in the cache.
74 | * The add and replace methods do the same, but with a slight difference.
75 | *
    76 | *
  • add -- will store the object only if the server does not have an entry for this key
    • 77 | *
  • replace -- will store the object only if the server already has an entry for this key
    • 78 | *
79 | *

To delete a cache entry:

80 | * 
  81 |  *	MemcachedClient mc = new MemcachedClient();
  82 |  *	String key   = "cacheKey1";	
  83 |  *	mc.delete(key);
  84 |  *
85 | *

To delete a cache entry using a custom hash code:

86 | * 
  87 |  *	MemcachedClient mc = new MemcachedClient();
  88 |  *	String key   = "cacheKey1";	
  89 |  *	Integer hash = new Integer(45);	
  90 |  *	mc.delete(key, hashCode);
  91 |  *
92 | *

To store a counter and then increment or decrement that counter:

93 | * 
  94 |  *	MemcachedClient mc = new MemcachedClient();
  95 |  *	String key   = "counterKey";	
  96 |  *	mc.storeCounter(key, new Integer(100));
  97 |  *	System.out.println("counter after adding      1: " mc.incr(key));	
  98 |  *	System.out.println("counter after adding      5: " mc.incr(key, 5));	
  99 |  *	System.out.println("counter after subtracting 4: " mc.decr(key, 4));	
 100 |  *	System.out.println("counter after subtracting 1: " mc.decr(key));	
 101 |  *
102 | *

To store a counter and then increment or decrement that counter with custom hash:

103 | * 
 104 |  *	MemcachedClient mc = new MemcachedClient();
 105 |  *	String key   = "counterKey";	
 106 |  *	Integer hash = new Integer(45);	
 107 |  *	mc.storeCounter(key, new Integer(100), hash);
 108 |  *	System.out.println("counter after adding      1: " mc.incr(key, 1, hash));	
 109 |  *	System.out.println("counter after adding      5: " mc.incr(key, 5, hash));	
 110 |  *	System.out.println("counter after subtracting 4: " mc.decr(key, 4, hash));	
 111 |  *	System.out.println("counter after subtracting 1: " mc.decr(key, 1, hash));	
 112 |  *
113 | *

To retrieve an object from the cache:

114 | * 
 115 |  *	MemcachedClient mc = new MemcachedClient();
 116 |  *	String key   = "key";	
 117 |  *	Object value = mc.get(key);	
 118 |  *
119 | *

To retrieve an object from the cache with custom hash:

120 | * 
 121 |  *	MemcachedClient mc = new MemcachedClient();
 122 |  *	String key   = "key";	
 123 |  *	Integer hash = new Integer(45);	
 124 |  *	Object value = mc.get(key, hash);	
 125 |  *
126 | *

To retrieve an multiple objects from the cache

127 | * 
 128 |  *	MemcachedClient mc = new MemcachedClient();
 129 |  *	String[] keys      = { "key", "key1", "key2" };
 130 |  *	Map<Object> values = mc.getMulti(keys);
 131 |  *
132 | *

To retrieve an multiple objects from the cache with custom hashing

133 | * 
 134 |  *	MemcachedClient mc = new MemcachedClient();
 135 |  *	String[] keys      = { "key", "key1", "key2" };
 136 |  *	Integer[] hashes   = { new Integer(45), new Integer(32), new Integer(44) };
 137 |  *	Map<Object> values = mc.getMulti(keys, hashes);
 138 |  *
139 | *

To flush all items in server(s)

140 | * 
 141 |  *	MemcachedClient mc = new MemcachedClient();
 142 |  *	mc.flushAll();
 143 |  *
144 | *

To get stats from server(s)

145 | * 
 146 |  *	MemcachedClient mc = new MemcachedClient();
 147 |  *	Map stats = mc.stats();
 148 |  *
149 | * 150 | * @author greg whalin 151 | * @author Richard 'toast' Russo 152 | * @author Kevin Burton 153 | * @author Robert Watts 154 | * @author Vin Chawla 155 | * @version 1.5 156 | */ 157 | public class MemcachedClient { 158 | 159 | // logger 160 | private static Logger log = 161 | Logger.getLogger( MemcachedClient.class.getName() ); 162 | 163 | // return codes 164 | private static final String VALUE = "VALUE"; // start of value line from server 165 | private static final String STATS = "STAT"; // start of stats line from server 166 | private static final String ITEM = "ITEM"; // start of item line from server 167 | private static final String DELETED = "DELETED"; // successful deletion 168 | private static final String NOTFOUND = "NOT_FOUND"; // record not found for delete or incr/decr 169 | private static final String STORED = "STORED"; // successful store of data 170 | private static final String NOTSTORED = "NOT_STORED"; // data not stored 171 | private static final String OK = "OK"; // success 172 | private static final String END = "END"; // end of data from server 173 | 174 | private static final String ERROR = "ERROR"; // invalid command name from client 175 | private static final String CLIENT_ERROR = "CLIENT_ERROR"; // client error in input line - invalid protocol 176 | private static final String SERVER_ERROR = "SERVER_ERROR"; // server error 177 | 178 | private static final byte[] B_END = "END\r\n".getBytes(); 179 | private static final byte[] B_NOTFOUND = "NOT_FOUND\r\n".getBytes(); 180 | private static final byte[] B_DELETED = "DELETED\r\r".getBytes(); 181 | private static final byte[] B_STORED = "STORED\r\r".getBytes(); 182 | 183 | // default compression threshold 184 | private static final int COMPRESS_THRESH = 30720; 185 | 186 | // values for cache flags 187 | public static final int MARKER_BYTE = 1; 188 | public static final int MARKER_BOOLEAN = 8192; 189 | public static final int MARKER_INTEGER = 4; 190 | public static final int MARKER_LONG = 16384; 191 | public static final int MARKER_CHARACTER = 16; 192 | public static final int MARKER_STRING = 32; 193 | public static final int MARKER_STRINGBUFFER = 64; 194 | public static final int MARKER_FLOAT = 128; 195 | public static final int MARKER_SHORT = 256; 196 | public static final int MARKER_DOUBLE = 512; 197 | public static final int MARKER_DATE = 1024; 198 | public static final int MARKER_STRINGBUILDER = 2048; 199 | public static final int MARKER_BYTEARR = 4096; 200 | public static final int F_COMPRESSED = 2; 201 | public static final int F_SERIALIZED = 8; 202 | 203 | // flags 204 | private boolean sanitizeKeys; 205 | private boolean primitiveAsString; 206 | private boolean compressEnable; 207 | private long compressThreshold; 208 | private String defaultEncoding; 209 | 210 | // pool instance 211 | private SockIOPool pool; 212 | 213 | // which pool to use 214 | private String poolName; 215 | 216 | // optional passed in classloader 217 | private ClassLoader classLoader; 218 | 219 | // optional error handler 220 | private ErrorHandler errorHandler; 221 | 222 | /** 223 | * Creates a new instance of MemCachedClient. 224 | */ 225 | public MemcachedClient() { 226 | init(); 227 | } 228 | 229 | /** 230 | * Creates a new instance of MemCachedClient 231 | * accepting a passed in pool name. 232 | * 233 | * @param poolName name of SockIOPool 234 | */ 235 | public MemcachedClient( String poolName ) { 236 | this.poolName = poolName; 237 | init(); 238 | } 239 | 240 | /** 241 | * Creates a new instance of MemCacheClient but 242 | * acceptes a passed in ClassLoader. 243 | * 244 | * @param classLoader ClassLoader object. 245 | */ 246 | public MemcachedClient( ClassLoader classLoader ) { 247 | this.classLoader = classLoader; 248 | init(); 249 | } 250 | 251 | /** 252 | * Creates a new instance of MemCacheClient but 253 | * acceptes a passed in ClassLoader and a passed 254 | * in ErrorHandler. 255 | * 256 | * @param classLoader ClassLoader object. 257 | * @param errorHandler ErrorHandler object. 258 | */ 259 | public MemcachedClient( ClassLoader classLoader, ErrorHandler errorHandler ) { 260 | this.classLoader = classLoader; 261 | this.errorHandler = errorHandler; 262 | init(); 263 | } 264 | 265 | /** 266 | * Creates a new instance of MemCacheClient but 267 | * acceptes a passed in ClassLoader, ErrorHandler, 268 | * and SockIOPool name. 269 | * 270 | * @param classLoader ClassLoader object. 271 | * @param errorHandler ErrorHandler object. 272 | * @param poolName SockIOPool name 273 | */ 274 | public MemcachedClient( ClassLoader classLoader, ErrorHandler errorHandler, String poolName ) { 275 | this.classLoader = classLoader; 276 | this.errorHandler = errorHandler; 277 | this.poolName = poolName; 278 | init(); 279 | } 280 | 281 | /** 282 | * Initializes client object to defaults. 283 | * 284 | * This enables compression and sets compression threshhold to 15 KB. 285 | */ 286 | private void init() { 287 | this.sanitizeKeys = true; 288 | this.primitiveAsString = false; 289 | this.compressEnable = true; 290 | this.compressThreshold = COMPRESS_THRESH; 291 | this.defaultEncoding = "UTF-8"; 292 | this.poolName = ( this.poolName == null ) ? "default" : this.poolName; 293 | 294 | // get a pool instance to work with for the life of this instance 295 | this.pool = SockIOPool.getInstance( poolName ); 296 | } 297 | 298 | /** 299 | * Sets an optional ClassLoader to be used for 300 | * serialization. 301 | * 302 | * @param classLoader 303 | */ 304 | public void setClassLoader( ClassLoader classLoader ) { 305 | this.classLoader = classLoader; 306 | } 307 | 308 | /** 309 | * Sets an optional ErrorHandler. 310 | * 311 | * @param errorHandler 312 | */ 313 | public void setErrorHandler( ErrorHandler errorHandler ) { 314 | this.errorHandler = errorHandler; 315 | } 316 | 317 | /** 318 | * Enables/disables sanitizing keys by URLEncoding. 319 | * 320 | * @param sanitizeKeys if true, then URLEncode all keys 321 | */ 322 | public void setSanitizeKeys( boolean sanitizeKeys ) { 323 | this.sanitizeKeys = sanitizeKeys; 324 | } 325 | 326 | /** 327 | * Enables storing primitive types as their String values. 328 | * 329 | * @param primitiveAsString if true, then store all primitives as their string value. 330 | */ 331 | public void setPrimitiveAsString( boolean primitiveAsString ) { 332 | this.primitiveAsString = primitiveAsString; 333 | } 334 | 335 | /** 336 | * Sets default String encoding when storing primitives as Strings. 337 | * Default is UTF-8. 338 | * 339 | * @param defaultEncoding 340 | */ 341 | public void setDefaultEncoding( String defaultEncoding ) { 342 | this.defaultEncoding = defaultEncoding; 343 | } 344 | 345 | /** 346 | * Enable storing compressed data, provided it meets the threshold requirements. 347 | * 348 | * If enabled, data will be stored in compressed form if it is
349 | * longer than the threshold length set with setCompressThreshold(int)
350 | *
351 | * The default is that compression is enabled.
352 | *
353 | * Even if compression is disabled, compressed data will be automatically
354 | * decompressed. 355 | * 356 | * @param compressEnable true to enable compression, false to disable compression 357 | */ 358 | public void setCompressEnable( boolean compressEnable ) { 359 | this.compressEnable = compressEnable; 360 | } 361 | 362 | /** 363 | * Sets the required length for data to be considered for compression. 364 | * 365 | * If the length of the data to be stored is not equal or larger than this value, it will 366 | * not be compressed. 367 | * 368 | * This defaults to 15 KB. 369 | * 370 | * @param compressThreshold required length of data to consider compression 371 | */ 372 | public void setCompressThreshold( long compressThreshold ) { 373 | this.compressThreshold = compressThreshold; 374 | } 375 | 376 | /** 377 | * Checks to see if key exists in cache. 378 | * 379 | * @param key the key to look for 380 | * @return true if key found in cache, false if not (or if cache is down) 381 | */ 382 | public boolean keyExists( String key ) { 383 | return ( this.get( key, null, true ) != null ); 384 | } 385 | 386 | /** 387 | * Deletes an object from cache given cache key. 388 | * 389 | * @param key the key to be removed 390 | * @return true, if the data was deleted successfully 391 | */ 392 | public boolean delete( String key ) { 393 | return delete( key, null, null ); 394 | } 395 | 396 | /** 397 | * Deletes an object from cache given cache key and expiration date. 398 | * 399 | * @param key the key to be removed 400 | * @param expiry when to expire the record. 401 | * @return true, if the data was deleted successfully 402 | */ 403 | public boolean delete( String key, Date expiry ) { 404 | return delete( key, null, expiry ); 405 | } 406 | 407 | /** 408 | * Deletes an object from cache given cache key, a delete time, and an optional hashcode. 409 | * 410 | * The item is immediately made non retrievable.
411 | * Keep in mind {@link #add(String, Object) add} and {@link #replace(String, Object) replace}
412 | * will fail when used with the same key will fail, until the server reaches the
413 | * specified time. However, {@link #set(String, Object) set} will succeed,
414 | * and the new value will not be deleted. 415 | * 416 | * @param key the key to be removed 417 | * @param hashCode if not null, then the int hashcode to use 418 | * @param expiry when to expire the record. 419 | * @return true, if the data was deleted successfully 420 | */ 421 | public boolean delete( String key, Integer hashCode, Date expiry ) { 422 | 423 | if ( key == null ) { 424 | log.error( "null value for key passed to delete()" ); 425 | return false; 426 | } 427 | 428 | try { 429 | key = sanitizeKey( key ); 430 | } 431 | catch ( UnsupportedEncodingException e ) { 432 | 433 | // if we have an errorHandler, use its hook 434 | if ( errorHandler != null ) 435 | errorHandler.handleErrorOnDelete( this, e, key ); 436 | 437 | log.error( "failed to sanitize your key!", e ); 438 | return false; 439 | } 440 | 441 | // get SockIO obj from hash or from key 442 | SockIOPool.SockIO sock = pool.getSock( key, hashCode ); 443 | 444 | // return false if unable to get SockIO obj 445 | if ( sock == null ) { 446 | if ( errorHandler != null ) 447 | errorHandler.handleErrorOnDelete( this, new IOException( "no socket to server available" ), key ); 448 | return false; 449 | } 450 | 451 | // build command 452 | StringBuilder command = new StringBuilder( "delete " ).append( key ); 453 | if ( expiry != null ) 454 | command.append( " " + expiry.getTime() / 1000 ); 455 | 456 | command.append( "\r\n" ); 457 | 458 | try { 459 | sock.write( command.toString().getBytes() ); 460 | sock.flush(); 461 | 462 | // if we get appropriate response back, then we return true 463 | String line = sock.readLine(); 464 | if ( DELETED.equals( line ) ) { 465 | if ( log.isInfoEnabled() ) 466 | log.info( "++++ deletion of key: " + key + " from cache was a success" ); 467 | 468 | // return sock to pool and bail here 469 | sock.close(); 470 | sock = null; 471 | return true; 472 | } 473 | else if ( NOTFOUND.equals( line ) ) { 474 | if ( log.isInfoEnabled() ) 475 | log.info( "++++ deletion of key: " + key + " from cache failed as the key was not found" ); 476 | } 477 | else { 478 | log.error( "++++ error deleting key: " + key ); 479 | log.error( "++++ server response: " + line ); 480 | } 481 | } 482 | catch ( IOException e ) { 483 | 484 | // if we have an errorHandler, use its hook 485 | if ( errorHandler != null ) 486 | errorHandler.handleErrorOnDelete( this, e, key ); 487 | 488 | // exception thrown 489 | log.error( "++++ exception thrown while writing bytes to server on delete" ); 490 | log.error( e.getMessage(), e ); 491 | 492 | try { 493 | sock.trueClose(); 494 | } 495 | catch ( IOException ioe ) { 496 | log.error( "++++ failed to close socket : " + sock.toString() ); 497 | } 498 | 499 | sock = null; 500 | } 501 | 502 | if ( sock != null ) { 503 | sock.close(); 504 | sock = null; 505 | } 506 | 507 | return false; 508 | } 509 | 510 | /** 511 | * Stores data on the server; only the key and the value are specified. 512 | * 513 | * @param key key to store data under 514 | * @param value value to store 515 | * @return true, if the data was successfully stored 516 | */ 517 | public boolean set( String key, Object value ) { 518 | return set( "set", key, value, null, null, primitiveAsString ); 519 | } 520 | 521 | /** 522 | * Stores data on the server; only the key and the value are specified. 523 | * 524 | * @param key key to store data under 525 | * @param value value to store 526 | * @param hashCode if not null, then the int hashcode to use 527 | * @return true, if the data was successfully stored 528 | */ 529 | public boolean set( String key, Object value, Integer hashCode ) { 530 | return set( "set", key, value, null, hashCode, primitiveAsString ); 531 | } 532 | 533 | /** 534 | * Stores data on the server; the key, value, and an expiration time are specified. 535 | * 536 | * @param key key to store data under 537 | * @param value value to store 538 | * @param expiry when to expire the record 539 | * @return true, if the data was successfully stored 540 | */ 541 | public boolean set( String key, Object value, Date expiry ) { 542 | return set( "set", key, value, expiry, null, primitiveAsString ); 543 | } 544 | 545 | /** 546 | * Stores data on the server; the key, value, and an expiration time are specified. 547 | * 548 | * @param key key to store data under 549 | * @param value value to store 550 | * @param expiry when to expire the record 551 | * @param hashCode if not null, then the int hashcode to use 552 | * @return true, if the data was successfully stored 553 | */ 554 | public boolean set( String key, Object value, Date expiry, Integer hashCode ) { 555 | return set( "set", key, value, expiry, hashCode, primitiveAsString ); 556 | } 557 | 558 | /** 559 | * Adds data to the server; only the key and the value are specified. 560 | * 561 | * @param key key to store data under 562 | * @param value value to store 563 | * @return true, if the data was successfully stored 564 | */ 565 | public boolean add( String key, Object value ) { 566 | return set( "add", key, value, null, null, primitiveAsString ); 567 | } 568 | 569 | /** 570 | * Adds data to the server; the key, value, and an optional hashcode are passed in. 571 | * 572 | * @param key key to store data under 573 | * @param value value to store 574 | * @param hashCode if not null, then the int hashcode to use 575 | * @return true, if the data was successfully stored 576 | */ 577 | public boolean add( String key, Object value, Integer hashCode ) { 578 | return set( "add", key, value, null, hashCode, primitiveAsString ); 579 | } 580 | 581 | /** 582 | * Adds data to the server; the key, value, and an expiration time are specified. 583 | * 584 | * @param key key to store data under 585 | * @param value value to store 586 | * @param expiry when to expire the record 587 | * @return true, if the data was successfully stored 588 | */ 589 | public boolean add( String key, Object value, Date expiry ) { 590 | return set( "add", key, value, expiry, null, primitiveAsString ); 591 | } 592 | 593 | /** 594 | * Adds data to the server; the key, value, and an expiration time are specified. 595 | * 596 | * @param key key to store data under 597 | * @param value value to store 598 | * @param expiry when to expire the record 599 | * @param hashCode if not null, then the int hashcode to use 600 | * @return true, if the data was successfully stored 601 | */ 602 | public boolean add( String key, Object value, Date expiry, Integer hashCode ) { 603 | return set( "add", key, value, expiry, hashCode, primitiveAsString ); 604 | } 605 | 606 | /** 607 | * Updates data on the server; only the key and the value are specified. 608 | * 609 | * @param key key to store data under 610 | * @param value value to store 611 | * @return true, if the data was successfully stored 612 | */ 613 | public boolean replace( String key, Object value ) { 614 | return set( "replace", key, value, null, null, primitiveAsString ); 615 | } 616 | 617 | /** 618 | * Updates data on the server; only the key and the value and an optional hash are specified. 619 | * 620 | * @param key key to store data under 621 | * @param value value to store 622 | * @param hashCode if not null, then the int hashcode to use 623 | * @return true, if the data was successfully stored 624 | */ 625 | public boolean replace( String key, Object value, Integer hashCode ) { 626 | return set( "replace", key, value, null, hashCode, primitiveAsString ); 627 | } 628 | 629 | /** 630 | * Updates data on the server; the key, value, and an expiration time are specified. 631 | * 632 | * @param key key to store data under 633 | * @param value value to store 634 | * @param expiry when to expire the record 635 | * @return true, if the data was successfully stored 636 | */ 637 | public boolean replace( String key, Object value, Date expiry ) { 638 | return set( "replace", key, value, expiry, null, primitiveAsString ); 639 | } 640 | 641 | /** 642 | * Updates data on the server; the key, value, and an expiration time are specified. 643 | * 644 | * @param key key to store data under 645 | * @param value value to store 646 | * @param expiry when to expire the record 647 | * @param hashCode if not null, then the int hashcode to use 648 | * @return true, if the data was successfully stored 649 | */ 650 | public boolean replace( String key, Object value, Date expiry, Integer hashCode ) { 651 | return set( "replace", key, value, expiry, hashCode, primitiveAsString ); 652 | } 653 | 654 | /** 655 | * Stores data to cache. 656 | * 657 | * If data does not already exist for this key on the server, or if the key is being
658 | * deleted, the specified value will not be stored.
659 | * The server will automatically delete the value when the expiration time has been reached.
660 | *
661 | * If compression is enabled, and the data is longer than the compression threshold
662 | * the data will be stored in compressed form.
663 | *
664 | * As of the current release, all objects stored will use java serialization. 665 | * 666 | * @param cmdname action to take (set, add, replace) 667 | * @param key key to store cache under 668 | * @param value object to cache 669 | * @param expiry expiration 670 | * @param hashCode if not null, then the int hashcode to use 671 | * @param asString store this object as a string? 672 | * @return true/false indicating success 673 | */ 674 | private boolean set( String cmdname, String key, Object value, Date expiry, Integer hashCode, boolean asString ) { 675 | 676 | if ( cmdname == null || cmdname.trim().equals( "" ) || key == null ) { 677 | log.error( "key is null or cmd is null/empty for set()" ); 678 | return false; 679 | } 680 | 681 | try { 682 | key = sanitizeKey( key ); 683 | } 684 | catch ( UnsupportedEncodingException e ) { 685 | 686 | // if we have an errorHandler, use its hook 687 | if ( errorHandler != null ) 688 | errorHandler.handleErrorOnSet( this, e, key ); 689 | 690 | log.error( "failed to sanitize your key!", e ); 691 | return false; 692 | } 693 | 694 | if ( value == null ) { 695 | log.error( "trying to store a null value to cache" ); 696 | return false; 697 | } 698 | 699 | // get SockIO obj 700 | SockIOPool.SockIO sock = pool.getSock( key, hashCode ); 701 | 702 | if ( sock == null ) { 703 | if ( errorHandler != null ) 704 | errorHandler.handleErrorOnSet( this, new IOException( "no socket to server available" ), key ); 705 | return false; 706 | } 707 | 708 | if ( expiry == null ) 709 | expiry = new Date(0); 710 | 711 | // store flags 712 | int flags = 0; 713 | 714 | // byte array to hold data 715 | byte[] val; 716 | 717 | if ( NativeHandler.isHandled( value ) ) { 718 | 719 | if ( asString ) { 720 | // useful for sharing data between java and non-java 721 | // and also for storing ints for the increment method 722 | try { 723 | if ( log.isInfoEnabled() ) 724 | log.info( "++++ storing data as a string for key: " + key + " for class: " + value.getClass().getName() ); 725 | val = value.toString().getBytes( defaultEncoding ); 726 | } 727 | catch ( UnsupportedEncodingException ue ) { 728 | 729 | // if we have an errorHandler, use its hook 730 | if ( errorHandler != null ) 731 | errorHandler.handleErrorOnSet( this, ue, key ); 732 | 733 | log.error( "invalid encoding type used: " + defaultEncoding, ue ); 734 | sock.close(); 735 | sock = null; 736 | return false; 737 | } 738 | } 739 | else { 740 | try { 741 | if ( log.isInfoEnabled() ) 742 | log.info( "Storing with native handler..." ); 743 | flags |= NativeHandler.getMarkerFlag( value ); 744 | val = NativeHandler.encode( value ); 745 | } 746 | catch ( Exception e ) { 747 | 748 | // if we have an errorHandler, use its hook 749 | if ( errorHandler != null ) 750 | errorHandler.handleErrorOnSet( this, e, key ); 751 | 752 | log.error( "Failed to native handle obj", e ); 753 | 754 | sock.close(); 755 | sock = null; 756 | return false; 757 | } 758 | } 759 | } 760 | else { 761 | // always serialize for non-primitive types 762 | try { 763 | if ( log.isInfoEnabled() ) 764 | log.info( "++++ serializing for key: " + key + " for class: " + value.getClass().getName() ); 765 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 766 | (new ObjectOutputStream( bos )).writeObject( value ); 767 | val = bos.toByteArray(); 768 | flags |= F_SERIALIZED; 769 | } 770 | catch ( IOException e ) { 771 | 772 | // if we have an errorHandler, use its hook 773 | if ( errorHandler != null ) 774 | errorHandler.handleErrorOnSet( this, e, key ); 775 | 776 | // if we fail to serialize, then 777 | // we bail 778 | log.error( "failed to serialize obj", e ); 779 | log.error( value.toString() ); 780 | 781 | // return socket to pool and bail 782 | sock.close(); 783 | sock = null; 784 | return false; 785 | } 786 | } 787 | 788 | // now try to compress if we want to 789 | // and if the length is over the threshold 790 | if ( compressEnable && val.length > compressThreshold ) { 791 | 792 | try { 793 | if ( log.isInfoEnabled() ) { 794 | log.info( "++++ trying to compress data" ); 795 | log.info( "++++ size prior to compression: " + val.length ); 796 | } 797 | ByteArrayOutputStream bos = new ByteArrayOutputStream( val.length ); 798 | GZIPOutputStream gos = new GZIPOutputStream( bos ); 799 | gos.write( val, 0, val.length ); 800 | gos.finish(); 801 | gos.close(); 802 | 803 | // store it and set compression flag 804 | val = bos.toByteArray(); 805 | flags |= F_COMPRESSED; 806 | 807 | if ( log.isInfoEnabled() ) 808 | log.info( "++++ compression succeeded, size after: " + val.length ); 809 | } 810 | catch ( IOException e ) { 811 | 812 | // if we have an errorHandler, use its hook 813 | if ( errorHandler != null ) 814 | errorHandler.handleErrorOnSet( this, e, key ); 815 | 816 | log.error( "IOException while compressing stream: " + e.getMessage() ); 817 | log.error( "storing data uncompressed" ); 818 | } 819 | } 820 | 821 | // now write the data to the cache server 822 | try { 823 | String cmd = String.format( "%s %s %d %d %d\r\n", cmdname, key, flags, (expiry.getTime() / 1000), val.length ); 824 | sock.write( cmd.getBytes() ); 825 | sock.write( val ); 826 | sock.write( "\r\n".getBytes() ); 827 | sock.flush(); 828 | 829 | // get result code 830 | String line = sock.readLine(); 831 | if ( log.isInfoEnabled() ) 832 | log.info( "++++ memcache cmd (result code): " + cmd + " (" + line + ")" ); 833 | 834 | if ( STORED.equals( line ) ) { 835 | if ( log.isInfoEnabled() ) 836 | log.info("++++ data successfully stored for key: " + key ); 837 | sock.close(); 838 | sock = null; 839 | return true; 840 | } 841 | else if ( NOTSTORED.equals( line ) ) { 842 | if ( log.isInfoEnabled() ) 843 | log.info( "++++ data not stored in cache for key: " + key ); 844 | } 845 | else { 846 | log.error( "++++ error storing data in cache for key: " + key + " -- length: " + val.length ); 847 | log.error( "++++ server response: " + line ); 848 | } 849 | } 850 | catch ( IOException e ) { 851 | 852 | // if we have an errorHandler, use its hook 853 | if ( errorHandler != null ) 854 | errorHandler.handleErrorOnSet( this, e, key ); 855 | 856 | // exception thrown 857 | log.error( "++++ exception thrown while writing bytes to server on set" ); 858 | log.error( e.getMessage(), e ); 859 | 860 | try { 861 | sock.trueClose(); 862 | } 863 | catch ( IOException ioe ) { 864 | log.error( "++++ failed to close socket : " + sock.toString() ); 865 | } 866 | 867 | sock = null; 868 | } 869 | 870 | if ( sock != null ) { 871 | sock.close(); 872 | sock = null; 873 | } 874 | 875 | return false; 876 | } 877 | 878 | /** 879 | * Store a counter to memcached given a key 880 | * 881 | * @param key cache key 882 | * @param counter number to store 883 | * @return true/false indicating success 884 | */ 885 | public boolean storeCounter( String key, long counter ) { 886 | return set( "set", key, new Long( counter ), null, null, true ); 887 | } 888 | 889 | /** 890 | * Store a counter to memcached given a key 891 | * 892 | * @param key cache key 893 | * @param counter number to store 894 | * @return true/false indicating success 895 | */ 896 | public boolean storeCounter( String key, Long counter ) { 897 | return set( "set", key, counter, null, null, true ); 898 | } 899 | 900 | /** 901 | * Store a counter to memcached given a key 902 | * 903 | * @param key cache key 904 | * @param counter number to store 905 | * @param hashCode if not null, then the int hashcode to use 906 | * @return true/false indicating success 907 | */ 908 | public boolean storeCounter( String key, Long counter, Integer hashCode ) { 909 | return set( "set", key, counter, null, hashCode, true ); 910 | } 911 | 912 | /** 913 | * Returns value in counter at given key as long. 914 | * 915 | * @param key cache ket 916 | * @return counter value or -1 if not found 917 | */ 918 | public long getCounter( String key ) { 919 | return getCounter( key, null ); 920 | } 921 | 922 | /** 923 | * Returns value in counter at given key as long. 924 | * 925 | * @param key cache ket 926 | * @param hashCode if not null, then the int hashcode to use 927 | * @return counter value or -1 if not found 928 | */ 929 | public long getCounter( String key, Integer hashCode ) { 930 | 931 | if ( key == null ) { 932 | log.error( "null key for getCounter()" ); 933 | return -1; 934 | } 935 | 936 | long counter = -1; 937 | try { 938 | counter = Long.parseLong( (String)get( key, hashCode, true ) ); 939 | } 940 | catch ( Exception ex ) { 941 | 942 | // if we have an errorHandler, use its hook 943 | if ( errorHandler != null ) 944 | errorHandler.handleErrorOnGet( this, ex, key ); 945 | 946 | // not found or error getting out 947 | if ( log.isInfoEnabled() ) 948 | log.info( String.format( "Failed to parse Long value for key: %s", key ) ); 949 | } 950 | 951 | return counter; 952 | } 953 | 954 | /** 955 | * Thread safe way to initialize and increment a counter. 956 | * 957 | * @param key key where the data is stored 958 | * @return value of incrementer 959 | */ 960 | public long addOrIncr( String key ) { 961 | return addOrIncr( key, 0, null ); 962 | } 963 | 964 | /** 965 | * Thread safe way to initialize and increment a counter. 966 | * 967 | * @param key key where the data is stored 968 | * @param inc value to set or increment by 969 | * @return value of incrementer 970 | */ 971 | public long addOrIncr( String key, long inc ) { 972 | return addOrIncr( key, inc, null ); 973 | } 974 | 975 | /** 976 | * Thread safe way to initialize and increment a counter. 977 | * 978 | * @param key key where the data is stored 979 | * @param inc value to set or increment by 980 | * @param hashCode if not null, then the int hashcode to use 981 | * @return value of incrementer 982 | */ 983 | public long addOrIncr( String key, long inc, Integer hashCode ) { 984 | boolean ret = set( "add", key, new Long( inc ), null, hashCode, true ); 985 | 986 | if ( ret ) { 987 | return inc; 988 | } 989 | else { 990 | return incrdecr( "incr", key, inc, hashCode ); 991 | } 992 | } 993 | 994 | /** 995 | * Thread safe way to initialize and decrement a counter. 996 | * 997 | * @param key key where the data is stored 998 | * @return value of incrementer 999 | */ 1000 | public long addOrDecr( String key ) { 1001 | return addOrDecr( key, 0, null ); 1002 | } 1003 | 1004 | /** 1005 | * Thread safe way to initialize and decrement a counter. 1006 | * 1007 | * @param key key where the data is stored 1008 | * @param inc value to set or increment by 1009 | * @return value of incrementer 1010 | */ 1011 | public long addOrDecr( String key, long inc ) { 1012 | return addOrDecr( key, inc, null ); 1013 | } 1014 | 1015 | /** 1016 | * Thread safe way to initialize and decrement a counter. 1017 | * 1018 | * @param key key where the data is stored 1019 | * @param inc value to set or increment by 1020 | * @param hashCode if not null, then the int hashcode to use 1021 | * @return value of incrementer 1022 | */ 1023 | public long addOrDecr( String key, long inc, Integer hashCode ) { 1024 | boolean ret = set( "add", key, new Long( inc ), null, hashCode, true ); 1025 | 1026 | if ( ret ) { 1027 | return inc; 1028 | } 1029 | else { 1030 | return incrdecr( "decr", key, inc, hashCode ); 1031 | } 1032 | } 1033 | 1034 | /** 1035 | * Increment the value at the specified key by 1, and then return it. 1036 | * 1037 | * @param key key where the data is stored 1038 | * @return -1, if the key is not found, the value after incrementing otherwise 1039 | */ 1040 | public long incr( String key ) { 1041 | return incrdecr( "incr", key, 1, null ); 1042 | } 1043 | 1044 | /** 1045 | * Increment the value at the specified key by passed in val. 1046 | * 1047 | * @param key key where the data is stored 1048 | * @param inc how much to increment by 1049 | * @return -1, if the key is not found, the value after incrementing otherwise 1050 | */ 1051 | public long incr( String key, long inc ) { 1052 | return incrdecr( "incr", key, inc, null ); 1053 | } 1054 | 1055 | /** 1056 | * Increment the value at the specified key by the specified increment, and then return it. 1057 | * 1058 | * @param key key where the data is stored 1059 | * @param inc how much to increment by 1060 | * @param hashCode if not null, then the int hashcode to use 1061 | * @return -1, if the key is not found, the value after incrementing otherwise 1062 | */ 1063 | public long incr( String key, long inc, Integer hashCode ) { 1064 | return incrdecr( "incr", key, inc, hashCode ); 1065 | } 1066 | 1067 | /** 1068 | * Decrement the value at the specified key by 1, and then return it. 1069 | * 1070 | * @param key key where the data is stored 1071 | * @return -1, if the key is not found, the value after incrementing otherwise 1072 | */ 1073 | public long decr( String key ) { 1074 | return incrdecr( "decr", key, 1, null ); 1075 | } 1076 | 1077 | /** 1078 | * Decrement the value at the specified key by passed in value, and then return it. 1079 | * 1080 | * @param key key where the data is stored 1081 | * @param inc how much to increment by 1082 | * @return -1, if the key is not found, the value after incrementing otherwise 1083 | */ 1084 | public long decr( String key, long inc ) { 1085 | return incrdecr( "decr", key, inc, null ); 1086 | } 1087 | 1088 | /** 1089 | * Decrement the value at the specified key by the specified increment, and then return it. 1090 | * 1091 | * @param key key where the data is stored 1092 | * @param inc how much to increment by 1093 | * @param hashCode if not null, then the int hashcode to use 1094 | * @return -1, if the key is not found, the value after incrementing otherwise 1095 | */ 1096 | public long decr( String key, long inc, Integer hashCode ) { 1097 | return incrdecr( "decr", key, inc, hashCode ); 1098 | } 1099 | 1100 | /** 1101 | * Increments/decrements the value at the specified key by inc. 1102 | * 1103 | * Note that the server uses a 32-bit unsigned integer, and checks for
1104 | * underflow. In the event of underflow, the result will be zero. Because
1105 | * Java lacks unsigned types, the value is returned as a 64-bit integer.
1106 | * The server will only decrement a value if it already exists;
1107 | * if a value is not found, -1 will be returned. 1108 | * 1109 | * @param cmdname increment/decrement 1110 | * @param key cache key 1111 | * @param inc amount to incr or decr 1112 | * @param hashCode if not null, then the int hashcode to use 1113 | * @return new value or -1 if not exist 1114 | */ 1115 | private long incrdecr( String cmdname, String key, long inc, Integer hashCode ) { 1116 | 1117 | if ( key == null ) { 1118 | log.error( "null key for incrdecr()" ); 1119 | return -1; 1120 | } 1121 | 1122 | try { 1123 | key = sanitizeKey( key ); 1124 | } 1125 | catch ( UnsupportedEncodingException e ) { 1126 | 1127 | // if we have an errorHandler, use its hook 1128 | if ( errorHandler != null ) 1129 | errorHandler.handleErrorOnGet( this, e, key ); 1130 | 1131 | log.error( "failed to sanitize your key!", e ); 1132 | return -1; 1133 | } 1134 | 1135 | // get SockIO obj for given cache key 1136 | SockIOPool.SockIO sock = pool.getSock( key, hashCode ); 1137 | 1138 | if ( sock == null ) { 1139 | if ( errorHandler != null ) 1140 | errorHandler.handleErrorOnSet( this, new IOException( "no socket to server available" ), key ); 1141 | return -1; 1142 | } 1143 | 1144 | try { 1145 | String cmd = String.format( "%s %s %d\r\n", cmdname, key, inc ); 1146 | if ( log.isDebugEnabled() ) 1147 | log.debug( "++++ memcache incr/decr command: " + cmd ); 1148 | 1149 | sock.write( cmd.getBytes() ); 1150 | sock.flush(); 1151 | 1152 | // get result back 1153 | String line = sock.readLine(); 1154 | 1155 | if ( line.matches( "\\d+" ) ) { 1156 | 1157 | // return sock to pool and return result 1158 | sock.close(); 1159 | try { 1160 | return Long.parseLong( line ); 1161 | } 1162 | catch ( Exception ex ) { 1163 | 1164 | // if we have an errorHandler, use its hook 1165 | if ( errorHandler != null ) 1166 | errorHandler.handleErrorOnGet( this, ex, key ); 1167 | 1168 | log.error( String.format( "Failed to parse Long value for key: %s", key ) ); 1169 | } 1170 | } 1171 | else if ( NOTFOUND.equals( line ) ) { 1172 | if ( log.isInfoEnabled() ) 1173 | log.info( "++++ key not found to incr/decr for key: " + key ); 1174 | } 1175 | else { 1176 | log.error( "++++ error incr/decr key: " + key ); 1177 | log.error( "++++ server response: " + line ); 1178 | } 1179 | } 1180 | catch ( IOException e ) { 1181 | 1182 | // if we have an errorHandler, use its hook 1183 | if ( errorHandler != null ) 1184 | errorHandler.handleErrorOnGet( this, e, key ); 1185 | 1186 | // exception thrown 1187 | log.error( "++++ exception thrown while writing bytes to server on incr/decr" ); 1188 | log.error( e.getMessage(), e ); 1189 | 1190 | try { 1191 | sock.trueClose(); 1192 | } 1193 | catch ( IOException ioe ) { 1194 | log.error( "++++ failed to close socket : " + sock.toString() ); 1195 | } 1196 | 1197 | sock = null; 1198 | } 1199 | 1200 | if ( sock != null ) { 1201 | sock.close(); 1202 | sock = null; 1203 | } 1204 | 1205 | return -1; 1206 | } 1207 | 1208 | /** 1209 | * Retrieve a key from the server, using a specific hash. 1210 | * 1211 | * If the data was compressed or serialized when compressed, it will automatically
1212 | * be decompressed or serialized, as appropriate. (Inclusive or)
1213 | *
1214 | * Non-serialized data will be returned as a string, so explicit conversion to
1215 | * numeric types will be necessary, if desired
1216 | * 1217 | * @param key key where data is stored 1218 | * @return the object that was previously stored, or null if it was not previously stored 1219 | */ 1220 | public Object get( String key ) { 1221 | return get( key, null, false ); 1222 | } 1223 | 1224 | /** 1225 | * Retrieve a key from the server, using a specific hash. 1226 | * 1227 | * If the data was compressed or serialized when compressed, it will automatically
1228 | * be decompressed or serialized, as appropriate. (Inclusive or)
1229 | *
1230 | * Non-serialized data will be returned as a string, so explicit conversion to
1231 | * numeric types will be necessary, if desired
1232 | * 1233 | * @param key key where data is stored 1234 | * @param hashCode if not null, then the int hashcode to use 1235 | * @return the object that was previously stored, or null if it was not previously stored 1236 | */ 1237 | public Object get( String key, Integer hashCode ) { 1238 | return get( key, hashCode, false ); 1239 | } 1240 | 1241 | /** 1242 | * Retrieve a key from the server, using a specific hash. 1243 | * 1244 | * If the data was compressed or serialized when compressed, it will automatically
1245 | * be decompressed or serialized, as appropriate. (Inclusive or)
1246 | *
1247 | * Non-serialized data will be returned as a string, so explicit conversion to
1248 | * numeric types will be necessary, if desired
1249 | * 1250 | * @param key key where data is stored 1251 | * @param hashCode if not null, then the int hashcode to use 1252 | * @param asString if true, then return string val 1253 | * @return the object that was previously stored, or null if it was not previously stored 1254 | */ 1255 | public Object get( String key, Integer hashCode, boolean asString ) { 1256 | 1257 | if ( key == null ) { 1258 | log.error( "key is null for get()" ); 1259 | return null; 1260 | } 1261 | 1262 | try { 1263 | key = sanitizeKey( key ); 1264 | } 1265 | catch ( UnsupportedEncodingException e ) { 1266 | 1267 | // if we have an errorHandler, use its hook 1268 | if ( errorHandler != null ) 1269 | errorHandler.handleErrorOnGet( this, e, key ); 1270 | 1271 | log.error( "failed to sanitize your key!", e ); 1272 | return null; 1273 | } 1274 | 1275 | // get SockIO obj using cache key 1276 | SockIOPool.SockIO sock = pool.getSock( key, hashCode ); 1277 | 1278 | if ( sock == null ) { 1279 | if ( errorHandler != null ) 1280 | errorHandler.handleErrorOnGet( this, new IOException( "no socket to server available" ), key ); 1281 | return null; 1282 | } 1283 | 1284 | try { 1285 | String cmd = "get " + key + "\r\n"; 1286 | 1287 | if ( log.isDebugEnabled() ) 1288 | log.debug("++++ memcache get command: " + cmd); 1289 | 1290 | sock.write( cmd.getBytes() ); 1291 | sock.flush(); 1292 | 1293 | // ready object 1294 | Object o = null; 1295 | 1296 | while ( true ) { 1297 | String line = sock.readLine(); 1298 | 1299 | if ( log.isDebugEnabled() ) 1300 | log.debug( "++++ line: " + line ); 1301 | 1302 | if ( line.startsWith( VALUE ) ) { 1303 | String[] info = line.split(" "); 1304 | int flag = Integer.parseInt( info[2] ); 1305 | int length = Integer.parseInt( info[3] ); 1306 | 1307 | if ( log.isDebugEnabled() ) { 1308 | log.debug( "++++ key: " + key ); 1309 | log.debug( "++++ flags: " + flag ); 1310 | log.debug( "++++ length: " + length ); 1311 | } 1312 | 1313 | // read obj into buffer 1314 | byte[] buf = new byte[length]; 1315 | sock.read( buf ); 1316 | sock.clearEOL(); 1317 | 1318 | if ( (flag & F_COMPRESSED) == F_COMPRESSED ) { 1319 | try { 1320 | // read the input stream, and write to a byte array output stream since 1321 | // we have to read into a byte array, but we don't know how large it 1322 | // will need to be, and we don't want to resize it a bunch 1323 | GZIPInputStream gzi = new GZIPInputStream( new ByteArrayInputStream( buf ) ); 1324 | ByteArrayOutputStream bos = new ByteArrayOutputStream( buf.length ); 1325 | 1326 | int count; 1327 | byte[] tmp = new byte[2048]; 1328 | while ( (count = gzi.read(tmp)) != -1 ) { 1329 | bos.write( tmp, 0, count ); 1330 | } 1331 | 1332 | // store uncompressed back to buffer 1333 | buf = bos.toByteArray(); 1334 | gzi.close(); 1335 | } 1336 | catch ( IOException e ) { 1337 | 1338 | // if we have an errorHandler, use its hook 1339 | if ( errorHandler != null ) 1340 | errorHandler.handleErrorOnGet( this, e, key ); 1341 | 1342 | log.error( "++++ IOException thrown while trying to uncompress input stream for key: " + key + " -- " + e.getMessage() ); 1343 | throw new NestedIOException( "++++ IOException thrown while trying to uncompress input stream for key: " + key, e ); 1344 | } 1345 | } 1346 | 1347 | // we can only take out serialized objects 1348 | if ( ( flag & F_SERIALIZED ) != F_SERIALIZED ) { 1349 | if ( primitiveAsString || asString ) { 1350 | // pulling out string value 1351 | if ( log.isInfoEnabled() ) 1352 | log.info( "++++ retrieving object and stuffing into a string." ); 1353 | o = new String( buf, defaultEncoding ); 1354 | } 1355 | else { 1356 | // decoding object 1357 | try { 1358 | o = NativeHandler.decode( buf, flag ); 1359 | } 1360 | catch ( Exception e ) { 1361 | 1362 | // if we have an errorHandler, use its hook 1363 | if ( errorHandler != null ) 1364 | errorHandler.handleErrorOnGet( this, e, key ); 1365 | 1366 | log.error( "++++ Exception thrown while trying to deserialize for key: " + key, e ); 1367 | throw new NestedIOException( e ); 1368 | } 1369 | } 1370 | } 1371 | else { 1372 | // deserialize if the data is serialized 1373 | ContextObjectInputStream ois = 1374 | new ContextObjectInputStream( new ByteArrayInputStream( buf ), classLoader ); 1375 | try { 1376 | o = ois.readObject(); 1377 | if ( log.isInfoEnabled() ) 1378 | log.info( "++++ deserializing " + o.getClass() ); 1379 | } 1380 | catch ( Exception e ) { 1381 | if ( errorHandler != null ) 1382 | errorHandler.handleErrorOnGet( this, e, key ); 1383 | 1384 | o = null; 1385 | log.error( "++++ Exception thrown while trying to deserialize for key: " + key + " -- " + e.getMessage() ); 1386 | } 1387 | } 1388 | } 1389 | else if ( END.equals( line ) ) { 1390 | if ( log.isDebugEnabled() ) 1391 | log.debug( "++++ finished reading from cache server" ); 1392 | break; 1393 | } 1394 | } 1395 | 1396 | sock.close(); 1397 | sock = null; 1398 | return o; 1399 | } 1400 | catch ( IOException e ) { 1401 | 1402 | // if we have an errorHandler, use its hook 1403 | if ( errorHandler != null ) 1404 | errorHandler.handleErrorOnGet( this, e, key ); 1405 | 1406 | // exception thrown 1407 | log.error( "++++ exception thrown while trying to get object from cache for key: " + key + " -- " + e.getMessage() ); 1408 | 1409 | try { 1410 | sock.trueClose(); 1411 | } 1412 | catch ( IOException ioe ) { 1413 | log.error( "++++ failed to close socket : " + sock.toString() ); 1414 | } 1415 | sock = null; 1416 | } 1417 | 1418 | if ( sock != null ) 1419 | sock.close(); 1420 | 1421 | return null; 1422 | } 1423 | 1424 | /** 1425 | * Retrieve multiple objects from the memcache. 1426 | * 1427 | * This is recommended over repeated calls to {@link #get(String) get()}, since it
1428 | * is more efficient.
1429 | * 1430 | * @param keys String array of keys to retrieve 1431 | * @return Object array ordered in same order as key array containing results 1432 | */ 1433 | public Object[] getMultiArray( String[] keys ) { 1434 | return getMultiArray( keys, null, false ); 1435 | } 1436 | 1437 | /** 1438 | * Retrieve multiple objects from the memcache. 1439 | * 1440 | * This is recommended over repeated calls to {@link #get(String) get()}, since it
1441 | * is more efficient.
1442 | * 1443 | * @param keys String array of keys to retrieve 1444 | * @param hashCodes if not null, then the Integer array of hashCodes 1445 | * @return Object array ordered in same order as key array containing results 1446 | */ 1447 | public Object[] getMultiArray( String[] keys, Integer[] hashCodes ) { 1448 | return getMultiArray( keys, hashCodes, false ); 1449 | } 1450 | 1451 | /** 1452 | * Retrieve multiple objects from the memcache. 1453 | * 1454 | * This is recommended over repeated calls to {@link #get(String) get()}, since it
1455 | * is more efficient.
1456 | * 1457 | * @param keys String array of keys to retrieve 1458 | * @param hashCodes if not null, then the Integer array of hashCodes 1459 | * @param asString if true, retrieve string vals 1460 | * @return Object array ordered in same order as key array containing results 1461 | */ 1462 | public Object[] getMultiArray( String[] keys, Integer[] hashCodes, boolean asString ) { 1463 | 1464 | Map data = getMulti( keys, hashCodes, asString ); 1465 | 1466 | if ( data == null ) 1467 | return null; 1468 | 1469 | Object[] res = new Object[ keys.length ]; 1470 | for ( int i = 0; i < keys.length; i++ ) { 1471 | res[i] = data.get( keys[i] ); 1472 | } 1473 | 1474 | return res; 1475 | } 1476 | 1477 | /** 1478 | * Retrieve multiple objects from the memcache. 1479 | * 1480 | * This is recommended over repeated calls to {@link #get(String) get()}, since it
1481 | * is more efficient.
1482 | * 1483 | * @param keys String array of keys to retrieve 1484 | * @return a hashmap with entries for each key is found by the server, 1485 | * keys that are not found are not entered into the hashmap, but attempting to 1486 | * retrieve them from the hashmap gives you null. 1487 | */ 1488 | public Map getMulti( String[] keys ) { 1489 | return getMulti( keys, null, false ); 1490 | } 1491 | 1492 | /** 1493 | * Retrieve multiple keys from the memcache. 1494 | * 1495 | * This is recommended over repeated calls to {@link #get(String) get()}, since it
1496 | * is more efficient.
1497 | * 1498 | * @param keys keys to retrieve 1499 | * @param hashCodes if not null, then the Integer array of hashCodes 1500 | * @return a hashmap with entries for each key is found by the server, 1501 | * keys that are not found are not entered into the hashmap, but attempting to 1502 | * retrieve them from the hashmap gives you null. 1503 | */ 1504 | public Map getMulti( String[] keys, Integer[] hashCodes ) { 1505 | return getMulti( keys, hashCodes, false ); 1506 | } 1507 | 1508 | /** 1509 | * Retrieve multiple keys from the memcache. 1510 | * 1511 | * This is recommended over repeated calls to {@link #get(String) get()}, since it
1512 | * is more efficient.
1513 | * 1514 | * @param keys keys to retrieve 1515 | * @param hashCodes if not null, then the Integer array of hashCodes 1516 | * @param asString if true then retrieve using String val 1517 | * @return a hashmap with entries for each key is found by the server, 1518 | * keys that are not found are not entered into the hashmap, but attempting to 1519 | * retrieve them from the hashmap gives you null. 1520 | */ 1521 | public Map getMulti( String[] keys, Integer[] hashCodes, boolean asString ) { 1522 | 1523 | if ( keys == null || keys.length == 0 ) { 1524 | log.error( "missing keys for getMulti()" ); 1525 | return null; 1526 | } 1527 | 1528 | Map cmdMap = 1529 | new HashMap(); 1530 | 1531 | for ( int i = 0; i < keys.length; ++i ) { 1532 | 1533 | String key = keys[i]; 1534 | if ( key == null ) { 1535 | log.error( "null key, so skipping" ); 1536 | continue; 1537 | } 1538 | 1539 | Integer hash = null; 1540 | if ( hashCodes != null && hashCodes.length > i ) 1541 | hash = hashCodes[ i ]; 1542 | 1543 | String cleanKey = key; 1544 | try { 1545 | cleanKey = sanitizeKey( key ); 1546 | } 1547 | catch ( UnsupportedEncodingException e ) { 1548 | 1549 | // if we have an errorHandler, use its hook 1550 | if ( errorHandler != null ) 1551 | errorHandler.handleErrorOnGet( this, e, key ); 1552 | 1553 | log.error( "failed to sanitize your key!", e ); 1554 | continue; 1555 | } 1556 | 1557 | // get SockIO obj from cache key 1558 | SockIOPool.SockIO sock = pool.getSock( cleanKey, hash ); 1559 | 1560 | if ( sock == null ) { 1561 | if ( errorHandler != null ) 1562 | errorHandler.handleErrorOnGet( this, new IOException( "no socket to server available" ), key ); 1563 | continue; 1564 | } 1565 | 1566 | // store in map and list if not already 1567 | if ( !cmdMap.containsKey( sock.getHost() ) ) 1568 | cmdMap.put( sock.getHost(), new StringBuilder( "get" ) ); 1569 | 1570 | cmdMap.get( sock.getHost() ).append( " " + cleanKey ); 1571 | 1572 | // return to pool 1573 | sock.close(); 1574 | } 1575 | 1576 | if ( log.isInfoEnabled() ) 1577 | log.info( "multi get socket count : " + cmdMap.size() ); 1578 | 1579 | // now query memcache 1580 | Map ret = 1581 | new HashMap( keys.length ); 1582 | 1583 | // now use new NIO implementation 1584 | (new NIOLoader( this )).doMulti( asString, cmdMap, keys, ret ); 1585 | 1586 | // fix the return array in case we had to rewrite any of the keys 1587 | for ( String key : keys ) { 1588 | 1589 | String cleanKey = key; 1590 | try { 1591 | cleanKey = sanitizeKey( key ); 1592 | } 1593 | catch ( UnsupportedEncodingException e ) { 1594 | 1595 | // if we have an errorHandler, use its hook 1596 | if ( errorHandler != null ) 1597 | errorHandler.handleErrorOnGet( this, e, key ); 1598 | 1599 | log.error( "failed to sanitize your key!", e ); 1600 | continue; 1601 | } 1602 | 1603 | if ( ! key.equals( cleanKey ) && ret.containsKey( cleanKey ) ) { 1604 | ret.put( key, ret.get( cleanKey ) ); 1605 | ret.remove( cleanKey ); 1606 | } 1607 | 1608 | // backfill missing keys w/ null value 1609 | if ( ! ret.containsKey( key ) ) 1610 | ret.put( key, null ); 1611 | } 1612 | 1613 | if ( log.isDebugEnabled() ) 1614 | log.debug( "++++ memcache: got back " + ret.size() + " results" ); 1615 | return ret; 1616 | } 1617 | 1618 | /** 1619 | * This method loads the data from cache into a Map. 1620 | * 1621 | * Pass a SockIO object which is ready to receive data and a HashMap
1622 | * to store the results. 1623 | * 1624 | * @param sock socket waiting to pass back data 1625 | * @param hm hashmap to store data into 1626 | * @param asString if true, and if we are using NativehHandler, return string val 1627 | * @throws IOException if io exception happens while reading from socket 1628 | */ 1629 | private void loadMulti( LineInputStream input, Map hm, boolean asString ) throws IOException { 1630 | 1631 | while ( true ) { 1632 | String line = input.readLine(); 1633 | if ( log.isDebugEnabled() ) 1634 | log.debug( "++++ line: " + line ); 1635 | 1636 | if ( line.startsWith( VALUE ) ) { 1637 | String[] info = line.split(" "); 1638 | String key = info[1]; 1639 | int flag = Integer.parseInt( info[2] ); 1640 | int length = Integer.parseInt( info[3] ); 1641 | 1642 | if ( log.isDebugEnabled() ) { 1643 | log.debug( "++++ key: " + key ); 1644 | log.debug( "++++ flags: " + flag ); 1645 | log.debug( "++++ length: " + length ); 1646 | } 1647 | 1648 | // read obj into buffer 1649 | byte[] buf = new byte[length]; 1650 | input.read( buf ); 1651 | input.clearEOL(); 1652 | 1653 | // ready object 1654 | Object o; 1655 | 1656 | // check for compression 1657 | if ( (flag & F_COMPRESSED) == F_COMPRESSED ) { 1658 | try { 1659 | // read the input stream, and write to a byte array output stream since 1660 | // we have to read into a byte array, but we don't know how large it 1661 | // will need to be, and we don't want to resize it a bunch 1662 | GZIPInputStream gzi = new GZIPInputStream( new ByteArrayInputStream( buf ) ); 1663 | ByteArrayOutputStream bos = new ByteArrayOutputStream( buf.length ); 1664 | 1665 | int count; 1666 | byte[] tmp = new byte[2048]; 1667 | while ( (count = gzi.read(tmp)) != -1 ) { 1668 | bos.write( tmp, 0, count ); 1669 | } 1670 | 1671 | // store uncompressed back to buffer 1672 | buf = bos.toByteArray(); 1673 | gzi.close(); 1674 | } 1675 | catch ( IOException e ) { 1676 | 1677 | // if we have an errorHandler, use its hook 1678 | if ( errorHandler != null ) 1679 | errorHandler.handleErrorOnGet( this, e, key ); 1680 | 1681 | log.error( "++++ IOException thrown while trying to uncompress input stream for key: " + key + " -- " + e.getMessage() ); 1682 | throw new NestedIOException( "++++ IOException thrown while trying to uncompress input stream for key: " + key, e ); 1683 | } 1684 | } 1685 | 1686 | // we can only take out serialized objects 1687 | if ( ( flag & F_SERIALIZED ) != F_SERIALIZED ) { 1688 | if ( primitiveAsString || asString ) { 1689 | // pulling out string value 1690 | if ( log.isInfoEnabled() ) 1691 | log.info( "++++ retrieving object and stuffing into a string." ); 1692 | o = new String( buf, defaultEncoding ); 1693 | } 1694 | else { 1695 | // decoding object 1696 | try { 1697 | o = NativeHandler.decode( buf, flag ); 1698 | } 1699 | catch ( Exception e ) { 1700 | 1701 | // if we have an errorHandler, use its hook 1702 | if ( errorHandler != null ) 1703 | errorHandler.handleErrorOnGet( this, e, key ); 1704 | 1705 | log.error( "++++ Exception thrown while trying to deserialize for key: " + key + " -- " + e.getMessage() ); 1706 | throw new NestedIOException( e ); 1707 | } 1708 | } 1709 | } 1710 | else { 1711 | // deserialize if the data is serialized 1712 | ContextObjectInputStream ois = 1713 | new ContextObjectInputStream( new ByteArrayInputStream( buf ), classLoader ); 1714 | try { 1715 | o = ois.readObject(); 1716 | if ( log.isInfoEnabled() ) 1717 | log.info( "++++ deserializing " + o.getClass() ); 1718 | } 1719 | catch ( InvalidClassException e ) { 1720 | /* Errors de-serializing are to be expected in the case of a 1721 | * long running server that spans client restarts with updated 1722 | * classes. 1723 | */ 1724 | // if we have an errorHandler, use its hook 1725 | if ( errorHandler != null ) 1726 | errorHandler.handleErrorOnGet( this, e, key ); 1727 | 1728 | o = null; 1729 | log.error( "++++ InvalidClassException thrown while trying to deserialize for key: " + key + " -- " + e.getMessage() ); 1730 | } 1731 | catch ( ClassNotFoundException e ) { 1732 | 1733 | // if we have an errorHandler, use its hook 1734 | if ( errorHandler != null ) 1735 | errorHandler.handleErrorOnGet( this, e, key ); 1736 | 1737 | o = null; 1738 | log.error( "++++ ClassNotFoundException thrown while trying to deserialize for key: " + key + " -- " + e.getMessage() ); 1739 | } 1740 | } 1741 | 1742 | // store the object into the cache 1743 | if ( o != null ) 1744 | hm.put( key, o ); 1745 | } 1746 | else if ( END.equals( line ) ) { 1747 | if ( log.isDebugEnabled() ) 1748 | log.debug( "++++ finished reading from cache server" ); 1749 | break; 1750 | } 1751 | } 1752 | } 1753 | 1754 | private String sanitizeKey( String key ) throws UnsupportedEncodingException { 1755 | return ( sanitizeKeys ) ? URLEncoder.encode( key, "UTF-8" ) : key; 1756 | } 1757 | 1758 | /** 1759 | * Invalidates the entire cache. 1760 | * 1761 | * Will return true only if succeeds in clearing all servers. 1762 | * 1763 | * @return success true/false 1764 | */ 1765 | public boolean flushAll() { 1766 | return flushAll( null ); 1767 | } 1768 | 1769 | /** 1770 | * Invalidates the entire cache. 1771 | * 1772 | * Will return true only if succeeds in clearing all servers. 1773 | * If pass in null, then will try to flush all servers. 1774 | * 1775 | * @param servers optional array of host(s) to flush (host:port) 1776 | * @return success true/false 1777 | */ 1778 | public boolean flushAll( String[] servers ) { 1779 | 1780 | // get SockIOPool instance 1781 | // return false if unable to get SockIO obj 1782 | if ( pool == null ) { 1783 | log.error( "++++ unable to get SockIOPool instance" ); 1784 | return false; 1785 | } 1786 | 1787 | // get all servers and iterate over them 1788 | servers = ( servers == null ) 1789 | ? pool.getServers() 1790 | : servers; 1791 | 1792 | // if no servers, then return early 1793 | if ( servers == null || servers.length <= 0 ) { 1794 | log.error( "++++ no servers to flush" ); 1795 | return false; 1796 | } 1797 | 1798 | boolean success = true; 1799 | 1800 | for ( int i = 0; i < servers.length; i++ ) { 1801 | 1802 | SockIOPool.SockIO sock = pool.getConnection( servers[i] ); 1803 | if ( sock == null ) { 1804 | log.error( "++++ unable to get connection to : " + servers[i] ); 1805 | success = false; 1806 | if ( errorHandler != null ) 1807 | errorHandler.handleErrorOnFlush( this, new IOException( "no socket to server available" ) ); 1808 | continue; 1809 | } 1810 | 1811 | // build command 1812 | String command = "flush_all\r\n"; 1813 | 1814 | try { 1815 | sock.write( command.getBytes() ); 1816 | sock.flush(); 1817 | 1818 | // if we get appropriate response back, then we return true 1819 | String line = sock.readLine(); 1820 | success = ( OK.equals( line ) ) 1821 | ? success && true 1822 | : false; 1823 | } 1824 | catch ( IOException e ) { 1825 | 1826 | // if we have an errorHandler, use its hook 1827 | if ( errorHandler != null ) 1828 | errorHandler.handleErrorOnFlush( this, e ); 1829 | 1830 | // exception thrown 1831 | log.error( "++++ exception thrown while writing bytes to server on flushAll" ); 1832 | log.error( e.getMessage(), e ); 1833 | 1834 | try { 1835 | sock.trueClose(); 1836 | } 1837 | catch ( IOException ioe ) { 1838 | log.error( "++++ failed to close socket : " + sock.toString() ); 1839 | } 1840 | 1841 | success = false; 1842 | sock = null; 1843 | } 1844 | 1845 | if ( sock != null ) { 1846 | sock.close(); 1847 | sock = null; 1848 | } 1849 | } 1850 | 1851 | return success; 1852 | } 1853 | 1854 | /** 1855 | * Retrieves stats for all servers. 1856 | * 1857 | * Returns a map keyed on the servername. 1858 | * The value is another map which contains stats 1859 | * with stat name as key and value as value. 1860 | * 1861 | * @return Stats map 1862 | */ 1863 | public Map stats() { 1864 | return stats( null ); 1865 | } 1866 | 1867 | /** 1868 | * Retrieves stats for passed in servers (or all servers). 1869 | * 1870 | * Returns a map keyed on the servername. 1871 | * The value is another map which contains stats 1872 | * with stat name as key and value as value. 1873 | * 1874 | * @param servers string array of servers to retrieve stats from, or all if this is null 1875 | * @return Stats map 1876 | */ 1877 | public Map stats( String[] servers ) { 1878 | return stats( servers, "stats\r\n", STATS ); 1879 | } 1880 | 1881 | /** 1882 | * Retrieves stats items for all servers. 1883 | * 1884 | * Returns a map keyed on the servername. 1885 | * The value is another map which contains item stats 1886 | * with itemname:number:field as key and value as value. 1887 | * 1888 | * @return Stats map 1889 | */ 1890 | public Map statsItems() { 1891 | return statsItems( null ); 1892 | } 1893 | 1894 | /** 1895 | * Retrieves stats for passed in servers (or all servers). 1896 | * 1897 | * Returns a map keyed on the servername. 1898 | * The value is another map which contains item stats 1899 | * with itemname:number:field as key and value as value. 1900 | * 1901 | * @param servers string array of servers to retrieve stats from, or all if this is null 1902 | * @return Stats map 1903 | */ 1904 | public Map statsItems( String[] servers ) { 1905 | return stats( servers, "stats items\r\n", STATS ); 1906 | } 1907 | 1908 | /** 1909 | * Retrieves stats items for all servers. 1910 | * 1911 | * Returns a map keyed on the servername. 1912 | * The value is another map which contains slabs stats 1913 | * with slabnumber:field as key and value as value. 1914 | * 1915 | * @return Stats map 1916 | */ 1917 | public Map statsSlabs() { 1918 | return statsSlabs( null ); 1919 | } 1920 | 1921 | /** 1922 | * Retrieves stats for passed in servers (or all servers). 1923 | * 1924 | * Returns a map keyed on the servername. 1925 | * The value is another map which contains slabs stats 1926 | * with slabnumber:field as key and value as value. 1927 | * 1928 | * @param servers string array of servers to retrieve stats from, or all if this is null 1929 | * @return Stats map 1930 | */ 1931 | public Map statsSlabs( String[] servers ) { 1932 | return stats( servers, "stats slabs\r\n", STATS ); 1933 | } 1934 | 1935 | /** 1936 | * Retrieves items cachedump for all servers. 1937 | * 1938 | * Returns a map keyed on the servername. 1939 | * The value is another map which contains cachedump stats 1940 | * with the cachekey as key and byte size and unix timestamp as value. 1941 | * 1942 | * @param slabNumber the item number of the cache dump 1943 | * @return Stats map 1944 | */ 1945 | public Map statsCacheDump( int slabNumber, int limit ) { 1946 | return statsCacheDump( null, slabNumber, limit ); 1947 | } 1948 | 1949 | /** 1950 | * Retrieves stats for passed in servers (or all servers). 1951 | * 1952 | * Returns a map keyed on the servername. 1953 | * The value is another map which contains cachedump stats 1954 | * with the cachekey as key and byte size and unix timestamp as value. 1955 | * 1956 | * @param servers string array of servers to retrieve stats from, or all if this is null 1957 | * @param slabNumber the item number of the cache dump 1958 | * @return Stats map 1959 | */ 1960 | public Map statsCacheDump( String[] servers, int slabNumber, int limit ) { 1961 | return stats( servers, String.format( "stats cachedump %d %d\r\n", slabNumber, limit ), ITEM ); 1962 | } 1963 | 1964 | private Map stats( String[] servers, String command, String lineStart ) { 1965 | 1966 | if ( command == null || command.trim().equals( "" ) ) { 1967 | log.error( "++++ invalid / missing command for stats()" ); 1968 | return null; 1969 | } 1970 | 1971 | // get all servers and iterate over them 1972 | servers = (servers == null) 1973 | ? pool.getServers() 1974 | : servers; 1975 | 1976 | // if no servers, then return early 1977 | if ( servers == null || servers.length <= 0 ) { 1978 | log.error( "++++ no servers to check stats" ); 1979 | return null; 1980 | } 1981 | 1982 | // array of stats Maps 1983 | Map statsMaps = 1984 | new HashMap(); 1985 | 1986 | for ( int i = 0; i < servers.length; i++ ) { 1987 | 1988 | SockIOPool.SockIO sock = pool.getConnection( servers[i] ); 1989 | if ( sock == null ) { 1990 | log.error( "++++ unable to get connection to : " + servers[i] ); 1991 | if ( errorHandler != null ) 1992 | errorHandler.handleErrorOnStats( this, new IOException( "no socket to server available" ) ); 1993 | continue; 1994 | } 1995 | 1996 | // build command 1997 | try { 1998 | sock.write( command.getBytes() ); 1999 | sock.flush(); 2000 | 2001 | // map to hold key value pairs 2002 | Map stats = new HashMap(); 2003 | 2004 | // loop over results 2005 | while ( true ) { 2006 | String line = sock.readLine(); 2007 | if ( log.isDebugEnabled() ) 2008 | log.debug( "++++ line: " + line ); 2009 | 2010 | if ( line.startsWith( lineStart ) ) { 2011 | String[] info = line.split( " ", 3 ); 2012 | String key = info[1]; 2013 | String value = info[2]; 2014 | 2015 | if ( log.isDebugEnabled() ) { 2016 | log.debug( "++++ key : " + key ); 2017 | log.debug( "++++ value: " + value ); 2018 | } 2019 | 2020 | stats.put( key, value ); 2021 | } 2022 | else if ( END.equals( line ) ) { 2023 | // finish when we get end from server 2024 | if ( log.isDebugEnabled() ) 2025 | log.debug( "++++ finished reading from cache server" ); 2026 | break; 2027 | } 2028 | else if ( line.startsWith( ERROR ) || line.startsWith( CLIENT_ERROR ) || line.startsWith( SERVER_ERROR ) ) { 2029 | log.error( "++++ failed to query stats" ); 2030 | log.error( "++++ server response: " + line ); 2031 | break; 2032 | } 2033 | 2034 | statsMaps.put( servers[i], stats ); 2035 | } 2036 | } 2037 | catch ( IOException e ) { 2038 | 2039 | // if we have an errorHandler, use its hook 2040 | if ( errorHandler != null ) 2041 | errorHandler.handleErrorOnStats( this, e ); 2042 | 2043 | // exception thrown 2044 | log.error( "++++ exception thrown while writing bytes to server on stats" ); 2045 | log.error( e.getMessage(), e ); 2046 | 2047 | try { 2048 | sock.trueClose(); 2049 | } 2050 | catch ( IOException ioe ) { 2051 | log.error( "++++ failed to close socket : " + sock.toString() ); 2052 | } 2053 | 2054 | sock = null; 2055 | } 2056 | 2057 | if ( sock != null ) { 2058 | sock.close(); 2059 | sock = null; 2060 | } 2061 | } 2062 | 2063 | return statsMaps; 2064 | } 2065 | 2066 | protected final class NIOLoader { 2067 | protected Selector selector; 2068 | protected int numConns = 0; 2069 | protected MemcachedClient mc; 2070 | protected Connection[] conns; 2071 | 2072 | public NIOLoader( MemcachedClient mc ) { 2073 | this.mc = mc; 2074 | } 2075 | 2076 | private final class Connection { 2077 | 2078 | public List incoming = new ArrayList(); 2079 | public ByteBuffer outgoing; 2080 | public SockIOPool.SockIO sock; 2081 | public SocketChannel channel; 2082 | private boolean isDone = false; 2083 | 2084 | public Connection( SockIOPool.SockIO sock, StringBuilder request ) throws IOException { 2085 | if ( log.isDebugEnabled() ) 2086 | log.debug( "setting up connection to "+sock.getHost() ); 2087 | 2088 | this.sock = sock; 2089 | outgoing = ByteBuffer.wrap( request.append( "\r\n" ).toString().getBytes() ); 2090 | 2091 | channel = sock.getChannel(); 2092 | if ( channel == null ) 2093 | throw new IOException( "dead connection to: " + sock.getHost() ); 2094 | 2095 | channel.configureBlocking( false ); 2096 | channel.register( selector, SelectionKey.OP_WRITE, this ); 2097 | } 2098 | 2099 | public void close() { 2100 | try { 2101 | if ( isDone ) { 2102 | // turn off non-blocking IO and return to pool 2103 | if ( log.isDebugEnabled() ) 2104 | log.debug( "++++ gracefully closing connection to "+sock.getHost() ); 2105 | 2106 | channel.configureBlocking( true ); 2107 | sock.close(); 2108 | return; 2109 | } 2110 | } 2111 | catch ( IOException e ) { 2112 | log.warn( "++++ memcache: unexpected error closing normally" ); 2113 | } 2114 | 2115 | try { 2116 | if ( log.isDebugEnabled() ) 2117 | log.debug("forcefully closing connection to "+sock.getHost()); 2118 | 2119 | channel.close(); 2120 | sock.trueClose(); 2121 | } 2122 | catch ( IOException ignoreMe ) { } 2123 | } 2124 | 2125 | public boolean isDone() { 2126 | // if we know we're done, just say so 2127 | if ( isDone ) 2128 | return true; 2129 | 2130 | // else find out the hard way 2131 | int strPos = B_END.length-1; 2132 | 2133 | int bi = incoming.size() - 1; 2134 | while ( bi >= 0 && strPos >= 0 ) { 2135 | ByteBuffer buf = incoming.get( bi ); 2136 | int pos = buf.position()-1; 2137 | while ( pos >= 0 && strPos >= 0 ) { 2138 | if ( buf.get( pos-- ) != B_END[strPos--] ) 2139 | return false; 2140 | } 2141 | 2142 | bi--; 2143 | } 2144 | 2145 | isDone = strPos < 0; 2146 | return isDone; 2147 | } 2148 | 2149 | public ByteBuffer getBuffer() { 2150 | int last = incoming.size()-1; 2151 | if ( last >= 0 && incoming.get( last ).hasRemaining() ) { 2152 | return incoming.get( last ); 2153 | } 2154 | else { 2155 | ByteBuffer newBuf = ByteBuffer.allocate( 8192 ); 2156 | incoming.add( newBuf ); 2157 | return newBuf; 2158 | } 2159 | } 2160 | 2161 | public String toString() { 2162 | return "Connection to " + sock.getHost() + " with " + incoming.size() + " bufs; done is " + isDone; 2163 | } 2164 | } 2165 | 2166 | public void doMulti( boolean asString, Map sockKeys, String[] keys, Map ret ) { 2167 | 2168 | long timeRemaining = 0; 2169 | try { 2170 | selector = Selector.open(); 2171 | 2172 | // get the sockets, flip them to non-blocking, and set up data 2173 | // structures 2174 | conns = new Connection[sockKeys.keySet().size()]; 2175 | numConns = 0; 2176 | for ( Iterator i = sockKeys.keySet().iterator(); i.hasNext(); ) { 2177 | // get SockIO obj from hostname 2178 | String host = i.next(); 2179 | 2180 | SockIOPool.SockIO sock = pool.getConnection( host ); 2181 | 2182 | if ( sock == null ) { 2183 | if ( errorHandler != null ) 2184 | errorHandler.handleErrorOnGet( this.mc, new IOException( "no socket to server available" ), keys ); 2185 | return; 2186 | } 2187 | 2188 | conns[numConns++] = new Connection( sock, sockKeys.get( host ) ); 2189 | } 2190 | 2191 | // the main select loop; ends when 2192 | // 1) we've received data from all the servers, or 2193 | // 2) we time out 2194 | long startTime = System.currentTimeMillis(); 2195 | 2196 | long timeout = pool.getMaxBusy(); 2197 | timeRemaining = timeout; 2198 | 2199 | while ( numConns > 0 && timeRemaining > 0 ) { 2200 | int n = selector.select( Math.min( timeout, 5000 ) ); 2201 | if ( n > 0 ) { 2202 | // we've got some activity; handle it 2203 | Iterator it = selector.selectedKeys().iterator(); 2204 | while ( it.hasNext() ) { 2205 | SelectionKey key = it.next(); 2206 | it.remove(); 2207 | handleKey( key ); 2208 | } 2209 | } 2210 | else { 2211 | // timeout likely... better check 2212 | // TODO: This seems like a problem area that we need to figure out how to handle. 2213 | log.error( "selector timed out waiting for activity" ); 2214 | } 2215 | 2216 | timeRemaining = timeout - (System.currentTimeMillis() - startTime); 2217 | } 2218 | } 2219 | catch ( IOException e ) { 2220 | // errors can happen just about anywhere above, from 2221 | // connection setup to any of the mechanics 2222 | handleError( e, keys ); 2223 | return; 2224 | } 2225 | finally { 2226 | if ( log.isDebugEnabled() ) 2227 | log.debug( "Disconnecting; numConns=" + numConns + " timeRemaining=" + timeRemaining ); 2228 | 2229 | // run through our conns and either return them to the pool 2230 | // or forcibly close them 2231 | try { 2232 | if ( selector != null ) 2233 | selector.close(); 2234 | } 2235 | catch ( IOException ignoreMe ) { } 2236 | 2237 | for ( Connection c : conns ) { 2238 | if ( c != null ) 2239 | c.close(); 2240 | } 2241 | } 2242 | 2243 | // Done! Build the list of results and return them. If we get 2244 | // here by a timeout, then some of the connections are probably 2245 | // not done. But we'll return what we've got... 2246 | for ( Connection c : conns ) { 2247 | try { 2248 | if ( c.incoming.size() > 0 && c.isDone() ) 2249 | loadMulti( new ByteBufArrayInputStream( c.incoming ), ret, asString ); 2250 | } 2251 | catch ( Exception e ) { 2252 | // shouldn't happen; we have all the data already 2253 | log.warn( "Caught the aforementioned exception on "+c ); 2254 | } 2255 | } 2256 | } 2257 | 2258 | private void handleError( Throwable e, String[] keys ) { 2259 | // if we have an errorHandler, use its hook 2260 | if ( errorHandler != null ) 2261 | errorHandler.handleErrorOnGet( MemcachedClient.this, e, keys ); 2262 | 2263 | // exception thrown 2264 | log.error( "++++ exception thrown while getting from cache on getMulti" ); 2265 | log.error( e.getMessage() ); 2266 | } 2267 | 2268 | private void handleKey( SelectionKey key ) throws IOException { 2269 | if ( log.isDebugEnabled() ) 2270 | log.debug( "handling selector op " + key.readyOps() + " for key " + key ); 2271 | 2272 | if ( key.isReadable() ) 2273 | readResponse( key ); 2274 | else if ( key.isWritable() ) 2275 | writeRequest( key ); 2276 | } 2277 | 2278 | public void writeRequest( SelectionKey key ) throws IOException { 2279 | ByteBuffer buf = ((Connection) key.attachment()).outgoing; 2280 | SocketChannel sc = (SocketChannel)key.channel(); 2281 | 2282 | if ( buf.hasRemaining() ) { 2283 | if ( log.isDebugEnabled() ) 2284 | log.debug( "writing " + buf.remaining() + "B to " + ((SocketChannel) key.channel()).socket().getInetAddress() ); 2285 | 2286 | sc.write( buf ); 2287 | } 2288 | 2289 | if ( !buf.hasRemaining() ) { 2290 | if ( log.isDebugEnabled() ) 2291 | log.debug( "switching to read mode for server " + ((SocketChannel)key.channel()).socket().getInetAddress() ); 2292 | 2293 | key.interestOps( SelectionKey.OP_READ ); 2294 | } 2295 | } 2296 | 2297 | public void readResponse( SelectionKey key ) throws IOException { 2298 | Connection conn = (Connection)key.attachment(); 2299 | ByteBuffer buf = conn.getBuffer(); 2300 | int count = conn.channel.read( buf ); 2301 | if ( count > 0 ) { 2302 | if ( log.isDebugEnabled() ) 2303 | log.debug( "read " + count + " from " + conn.channel.socket().getInetAddress() ); 2304 | 2305 | if ( conn.isDone() ) { 2306 | if ( log.isDebugEnabled() ) 2307 | log.debug( "connection done to " + conn.channel.socket().getInetAddress() ); 2308 | 2309 | key.cancel(); 2310 | numConns--; 2311 | return; 2312 | } 2313 | } 2314 | } 2315 | } 2316 | } 2317 | --------------------------------------------------------------------------------